feat: add HTTP catalog for Python packages and repository metrics

.gitignore

@@ -8,6 +8,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
+.venv/
 build/
 develop-eggs/
 dist/
@@ -61,3 +62,6 @@ target/
 
 # PyCharm
 .idea
+
+# VS Code / Cursor
+.vscode/

CHANGES/1358.feature

@@ -0,0 +1 @@
+Added a repository endpoint that lists Python packages with limit/offset pagination.

docs/index.md

@@ -12,6 +12,7 @@ The REST API documentation for `pulp_python` is available [here](site:pulp_pytho
 
 - [Create local mirrors of PyPI](site:pulp_python/docs/user/guides/sync/) that you have full control over
 - [Upload your own Python packages](site:pulp_python/docs/user/guides/upload/)
+- [Browse the package catalog](site:pulp_python/docs/user/guides/catalog/) over the REST API
 - [Perform pip install](site:pulp_python/docs/user/guides/host/) from your Pulp Python repositories
 - Download packages on-demand to reduce disk usage
 - Every operation creates a restorable snapshot with Versioned Repositories
@@ -34,5 +35,4 @@ Users may also find pulpcore’s conceptual docs useful.
 This documentation falls into two main categories:
 
 1. `How-to Guides` shows the **major features** of the Python plugin, with links to reference docs.
-2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each 
-minor feature, including all fields and options.
+2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each minor feature, including all fields and options.

docs/user/guides/_SUMMARY.md

@@ -1,6 +1,7 @@
 * [Set up your own PyPI](pypi.md)
 * [Sync from Remote Repositories](sync.md)
 * [Upload and Manage Content](upload.md)
+* [Browse the package catalog](catalog.md)
 * [Host Python Content](host.md)
 * [Vulnerability Report](vulnerability_report.md)
 * [Attestation Hosting](attestation.md)

docs/user/guides/catalog.md

@@ -0,0 +1,44 @@
+# Browse the package catalog
+
+The package list returns **one row per package name**. It defaults to the
+repository's latest complete version. Pass `repository_version` (HREF or PRN)
+to read an older snapshot. `{pulp_id}` is the repository UUID.
+
+## List packages
+
+```bash
+http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/?limit=10"
+```
+
+`count` is the number of distinct packages, not files. Each row is one package
+name. `summary`, `description`, `author`, `author_email`, and `license` come
+from the newest version. `versions` has one entry per stored version string,
+newest first, with that version's `license_expression`. A wheel and an sdist
+of the same version are one entry. A rebuild such as `1.0.0+test.1` stays its
+own entry because the stored version differs.
+
+```json
+{
+  "count": 1,
+  "next": null,
+  "previous": null,
+  "results": [
+    {
+      "name": "shelf-reader",
+      "name_normalized": "shelf-reader",
+      "summary": "A small example package",
+      "description": "",
+      "author": "",
+      "author_email": "",
+      "license": "",
+      "versions": [
+        {"version": "0.1", "license_expression": ""}
+      ]
+    }
+  ]
+}
+```
+
+Results are ordered by `name_normalized`. Within a package, versions are newest
+first. Page with `limit` and `offset` to read the whole catalog. Pass
+`repository_version` when the export must stay on one snapshot.

pulp_python/app/catalog.py

@@ -0,0 +1,186 @@
+"""Helpers for the repository package list."""
+
+from collections import defaultdict
+from urllib.parse import urlparse
+
+from django.core.exceptions import FieldError
+from django.core.exceptions import ValidationError as DjangoValidationError
+from django.db.models import Case, IntegerField, Q, Value, When
+from django.urls import Resolver404, resolve
+from packaging.version import InvalidVersion, Version
+
+from pulpcore.plugin.models import Repository, RepositoryContent, RepositoryVersion
+from pulpcore.plugin.util import resolve_prn
+
+from pulp_python.app.models import PythonPackageContent
+
+
+def _version_sort_key(version):
+    """PEP 440 sort key. Use with ``reverse=True`` for newest first.
+
+    Invalid versions sort after all valid ones when ``reverse=True``.
+    """
+    if not version:
+        return (-1, "")
+    try:
+        return (0, Version(version))
+    except InvalidVersion:
+        return (-1, str(version))
+
+
+def _in_version_bounds_q(repository_version, prefix=""):
+    """Version-added/removed bounds for rows present in ``repository_version``.
+
+    ``prefix`` is empty for ``RepositoryContent`` and ``in_repo__`` for the
+    filtered membership join. Both catalog paths use this helper.
+    """
+    return Q(**{f"{prefix}version_added__number__lte": repository_version.number}) & (
+        Q(**{f"{prefix}version_removed__isnull": True})
+        | Q(**{f"{prefix}version_removed__number__gt": repository_version.number})
+    )
+
+
+def memberships_in_version(repository_version):
+    """RepositoryContent rows contained in ``repository_version``.
+
+    A subquery against this queryset keeps the content-id list in the database.
+    ``RepositoryVersion.content`` inlines ``content_ids`` as one bound UUID per
+    content unit whenever that array is shorter than 65535.
+    """
+    return RepositoryContent.objects.filter(
+        repository_id=repository_version.repository_id,
+    ).filter(_in_version_bounds_q(repository_version))
+
+
+def python_packages_in_version(repository_version):
+    """Python package content contained in ``repository_version``."""
+    if repository_version is None:
+        return PythonPackageContent.objects.none()
+    content_ids = (
+        memberships_in_version(repository_version)
+        .filter(content__pulp_type=PythonPackageContent.get_pulp_type())
+        .order_by()
+        .values("content_id")
+    )
+    return PythonPackageContent.objects.filter(pk__in=content_ids)
+
+
+def resolve_repository_version(href):
+    """Load a repository version without selecting ``content_ids``.
+
+    Accepts a repository-version or repository HREF/PRN. Domain-prefixed hrefs
+    include ``pulp_domain`` in the resolved kwargs; that field lives on the
+    repository, not ``RepositoryVersion``.
+    """
+    if not href:
+        raise ValueError("No value supplied for repository version.")
+
+    if href.startswith("prn:"):
+        model, pk = resolve_prn(href)
+        found_kwargs = {"pk": pk}
+    else:
+        try:
+            match = resolve(urlparse(href).path)
+        except Resolver404:
+            raise ValueError(f"URI not valid: {href}") from None
+        model = match.func.cls.queryset.model
+        found_kwargs = match.kwargs
+
+    if "pk" in found_kwargs:
+        lookup = {"pk": found_kwargs["pk"]}
+    else:
+        lookup = {}
+        for key, value in found_kwargs.items():
+            if key.endswith("_pk"):
+                lookup[f"{key[:-3]}__pk"] = value
+            elif key == "pulp_domain":
+                if hasattr(model, "pulp_domain"):
+                    lookup["pulp_domain__name"] = value
+            elif key in ("api_root", "version"):
+                continue
+            else:
+                lookup[key] = value
+
+    qs = model.objects.all()
+    if issubclass(model, RepositoryVersion):
+        qs = qs.defer("content_ids")
+    try:
+        obj = qs.get(**lookup)
+    except model.DoesNotExist:
+        raise ValueError(f"URI {href} not found.") from None
+    except (model.MultipleObjectsReturned, DjangoValidationError, FieldError):
+        raise ValueError(f"URI {href} is not a valid repository version.") from None
+
+    if isinstance(obj, RepositoryVersion):
+        return obj
+    if isinstance(obj, Repository):
+        try:
+            return obj.versions.complete().defer("content_ids").latest()
+        except RepositoryVersion.DoesNotExist:
+            return None
+    raise ValueError("Must be a repository version.")
+
+
+def distinct_package_names_qs(content_qs):
+    """One row per distinct ``name_normalized``, ordered for stable pagination."""
+    return content_qs.order_by().values("name_normalized").distinct().order_by("name_normalized")
+
+
+def assemble_package_index(content_qs, name_rows, repository_version):
+    """One catalog row per package name, with one content unit per stored version.
+
+    Wheel and sdist of the same ``(name_normalized, version)`` collapse to one
+    row. An sdist is preferred; otherwise the newest ``pulp_created`` is kept.
+    Version strings are the values stored on the content unit, newest first.
+    Summary, description, author, author email, and license are taken from the
+    newest version because they do not vary by version.
+    """
+    if not name_rows or repository_version is None:
+        return []
+
+    names = [row["name_normalized"] for row in name_rows]
+    by_name = defaultdict(list)
+    # DISTINCT ON keeps the first row for each name and stored version.
+    units = (
+        content_qs.filter(name_normalized__in=names)
+        .order_by(
+            "name_normalized",
+            "version",
+            Case(
+                When(packagetype="sdist", then=Value(0)),
+                default=Value(1),
+                output_field=IntegerField(),
+            ),
+            "-pulp_created",
+            "pk",
+        )
+        .distinct("name_normalized", "version")
+    )
+    for unit in units:
+        by_name[unit.name_normalized].append(unit)
+
+    result = []
+    for row in name_rows:
+        normalized = row["name_normalized"]
+        versions = sorted(
+            by_name.get(normalized, []),
+            key=lambda unit: (_version_sort_key(unit.version), unit.filename),
+            reverse=True,
+        )
+        source = versions[0] if versions else None
+        result.append(
+            {
+                "name": source.name if source else normalized,
+                "name_normalized": normalized,
+                "summary": source.summary if source else "",
+                "description": source.description if source else "",
+                "author": source.author if source else "",
+                "author_email": source.author_email if source else "",
+                "license": source.license if source else "",
+                "versions": [
+                    {"version": unit.version, "license_expression": unit.license_expression}
+                    for unit in versions
+                ],
+            }
+        )
+    return result

pulp_python/app/pypi/views.py

@@ -340,7 +340,7 @@ def list(self, request, path):
         if self.should_redirect(repo_version=repo_version):
             return redirect(urljoin(self.base_content_url, f"{path}/simple/"))
         names = (
-            content.order_by("name_normalized")
+            content.order_by("name_normalized", "name")
             .values_list("name", flat=True)
             .distinct("name_normalized")
             .iterator()

pulp_python/app/serializers.py

@@ -641,6 +641,57 @@ class Meta:
         model = python_models.PythonPackageContent
 
 
+class PythonRepositoryPackageVersionSerializer(serializers.Serializer):
+    """One stored version of a package."""
+
+    version = serializers.CharField(help_text=_("The package's version number."))
+    license_expression = serializers.CharField(
+        allow_blank=True,
+        help_text=_("Text string that is a valid SPDX license expression."),
+    )
+
+
+class PythonRepositoryPackageSerializer(serializers.Serializer):
+    """One distinct package name and its stored versions."""
+
+    name = serializers.CharField(
+        help_text=_("A project name stored on one of this package's files."),
+    )
+    name_normalized = serializers.CharField(
+        help_text=_("PEP 503 normalized package name. Index rows are unique on this field."),
+    )
+    summary = serializers.CharField(
+        allow_blank=True,
+        help_text=_("A one-line summary of what the package does."),
+    )
+    description = serializers.CharField(
+        allow_blank=True,
+        help_text=_("A longer description of the package that can run to several paragraphs."),
+    )
+    author = serializers.CharField(
+        allow_blank=True,
+        help_text=_(
+            "Text containing the author's name. Contact information can also be added,"
+            " separated with newlines."
+        ),
+    )
+    author_email = serializers.CharField(
+        allow_blank=True,
+        help_text=_("The author's e-mail address."),
+    )
+    license = serializers.CharField(
+        allow_blank=True,
+        help_text=_("Text indicating the license covering the distribution."),
+    )
+    versions = PythonRepositoryPackageVersionSerializer(
+        many=True,
+        help_text=_(
+            "One entry per stored version, newest first. A wheel and an sdist of the "
+            "same version are one entry. Version strings are unchanged."
+        ),
+    )
+
+
 class PackageProvenanceSerializer(core_serializers.NoArtifactContentUploadSerializer):
     """
     A Serializer for PackageProvenance.

pulp_python/app/tasks/publish.py

@@ -56,11 +56,14 @@ def write_simple_api(publication):
     domain = get_domain()
     simple_dir = "simple/"
     os.mkdir(simple_dir)
+    # Secondary ORDER BY makes DISTINCT ON pick a stable display name when
+    # metadata names differ but canonicalize to the same name_normalized
+    # (e.g. msg-parser vs msg_parser).
     project_names = (
         python_models.PythonPackageContent.objects.filter(
             pk__in=publication.repository_version.content, _pulp_domain=domain
         )
-        .order_by("name_normalized")
+        .order_by("name_normalized", "name")
         .values_list("name", flat=True)
         .distinct("name_normalized")
     )
@@ -81,7 +84,7 @@ def write_simple_api(publication):
     packages = python_models.PythonPackageContent.objects.filter(
         pk__in=publication.repository_version.content, _pulp_domain=domain
     )
-    releases = packages.order_by("name_normalized").values("name", "filename", "sha256")
+    releases = packages.order_by("name_normalized", "filename").values("name", "filename", "sha256")
 
     ind = 0
     current_name = canonicalize_name(project_names[ind])

pulp_python/app/utils.py

@@ -26,7 +26,6 @@
 
 log = logging.getLogger(__name__)
 
-
 PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL"
 """TODO This serial constant is temporary until Python repositories implements serials"""
 PYPI_SERIAL_CONSTANT = 1000000000