From bc2c3d881b9801dc5d6c3f21ef95dba0cc123660 Mon Sep 17 00:00:00 2001 From: TenSt Date: Tue, 6 Oct 2026 14:23:25 +0200 Subject: [PATCH] Add a paginated repository endpoint that lists Python packages. Clients can walk every package in a repository version. Each row is one package name, and each stored version is one entry. Closes #1358. Assisted-By: Cursor --- .gitignore | 4 + CHANGES/1358.feature | 1 + docs/index.md | 4 +- docs/user/guides/_SUMMARY.md | 1 + docs/user/guides/catalog.md | 44 ++++ pulp_python/app/catalog.py | 186 +++++++++++++++ pulp_python/app/pypi/views.py | 2 +- pulp_python/app/serializers.py | 51 +++++ pulp_python/app/tasks/publish.py | 7 +- pulp_python/app/utils.py | 1 - pulp_python/app/viewsets.py | 97 +++++++- .../tests/functional/api/test_catalog.py | 211 ++++++++++++++++++ .../functional/api/test_crud_publications.py | 64 ++++-- .../tests/functional/api/test_domains.py | 34 ++- pulp_python/tests/unit/test_catalog.py | 125 +++++++++++ 15 files changed, 807 insertions(+), 25 deletions(-) create mode 100644 CHANGES/1358.feature create mode 100644 docs/user/guides/catalog.md create mode 100644 pulp_python/app/catalog.py create mode 100644 pulp_python/tests/functional/api/test_catalog.py create mode 100644 pulp_python/tests/unit/test_catalog.py diff --git a/.gitignore b/.gitignore index 29c94b654..444a89de7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python env/ +.venv/ build/ develop-eggs/ dist/ @@ -61,3 +62,6 @@ target/ # PyCharm .idea + +# VS Code / Cursor +.vscode/ diff --git a/CHANGES/1358.feature b/CHANGES/1358.feature new file mode 100644 index 000000000..5cb68bb99 --- /dev/null +++ b/CHANGES/1358.feature @@ -0,0 +1 @@ +Added a repository endpoint that lists Python packages with limit/offset pagination. diff --git a/docs/index.md b/docs/index.md index 80c26e434..ff9bb16d1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,6 +12,7 @@ The REST API documentation for `pulp_python` is available [here](site:pulp_pytho - [Create local mirrors of PyPI](site:pulp_python/docs/user/guides/sync/) that you have full control over - [Upload your own Python packages](site:pulp_python/docs/user/guides/upload/) +- [Browse the package catalog](site:pulp_python/docs/user/guides/catalog/) over the REST API - [Perform pip install](site:pulp_python/docs/user/guides/host/) from your Pulp Python repositories - Download packages on-demand to reduce disk usage - Every operation creates a restorable snapshot with Versioned Repositories @@ -34,5 +35,4 @@ Users may also find pulpcore’s conceptual docs useful. This documentation falls into two main categories: 1. `How-to Guides` shows the **major features** of the Python plugin, with links to reference docs. -2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each -minor feature, including all fields and options. +2. The [REST API Docs](site:pulp_python/restapi/) are automatically generated and provide more detailed information for each minor feature, including all fields and options. diff --git a/docs/user/guides/_SUMMARY.md b/docs/user/guides/_SUMMARY.md index d9c01bdb2..36699ede5 100644 --- a/docs/user/guides/_SUMMARY.md +++ b/docs/user/guides/_SUMMARY.md @@ -1,6 +1,7 @@ * [Set up your own PyPI](pypi.md) * [Sync from Remote Repositories](sync.md) * [Upload and Manage Content](upload.md) +* [Browse the package catalog](catalog.md) * [Host Python Content](host.md) * [Vulnerability Report](vulnerability_report.md) * [Attestation Hosting](attestation.md) diff --git a/docs/user/guides/catalog.md b/docs/user/guides/catalog.md new file mode 100644 index 000000000..1632c7a7d --- /dev/null +++ b/docs/user/guides/catalog.md @@ -0,0 +1,44 @@ +# Browse the package catalog + +The package list returns **one row per package name**. It defaults to the +repository's latest complete version. Pass `repository_version` (HREF or PRN) +to read an older snapshot. `{pulp_id}` is the repository UUID. + +## List packages + +```bash +http GET "${BASE_ADDR}/pulp/api/v3/repositories/python/python/${REPO_PK}/packages/?limit=10" +``` + +`count` is the number of distinct packages, not files. Each row is one package +name. `summary`, `description`, `author`, `author_email`, and `license` come +from the newest version. `versions` has one entry per stored version string, +newest first, with that version's `license_expression`. A wheel and an sdist +of the same version are one entry. A rebuild such as `1.0.0+test.1` stays its +own entry because the stored version differs. + +```json +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "name": "shelf-reader", + "name_normalized": "shelf-reader", + "summary": "A small example package", + "description": "", + "author": "", + "author_email": "", + "license": "", + "versions": [ + {"version": "0.1", "license_expression": ""} + ] + } + ] +} +``` + +Results are ordered by `name_normalized`. Within a package, versions are newest +first. Page with `limit` and `offset` to read the whole catalog. Pass +`repository_version` when the export must stay on one snapshot. diff --git a/pulp_python/app/catalog.py b/pulp_python/app/catalog.py new file mode 100644 index 000000000..5ec58b575 --- /dev/null +++ b/pulp_python/app/catalog.py @@ -0,0 +1,186 @@ +"""Helpers for the repository package list.""" + +from collections import defaultdict +from urllib.parse import urlparse + +from django.core.exceptions import FieldError +from django.core.exceptions import ValidationError as DjangoValidationError +from django.db.models import Case, IntegerField, Q, Value, When +from django.urls import Resolver404, resolve +from packaging.version import InvalidVersion, Version + +from pulpcore.plugin.models import Repository, RepositoryContent, RepositoryVersion +from pulpcore.plugin.util import resolve_prn + +from pulp_python.app.models import PythonPackageContent + + +def _version_sort_key(version): + """PEP 440 sort key. Use with ``reverse=True`` for newest first. + + Invalid versions sort after all valid ones when ``reverse=True``. + """ + if not version: + return (-1, "") + try: + return (0, Version(version)) + except InvalidVersion: + return (-1, str(version)) + + +def _in_version_bounds_q(repository_version, prefix=""): + """Version-added/removed bounds for rows present in ``repository_version``. + + ``prefix`` is empty for ``RepositoryContent`` and ``in_repo__`` for the + filtered membership join. Both catalog paths use this helper. + """ + return Q(**{f"{prefix}version_added__number__lte": repository_version.number}) & ( + Q(**{f"{prefix}version_removed__isnull": True}) + | Q(**{f"{prefix}version_removed__number__gt": repository_version.number}) + ) + + +def memberships_in_version(repository_version): + """RepositoryContent rows contained in ``repository_version``. + + A subquery against this queryset keeps the content-id list in the database. + ``RepositoryVersion.content`` inlines ``content_ids`` as one bound UUID per + content unit whenever that array is shorter than 65535. + """ + return RepositoryContent.objects.filter( + repository_id=repository_version.repository_id, + ).filter(_in_version_bounds_q(repository_version)) + + +def python_packages_in_version(repository_version): + """Python package content contained in ``repository_version``.""" + if repository_version is None: + return PythonPackageContent.objects.none() + content_ids = ( + memberships_in_version(repository_version) + .filter(content__pulp_type=PythonPackageContent.get_pulp_type()) + .order_by() + .values("content_id") + ) + return PythonPackageContent.objects.filter(pk__in=content_ids) + + +def resolve_repository_version(href): + """Load a repository version without selecting ``content_ids``. + + Accepts a repository-version or repository HREF/PRN. Domain-prefixed hrefs + include ``pulp_domain`` in the resolved kwargs; that field lives on the + repository, not ``RepositoryVersion``. + """ + if not href: + raise ValueError("No value supplied for repository version.") + + if href.startswith("prn:"): + model, pk = resolve_prn(href) + found_kwargs = {"pk": pk} + else: + try: + match = resolve(urlparse(href).path) + except Resolver404: + raise ValueError(f"URI not valid: {href}") from None + model = match.func.cls.queryset.model + found_kwargs = match.kwargs + + if "pk" in found_kwargs: + lookup = {"pk": found_kwargs["pk"]} + else: + lookup = {} + for key, value in found_kwargs.items(): + if key.endswith("_pk"): + lookup[f"{key[:-3]}__pk"] = value + elif key == "pulp_domain": + if hasattr(model, "pulp_domain"): + lookup["pulp_domain__name"] = value + elif key in ("api_root", "version"): + continue + else: + lookup[key] = value + + qs = model.objects.all() + if issubclass(model, RepositoryVersion): + qs = qs.defer("content_ids") + try: + obj = qs.get(**lookup) + except model.DoesNotExist: + raise ValueError(f"URI {href} not found.") from None + except (model.MultipleObjectsReturned, DjangoValidationError, FieldError): + raise ValueError(f"URI {href} is not a valid repository version.") from None + + if isinstance(obj, RepositoryVersion): + return obj + if isinstance(obj, Repository): + try: + return obj.versions.complete().defer("content_ids").latest() + except RepositoryVersion.DoesNotExist: + return None + raise ValueError("Must be a repository version.") + + +def distinct_package_names_qs(content_qs): + """One row per distinct ``name_normalized``, ordered for stable pagination.""" + return content_qs.order_by().values("name_normalized").distinct().order_by("name_normalized") + + +def assemble_package_index(content_qs, name_rows, repository_version): + """One catalog row per package name, with one content unit per stored version. + + Wheel and sdist of the same ``(name_normalized, version)`` collapse to one + row. An sdist is preferred; otherwise the newest ``pulp_created`` is kept. + Version strings are the values stored on the content unit, newest first. + Summary, description, author, author email, and license are taken from the + newest version because they do not vary by version. + """ + if not name_rows or repository_version is None: + return [] + + names = [row["name_normalized"] for row in name_rows] + by_name = defaultdict(list) + # DISTINCT ON keeps the first row for each name and stored version. + units = ( + content_qs.filter(name_normalized__in=names) + .order_by( + "name_normalized", + "version", + Case( + When(packagetype="sdist", then=Value(0)), + default=Value(1), + output_field=IntegerField(), + ), + "-pulp_created", + "pk", + ) + .distinct("name_normalized", "version") + ) + for unit in units: + by_name[unit.name_normalized].append(unit) + + result = [] + for row in name_rows: + normalized = row["name_normalized"] + versions = sorted( + by_name.get(normalized, []), + key=lambda unit: (_version_sort_key(unit.version), unit.filename), + reverse=True, + ) + source = versions[0] if versions else None + result.append( + { + "name": source.name if source else normalized, + "name_normalized": normalized, + "summary": source.summary if source else "", + "description": source.description if source else "", + "author": source.author if source else "", + "author_email": source.author_email if source else "", + "license": source.license if source else "", + "versions": [ + {"version": unit.version, "license_expression": unit.license_expression} + for unit in versions + ], + } + ) + return result diff --git a/pulp_python/app/pypi/views.py b/pulp_python/app/pypi/views.py index daa51b0d1..c8fa9976d 100644 --- a/pulp_python/app/pypi/views.py +++ b/pulp_python/app/pypi/views.py @@ -340,7 +340,7 @@ def list(self, request, path): if self.should_redirect(repo_version=repo_version): return redirect(urljoin(self.base_content_url, f"{path}/simple/")) names = ( - content.order_by("name_normalized") + content.order_by("name_normalized", "name") .values_list("name", flat=True) .distinct("name_normalized") .iterator() diff --git a/pulp_python/app/serializers.py b/pulp_python/app/serializers.py index 038bb3584..9f6696e57 100644 --- a/pulp_python/app/serializers.py +++ b/pulp_python/app/serializers.py @@ -641,6 +641,57 @@ class Meta: model = python_models.PythonPackageContent +class PythonRepositoryPackageVersionSerializer(serializers.Serializer): + """One stored version of a package.""" + + version = serializers.CharField(help_text=_("The package's version number.")) + license_expression = serializers.CharField( + allow_blank=True, + help_text=_("Text string that is a valid SPDX license expression."), + ) + + +class PythonRepositoryPackageSerializer(serializers.Serializer): + """One distinct package name and its stored versions.""" + + name = serializers.CharField( + help_text=_("A project name stored on one of this package's files."), + ) + name_normalized = serializers.CharField( + help_text=_("PEP 503 normalized package name. Index rows are unique on this field."), + ) + summary = serializers.CharField( + allow_blank=True, + help_text=_("A one-line summary of what the package does."), + ) + description = serializers.CharField( + allow_blank=True, + help_text=_("A longer description of the package that can run to several paragraphs."), + ) + author = serializers.CharField( + allow_blank=True, + help_text=_( + "Text containing the author's name. Contact information can also be added," + " separated with newlines." + ), + ) + author_email = serializers.CharField( + allow_blank=True, + help_text=_("The author's e-mail address."), + ) + license = serializers.CharField( + allow_blank=True, + help_text=_("Text indicating the license covering the distribution."), + ) + versions = PythonRepositoryPackageVersionSerializer( + many=True, + help_text=_( + "One entry per stored version, newest first. A wheel and an sdist of the " + "same version are one entry. Version strings are unchanged." + ), + ) + + class PackageProvenanceSerializer(core_serializers.NoArtifactContentUploadSerializer): """ A Serializer for PackageProvenance. diff --git a/pulp_python/app/tasks/publish.py b/pulp_python/app/tasks/publish.py index 7c872159d..f3db2349c 100644 --- a/pulp_python/app/tasks/publish.py +++ b/pulp_python/app/tasks/publish.py @@ -56,11 +56,14 @@ def write_simple_api(publication): domain = get_domain() simple_dir = "simple/" os.mkdir(simple_dir) + # Secondary ORDER BY makes DISTINCT ON pick a stable display name when + # metadata names differ but canonicalize to the same name_normalized + # (e.g. msg-parser vs msg_parser). project_names = ( python_models.PythonPackageContent.objects.filter( pk__in=publication.repository_version.content, _pulp_domain=domain ) - .order_by("name_normalized") + .order_by("name_normalized", "name") .values_list("name", flat=True) .distinct("name_normalized") ) @@ -81,7 +84,7 @@ def write_simple_api(publication): packages = python_models.PythonPackageContent.objects.filter( pk__in=publication.repository_version.content, _pulp_domain=domain ) - releases = packages.order_by("name_normalized").values("name", "filename", "sha256") + releases = packages.order_by("name_normalized", "filename").values("name", "filename", "sha256") ind = 0 current_name = canonicalize_name(project_names[ind]) diff --git a/pulp_python/app/utils.py b/pulp_python/app/utils.py index 9e08c77c6..091c6b69e 100644 --- a/pulp_python/app/utils.py +++ b/pulp_python/app/utils.py @@ -26,7 +26,6 @@ log = logging.getLogger(__name__) - PYPI_LAST_SERIAL = "X-PYPI-LAST-SERIAL" """TODO This serial constant is temporary until Python repositories implements serials""" PYPI_SERIAL_CONSTANT = 1000000000 diff --git a/pulp_python/app/viewsets.py b/pulp_python/app/viewsets.py index 6c73575f4..5d81ef70e 100644 --- a/pulp_python/app/viewsets.py +++ b/pulp_python/app/viewsets.py @@ -4,7 +4,8 @@ from django.db import transaction from django_filters import CharFilter from django_filters.rest_framework import filters as drf_filters -from drf_spectacular.utils import extend_schema, extend_schema_view +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.utils import canonicalize_name from rest_framework import status @@ -32,6 +33,12 @@ from pulp_python.app import models as python_models from pulp_python.app import serializers as python_serializers from pulp_python.app import tasks +from pulp_python.app.catalog import ( + assemble_package_index, + distinct_package_names_qs, + python_packages_in_version, + resolve_repository_version, +) class PythonRepositoryViewSet( @@ -64,7 +71,7 @@ class PythonRepositoryViewSet( ], }, { - "action": ["retrieve"], + "action": ["retrieve", "packages"], "principal": "authenticated", "effect": "allow", "condition": "has_model_or_domain_or_obj_perms:python.view_pythonrepository", @@ -138,6 +145,46 @@ class PythonRepositoryViewSet( "python.pythonrepository_viewer": ["python.view_pythonrepository"], } + @property + def filter_backends(self): + """Repository filters apply to the repository list, not the package index. + + ``packages`` is a detail GET that returns a list. Spectacular treats that + as a list operation and would otherwise document ``RepositoryFilter``. + """ + if getattr(self, "action", None) == "packages": + return [] + return super().filter_backends + + def filter_queryset(self, queryset): + """Do not apply the repository FilterSet to package-index query params.""" + if getattr(self, "action", None) == "packages": + return queryset + return super().filter_queryset(queryset) + + def _repository_version_from_href(self, href): + """Load a repository version without selecting ``content_ids``.""" + try: + return resolve_repository_version(href) + except ValueError as exc: + raise ValidationError({"repository_version": str(exc)}) from exc + + def _requested_repository_version(self, repository): + """Resolve optional ``repository_version`` href/PRN, else latest complete version.""" + href = self.request.query_params.get("repository_version") + if not href: + try: + # content_ids is a UUID per content unit. The catalog never reads it. + return repository.versions.complete().defer("content_ids").latest() + except RepositoryVersion.DoesNotExist: + return None + repo_version = self._repository_version_from_href(href) + if repo_version is None: + return None + if repo_version.repository_id != repository.pk: + raise ValidationError({"repository_version": "Must be a version of this repository."}) + return repo_version + @extend_schema( description="Trigger an asynchronous task to create a new repository version.", summary="Modify Repository Content", @@ -247,6 +294,52 @@ def sync(self, request, pk, **kwargs): ) return core_viewsets.OperationPostponedResponse(result, request) + @extend_schema( + summary="List packages", + description=( + "Return one row per distinct package name in a repository version " + "(latest complete version if repository_version is omitted). " + "Each versions entry is the stored version and its license_expression " + "(wheel and sdist of the same version are a single entry). Results are " + "ordered by name_normalized. Pagination count is the number of " + "distinct packages, not files." + ), + parameters=[ + OpenApiParameter( + name="repository_version", + type=OpenApiTypes.URI, + location=OpenApiParameter.QUERY, + required=False, + description=( + "HREF or PRN of a version of this repository. " + "Defaults to the latest complete version." + ), + ), + ], + responses={200: python_serializers.PythonRepositoryPackageSerializer(many=True)}, + ) + @action( + detail=True, + methods=["get"], + serializer_class=python_serializers.PythonRepositoryPackageSerializer, + ) + def packages(self, request, pk, **kwargs): + """List distinct packages in a repository version.""" + repository = self.get_object() + repo_version = self._requested_repository_version(repository) + content_qs = python_packages_in_version(repo_version) + names_qs = distinct_package_names_qs(content_qs) + page = self.paginate_queryset(names_qs) + rows = assemble_package_index( + content_qs, + page if page is not None else list(names_qs), + repo_version, + ) + serializer = self.get_serializer(rows, many=True) + if page is not None: + return self.get_paginated_response(serializer.data) + return Response(serializer.data) + class PythonBlocklistEntryViewSet( core_viewsets.NamedModelViewSet, diff --git a/pulp_python/tests/functional/api/test_catalog.py b/pulp_python/tests/functional/api/test_catalog.py new file mode 100644 index 000000000..6f5d167f7 --- /dev/null +++ b/pulp_python/tests/functional/api/test_catalog.py @@ -0,0 +1,211 @@ +"""Catalog API tests. + +Generated client methods are unavailable until `oci-env generate-client` is rerun. +""" + +import io +import tarfile +import uuid +from urllib.parse import urljoin + +import pytest +import requests + +from pulp_python.tests.functional.constants import PYTHON_SM_PROJECT_SPECIFIER + + +def _api_get(bindings_cfg, path, **params): + url = urljoin(bindings_cfg.host + "/", path.lstrip("/")) + response = requests.get(url, params=params, auth=(bindings_cfg.username, bindings_cfg.password)) + assert response.status_code == 200, response.text + return response.json() + + +_PACKAGE_FIELDS = { + "name", + "name_normalized", + "summary", + "description", + "author", + "author_email", + "license", + "versions", +} + + +def _assert_package_row(pkg): + assert set(pkg) == _PACKAGE_FIELDS + assert pkg["name"] + assert pkg["name_normalized"] + versions = [item["version"] for item in pkg["versions"]] + assert versions + assert len(versions) == len(set(versions)) + for item in pkg["versions"]: + assert set(item) == {"version", "license_expression"} + assert item["version"] + + +def _write_sdist(directory, name, version): + """Write a minimal sdist whose PKG-INFO Name/Version pkginfo can read.""" + pkg_dir = f"{name}-{version}" + filename = f"{pkg_dir}.tar.gz" + path = directory / filename + pkg_info = f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n".encode() + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo(name=f"{pkg_dir}/PKG-INFO") + info.size = len(pkg_info) + tar.addfile(info, io.BytesIO(pkg_info)) + return filename, str(path) + + +def _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, version): + filename, path = _write_sdist(tmp_path, name, version) + python_content_factory(relative_path=filename, file=path, repository=repo) + return python_bindings.RepositoriesPythonApi.read(repo.pulp_href) + + +@pytest.fixture +def sm_repo(python_repo_with_sync, python_remote_factory): + remote = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER) + return python_repo_with_sync(remote) + + +@pytest.mark.parallel +def test_package_list_grouping_and_pagination(bindings_cfg, sm_repo): + """Package index is one row per name, and count is distinct packages not files.""" + data = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1) + assert data["count"] == 3 + assert len(data["results"]) == 1 + _assert_package_row(data["results"][0]) + + page2 = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=1, offset=1) + assert page2["count"] == 3 + assert page2["results"][0]["name_normalized"] != data["results"][0]["name_normalized"] + + all_rows = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", limit=100)["results"] + names = [pkg["name_normalized"] for pkg in all_rows] + assert names == sorted(names) + assert set(names) == {"aiohttp", "celery", "django"} + django = next(pkg for pkg in all_rows if pkg["name_normalized"] == "django") + assert {item["version"] for item in django["versions"]} >= { + "1.10.4", + "1.10.3", + "1.10.2", + "1.10.1", + } + assert all("+" not in item["version"] for item in django["versions"]) + + +@pytest.mark.parallel +def test_package_list_empty_repository(bindings_cfg, python_repo_factory): + repo = python_repo_factory() + data = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert data["count"] == 0 + assert data["results"] == [] + + +@pytest.mark.parallel +def test_packages_repository_version(bindings_cfg, sm_repo, python_repo_factory): + """repository_version selects a snapshot; omitted uses the latest complete version.""" + latest_href = sm_repo.latest_version_href + v0_href = f"{sm_repo.pulp_href}versions/0/" + + default_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/") + explicit_pkgs = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=latest_href + ) + assert default_pkgs["count"] == explicit_pkgs["count"] == 3 + + version = _api_get(bindings_cfg, latest_href) + prn_pkgs = _api_get( + bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=version["prn"] + ) + assert prn_pkgs["count"] == 3 + assert version["prn"].startswith("prn:core.repositoryversion:") + + v0_pkgs = _api_get(bindings_cfg, f"{sm_repo.pulp_href}packages/", repository_version=v0_href) + assert v0_pkgs["count"] == 0 + assert v0_pkgs["results"] == [] + + other = python_repo_factory() + url = urljoin(bindings_cfg.host + "/", f"{sm_repo.pulp_href}packages/".lstrip("/")) + response = requests.get( + url, + params={"repository_version": other.latest_version_href}, + auth=(bindings_cfg.username, bindings_cfg.password), + ) + assert response.status_code == 400, response.text + + +@pytest.mark.parallel +def test_package_list_one_entry_per_stored_version( + bindings_cfg, python_bindings, python_repo_with_sync +): + """A wheel and an sdist of the same version are one version string.""" + repo = python_repo_with_sync() + files = python_bindings.ContentPackagesApi.list(repository_version=repo.latest_version_href) + assert files.count == 2 + assert {item.packagetype for item in files.results} == {"sdist", "bdist_wheel"} + + pkgs = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 1 + row = pkgs["results"][0] + _assert_package_row(row) + assert row["name_normalized"] == "shelf-reader" + assert [item["version"] for item in row["versions"]] == ["0.1"] + + +@pytest.mark.parallel +def test_package_list_version_order( + bindings_cfg, python_bindings, python_content_factory, python_repo_factory, tmp_path +): + """Stored versions are newest-first by PEP 440, not lexicographically.""" + repo = python_repo_factory() + name = f"ordered-{uuid.uuid4().hex[:8]}" + for version in ("1.10", "1.9", "1.2"): + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, version) + data = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert data["count"] == 1 + pkg = data["results"][0] + _assert_package_row(pkg) + assert [item["version"] for item in pkg["versions"]] == ["1.10", "1.9", "1.2"] + + +@pytest.mark.parallel +def test_package_list_keeps_stored_versions( + bindings_cfg, python_bindings, python_content_factory, python_repo_factory, tmp_path +): + """A rebuild stays its own file, with the version string stored in Pulp.""" + repo = python_repo_factory() + suffix = uuid.uuid4().hex[:8] + name = f"pkg-{suffix}" + + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, "2.0.0") + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, "1.0.0+test.3") + row = _api_get(bindings_cfg, f"{repo.pulp_href}packages/")["results"][0] + _assert_package_row(row) + assert [item["version"] for item in row["versions"]] == ["2.0.0", "1.0.0+test.3"] + + +@pytest.mark.parallel +def test_package_list_keeps_rebuild_version_string( + bindings_cfg, python_bindings, python_content_factory, python_repo_factory, tmp_path +): + """A rebuild is a separate entry because its stored version string differs.""" + repo = python_repo_factory() + name = f"rebuild-{uuid.uuid4().hex[:8]}" + repo = _add_sdist(python_content_factory, python_bindings, tmp_path, repo, name, "5.3.17") + repo = _add_sdist( + python_content_factory, + python_bindings, + tmp_path, + repo, + name, + "5.3.17+test.1.n1", + ) + + pkgs = _api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 1 + pkg = pkgs["results"][0] + _assert_package_row(pkg) + assert {item["version"] for item in pkg["versions"]} == {"5.3.17", "5.3.17+test.1.n1"} diff --git a/pulp_python/tests/functional/api/test_crud_publications.py b/pulp_python/tests/functional/api/test_crud_publications.py index e7941ac18..fb65cbd15 100644 --- a/pulp_python/tests/functional/api/test_crud_publications.py +++ b/pulp_python/tests/functional/api/test_crud_publications.py @@ -1,8 +1,10 @@ +import io import random +import tarfile +import zipfile from urllib.parse import urljoin import pytest -from pypi_simple import PyPISimple from pulp_python.tests.functional.constants import ( PYTHON_EGG_FILENAME, @@ -113,27 +115,57 @@ def test_new_content_is_published(python_publication_workflow, python_distributi assert proper is True, msgs +def _write_sdist_with_name(directory, name, version): + """Write a minimal sdist whose PKG-INFO Name differs from PEP 503 canonical form.""" + pkg_dir = f"{name}-{version}" + filename = f"{pkg_dir}.tar.gz" + path = directory / filename + pkg_info = f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n".encode() + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo(name=f"{pkg_dir}/PKG-INFO") + info.size = len(pkg_info) + tar.addfile(info, io.BytesIO(pkg_info)) + return filename, str(path) + + +def _write_wheel_with_name(directory, metadata_name, version, filename): + """Write a minimal wheel whose METADATA Name can differ from the sdist Name.""" + dist_info = f"{filename.split('-')[0]}-{version}.dist-info" + metadata = f"Metadata-Version: 2.1\nName: {metadata_name}\nVersion: {version}\n" + wheel = ( + "Wheel-Version: 1.0\nGenerator: pulp-python-test\nRoot-Is-Purelib: true\n" + "Tag: py2.py3-none-any\n" + ) + path = directory / filename + with zipfile.ZipFile(path, "w") as zf: + zf.writestr(f"{dist_info}/METADATA", metadata) + zf.writestr(f"{dist_info}/WHEEL", wheel) + return filename, str(path) + + @pytest.mark.parallel def test_non_matching_canonicalized_name( - python_repo, python_content_factory, python_publication_factory, python_distribution_factory + python_repo, + python_content_factory, + python_publication_factory, + python_distribution_factory, + tmp_path, ): """Ensures a package with dists that have non-matching canonicalized names is published.""" - packages = [] - filenames = ["msg_parser-1.0.0-py2.py3-none-any.whl", "msg_parser-1.0.0.tar.gz"] - with PyPISimple() as client: - page = client.get_project_page("msg-parser") - for pkg in page.packages: - if pkg.filename in filenames: - c = python_content_factory(pkg.filename, url=pkg.url, repository=python_repo) - if c.filename.endswith(".tar.gz"): - # The metadata name in the SDist is not the same as the Wheel's name - assert c.name == "msg_parser" - else: - assert c.name == "msg-parser" - packages.append(c) + version = "1.0.0" + wheel_filename = "msg_parser-1.0.0-py2.py3-none-any.whl" + sdist_filename, sdist_path = _write_sdist_with_name(tmp_path, "msg_parser", version) + _, wheel_path = _write_wheel_with_name(tmp_path, "msg-parser", version, wheel_filename) + + wheel = python_content_factory(wheel_filename, file=wheel_path, repository=python_repo) + sdist = python_content_factory(sdist_filename, file=sdist_path, repository=python_repo) + # The metadata name in the SDist is not the same as the Wheel's name + assert wheel.name == "msg-parser" + assert sdist.name == "msg_parser" + pub = python_publication_factory(repository=python_repo) distro = python_distribution_factory(publication=pub) url = urljoin(distro.base_url, "simple/") - proper, msgs = ensure_simple(url, {"msg-parser": filenames}) + proper, msgs = ensure_simple(url, {"msg-parser": [wheel_filename, sdist_filename]}) assert proper is True, msgs diff --git a/pulp_python/tests/functional/api/test_domains.py b/pulp_python/tests/functional/api/test_domains.py index 391af15c5..2a2e54865 100644 --- a/pulp_python/tests/functional/api/test_domains.py +++ b/pulp_python/tests/functional/api/test_domains.py @@ -1,9 +1,10 @@ import json import subprocess import uuid -from urllib.parse import urlsplit +from urllib.parse import urljoin, urlsplit import pytest +import requests from pulpcore.app import settings # noqa: TID251 @@ -83,6 +84,37 @@ def test_domain_object_creation( } +def _catalog_api_get(bindings_cfg, path, **params): + url = urljoin(bindings_cfg.host + "/", path.lstrip("/")) + response = requests.get(url, params=params, auth=(bindings_cfg.username, bindings_cfg.password)) + assert response.status_code == 200, response.text + return response.json() + + +@pytest.mark.parallel +def test_domain_catalog_packages(bindings_cfg, domain_factory, python_repo_factory): + """Domain-prefixed repository and repository_version hrefs, including PRN.""" + domain = domain_factory() + repo = python_repo_factory(pulp_domain=domain.name) + assert f"{domain.name}/api/" in repo.pulp_href + assert f"{domain.name}/api/" in repo.latest_version_href + + pkgs = _catalog_api_get(bindings_cfg, f"{repo.pulp_href}packages/") + assert pkgs["count"] == 0 + assert pkgs["results"] == [] + + by_href = _catalog_api_get( + bindings_cfg, f"{repo.pulp_href}packages/", repository_version=repo.latest_version_href + ) + assert by_href["count"] == 0 + version = _catalog_api_get(bindings_cfg, repo.latest_version_href) + by_prn = _catalog_api_get( + bindings_cfg, f"{repo.pulp_href}packages/", repository_version=version["prn"] + ) + assert by_prn["count"] == 0 + assert version["prn"].startswith("prn:core.repositoryversion:") + + @pytest.mark.parallel def test_domain_content_upload( domain_factory, diff --git a/pulp_python/tests/unit/test_catalog.py b/pulp_python/tests/unit/test_catalog.py new file mode 100644 index 000000000..a254853d3 --- /dev/null +++ b/pulp_python/tests/unit/test_catalog.py @@ -0,0 +1,125 @@ +"""Unit tests for the repository package index helpers.""" + +import re +import uuid + +import pytest +from django.db import connection + +from pulpcore.plugin.models import RepositoryContent + +from pulp_python.app.catalog import ( + assemble_package_index, + distinct_package_names_qs, + python_packages_in_version, +) +from pulp_python.app.models import PythonPackageContent, PythonRepository + +_UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + +_EMPTY_TEXT = ( + "author", + "author_email", + "description", + "home_page", + "keywords", + "license", + "metadata_version", + "platform", + "summary", + "download_url", + "supported_platform", + "maintainer", + "maintainer_email", + "project_url", + "requires_python", + "description_content_type", + "license_expression", + "python_version", +) + + +def _bound_id_count(sql): + """How many ids the statement binds, whether mogrified or left as ``%s``.""" + lowered = sql.lower() + return max(len(_UUID_RE.findall(lowered)), lowered.count("%s")) + + +def _create_package(name, version): + fields = {field: "" for field in _EMPTY_TEXT} + fields.update( + name=name, + version=version, + filename=f"{name}-{version}.tar.gz", + packagetype="sdist", + sha256=uuid.uuid4().hex + uuid.uuid4().hex, + ) + return PythonPackageContent.objects.create(**fields) + + +@pytest.mark.django_db +def test_package_list_sql_does_not_expand_content_ids(): + """The package index must filter through RepositoryContent, not content_ids. + + A paged list used to inline every content UUID in the version. The name + page is a distinct name_normalized query. File rows for that page are + loaded through the membership subquery. + """ + repository = PythonRepository.objects.create(name=str(uuid.uuid4())) + created = [ + _create_package(name, f"1.0.{number}") + for name in ("alpha", "beta", "gamma", "delta") + for number in range(5) + ] + with repository.new_version() as version: + version.add_content( + PythonPackageContent.objects.filter(pk__in=[item.pk for item in created]) + ) + + repo_version = repository.versions.complete().defer("content_ids").latest() + content_count = RepositoryContent.objects.filter( + repository_id=repository.pk, version_removed__isnull=True + ).count() + content_qs = python_packages_in_version(repo_version) + default_qs = distinct_package_names_qs(content_qs) + count_qs = content_qs.order_by().values("name_normalized").distinct() + + connection.force_debug_cursor = True + start = len(connection.queries) + + def _since(): + nonlocal start + sqls = [query["sql"] for query in connection.queries[start:]] + start = len(connection.queries) + return sqls + + try: + count_qs.count() + count_sqls = _since() + page = list(default_qs[:20]) + default_sqls = _since() + rows = assemble_package_index(content_qs, page, repo_version) + assembled = _since() + finally: + connection.force_debug_cursor = False + + assert rows + assert rows[0]["versions"] + assert content_count == len(created) + membership_sql = [ + sql + for sql in count_sqls + default_sqls + assembled + if "core_repositorycontent" in sql.lower() + ] + assert membership_sql + for sql in membership_sql: + lowered = sql.lower() + assert "content_id" in lowered, sql + assert _bound_id_count(sql) < content_count, sql + assert len(count_sqls) == 1, count_sqls + assert len(default_sqls) == 1, default_sqls + count_sql, default_sql = count_sqls[0], default_sqls[0] + assert "max(" not in count_sql.lower() + assert "max(" not in default_sql.lower() + assert "distinct" in default_sql.lower() + assert "limit" in default_sql.lower()