Add JSON-based Simple API

Author: jobselko

pulp_python/app/pypi/views.py

@@ -3,7 +3,9 @@
 
 from aiohttp.client_exceptions import ClientError
 from rest_framework.viewsets import ViewSet
+from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer, TemplateHTMLRenderer
 from rest_framework.response import Response
+from rest_framework.exceptions import NotAcceptable, NotFound
 from django.core.exceptions import ObjectDoesNotExist
 from django.shortcuts import redirect
 from datetime import datetime, timezone, timedelta
@@ -17,6 +19,7 @@
     HttpResponseBadRequest,
     StreamingHttpResponse,
     HttpResponse,
+    JsonResponse,
 )
 from drf_spectacular.utils import extend_schema
 from dynaconf import settings
@@ -43,7 +46,9 @@
 )
 from pulp_python.app.utils import (
     write_simple_index,
+    write_simple_index_json,
     write_simple_detail,
+    write_simple_detail_json,
     python_content_to_json,
     PYPI_LAST_SERIAL,
     PYPI_SERIAL_CONSTANT,
@@ -57,6 +62,18 @@
 ORIGIN_HOST = settings.CONTENT_ORIGIN if settings.CONTENT_ORIGIN else settings.PYPI_API_HOSTNAME
 BASE_CONTENT_URL = urljoin(ORIGIN_HOST, settings.CONTENT_PATH_PREFIX)
 
+# PYPI_TEXT_HTML = "text/html"
+PYPI_SIMPLE_V1_HTML = "application/vnd.pypi.simple.v1+html"
+PYPI_SIMPLE_V1_JSON = "application/vnd.pypi.simple.v1+json"
+
+
+class PyPISimpleHTMLRenderer(TemplateHTMLRenderer):
+    media_type = PYPI_SIMPLE_V1_HTML
+
+
+class PyPISimpleJSONRenderer(JSONRenderer):
+    media_type = PYPI_SIMPLE_V1_JSON
+
 
 class PyPIMixin:
     """Mixin to get index specific info."""
@@ -235,14 +252,55 @@ class SimpleView(PackageUploadMixin, ViewSet):
         ],
     }
 
+    def perform_content_negotiation(self, request, force=False):
+        """
+        Uses standard content negotiation, defaulting to HTML if no acceptable renderer is found.
+        """
+        try:
+            return super().perform_content_negotiation(request, force)
+        except NotAcceptable:
+            return TemplateHTMLRenderer(), "text/html"
+
+    def get_renderers(self):
+        """
+        Uses custom renderers for PyPI Simple API endpoints, defaulting to standard ones.
+        """
+        if self.action in ["list", "retrieve"]:
+            # Ordered by priority if multiple content types are present
+            return [
+                TemplateHTMLRenderer(),
+                PyPISimpleHTMLRenderer(),
+                PyPISimpleJSONRenderer(),
+            ]
+        else:
+            return [JSONRenderer(), BrowsableAPIRenderer()]
+
+    def handle_exception(self, exc):
+        # todo: fixes test_pull_through_filter?
+        if isinstance(exc, (Http404, NotFound)):
+            # Force JSON response since JSONRenderer is not in get_renderers()
+            return JsonResponse({"detail": str(exc)}, status=404)
+        return super().handle_exception(exc)
+
     @extend_schema(summary="Get index simple page")
     def list(self, request, path):
         """Gets the simple api html page for the index."""
         repo_version, content = self.get_rvc()
         if self.should_redirect(repo_version=repo_version):
             return redirect(urljoin(self.base_content_url, f"{path}/simple/"))
-        names = content.order_by("name").values_list("name", flat=True).distinct().iterator()
-        return StreamingHttpResponse(write_simple_index(names, streamed=True))
+
+        names = content.order_by("name").values_list("name", flat=True).distinct()
+        media_type = request.accepted_renderer.media_type
+
+        if media_type == PYPI_SIMPLE_V1_JSON:
+            index_data = write_simple_index_json(list(names))
+            headers = {"X-PyPI-Last-Serial": str(PYPI_SERIAL_CONSTANT)}
+            return Response(index_data, headers=headers)
+        else:
+            index_data = write_simple_index(names.iterator(), streamed=True)
+            kwargs = {"content_type": media_type}
+            response = StreamingHttpResponse(index_data, **kwargs)
+            return response
 
     def pull_through_package_simple(self, package, path, remote):
         """Gets the package's simple page from remote."""
@@ -301,7 +359,16 @@ def retrieve(self, request, path, package):
             packages = chain([present], packages)
             name = present[2]
         releases = ((f, urljoin(self.base_content_url, f"{path}/{f}"), d) for f, d, _ in packages)
-        return StreamingHttpResponse(write_simple_detail(name, releases, streamed=True))
+        media_type = request.accepted_renderer.media_type
+
+        if media_type == PYPI_SIMPLE_V1_JSON:
+            detail_data = write_simple_detail_json(name, list(releases))
+            headers = {"X-PyPI-Last-Serial": str(PYPI_SERIAL_CONSTANT)}
+            return Response(detail_data, headers=headers)
+        else:
+            detail_data = write_simple_detail(name, releases, streamed=True)
+            kwargs = {"content_type": media_type}
+            return StreamingHttpResponse(detail_data, kwargs)
 
     @extend_schema(
         request=PackageUploadSerializer,

pulp_python/app/utils.py

@@ -16,6 +16,10 @@
 """TODO This serial constant is temporary until Python repositories implements serials"""
 PYPI_SERIAL_CONSTANT = 1000000000
 
+# todo: update to reflect supported version
+# https://packaging.python.org/en/latest/specifications/simple-repository-api/#api-version-history
+PYPI_API_VERSION = "1.4"
+
 simple_index_template = """<!DOCTYPE html>
 <html>
   <head>
@@ -414,6 +418,46 @@ def write_simple_detail(project_name, project_packages, streamed=False):
     return detail.stream(**context) if streamed else detail.render(**context)
 
 
+def write_simple_index_json(project_names):
+    """Writes the simple index in JSON format."""
+    projects = [{"_last-serial": PYPI_SERIAL_CONSTANT, "name": name} for name in project_names]
+
+    return {
+        "meta": {"api-version": PYPI_API_VERSION, "_last-serial": PYPI_SERIAL_CONSTANT},
+        "projects": projects,
+    }
+
+
+# todo: fields
+def write_simple_detail_json(project_name, project_packages):
+    """Writes the simple detail page in JSON format."""
+    files = []
+    for filename, url, sha256 in project_packages:
+        files.append(
+            {
+                "filename": filename,
+                "url": url,
+                "hashes": {"sha256": sha256},
+                # requires_python # todo
+                # size / version 1.1
+                # upload-time / 1.1
+                # yanked # todo
+                # data-dist-info-metadata
+                # core-metadata
+                # provenance / 1.3
+            }
+        )
+
+    return {
+        "meta": {"api-version": PYPI_API_VERSION, "_last-serial": PYPI_SERIAL_CONSTANT},
+        "name": project_name,  # should be normalized
+        # project-status / 1.4
+        # versions / 1.1
+        # alternate-locations
+        "files": files,
+    }
+
+
 class PackageIncludeFilter:
     """A special class to help filter Package's based on a remote's include/exclude"""
 

pulp_python/tests/functional/api/test_pypi_simple_json_api.py

@@ -0,0 +1,71 @@
+from urllib.parse import urljoin
+
+import pytest
+import requests
+
+from pulp_python.tests.functional.constants import PYTHON_SM_PROJECT_SPECIFIER
+
+
+@pytest.mark.parallel
+def test_simple_json_api(python_remote_factory, python_repo_with_sync, python_distribution_factory):
+    remote = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER)
+    repo = python_repo_with_sync(remote)
+    distro = python_distribution_factory(repository=repo)
+
+    index_url = urljoin(distro.base_url, "simple/")
+    detail_url = f"{index_url}aiohttp"
+    headers = {"Accept": "application/vnd.pypi.simple.v1+json"}
+
+    response_index = requests.get(index_url, headers=headers)
+    data_index = response_index.json()
+    assert data_index["meta"] == {"api-version": "1.4", "_last-serial": 1000000000}
+    assert data_index["projects"]
+    for project in data_index["projects"]:
+        for i in ["_last-serial", "name"]:
+            assert project[i]
+    assert response_index.headers["Content-Type"] == "application/vnd.pypi.simple.v1+json"
+    assert response_index.headers["X-PyPI-Last-Serial"] == "1000000000"
+
+    response_detail = requests.get(detail_url, headers=headers)
+    data_detail = response_detail.json()
+    assert data_detail["meta"] == {"api-version": "1.4", "_last-serial": 1000000000}
+    assert data_detail["name"] == "aiohttp"
+    assert data_detail["files"]
+    for file in data_detail["files"]:
+        for i in ["filename", "url", "hashes"]:
+            assert file[i]
+    assert response_detail.headers["Content-Type"] == "application/vnd.pypi.simple.v1+json"
+    assert response_detail.headers["X-PyPI-Last-Serial"] == "1000000000"
+
+
+@pytest.mark.parametrize(
+    "header, result",
+    [
+        ("text/html", "text/html"),
+        ("application/vnd.pypi.simple.v1+html", "application/vnd.pypi.simple.v1+html"),
+        ("application/vnd.pypi.simple.v1+json", "application/vnd.pypi.simple.v1+json"),
+        # Follows defined ordering (html, pypi html, pypi json)
+        (
+            "application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html",
+            "application/vnd.pypi.simple.v1+html",
+        ),
+        # Everything else should be html
+        ("", "text/html"),
+        ("application/json", "text/html"),
+        ("sth/else", "text/html"),
+    ],
+)
+def test_simple_json_api_headers(
+    python_remote_factory, python_repo_with_sync, python_distribution_factory, header, result
+):
+    remote = python_remote_factory(includes=PYTHON_SM_PROJECT_SPECIFIER)
+    repo = python_repo_with_sync(remote)
+    distro = python_distribution_factory(repository=repo)
+
+    index_url = urljoin(distro.base_url, "simple/")
+    detail_url = f"{index_url}aiohttp"
+
+    for url in [index_url, detail_url]:
+        response = requests.get(url, headers={"Accept": header})
+        assert response.status_code == 200
+        assert result in response.headers["Content-Type"]