From d2354585db5ea7065eade011c4be6281186a9320 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Wed, 16 Oct 2024 15:33:18 +0200 Subject: [PATCH] fix: find images when there is an oci index (#462) This was causing the API to consider some images as non-existing. Whereas in reality they exist. I noticed this with this image: ghcr.io/salimkayal/renku-devcontainer-wizard:latest. --- .../notebooks/api/classes/image.py | 16 +- .../notebooks/test_notebooks_image_checks.py | 214 ++++++++++++++++++ 2 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 test/components/renku_data_services/notebooks/test_notebooks_image_checks.py diff --git a/components/renku_data_services/notebooks/api/classes/image.py b/components/renku_data_services/notebooks/api/classes/image.py index 40e6ee850..e404699c6 100644 --- a/components/renku_data_services/notebooks/api/classes/image.py +++ b/components/renku_data_services/notebooks/api/classes/image.py @@ -17,7 +17,8 @@ class ManifestTypes(Enum): """The mime types for docker image manifests.""" docker_v2: str = "application/vnd.docker.distribution.manifest.v2+json" - oci_v1: str = "application/vnd.oci.image.manifest.v1+json" + oci_v1_manifest: str = "application/vnd.oci.image.manifest.v1+json" + oci_v1_index: str = "application/vnd.oci.image.index.v1+json" @dataclass @@ -74,8 +75,19 @@ async def get_image_manifest(self, image: "Image") -> Optional[dict[str, Any]]: headers["Authorization"] = f"Bearer {token}" res = await self.client.get(image_digest_url, headers=headers) if res.status_code != 200: - headers["Accept"] = ManifestTypes.oci_v1.value + headers["Accept"] = ManifestTypes.oci_v1_manifest.value res = await self.client.get(image_digest_url, headers=headers) + if res.status_code != 200: + headers["Accept"] = ManifestTypes.oci_v1_index.value + res = await self.client.get(image_digest_url, headers=headers) + if res.status_code == 200: + index_parsed = res.json() + manifest = next( + (man for man in index_parsed.get("manifests", []) if man.get("platform", {}).get("os") == "linux"), + None, + ) + manifest = cast(dict[str, Any] | None, manifest) + return manifest if res.status_code != 200: return None return cast(dict[str, Any], res.json()) diff --git a/test/components/renku_data_services/notebooks/test_notebooks_image_checks.py b/test/components/renku_data_services/notebooks/test_notebooks_image_checks.py new file mode 100644 index 000000000..68d694a73 --- /dev/null +++ b/test/components/renku_data_services/notebooks/test_notebooks_image_checks.py @@ -0,0 +1,214 @@ +from dataclasses import asdict +from pathlib import PurePosixPath + +import pytest + +from renku_data_services.notebooks.api.classes.image import Image + + +@pytest.mark.parametrize( + "name,expected", + [ + ( + "nginx", + { + "hostname": "registry-1.docker.io", + "name": "library/nginx", + "tag": "latest", + }, + ), + ( + "nginx:1.28", + { + "hostname": "registry-1.docker.io", + "name": "library/nginx", + "tag": "1.28", + }, + ), + ( + "nginx@sha256:24235rt2rewg345ferwf", + { + "hostname": "registry-1.docker.io", + "name": "library/nginx", + "tag": "sha256:24235rt2rewg345ferwf", + }, + ), + ( + "username/image", + { + "hostname": "registry-1.docker.io", + "name": "username/image", + "tag": "latest", + }, + ), + ( + "username/image:1.0.0", + { + "hostname": "registry-1.docker.io", + "name": "username/image", + "tag": "1.0.0", + }, + ), + ( + "username/image@sha256:fdsaf345tre3412t1413r", + { + "hostname": "registry-1.docker.io", + "name": "username/image", + "tag": "sha256:fdsaf345tre3412t1413r", + }, + ), + ( + "gitlab.smth.com/username/project", + { + "hostname": "gitlab.smth.com", + "name": "username/project", + "tag": "latest", + }, + ), + ( + "gitlab.smth.com:443/username/project", + { + "hostname": "gitlab.smth.com:443", + "name": "username/project", + "tag": "latest", + }, + ), + ( + "gitlab.smth.com/username/project/image/subimage", + { + "hostname": "gitlab.smth.com", + "name": "username/project/image/subimage", + "tag": "latest", + }, + ), + ( + "gitlab.smth.com:443/username/project/image/subimage", + { + "hostname": "gitlab.smth.com:443", + "name": "username/project/image/subimage", + "tag": "latest", + }, + ), + ( + "gitlab.smth.com/username/project:1.2.3", + { + "hostname": "gitlab.smth.com", + "name": "username/project", + "tag": "1.2.3", + }, + ), + ( + "gitlab.smth.com:443/username/project:1.2.3", + { + "hostname": "gitlab.smth.com:443", + "name": "username/project", + "tag": "1.2.3", + }, + ), + ( + "gitlab.smth.com/username/project/image/subimage:1.2.3", + { + "hostname": "gitlab.smth.com", + "name": "username/project/image/subimage", + "tag": "1.2.3", + }, + ), + ( + "gitlab.smth.com:443/username/project/image/subimage:1.2.3", + { + "hostname": "gitlab.smth.com:443", + "name": "username/project/image/subimage", + "tag": "1.2.3", + }, + ), + ( + "gitlab.smth.com/username/project@sha256:324fet13t4", + { + "hostname": "gitlab.smth.com", + "name": "username/project", + "tag": "sha256:324fet13t4", + }, + ), + ( + "gitlab.smth.com:443/username/project@sha256:324fet13t4", + { + "hostname": "gitlab.smth.com:443", + "name": "username/project", + "tag": "sha256:324fet13t4", + }, + ), + ( + "gitlab.smth.com/username/project/image/subimage@sha256:324fet13t4", + { + "hostname": "gitlab.smth.com", + "name": "username/project/image/subimage", + "tag": "sha256:324fet13t4", + }, + ), + ( + "gitlab.smth.com:443/username/project/image/subimage@sha256:324fet13t4", + { + "hostname": "gitlab.smth.com:443", + "name": "username/project/image/subimage", + "tag": "sha256:324fet13t4", + }, + ), + ( + "us.gcr.io/image/subimage@sha256:324fet13t4", + { + "hostname": "us.gcr.io", + "name": "image/subimage", + "tag": "sha256:324fet13t4", + }, + ), + ( + "us.gcr.io/proj/image", + {"hostname": "us.gcr.io", "name": "proj/image", "tag": "latest"}, + ), + ( + "us.gcr.io/proj/image/subimage", + {"hostname": "us.gcr.io", "name": "proj/image/subimage", "tag": "latest"}, + ), + ], +) +def test_public_image_name_parsing(name: str, expected: dict[str, str]) -> None: + assert asdict(Image.from_path(name)) == expected + + +@pytest.mark.parametrize( + "image,exists_expected", + [ + ("nginx:1.19.3", True), + ("nginx", True), + ("renku/singleuser:cb70d7e", True), + ("renku/singleuser", True), + ("madeuprepo/madeupproject:tag", False), + ("olevski90/oci-image:0.0.1", True), + ("ghcr.io/linuxserver/nginx:latest", True), + ], +) +@pytest.mark.asyncio +@pytest.mark.integration +async def test_public_image_check(image: str, exists_expected: bool) -> None: + parsed_image = Image.from_path(image) + exists_observed = await parsed_image.repo_api().image_exists(parsed_image) + assert exists_expected == exists_observed + + +@pytest.mark.parametrize( + "image,expected_path", + [ + ("jupyter/minimal-notebook", PurePosixPath("/home/jovyan")), + ("nginx", None), + ("madeuprepo/madeupproject:tag", None), + ], +) +@pytest.mark.asyncio +@pytest.mark.integration +async def test_image_workdir_check(image: str, expected_path: PurePosixPath | None) -> None: + parsed_image = Image.from_path(image) + workdir = await parsed_image.repo_api().image_workdir(parsed_image) + if expected_path is None: + assert workdir is None, f"The image workdir should be None but instead it is {workdir}" + else: + assert workdir == expected_path