From b0372e44ca19307fe1ab83524f014d3dc417c242 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Fri, 8 Sep 2023 02:58:40 -0400 Subject: [PATCH] Add field `disk_size` to Repository & RepositoryVersion fixes: #2079 --- CHANGES/2079.feature | 1 + pulpcore/app/models/repository.py | 20 ++++++++ pulpcore/app/serializers/repository.py | 27 ++++++++++ pulpcore/tests/functional/api/test_repos.py | 55 +++++++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100644 CHANGES/2079.feature diff --git a/CHANGES/2079.feature b/CHANGES/2079.feature new file mode 100644 index 0000000000..c19d84a5e3 --- /dev/null +++ b/CHANGES/2079.feature @@ -0,0 +1 @@ +Added field ``disk_size`` to Repository and RepositoryVersion that shows current total size in bytes of files stored on disk. diff --git a/pulpcore/app/models/repository.py b/pulpcore/app/models/repository.py index a73bfaac93..d436595ab1 100644 --- a/pulpcore/app/models/repository.py +++ b/pulpcore/app/models/repository.py @@ -82,6 +82,21 @@ class Meta: unique_together = ("name", "pulp_domain") verbose_name_plural = "repositories" + @property + def disk_size(self): + """Returns the approximate size on disk for all artifacts stored across all versions.""" + all_content = ( + RepositoryContent.objects.filter(repository=self) + .distinct("content") + .values_list("content") + ) + return ( + Artifact.objects.filter(content__pk__in=all_content) + .distinct() + .aggregate(size=models.Sum("size"))["size"] + or 0 + ) + def on_new_version(self, version): """Called after a new repository version has been created. @@ -795,6 +810,11 @@ def artifacts(self): """ return self.repository.cast().artifacts_for_version(self) + @property + def disk_size(self): + """Returns the size on disk of all the artifacts in this repository version.""" + return self.artifacts.distinct().aggregate(size=models.Sum("size"))["size"] or 0 + def added(self, base_version=None): """ Args: diff --git a/pulpcore/app/serializers/repository.py b/pulpcore/app/serializers/repository.py index 6d9d2c6f5b..d85cc66b5f 100644 --- a/pulpcore/app/serializers/repository.py +++ b/pulpcore/app/serializers/repository.py @@ -4,6 +4,7 @@ from rest_framework import fields, serializers from rest_framework_nested.serializers import NestedHyperlinkedModelSerializer +from drf_spectacular.utils import extend_schema_field from pulpcore.app import models, settings from pulpcore.app.serializers import ( @@ -49,6 +50,18 @@ class RepositorySerializer(ModelSerializer): required=False, allow_null=True, ) + disk_size = serializers.SerializerMethodField( + help_text=_( + "Approximate size on disk of all artifacts in the repository across all its versions." + ), + ) + + @extend_schema_field(serializers.IntegerField(read_only=True, allow_null=True)) + def get_disk_size(self, obj): + if view := self.context.get("view", None): + if getattr(view, "action", "") == "retrieve": + return obj.disk_size + return None def validate_remote(self, value): if value and type(value) not in self.Meta.model.REMOTE_TYPES: @@ -66,6 +79,7 @@ class Meta: "latest_version_href", "name", "description", + "disk_size", "retain_repo_versions", "remote", ) @@ -420,12 +434,25 @@ class RepositoryVersionSerializer(ModelSerializer, NestedHyperlinkedModelSeriali source="*", read_only=True, ) + disk_size = serializers.SerializerMethodField( + help_text=_( + "Approximate size on disk of all artifacts in the repository version." + ), + ) + + @extend_schema_field(serializers.IntegerField(read_only=True, allow_null=True)) + def get_disk_size(self, obj): + if view := self.context.get("view", None): + if getattr(view, "action", "") == "retrieve": + return obj.disk_size + return None class Meta: model = models.RepositoryVersion fields = ModelSerializer.Meta.fields + ( "pulp_href", "number", + "disk_size", "repository", "base_version", "content_summary", diff --git a/pulpcore/tests/functional/api/test_repos.py b/pulpcore/tests/functional/api/test_repos.py index 91d4a518e8..d02aa47f8a 100644 --- a/pulpcore/tests/functional/api/test_repos.py +++ b/pulpcore/tests/functional/api/test_repos.py @@ -48,3 +48,58 @@ def test_repository_content_filters( # but not in its latest version anymore results = file_repository_api_client.list(latest_with_content=content.pulp_href).results assert results == [] + + +@pytest.mark.parallel +def test_repo_size( + file_repo, + file_repository_api_client, + file_repository_version_api_client, + file_remote_factory, + basic_manifest_path, + random_artifact_factory, + file_content_api_client, + monitor_task +): + # Check that 'disk_size' is only added on retrieve operations + file_repo = file_repository_api_client.read(file_repo.pulp_href) + assert file_repo.disk_size == 0 + repo_ver0 = file_repository_version_api_client.read(file_repo.latest_version_href) + assert repo_ver0.disk_size == 0 + + listed_repo = file_repository_api_client.list().results[0] + assert listed_repo.disk_size is None + listed_ver = file_repository_version_api_client.list(file_repo.pulp_href).results[0] + assert listed_ver.disk_size is None + + # Sync repository with on_demand + remote = file_remote_factory(manifest_path=basic_manifest_path, policy="on_demand") + body = {"remote": remote.pulp_href} + monitor_task(file_repository_api_client.sync(file_repo.pulp_href, body).task) + file_repo = file_repository_api_client.read(file_repo.pulp_href) + + # disk_size should still be 0 + assert file_repo.disk_size == 0 + repo_ver1 = file_repository_version_api_client.read(file_repo.latest_version_href) + assert repo_ver1.disk_size == 0 + + # Resync with immediate + remote = file_remote_factory(manifest_path=basic_manifest_path, policy="immediate") + body = {"remote": remote.pulp_href} + monitor_task(file_repository_api_client.sync(file_repo.pulp_href, body).task) + file_repo = file_repository_api_client.read(file_repo.pulp_href) + + assert file_repo.disk_size == 3072 # 3 * 1024 + repo_ver1 = file_repository_version_api_client.read(file_repo.latest_version_href) + assert repo_ver1.disk_size == 3072 + + # Add content unit w/ same name, but different artifact + art1 = random_artifact_factory() + body = {"repository": file_repo.pulp_href, "artifact": art1.pulp_href, "relative_path": "1.iso"} + monitor_task(file_content_api_client.create(**body).task) + file_repo = file_repository_api_client.read(file_repo.pulp_href) + + assert file_repo.disk_size == 3072 + art1.size # All 4 artifacts in repo + repo_ver2 = file_repository_version_api_client.read(file_repo.latest_version_href) + assert repo_ver2.content_summary.present["file.file"]["count"] == 3 + assert repo_ver2.disk_size == 2048 + art1.size # New size of 3 artifacts in version