From 301c958341be6c583bc1bac751d878130965330e Mon Sep 17 00:00:00 2001 From: Arthur Deierlein Date: Tue, 16 Jan 2024 15:07:14 +0100 Subject: [PATCH] feat!: added dependency sources to api --- api/outdated/conftest.py | 1 + api/outdated/outdated/factories.py | 20 +++-- .../outdated/migrations/0001_initial.py | 82 +++++++++++++------ api/outdated/outdated/models.py | 25 ++++-- api/outdated/outdated/parser.py | 16 ++-- api/outdated/outdated/serializers.py | 35 ++++---- api/outdated/outdated/tests/test_api.py | 57 +++++++------ api/outdated/outdated/tests/test_parser.py | 9 +- api/outdated/outdated/tests/test_tracking.py | 17 ++-- api/outdated/outdated/tracking.py | 3 +- api/outdated/outdated/views.py | 8 +- .../tests/test_unique_boolean_field.py | 2 +- 12 files changed, 176 insertions(+), 99 deletions(-) diff --git a/api/outdated/conftest.py b/api/outdated/conftest.py index 0fc25b0e..9c5be29c 100644 --- a/api/outdated/conftest.py +++ b/api/outdated/conftest.py @@ -25,6 +25,7 @@ register(factories.ReleaseVersionFactory) register(factories.ProjectFactory) register(factories.MaintainerFactory) +register(factories.DependencySourceFactory) register(UserFactory) diff --git a/api/outdated/outdated/factories.py b/api/outdated/outdated/factories.py index c2f0feff..5dafdd01 100644 --- a/api/outdated/outdated/factories.py +++ b/api/outdated/outdated/factories.py @@ -56,20 +56,30 @@ class ProjectFactory(DjangoModelFactory): repo = Sequence(lambda n: "github.com/userorcompany/%s/" % n) repo_type = "public" + class Meta: + model = models.Project + + +class DependencySourceFactory(DjangoModelFactory): + project = SubFactory(ProjectFactory) + path = random.choice( + ["/pyproject.toml", "/api/pyproject.toml", "/ember/pnpm-lock.yaml"] + ) + @post_generation - def versioned_dependencies(self, create, extracted, **kwargs): + def versions(self, create, extracted, **kwargs): if not create: return # pragma: no cover if extracted: - for versioned_dependency in extracted: - self.versioned_dependencies.add(versioned_dependency) + for version in extracted: + self.versions.add(version) class Meta: - model = models.Project + model = models.DependencySource class MaintainerFactory(DjangoModelFactory): - project = SubFactory(ProjectFactory) + source = SubFactory(DependencySourceFactory) user = SubFactory(UserFactory) class Meta: diff --git a/api/outdated/outdated/migrations/0001_initial.py b/api/outdated/outdated/migrations/0001_initial.py index 18cc9af2..59210187 100644 --- a/api/outdated/outdated/migrations/0001_initial.py +++ b/api/outdated/outdated/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.6 on 2024-01-09 09:41 +# Generated by Django 4.2.6 on 2024-01-12 15:45 from django.db import migrations, models import django.db.models.deletion @@ -25,35 +25,24 @@ class Migration(migrations.Migration): ], options={ 'ordering': ['name', 'id'], - 'unique_together': {('name', 'provider')}, }, ), migrations.CreateModel( - name='ReleaseVersion', + name='DependencySource', fields=[ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('major_version', models.IntegerField()), - ('minor_version', models.IntegerField()), - ('end_of_life', models.DateField(blank=True, null=True)), - ('dependency', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='outdated.dependency')), + ('path', models.CharField()), ], options={ - 'ordering': ['end_of_life', 'dependency__name', 'major_version', 'minor_version'], - 'unique_together': {('dependency', 'major_version', 'minor_version')}, + 'abstract': False, }, ), migrations.CreateModel( - name='Version', + name='Maintainer', fields=[ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('patch_version', models.IntegerField()), - ('release_date', models.DateField(blank=True, null=True)), - ('release_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='outdated.releaseversion')), + ('is_primary', outdated.models.UniqueBooleanField(default=False)), ], - options={ - 'ordering': ['release_version__end_of_life', 'release_version__dependency__name', 'release_version__major_version', 'release_version__minor_version', 'patch_version'], - 'unique_together': {('release_version', 'patch_version')}, - }, ), migrations.CreateModel( name='Project', @@ -62,20 +51,35 @@ class Migration(migrations.Migration): ('name', models.CharField(db_index=True, max_length=100)), ('repo', outdated.models.RepositoryURLField(max_length=100)), ('repo_type', models.CharField(choices=[('public', 'public'), ('access-token', 'access-token')], max_length=25)), - ('versioned_dependencies', models.ManyToManyField(blank=True, to='outdated.version')), ], options={ 'ordering': ['name', 'id'], }, ), migrations.CreateModel( - name='Maintainer', + name='ReleaseVersion', fields=[ ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('is_primary', outdated.models.UniqueBooleanField(default=False)), - ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='maintainers', to='outdated.project')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='user.user')), + ('major_version', models.IntegerField()), + ('minor_version', models.IntegerField()), + ('end_of_life', models.DateField(blank=True, null=True)), + ('dependency', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='outdated.dependency')), ], + options={ + 'ordering': ['end_of_life', 'dependency__name', 'major_version', 'minor_version'], + }, + ), + migrations.CreateModel( + name='Version', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('patch_version', models.IntegerField()), + ('release_date', models.DateField(blank=True, null=True)), + ('release_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='outdated.releaseversion')), + ], + options={ + 'ordering': ['release_version__end_of_life', 'release_version__dependency__name', 'release_version__major_version', 'release_version__minor_version', 'patch_version'], + }, ), migrations.AddConstraint( model_name='project', @@ -85,8 +89,40 @@ class Migration(migrations.Migration): model_name='project', constraint=models.UniqueConstraint(django.db.models.functions.text.Lower('repo'), name='unique_project_repo'), ), + migrations.AddField( + model_name='maintainer', + name='source', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='maintainers', to='outdated.dependencysource'), + ), + migrations.AddField( + model_name='maintainer', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='user.user'), + ), + migrations.AddField( + model_name='dependencysource', + name='project', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='sources', to='outdated.project'), + ), + migrations.AddField( + model_name='dependencysource', + name='versions', + field=models.ManyToManyField(blank=True, to='outdated.version'), + ), + migrations.AlterUniqueTogether( + name='dependency', + unique_together={('name', 'provider')}, + ), + migrations.AlterUniqueTogether( + name='version', + unique_together={('release_version', 'patch_version')}, + ), + migrations.AlterUniqueTogether( + name='releaseversion', + unique_together={('dependency', 'major_version', 'minor_version')}, + ), migrations.AlterUniqueTogether( name='maintainer', - unique_together={('user', 'project')}, + unique_together={('user', 'source')}, ), ] diff --git a/api/outdated/outdated/models.py b/api/outdated/outdated/models.py index 866c6ab1..1c6ed1fb 100644 --- a/api/outdated/outdated/models.py +++ b/api/outdated/outdated/models.py @@ -122,8 +122,6 @@ def version(self) -> str: class Project(UUIDModel): name = models.CharField(max_length=100, db_index=True) - - versioned_dependencies = models.ManyToManyField(Version, blank=True) repo = RepositoryURLField(max_length=100) repo_type = models.CharField(max_length=25, choices=REPO_TYPES) @@ -202,21 +200,34 @@ class Meta: @property def status(self) -> str: - first = self.versioned_dependencies.first() + first = self.sources.all().values_list("versions", flat=True).first() return first.release_version.status if first else STATUS_OPTIONS["undefined"] def __str__(self): return self.name +class DependencySource(UUIDModel): + path = models.CharField() + project = models.ForeignKey( + Project, on_delete=models.CASCADE, related_name="sources" + ) + versions = models.ManyToManyField(Version, blank=True) + + @property + def status(self) -> str: + first = self.versions.first() + return first.release_version.status if first else STATUS_OPTIONS["undefined"] + + class Maintainer(UUIDModel): user = models.ForeignKey(User, on_delete=models.CASCADE) - project = models.ForeignKey( - Project, + source = models.ForeignKey( + DependencySource, on_delete=models.CASCADE, related_name="maintainers", ) - is_primary = UniqueBooleanField(default=False, together=["project"]) + is_primary = UniqueBooleanField(default=False, together=["source"]) class Meta: - unique_together = ("user", "project") + unique_together = ("user", "source") diff --git a/api/outdated/outdated/parser.py b/api/outdated/outdated/parser.py index 2c28a8e0..aa7a0ea5 100644 --- a/api/outdated/outdated/parser.py +++ b/api/outdated/outdated/parser.py @@ -20,8 +20,9 @@ class LockfileParser: """Parse a lockfile and return a list of dependencies.""" - def __init__(self, lockfiles: list[Path]) -> None: + def __init__(self, project: models.Project, lockfiles: list[Path]) -> None: self.lockfiles = lockfiles + self.project = project def _get_provider(self, name: str) -> str: """Get the provider of the lockfile.""" @@ -102,10 +103,8 @@ def _get_release_date(self, version: models.Version) -> date: return parse_date(release_date).date() - def parse(self) -> list[models.Version]: + def parse(self) -> None: """Parse the lockfile and return a dictionary of dependencies.""" - versions = [] - for lockfile in self.lockfiles: name = lockfile.name data = lockfile.read_text() @@ -139,8 +138,9 @@ def parse(self) -> list[models.Version]: and requirements[0][0] in settings.TRACKED_DEPENDENCIES ] - versions.extend( - self._get_version(dependency, provider) for dependency in dependencies + source, _ = models.DependencySource.objects.get_or_create( + path=name, project=self.project + ) + source.versions.set( + [self._get_version(dependency, provider) for dependency in dependencies] ) - - return versions diff --git a/api/outdated/outdated/serializers.py b/api/outdated/outdated/serializers.py index b900e86d..ab6a2975 100644 --- a/api/outdated/outdated/serializers.py +++ b/api/outdated/outdated/serializers.py @@ -42,7 +42,7 @@ class Meta: class MaintainerSerializer(serializers.ModelSerializer): included_serializers = { "user": "outdated.user.serializers.UserSerializer", - "project": "outdated.outdated.serializers.ProjectSerializer", + "source": "outdated.outdated.serializers.DependencySourceSerializer", } class Meta: @@ -50,13 +50,23 @@ class Meta: fields = "__all__" -class ProjectSerializer(serializers.ModelSerializer): +class DependencySourceSerializer(serializers.ModelSerializer): maintainers = serializers.ResourceRelatedField( many=True, read_only=True, - required=False, ) + included_serializers = { + "versions": VersionSerializer, + "maintainers": MaintainerSerializer, + } + + class Meta: + model = models.DependencySource + fields = "__all__" + + +class ProjectSerializer(serializers.ModelSerializer): access_token = serializers.CharField( max_length=100, write_only=True, @@ -64,6 +74,12 @@ class ProjectSerializer(serializers.ModelSerializer): allow_blank=True, validators=[RegexValidator(r"[-_a-zA-Z\d]+")], ) + + sources = serializers.ResourceRelatedField( + many=True, + read_only=True, + ) + repo = serializers.CharField( validators=[ UniqueValidator(queryset=models.Project.objects.all(), lookup="iexact") @@ -76,8 +92,7 @@ class ProjectSerializer(serializers.ModelSerializer): ) included_serializers = { - "versioned_dependencies": "outdated.outdated.serializers.VersionSerializer", - "maintainers": "outdated.outdated.serializers.MaintainerSerializer", + "sources": "outdated.outdated.serializers.DependencySourceSerializer" } class Meta: @@ -87,15 +102,7 @@ class Meta: validate_access_token_required, validate_no_access_token_when_public, ] - fields = ( - "name", - "repo", - "repo_type", - "access_token", - "status", - "versioned_dependencies", - "maintainers", - ) + fields = ("name", "repo", "repo_type", "access_token", "status", "sources") def create(self, validated_data: dict) -> models.Project: access_token = None diff --git a/api/outdated/outdated/tests/test_api.py b/api/outdated/outdated/tests/test_api.py index 24a20d53..4fc9ec39 100644 --- a/api/outdated/outdated/tests/test_api.py +++ b/api/outdated/outdated/tests/test_api.py @@ -120,10 +120,15 @@ def test_version(client, version_factory): @pytest.mark.parametrize("defined", [True, False]) -def test_project(client, project_factory, version_factory, defined): - generated_project = project_factory( - versioned_dependencies=[version_factory()] if defined else [], +def test_project( + client, project_factory, dependency_source_factory, version_factory, defined +): + generated_project = project_factory() + + dependency_source_factory( + versions=[version_factory()] if defined else [], project=generated_project ) + url = reverse("project-list") resp = client.get(url) assert resp.status_code == http_status.HTTP_200_OK @@ -152,20 +157,20 @@ def test_project(client, project_factory, version_factory, defined): ) if defined: for gen_dep_version, resp_dep_version in zip( - resp_detailed.json()["data"]["relationships"]["versioned-dependencies"][ - "data" - ], - generated_project.versioned_dependencies.all(), + resp_detailed.json()["data"]["relationships"]["sources"]["data"], + generated_project.sources.all(), ): assert gen_dep_version["id"] == str(resp_dep_version.id) else: - assert not generated_project.versioned_dependencies.first() + assert not generated_project.sources.first().versions.first() + assert generated_project.sources.first().status == "UNDEFINED" assert generated_project.status == "UNDEFINED" def test_project_ordered_by_eol( client, project_factory, + dependency_source_factory, release_version_factory, version_factory, ): @@ -179,26 +184,24 @@ def test_project_ordered_by_eol( release_version=release_version_factory(up_to_date=True), ) - project_last = project_factory( - name="A project", - versioned_dependencies=[up_to_date_version], - ) - project_middle = project_factory( - name="B project", - versioned_dependencies=[warning_version], - ) - project_first = project_factory( - name="C project", - versioned_dependencies=[outdated_version], - ) + project_up_to_date = project_factory(name="A project") + dependency_source_factory(versions=[up_to_date_version], project=project_up_to_date) + project_warning = project_factory(name="B project") + dependency_source_factory(versions=[warning_version], project=project_warning) + project_outdated = project_factory(name="C project") + dependency_source_factory(versions=[outdated_version], project=project_outdated) + project_undefined = project_factory(name="D project") url = reverse("project-list") resp = client.get(url) + + assert resp.status_code == http_status.HTTP_200_OK json = resp.json() - assert json["data"][0]["id"] == str(project_first.pk) - assert json["data"][1]["id"] == str(project_middle.pk) - assert json["data"][2]["id"] == str(project_last.pk) + assert json["data"][0]["id"] == str(project_outdated.pk) + assert json["data"][1]["id"] == str(project_warning.pk) + assert json["data"][2]["id"] == str(project_up_to_date.pk) + assert json["data"][3]["id"] == str(project_undefined.pk) def test_maintainer(client, maintainer): @@ -216,15 +219,15 @@ def test_maintainer(client, maintainer): == str(maintainer.user.id) ) assert ( - relationships["project"]["data"]["id"] - == detailed_relationships["project"]["data"]["id"] - == str(maintainer.project.id) + relationships["source"]["data"]["id"] + == detailed_relationships["source"]["data"]["id"] + == str(maintainer.source.id) ) assert ( resp.json()["data"][0]["attributes"] == resp_detailed.json()["data"]["attributes"] ) - assert maintainer.project.maintainers.all()[0] == maintainer + assert maintainer.source.maintainers.all()[0] == maintainer @pytest.mark.django_db(transaction=True) diff --git a/api/outdated/outdated/tests/test_parser.py b/api/outdated/outdated/tests/test_parser.py index 8fc36e54..3e74d2da 100644 --- a/api/outdated/outdated/tests/test_parser.py +++ b/api/outdated/outdated/tests/test_parser.py @@ -2,6 +2,7 @@ import pytest +from outdated.outdated.models import Version from outdated.outdated.parser import LockfileParser from outdated.outdated.tracking import Tracker @@ -110,9 +111,9 @@ def test_parser(db, tmp_repo_root, project, lockfile, content, expected): assert len(lockfiles) == 1 assert lockfiles[0].name == lockfile - results = LockfileParser(lockfiles).parse() + LockfileParser(project, lockfiles).parse() - assert len(results) == len(expected) + assert len(project.sources.values_list("versions", flat=True)) == len(expected) - for result in results: - assert str(result) in expected + for result in project.sources.values_list("versions", flat=True): + assert str(Version.objects.get(id=result)) in expected diff --git a/api/outdated/outdated/tests/test_tracking.py b/api/outdated/outdated/tests/test_tracking.py index a7e8a5fa..e10ac490 100644 --- a/api/outdated/outdated/tests/test_tracking.py +++ b/api/outdated/outdated/tests/test_tracking.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from unittest.mock import PropertyMock, call import pytest @@ -214,6 +216,7 @@ def test_sync( exists, mocker, version_factory, + dependency_source_factory, ): project_path = tmp_repo_root / project.clone_path @@ -233,15 +236,17 @@ def test_sync( ) versions = version_factory.create_batch(5) + + def side_effect() -> None: + dependency_source_factory(versions=versions, project=project) + lockfile_parser_parser_mock = mocker.patch.object( - LockfileParser, - "parse", - return_value=versions, + LockfileParser, "parse", side_effect=side_effect ) tracker = Tracker(project) assert tracker.local_path == project_path - assert not project.versioned_dependencies.all() + assert not project.sources.all() if exists: project_path.mkdir(parents=True, exist_ok=False) @@ -254,13 +259,13 @@ def test_sync( tracker_checkout_mock.assert_called_once() - lockfile_parser_init_mock.assert_called_once_with([]) + lockfile_parser_init_mock.assert_called_once_with(project, []) lockfile_parser_parser_mock.assert_called_once_with() tracker_lockfile_mock.assert_called_once() - assert set(project.versioned_dependencies.all()) == set(versions) + assert set(project.sources.first().versions.all()) == set(versions) @pytest.mark.parametrize("exists", [True, False]) diff --git a/api/outdated/outdated/tracking.py b/api/outdated/outdated/tracking.py index 6a6db517..eb24fcb4 100644 --- a/api/outdated/outdated/tracking.py +++ b/api/outdated/outdated/tracking.py @@ -101,8 +101,7 @@ def sync(self): if not self.local_path.exists(): self.clone() self.checkout() - dependencies = LockfileParser(self.lockfiles).parse() - self.project.versioned_dependencies.set(dependencies) + LockfileParser(self.project, self.lockfiles).parse() def setup(self): # pragma: no cover self.clone() diff --git a/api/outdated/outdated/views.py b/api/outdated/outdated/views.py index 43f155df..36820b69 100644 --- a/api/outdated/outdated/views.py +++ b/api/outdated/outdated/views.py @@ -1,7 +1,7 @@ from django.db.models import Max from rest_framework.decorators import action from rest_framework.response import Response -from rest_framework.viewsets import ModelViewSet +from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from . import models, serializers from .tracking import Tracker @@ -11,7 +11,7 @@ class ProjectViewSet(ModelViewSet): queryset = ( models.Project.objects.all() .annotate( - latest_eol=Max("versioned_dependencies__release_version__end_of_life"), + latest_eol=Max("sources__versions__release_version__end_of_life"), ) .order_by("latest_eol") ) @@ -45,6 +45,10 @@ class DependencyViewSet(ModelViewSet): serializer_class = serializers.DependencySerializer +class DependencySourceViewSet(ReadOnlyModelViewSet): + queryset = models.DependencySource.objects.all() + + class MaintainerViewset(ModelViewSet): queryset = models.Maintainer.objects.all() serializer_class = serializers.MaintainerSerializer diff --git a/api/outdated/tests/test_unique_boolean_field.py b/api/outdated/tests/test_unique_boolean_field.py index d028fb8d..a7880a9d 100644 --- a/api/outdated/tests/test_unique_boolean_field.py +++ b/api/outdated/tests/test_unique_boolean_field.py @@ -6,7 +6,7 @@ def test_unique_boolean_field(db, maintainer_factory): assert Maintainer.objects.count() == 1 assert maintainer.is_primary - other_maintainer = maintainer_factory(project=maintainer.project) + other_maintainer = maintainer_factory(source=maintainer.source) assert Maintainer.objects.count() == 2 assert not other_maintainer.is_primary