diff --git a/backend/core/admin.py b/backend/core/admin.py index ec23519..deb1193 100644 --- a/backend/core/admin.py +++ b/backend/core/admin.py @@ -4,19 +4,30 @@ from django.contrib import admin from django.contrib.auth.admin import UserAdmin from django.contrib.auth.models import User +from orchestration.models import Pipeline, Process + + +@admin.register(Pipeline) +class PipelineAdmin(admin.ModelAdmin): + list_display = ("id", "name", "display_name", "created_at") + search_fields = ("name", "display_name") + + +@admin.register(Process) +class ProcessAdmin(admin.ModelAdmin): + list_display = ("id", "task_id", "status", "started_at", "ended_at", "created_at") + search_fields = ("task_id", "status", "id") @admin.register(ProductType) class ProductTypeAdmin(admin.ModelAdmin): list_display = ("id", "name", "display_name", "created_at") - search_fields = ("name", "display_name") @admin.register(Release) class ReleaseAdmin(admin.ModelAdmin): list_display = ("id", "name", "display_name", "created_at") - search_fields = ("name", "display_name") @@ -44,7 +55,6 @@ class ProductAdmin(admin.ModelAdmin): ) search_fields = ("name", "display_name") - form = ProductAdminForm # This will help you to disbale add functionality @@ -68,8 +78,7 @@ def has_delete_permission(self, request, obj=None): @admin.register(ProductFile) class ProductFileAdmin(admin.ModelAdmin): - list_display = ("id", "product", "file", "role", - "type", "size", "extension") + list_display = ("id", "product", "file", "role", "type", "size", "extension") def has_add_permission(self, request): return False diff --git a/backend/core/serializers/__init__.py b/backend/core/serializers/__init__.py index ec9acd3..6730ca5 100644 --- a/backend/core/serializers/__init__.py +++ b/backend/core/serializers/__init__.py @@ -1,6 +1,6 @@ -from core.serializers.release import ReleaseSerializer -from core.serializers.product_type import ProductTypeSerializer -from core.serializers.product import ProductSerializer +from core.serializers.product import ProductSerializer, ProductSimpleSerializer from core.serializers.product_content import ProductContentSerializer from core.serializers.product_file import ProductFileSerializer +from core.serializers.product_type import ProductTypeSerializer +from core.serializers.release import ReleaseSerializer from core.serializers.user import UserSerializer diff --git a/backend/core/serializers/product.py b/backend/core/serializers/product.py index 3dd7c44..e6e6346 100644 --- a/backend/core/serializers/product.py +++ b/backend/core/serializers/product.py @@ -1,6 +1,6 @@ +from core.models import Product, ProductType, Release from pkg_resources import require from rest_framework import serializers -from core.models import Release, ProductType, Product class ProductSerializer(serializers.ModelSerializer): @@ -43,3 +43,11 @@ def get_is_owner(self, obj): return True else: return False + + +class ProductSimpleSerializer(ProductSerializer): + + class Meta: + model = Product + fields = ("id", "display_name", "internal_name") + diff --git a/backend/orchestration/__init__.py b/backend/orchestration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/orchestration/admin.py b/backend/orchestration/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/backend/orchestration/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/backend/orchestration/apps.py b/backend/orchestration/apps.py new file mode 100644 index 0000000..64b527b --- /dev/null +++ b/backend/orchestration/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class OrchestrationConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "orchestration" diff --git a/backend/orchestration/filters/__init__.py b/backend/orchestration/filters/__init__.py new file mode 100644 index 0000000..4bbc381 --- /dev/null +++ b/backend/orchestration/filters/__init__.py @@ -0,0 +1 @@ +from orchestration.filters.process import ProcessFilter diff --git a/backend/orchestration/filters/process.py b/backend/orchestration/filters/process.py new file mode 100644 index 0000000..b85b32a --- /dev/null +++ b/backend/orchestration/filters/process.py @@ -0,0 +1,48 @@ +from django.db.models import Q +from django_filters import rest_framework as filters +from orchestration.models import Process + + +class ProcessFilter(filters.FilterSet): + owner__or = filters.CharFilter(method="filter_user") + owner = filters.CharFilter(method="filter_user") + name__or = filters.CharFilter(method="filter_name") + name = filters.CharFilter(method="filter_name") + pipeline_name__or = filters.CharFilter(method="filter_pipeline_name") + pipeline_name = filters.CharFilter(method="filter_pipeline_name") + + class Meta: + model = Process + fields = ["pipeline_name", "name", "owner"] + + def filter_user(self, queryset, name, value): + query = self.format_query_to_char( + name, value, ["user__username", "user__first_name", "user__last_name"] + ) + + return queryset.filter(query) + + def filter_name(self, queryset, name, value): + query = self.format_query_to_char(name, value, ["name"]) + + return queryset.filter(query) + + def filter_pipeline_name(self, queryset, name, value): + query = self.format_query_to_char(name, value, ["pipeline_name"]) + + return queryset.filter(query) + + @staticmethod + def format_query_to_char(key, value, fields): + condition = Q.OR if key.endswith("__or") else Q.AND + values = value.split(",") + query = Q() + + for value in values: + subfilter = Q() + for field in fields: + subfilter.add(Q(**{f"{field}__icontains": value}), Q.OR) + + query.add(subfilter, condition) + + return query diff --git a/backend/orchestration/migrations/0001_initial.py b/backend/orchestration/migrations/0001_initial.py new file mode 100644 index 0000000..01ee506 --- /dev/null +++ b/backend/orchestration/migrations/0001_initial.py @@ -0,0 +1,36 @@ +# Generated by Django 4.0.2 on 2023-06-28 16:30 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Process', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('pipeline_name', models.CharField(max_length=255)), + ('pipeline_version', models.CharField(max_length=255)), + ('pipeline_config', models.JSONField()), + ('used_config', models.JSONField(blank=True, null=True)), + ('description', models.TextField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('started_at', models.DateTimeField(blank=True, null=True)), + ('stoped_at', models.DateTimeField(blank=True, null=True)), + ('task_id', models.CharField(max_length=255)), + ('status', models.IntegerField(choices=[(0, 'Successful'), (1, 'Pending'), (2, 'Running'), (3, 'Revoked'), (4, 'Failed')], default=1, verbose_name='Status')), + ('path', models.FilePathField(blank=True, default=None, null=True, verbose_name='Path')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='processes', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/backend/orchestration/migrations/0002_rename_stoped_at_process_ended_at_alter_process_name_and_more.py b/backend/orchestration/migrations/0002_rename_stoped_at_process_ended_at_alter_process_name_and_more.py new file mode 100644 index 0000000..5f32627 --- /dev/null +++ b/backend/orchestration/migrations/0002_rename_stoped_at_process_ended_at_alter_process_name_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.0.2 on 2023-06-28 17:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orchestration', '0001_initial'), + ] + + operations = [ + migrations.RenameField( + model_name='process', + old_name='stoped_at', + new_name='ended_at', + ), + migrations.AlterField( + model_name='process', + name='name', + field=models.CharField(blank=True, default=None, max_length=255, null=True), + ), + migrations.AlterField( + model_name='process', + name='pipeline_config', + field=models.JSONField(blank=True, null=True), + ), + migrations.AlterField( + model_name='process', + name='pipeline_version', + field=models.CharField(blank=True, default=None, max_length=255, null=True), + ), + migrations.AlterField( + model_name='process', + name='task_id', + field=models.CharField(blank=True, default=None, max_length=255, null=True), + ), + ] diff --git a/backend/orchestration/migrations/0003_rename_description_process_comment_and_more.py b/backend/orchestration/migrations/0003_rename_description_process_comment_and_more.py new file mode 100644 index 0000000..570a9d3 --- /dev/null +++ b/backend/orchestration/migrations/0003_rename_description_process_comment_and_more.py @@ -0,0 +1,62 @@ +# Generated by Django 4.0.2 on 2023-06-28 18:53 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0025_remove_product_survey'), + ('orchestration', '0002_rename_stoped_at_process_ended_at_alter_process_name_and_more'), + ] + + operations = [ + migrations.RenameField( + model_name='process', + old_name='description', + new_name='comment', + ), + migrations.RemoveField( + model_name='process', + name='name', + ), + migrations.RemoveField( + model_name='process', + name='pipeline_config', + ), + migrations.RemoveField( + model_name='process', + name='pipeline_name', + ), + migrations.AddField( + model_name='process', + name='inputs', + field=models.ManyToManyField(blank=True, default=None, null=True, related_name='processes', to='core.Product'), + ), + migrations.AddField( + model_name='process', + name='release', + field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='processes', to='core.release'), + ), + migrations.CreateModel( + name='Pipeline', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ('display_name', models.CharField(blank=True, max_length=255, null=True)), + ('version', models.CharField(blank=True, max_length=255, null=True)), + ('config', models.JSONField(blank=True, null=True)), + ('description', models.TextField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('use_release', models.BooleanField(default=False)), + ('product_types', models.ManyToManyField(blank=True, default=None, null=True, related_name='pipelines', to='core.ProductType')), + ], + ), + migrations.AddField( + model_name='process', + name='pipeline', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='processes', to='orchestration.pipeline'), + preserve_default=False, + ), + ] diff --git a/backend/orchestration/migrations/0004_remove_pipeline_config_remove_pipeline_version.py b/backend/orchestration/migrations/0004_remove_pipeline_config_remove_pipeline_version.py new file mode 100644 index 0000000..f065403 --- /dev/null +++ b/backend/orchestration/migrations/0004_remove_pipeline_config_remove_pipeline_version.py @@ -0,0 +1,21 @@ +# Generated by Django 4.0.2 on 2023-06-28 23:02 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('orchestration', '0003_rename_description_process_comment_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='pipeline', + name='config', + ), + migrations.RemoveField( + model_name='pipeline', + name='version', + ), + ] diff --git a/backend/orchestration/migrations/0005_alter_pipeline_product_types_alter_process_inputs.py b/backend/orchestration/migrations/0005_alter_pipeline_product_types_alter_process_inputs.py new file mode 100644 index 0000000..b23b912 --- /dev/null +++ b/backend/orchestration/migrations/0005_alter_pipeline_product_types_alter_process_inputs.py @@ -0,0 +1,24 @@ +# Generated by Django 4.0.2 on 2023-06-28 23:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0025_remove_product_survey'), + ('orchestration', '0004_remove_pipeline_config_remove_pipeline_version'), + ] + + operations = [ + migrations.AlterField( + model_name='pipeline', + name='product_types', + field=models.ManyToManyField(related_name='pipelines', to='core.ProductType'), + ), + migrations.AlterField( + model_name='process', + name='inputs', + field=models.ManyToManyField(related_name='processes', to='core.Product'), + ), + ] diff --git a/backend/orchestration/migrations/__init__.py b/backend/orchestration/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/orchestration/models/__init__.py b/backend/orchestration/models/__init__.py new file mode 100644 index 0000000..842e6ec --- /dev/null +++ b/backend/orchestration/models/__init__.py @@ -0,0 +1,2 @@ +from orchestration.models.pipeline import Pipeline +from orchestration.models.process import Process diff --git a/backend/orchestration/models/pipeline.py b/backend/orchestration/models/pipeline.py new file mode 100644 index 0000000..88cc8ba --- /dev/null +++ b/backend/orchestration/models/pipeline.py @@ -0,0 +1,14 @@ +from core.models import ProductType +from django.db import models + + +class Pipeline(models.Model): + name = models.CharField(max_length=255, unique=True) + display_name = models.CharField(max_length=255, null=True, blank=True) + description = models.TextField(null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + use_release = models.BooleanField(default=False) + product_types = models.ManyToManyField(ProductType, related_name="pipelines") + + def __str__(self): + return f"{self.name}" diff --git a/backend/orchestration/models/process.py b/backend/orchestration/models/process.py new file mode 100644 index 0000000..d4e526d --- /dev/null +++ b/backend/orchestration/models/process.py @@ -0,0 +1,67 @@ +import pathlib +import shutil + +from core.models import Product, Release +from django.conf import settings +from django.contrib.auth.models import User +from django.db import models +from orchestration.models import Pipeline + + +class ProcessStatus(models.IntegerChoices): + SUCCESSFUL = 0, "Successful" + PENDING = 1, "Pending" + RUNNING = 2, "Running" + REVOKED = 3, "Revoked" + FAILED = 4, "Failed" + + +class Process(models.Model): + pipeline = models.ForeignKey( + Pipeline, on_delete=models.CASCADE, related_name="processes" + ) + pipeline_version = models.CharField( + max_length=255, null=True, blank=True, default=None + ) + used_config = models.JSONField(null=True, blank=True) + inputs = models.ManyToManyField(Product, related_name="processes") + release = models.ForeignKey( + Release, + on_delete=models.CASCADE, + related_name="processes", + null=True, + blank=True, + default=None, + ) + user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="processes") + created_at = models.DateTimeField(auto_now_add=True) + started_at = models.DateTimeField(null=True, blank=True) + ended_at = models.DateTimeField(null=True, blank=True) + task_id = models.CharField(max_length=255, null=True, blank=True, default=None) + status = models.IntegerField( + verbose_name="Status", + default=ProcessStatus.PENDING, + choices=ProcessStatus.choices, + ) + path = models.FilePathField( + verbose_name="Path", null=True, blank=True, default=None + ) + comment = models.TextField(null=True, blank=True) + + def __str__(self): + return f"{self.pk}" + + def delete(self, *args, **kwargs): + process_path = pathlib.Path(settings.PROCESSING_DIR, str(self.path)) + if process_path.exists(): + self.rmtree(process_path) + + super().delete(*args, **kwargs) + + @staticmethod + def rmtree(process_path): + try: + # WARN: is not run by admin + shutil.rmtree(process_path) + except OSError as e: + raise OSError("Failed to remove directory: [ %s ] %s" % (process_path, e)) diff --git a/backend/orchestration/serializers/__init__.py b/backend/orchestration/serializers/__init__.py new file mode 100644 index 0000000..a84e9e9 --- /dev/null +++ b/backend/orchestration/serializers/__init__.py @@ -0,0 +1,4 @@ +from orchestration.serializers.pipeline import (PipelineDetailSerializer, + PipelineSerializer, + PipelineSimpleSerializer) +from orchestration.serializers.process import ProcessSerializer diff --git a/backend/orchestration/serializers/pipeline.py b/backend/orchestration/serializers/pipeline.py new file mode 100644 index 0000000..84eff29 --- /dev/null +++ b/backend/orchestration/serializers/pipeline.py @@ -0,0 +1,36 @@ +from core.models import ProductType +from orchestration.models import Pipeline +from orchestration.utils import get_pipeline_config, get_pipeline_version +from rest_framework import serializers + + +class PipelineSerializer(serializers.ModelSerializer): + product_types = serializers.PrimaryKeyRelatedField( + queryset=ProductType.objects.all(), many=True + ) + + class Meta: + model = Pipeline + read_only_fields = ( + "display_name", + "created_at", + ) + exclude = ["description"] + + +class PipelineDetailSerializer(PipelineSerializer): + config = serializers.SerializerMethodField() + version = serializers.SerializerMethodField() + + def get_config(self, obj): + return get_pipeline_config(obj.name) + + def get_version(self, obj): + return get_pipeline_version(obj.name) + + +class PipelineSimpleSerializer(PipelineDetailSerializer): + + class Meta: + model = Pipeline + fields = ("id", "name", "display_name", "version") diff --git a/backend/orchestration/serializers/process.py b/backend/orchestration/serializers/process.py new file mode 100644 index 0000000..3689347 --- /dev/null +++ b/backend/orchestration/serializers/process.py @@ -0,0 +1,32 @@ +from core.models import Product +from orchestration.models import Pipeline, Process +from orchestration.serializers import PipelineSimpleSerializer +from rest_framework import serializers + + +class ProcessSerializer(serializers.ModelSerializer): + owner = serializers.SerializerMethodField() + pipeline_name = serializers.SerializerMethodField() + inputs = serializers.PrimaryKeyRelatedField( + queryset=Product.objects.all(), many=True + ) + deph = 1 + + class Meta: + model = Process + read_only_fields = ( + "pipeline_version", + "started_at", + "ended_at", + "status", + "user", + "task_id", + "comment", + ) + exclude = ["path"] + + def get_owner(self, obj): + return obj.user.username + + def get_pipeline_name(self, obj): + return obj.pipeline.name \ No newline at end of file diff --git a/backend/orchestration/tests.py b/backend/orchestration/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/backend/orchestration/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/backend/orchestration/urls.py b/backend/orchestration/urls.py new file mode 100644 index 0000000..4b6009b --- /dev/null +++ b/backend/orchestration/urls.py @@ -0,0 +1,3 @@ +from django.urls import include, path +from orchestration.views import ProcessViewSet +from orchestration.views.process import index diff --git a/backend/orchestration/utils.py b/backend/orchestration/utils.py new file mode 100644 index 0000000..830d49a --- /dev/null +++ b/backend/orchestration/utils.py @@ -0,0 +1,71 @@ +import json +import logging +import pathlib +from collections import namedtuple +from typing import Any, Dict, List, NamedTuple, Type, Union + +import yaml +from django.conf import settings + +PIPE_PATHFILE = pathlib.Path(settings.PIPELINES_DIR, settings.PIPELINES_FILE) + +Json = Union[Dict[str, Any], List[Any], int, str, float, bool, Type[None]] + + +def read_yaml(yaml_file: str) -> Dict[str, Any]: + with open(yaml_file, encoding="UTF-8") as _file: + data = yaml.load(_file, Loader=yaml.loader.SafeLoader) + return data + + +def dict_to_namedtuple(data: Dict[str, Any], label="Data") -> NamedTuple: + _data = namedtuple(label, data) + return _data(**data) + + +def dict_to_json(data: Dict[str, Any], indent: int = 4) -> Json: + _data = json.dumps(data, indent=indent) + return _data + + +def get_pipelines() -> Dict[str, Any]: + if not PIPE_PATHFILE.is_file(): + raise FileNotFoundError(f"Pipeline list not found: {str(PIPE_PATHFILE)}") + + with open(PIPE_PATHFILE, encoding="UTF-8") as _file: + pipelines = yaml.load(_file, Loader=yaml.loader.SafeLoader) + return pipelines + + +def get_pipeline(name: str) -> Dict[str, Any]: + pipelines = get_pipelines() + pipeline = pipelines.get(name, None) + if not pipeline: + raise NotImplemented(f"Pipeline {name} not implemented") + pipeline["basepath"] = pathlib.Path(settings.PIPELINES_DIR, name) + return pipeline + + +def get_pipeline_config(pipeline_name: str) -> Dict[str, Any]: + pipeline = get_pipeline(pipeline_name) + filename = pipeline.get("config", "") + basepath = pipeline.get("basepath", "") + config_path = pathlib.Path(basepath, filename) + if not config_path.is_file(): + raise NotImplemented(f"Pipeline {pipeline_name}: config not implemented") + return read_yaml(str(config_path)) + + +def merge_config(config_file: str, user_config: Dict[str, Any]) -> Dict[str, Any]: + config = read_yaml(config_file) + logging.debug("User config: %s", user_config) + logging.debug("Default config: %s", config) + config.update(user_config) + logging.debug("Merge config: %s", config) + + return config + + +def get_pipeline_version(pipeline_name: str) -> str: + pipeline = get_pipeline(pipeline_name) + return pipeline.get("version", "") diff --git a/backend/orchestration/views/__init__.py b/backend/orchestration/views/__init__.py new file mode 100644 index 0000000..d7c1ac0 --- /dev/null +++ b/backend/orchestration/views/__init__.py @@ -0,0 +1,2 @@ +from orchestration.views.pipeline import PipelineViewSet +from orchestration.views.process import ProcessViewSet diff --git a/backend/orchestration/views/pipeline.py b/backend/orchestration/views/pipeline.py new file mode 100644 index 0000000..e9d20d5 --- /dev/null +++ b/backend/orchestration/views/pipeline.py @@ -0,0 +1,60 @@ +from django.db import transaction +from orchestration.models import Pipeline +from orchestration.serializers import (PipelineDetailSerializer, + PipelineSerializer) +from orchestration.utils import get_pipeline, get_pipeline_config +from rest_framework import status, viewsets +from rest_framework.response import Response + + +class PipelineViewSet(viewsets.ModelViewSet): + queryset = Pipeline.objects.all() + serializer_class = PipelineSerializer + search_fields = ["name", "display_name"] + ordering_fields = [ + "id", + "name", + "display_name", + "created_at", + ] + ordering = ["-created_at"] + + def get_serializer_class(self): + if self.action == "retrieve": + return PipelineDetailSerializer + return PipelineSerializer + + def create(self, request): + if not request.user.profile.is_admin(): + content = {"error": "User is not admin"} + return Response(content, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + pipeline_name = request.data["name"] + + # WARN: the pipeline must be installed and tested in the infra + # before registering in the database + try: + # gets pipeline information from $PIPELINES_DIR + pipeline_system = get_pipeline(pipeline_name) + except Exception as err: + content = { + "error": str(err), + "detail": f"Pipeline {pipeline_name} not instaled", + } + return Response(content, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + serializer = self.get_serializer(data=request.data) + + try: + with transaction.atomic(): + serializer.is_valid(raise_exception=True) + instance = serializer.save() + pipeline = Pipeline.objects.get(pk=instance.pk) + pipeline.display_name = pipeline_system.get("dname", None) + pipeline.save() + + data = self.get_serializer(instance=pipeline).data + return Response(data, status=status.HTTP_201_CREATED) + except Exception as err: + content = {"error": str(err), "detail": ""} + return Response(content, status=status.HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/backend/orchestration/views/process.py b/backend/orchestration/views/process.py new file mode 100644 index 0000000..10e30d8 --- /dev/null +++ b/backend/orchestration/views/process.py @@ -0,0 +1,66 @@ +import pathlib + +from django.conf import settings +from django.db import transaction +from orchestration.filters import ProcessFilter +from orchestration.models import Process +from orchestration.serializers import ProcessSerializer +from orchestration.utils import get_pipeline_version +from rest_framework import exceptions, status, viewsets +from rest_framework.response import Response + + +class ProcessViewSet(viewsets.ModelViewSet): + queryset = Process.objects.all() + serializer_class = ProcessSerializer + search_fields = [ + "pipeline__name", + "pipeline__display_name", + "user__username", + "user__first_name", + "user__last_name", + ] + filterset_class = ProcessFilter + ordering_fields = [ + "id", + "status", + "created_at", + "started_at", + "ended_at", + ] + ordering = ["-created_at"] + + def perform_create(self, serializer): + return serializer.save(user=self.request.user) + + def create(self, request): + serializer = self.get_serializer(data=request.data) + + try: + with transaction.atomic(): + serializer.is_valid(raise_exception=True) + instance = self.perform_create(serializer) + process = Process.objects.get(pk=instance.pk) + + # fill the current pipeline version + process.pipeline_version = get_pipeline_version(process.pipeline.name) + + # create process path + process.path = str(instance.pk).zfill(8) + path = pathlib.Path(settings.PROCESSING_DIR, process.path) + path.mkdir(parents=True, exist_ok=True) + + process.save() + + data = self.get_serializer(instance=process).data + return Response(data, status=status.HTTP_201_CREATED) + except Exception as e: + content = {"error": str(e)} + return Response(content, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def destroy(self, request, pk=None, *args, **kwargs): + instance = self.get_object() + if request.user.id == instance.user.pk: + return super(ProcessViewSet, self).destroy(request, pk, *args, **kwargs) + else: + raise exceptions.PermissionDenied() diff --git a/backend/pzserver/settings.py b/backend/pzserver/settings.py index 8ceec74..90d42fe 100644 --- a/backend/pzserver/settings.py +++ b/backend/pzserver/settings.py @@ -51,6 +51,7 @@ "shibboleth", # Apps "core", + "orchestration", ] MIDDLEWARE = [ @@ -135,6 +136,10 @@ MEDIA_URL = "/archive/data/" MEDIA_ROOT = "/archive/data/" +# pipelines processing +PROCESSING_DIR = os.getenv("PROCESSING_DIR", "/archive/processing") +PIPELINES_DIR = os.getenv("PIPELINES_DIR", "/archive/pipelines") +PIPELINES_FILE = os.getenv("PIPELINES_FILE", "pipelines.yml") # Default primary key field type # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field @@ -213,8 +218,7 @@ } SHIBBOLETH_GROUP_ATTRIBUTES = "Shibboleth" # Including Shibboleth authentication: - AUTHENTICATION_BACKENDS += ( - "shibboleth.backends.ShibbolethRemoteUserBackend",) + AUTHENTICATION_BACKENDS += ("shibboleth.backends.ShibbolethRemoteUserBackend",) SHIBBOLETH_ENABLED = True diff --git a/backend/pzserver/test_settings.py b/backend/pzserver/test_settings.py index 451943c..04571e7 100644 --- a/backend/pzserver/test_settings.py +++ b/backend/pzserver/test_settings.py @@ -51,6 +51,7 @@ "shibboleth", # Apps "core", + "orchestration", ] MIDDLEWARE = [ @@ -128,8 +129,13 @@ STATIC_URL = "/django_static/" STATIC_ROOT = os.path.join(BASE_DIR, "django_static") -MEDIA_URL = os.path.join(BASE_DIR, "archive/data/") -MEDIA_ROOT = os.path.join(BASE_DIR, "archive/data/") +MEDIA_URL = os.path.join(BASE_DIR, "/archive/data/") +MEDIA_ROOT = os.path.join(BASE_DIR, "/archive/data/") + +# pipelines processing +PROCESSING_DIR = os.getenv("PROCESSING_DIR", "/archive/processing") +PIPELINES_DIR = os.getenv("PIPELINES_DIR", "/archive/pipelines") +PIPELINES_FILE = os.getenv("PIPELINES_FILE", "pipelines.yml") # Default primary key field type @@ -200,8 +206,7 @@ } SHIBBOLETH_GROUP_ATTRIBUTES = "Shibboleth" # Including Shibboleth authentication: - AUTHENTICATION_BACKENDS += ( - "shibboleth.backends.ShibbolethRemoteUserBackend",) + AUTHENTICATION_BACKENDS += ("shibboleth.backends.ShibbolethRemoteUserBackend",) SHIBBOLETH_ENABLED = True diff --git a/backend/pzserver/urls.py b/backend/pzserver/urls.py index 9bbc18f..c90b0e0 100644 --- a/backend/pzserver/urls.py +++ b/backend/pzserver/urls.py @@ -13,7 +13,6 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ -# from core.api import viewsets as products_viewsets from core.views import ( CsrfToOauth, GetToken, @@ -33,8 +32,12 @@ SpectacularRedocView, SpectacularSwaggerView, ) +from orchestration.views import PipelineViewSet, ProcessViewSet from rest_framework import routers +# from core.api import viewsets as products_viewsets +from rest_framework.authtoken import views + route = routers.DefaultRouter() route.register(r"users", UserViewSet, basename="users") @@ -43,10 +46,10 @@ route.register(r"products", ProductViewSet, basename="products") route.register(r"product-contents", ProductContentViewSet, basename="product_contents") route.register(r"product-files", ProductFileViewSet, basename="product_files") +route.register(r"processes", ProcessViewSet, basename="processes") +route.register(r"pipelines", PipelineViewSet, basename="pipelines") -from rest_framework.authtoken import views - urlpatterns = [ path("admin/", admin.site.urls), path("api/", include(route.urls)),