Starting integration with orchestration

.devcontainer/docker-compose.yml

@@ -9,6 +9,9 @@ services:
       - ..:/workspaces:cached
       - ./archive/log/backend:/archive/log
       - ./archive/data:/archive/data
+      - ./orchestration/pipelines:/pipelines
+      - ./orchestration/processes:/processes
+      - ./orchestration/datasets:/datasets
 
     # Overrides default command so things don't shut down after the process ends.
     command: sleep infinity

.gitignore

@@ -18,3 +18,11 @@ nginx.conf
 # Docker Compose 
 docker-compose.yml
 .env.local
+
+# Orchestration
+orchestration/db
+orchestration/processes
+orchestration/logs
+orchestration/rabbitmq/*
+!orchestration/rabbitmq/enabled_plugins
+

README.md

@@ -98,9 +98,51 @@ In the development environment it is not necessary to change Ngnix settings.
 But if a local change is needed, copy the `nginx_development.conf` file to `nginx.conf`
 Also change the `docker-compose.yml` file in the ngnix service at the line `- ./nginx_development.conf:/etc/nginx/conf.d/default.conf:ro`. In this way, the ngnix.conf file represents your local environment, if you make any modifications that are necessary for the project, copy this modification to the template file, as the nginx.conf file is not part of the repository.
 
+
+### Orchestration setup
+
+The Pz Server uses [orchestration](https://github.com/linea-it/orchestration/) to process its pipelines and for this you need to configure it:
+
+``` bash
+mkdir orchestration/db orchestration/logs orchestration/processes
+```
+
+The next step is to add a virtual host to your local machine. On Linux, this must be done by adding the line `127.0.0.1 orchestration` in the `/etc/host`. The file should look like this:
+
+``` bash
+127.0.0.1 localhost
+127.0.0.1 orchestration
+
+# The following lines are desirable for IPv6 capable hosts
+::1 ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+```
+
+Start the orchestration with the command:
+
+``` bash
+docker-compose up orchestration
+```
+
+And follow the procedure to add an authentication app in this [link](https://github.com/linea-it/orchestration?tab=readme-ov-file#how-to-use-using-client-credential). But be careful because when integrating with the Pz Server, the orchestration will have a different url than `http://localhost`, in this case it will be [http://orchestration/admin/oauth2_provider/application/add/](http://orchestration/admin/oauth2_provider/application/add/).
+
+Another important detail is that the `CLIENT ID` and `SECRET KEY` value from the previous procedure must be changed in the `.env` of the Pz Server, looking similar to this:
+
+``` bash
+# Client ID and Client Secret must be registered in Django Admin
+# after backend Setup, in the Django Oauth Applications interface
+ORC_CLIENT_ID=wD85gkYeqGEQvVWv5o3Cx6ppBlfDl2S88dek8Exp
+ORC_CLIENT_SECRET=eM2dhhxa2vovfaAXmMwqR1M8TdGhVmBjT7co5uaA9pI4aKPDZGxtBtDG5LHfhHvZUabbSP5aUDRpTLpUJAiGS0ScNuhktbuCwuSPiz0bmEftEROJ3ZzzKp2aDNO7Vx0k
+```
+
+This is enough to have orchestration working with an image pinned to `orchestration/docker-compose.yml`. If you want to change the orchestration version, just change the image in `orchestration/docker-compose.yml`
+
 Once this is done, the development environment setup process is complete.
 
-Finally, to start the whole application: 
+Finally, to start the whole application:
 
 ``` bash
 docker-compose up 
@@ -125,6 +167,7 @@ Go to Django ADMIN (for local installation, open a web browser and go to the URL
 
 The installation is done, you can now test the newly configured application.
 
+
 ### Some example commands
 
 Turn on background environment (if you have the application already running on the terminal, stop it with `CTRL + C` keys and up ir again, but in the background using `-d` argument):

backend/core/admin.py

@@ -1,11 +1,18 @@
-from core.models import (Product, ProductContent, ProductFile, ProductType,
-                         Profile, Release)
+from core.models import (Pipeline, Process, Product, ProductContent,
+                         ProductFile, ProductType, Profile, Release)
 from django import forms
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin
 from django.contrib.auth.models import User
 
 
+@admin.register(Process)
+class ProcessAdmin(admin.ModelAdmin):
+    list_display = ("id", "pipeline", "status", "user", "created_at")
+    exclude = ["path"]
+    search_fields = ("pipeline", "status")
+
+
 @admin.register(ProductType)
 class ProductTypeAdmin(admin.ModelAdmin):
     list_display = ("id", "name", "display_name", "created_at")
@@ -20,6 +27,13 @@ class ReleaseAdmin(admin.ModelAdmin):
     search_fields = ("name", "display_name")
 
 
+@admin.register(Pipeline)
+class PipelineAdmin(admin.ModelAdmin):
+    list_display = ("id", "name", "display_name", "created_at")
+
+    search_fields = ("name", "display_name")
+
+
 class ProductAdminForm(forms.ModelForm):
     class Meta:
         model = Product

backend/core/migrations/0029_pipeline.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.0.6 on 2024-05-27 21:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0028_productfile_created_productfile_updated'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Pipeline',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255)),
+                ('display_name', models.CharField(max_length=255)),
+                ('version', models.CharField(max_length=55)),
+                ('description', models.TextField(blank=True, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('system_config', models.JSONField(blank=True, null=True)),
+            ],
+        ),
+    ]

backend/core/migrations/0030_process.py

@@ -0,0 +1,35 @@
+# Generated by Django 5.0.6 on 2024-05-28 15:39
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0029_pipeline'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Process',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('pipeline_version', models.CharField(blank=True, default=None, max_length=255, null=True)),
+                ('used_config', models.JSONField(blank=True, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('started_at', models.DateTimeField(blank=True, null=True)),
+                ('ended_at', models.DateTimeField(blank=True, null=True)),
+                ('task_id', models.CharField(blank=True, default=None, max_length=255, null=True)),
+                ('status', models.CharField(default='Pending', max_length=55)),
+                ('path', models.FilePathField(blank=True, default=None, null=True, verbose_name='Path')),
+                ('comment', models.TextField(blank=True, null=True)),
+                ('inputs', models.ManyToManyField(related_name='processes', to='core.product')),
+                ('pipeline', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='processes', to='core.pipeline')),
+                ('release', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='processes', to='core.release')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='processes', to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

backend/core/migrations/0031_pipeline_product_types_accepted.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-05-29 15:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0030_process'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='pipeline',
+            name='product_types_accepted',
+            field=models.ManyToManyField(related_name='pipelines', to='core.producttype'),
+        ),
+    ]

backend/core/migrations/0032_process_upload_alter_process_inputs.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.0.6 on 2024-05-29 17:45
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0031_pipeline_product_types_accepted'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='process',
+            name='upload',
+            field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.CASCADE, related_name='upload', to='core.product'),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='process',
+            name='inputs',
+            field=models.ManyToManyField(related_name='inputs', to='core.product'),
+        ),
+    ]

backend/core/migrations/0033_alter_process_upload.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-05-29 17:55
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0032_process_upload_alter_process_inputs'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='process',
+            name='upload',
+            field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='upload', to='core.product'),
+        ),
+    ]

backend/core/migrations/0034_process_upload_product_type.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.0.6 on 2024-05-29 22:01
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0033_alter_process_upload'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='process',
+            name='upload_product_type',
+            field=models.ForeignKey(default=2, on_delete=django.db.models.deletion.CASCADE, related_name='upload_product_type', to='core.producttype'),
+            preserve_default=False,
+        ),
+    ]

backend/core/migrations/0035_remove_process_upload_product_type_and_more.py

@@ -0,0 +1,30 @@
+# Generated by Django 5.0.6 on 2024-05-31 14:53
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0034_process_upload_product_type'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='process',
+            name='upload_product_type',
+        ),
+        migrations.AddField(
+            model_name='pipeline',
+            name='output_product_type',
+            field=models.ForeignKey(default=2, on_delete=django.db.models.deletion.CASCADE, related_name='output_product_type', to='core.producttype'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='process',
+            name='display_name',
+            field=models.CharField(default='Test DN', max_length=255),
+            preserve_default=False,
+        ),
+    ]

backend/core/models/__init__.py

@@ -1,6 +1,8 @@
 from core.models.release import Release
 from core.models.product_type import ProductType
-from core.models.product import Product
+from core.models.product import Product, ProductStatus
 from core.models.product_content import ProductContent
 from core.models.product_file import ProductFile
 from core.models.user_profile import Profile
+from core.models.pipeline import Pipeline
+from core.models.process import Process

backend/core/models/pipeline.py

@@ -0,0 +1,24 @@
+from core.models import ProductType
+from django.db import models
+
+
+class Pipeline(models.Model):
+
+    name = models.CharField(max_length=255)
+    display_name = models.CharField(max_length=255)
+    version = models.CharField(max_length=55)
+    description = models.TextField(null=True, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+    system_config = models.JSONField(null=True, blank=True)
+    product_types_accepted= models.ManyToManyField(
+        ProductType, related_name="pipelines"
+    )
+    output_product_type = models.ForeignKey(
+        ProductType,
+        on_delete=models.CASCADE,
+        related_name="output_product_type",
+    )
+
+    def __str__(self):
+        return f"{self.display_name}"
+

backend/core/models/process.py

@@ -0,0 +1,64 @@
+import pathlib
+import shutil
+
+from core.models import Pipeline, Product, ProductStatus, Release
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.db import models
+
+
+class Process(models.Model):
+    display_name = models.CharField(max_length=255)
+    pipeline = models.ForeignKey(
+        Pipeline, on_delete=models.CASCADE, related_name="processes"
+    )
+    pipeline_version = models.CharField(
+        max_length=255, null=True, blank=True, default=None
+    )
+    used_config = models.JSONField(null=True, blank=True)
+    inputs = models.ManyToManyField(Product, related_name="inputs")
+    release = models.ForeignKey(
+        Release,
+        on_delete=models.CASCADE,
+        related_name="processes",
+        null=True,
+        blank=True,
+        default=None,
+    )
+    upload = models.OneToOneField(
+        Product,
+        on_delete=models.CASCADE,
+        related_name="upload",
+    )
+    user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="processes")
+    created_at = models.DateTimeField(auto_now_add=True)
+    started_at = models.DateTimeField(null=True, blank=True)
+    ended_at = models.DateTimeField(null=True, blank=True)
+    task_id = models.CharField(max_length=255, null=True, blank=True, default=None)
+    status = models.IntegerField(
+        verbose_name="Status",
+        default=ProductStatus.REGISTERING,
+        choices=ProductStatus.choices,
+    )
+    path = models.FilePathField(
+        verbose_name="Path", null=True, blank=True, default=None
+    )
+    comment = models.TextField(null=True, blank=True)
+
+    def __str__(self):
+        return f"{self.pipeline}-{str(self.pk).zfill(8)}"
+
+    def delete(self, *args, **kwargs):
+        process_path = pathlib.Path(settings.PROCESSING_DIR, str(self.path))
+        if process_path.exists():
+            self.rmtree(process_path)
+
+        super().delete(*args, **kwargs)
+
+    @staticmethod
+    def rmtree(process_path):
+        try:
+            # WARN: is not run by admin
+            shutil.rmtree(process_path)
+        except OSError as e:
+            raise OSError("Failed to remove directory: [ %s ] %s" % (process_path, e))