actinia_parallel_plugin User defined

ideas for a plugin for parallel processing chain organization and execution

actinia should be able to receive a list of PCs and execute them in parallel, e.g. to perform the same processing chain for different regions, or execute independent PCs at the same time

Main part is the jobs array. It is an array of objects with each object containing a "normal" process chain with list, version, optionally webhooks and additionally the attribute parallel. The value of parallel can either be true or false. The jobs array is read sequentially and will execute jobs in parallel if possible until an object is "parallel": "false". There it waits for the parallel jobs above and proceeds when all of them are finished. If after a non parallel job jobs with "parallel": "true" are following, they will be executed in parallel again.

Sample process chain list

{
    "processing_host": "{{ processing_host }}",
    "processing_platform_name": "{{ processing_platform_name }}",
    "jobs": [
        {
            "list": [
                {
                    "module": "my.module.1",
                    "id": "dummy_1",
                    "inputs":[
                        {"param": "mykey", "value": "myval"},
                        {"param": "mykey2", "value": "myval2"}
                    ]
                }
            ],
            "parallel": "true",
            "version": "1"
        },
        {
            "list": [
                {
                    "module": "my.module.2",
                    "id": "dummy_2",
                    "inputs":[
                        {"param": "mykey", "value": "myval"},
                        {"param": "mykey2", "value": "myval2"}
                    ]
                }
            ],
            "parallel": "true",
            "version": "1"
        },
        {
            "list": [
                {
                    "module": "my.module.3",
                    "id": "dummy_3",
                    "inputs":[
                        {"param": "mykey", "value": "myval"},
                        {"param": "mykey2", "value": "myval2"}
                    ]
                },
                {
                    "module": "my.module.4",
                    "id": "dummy_4",
                    "inputs":[
                        {"param": "mykey", "value": "myval"},
                        {"param": "mykey2", "value": "myval2"}
                    ]
                }
            ],
            "parallel": "false",
            "version": "1"
        }
    ]
}

Sample "actinia-gdi" response

{
	"actinia_core_jobid": "resource_id-30c4ea6b-599c-45d6-b904-aca852528fd7, resource_id-sfhjpfjds-599c-45d6-b904-dskfsads",
	"actinia_core_platform": "vm",
	"actinia_core_platform_name": "attm6arnrlmg33grn3tq",
	"actinia_core_response": {
        "summary": {
            "total": 3,
            "processed": {
                "accepted": 0,
                "running": 1,
                "finished": 1,
                "error": 0,
                "terminated": 0
            },
            "batch": [
                {
                    "parallel": 2,
                    "accepted": 0,
                    "running": 1,
                    "finished": 1,
                    "error": 0,
                    "terminated": 0
                },
                // {
                //     "parallel": 1,
                //     "accepted": 0,
                //     "running": 0,
                //     "finished": 0,
                //     "error": 0,
                //     "terminated": 0
                // }
            ]
        },
        "resource_id-30c4ea6b-599c-45d6-b904-aca852528fd7": {
            "accept_datetime": "2021-08-04 08:08:39.839136",
            "accept_timestamp": 1628064519.839131,
            "api_info": {
                "endpoint": "asyncephemeralexportresource",
                "method": "POST",
                "path": "/api/v1/locations/utm32n/processing_async_export",
                "request_url": "http://localhost:8088/api/v1/locations/utm32n/processing_async_export"
            },
            "datetime": "2021-08-04 08:12:31.154253",
            "exception": {},
            "http_code": 200,
            "message": "",
            "process_chain_list": [],
            "process_log": [],
            "progress": {
                "num_of_steps": 22,
                "step": 9
            },
            "resource_id": "resource_id-30c4ea6b-599c-45d6-b904-aca852528fd7",
            "status": "finished",
            "time_delta": 231.3151397705078,
            "timestamp": 1628064751.154248,
            "urls": {
                "resources": [],
                "status": "http://localhost:8088/api/v1/resources/actinia-gdi/resource_id-30c4ea6b-599c-45d6-b904-aca852528fd7"
            },
            "user_id": "actinia-gdi"
        },
        "resource_id-sfhjpfjds-599c-45d6-b904-dskfsads": {
            "accept_datetime": "2021-08-04 08:08:39.839136",
            "accept_timestamp": 1628064519.839131,
            "api_info": {
                "endpoint": "asyncephemeralexportresource",
                "method": "POST",
                "path": "/api/v1/locations/utm32n/processing_async_export",
                "request_url": "http://localhost:8088/api/v1/locations/utm32n/processing_async_export"
            },
            "datetime": "2021-08-04 08:12:31.154253",
            "exception": {},
            "http_code": 400,
            "message": "",
            "process_chain_list": [],
            "process_log": [],
            "progress": {
                "num_of_steps": 22,
                "step": 9
            },
            "resource_id": "resource_id-30c4ea6b-599c-45d6-b904-aca852528fd7",
            "status": "running",
            "time_delta": 231.3151397705078,
            "timestamp": 1628064751.154248,
            "urls": {
                "resources": [],
                "status": "http://localhost:8088/api/v1/resources/actinia-gdi/resource_id-30c4ea6b-599c-45d6-b904-aca852528fd7"
            },
            "user_id": "actinia-gdi"

        }
	},
	"actinia_core_url": "http://localhost:8088",
	"creation_uuid": "72d064ab-1083-4dc9-8494-a1a1c627f7cd",
	"feature_type": "",
	"idpk_jobs": 2704,
	"job_description": {},
	"metadata": null,
	"process": "automatedorthophotoclassification",
	"rule_configuration": {},
	"status": "RUNNING",
	"terraformer_id": null,
	"terraformer_response": null,
	"time_created": "Wed, 04 Aug 2021 08:08:29 GMT",
	"time_ended": "Wed, 04 Aug 2021 08:12:31 GMT",
	"time_estimated": null,
	"time_started": "Wed, 04 Aug 2021 08:08:46 GMT"
}

unsorted further ideas

• create new plugin and new endpoint
• main process chain must be able to communicate with the parallel processes --> use rabbitMQ/redis queue?
• In the first version the parallel parameter is needed for each process chain, later we maybe want to change it to: "If parallel is not set the default value false is used."