Skip to content

Commit

Permalink
feat: Add alternative methods to create tasks and results separately
Browse files Browse the repository at this point in the history
  • Loading branch information
aneojgurhem committed May 4, 2023
1 parent f2c2378 commit 46cab83
Show file tree
Hide file tree
Showing 11 changed files with 652 additions and 23 deletions.
153 changes: 144 additions & 9 deletions Protos/V1/agent_common.proto
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ syntax = "proto3";

package armonik.api.grpc.v1.agent;

import "google/protobuf/timestamp.proto";
import "objects.proto";
import "result_status.proto";

option csharp_namespace = "ArmoniK.Api.gRPC.V1.Agent";

Expand All @@ -16,14 +18,15 @@ message CreateTaskRequest {
DataChunk task_payload = 3;
}

string communication_token = 4;
string communication_token = 4; /** Communication token received by the worker during task processing */
}

message CreateTaskReply {
message TaskInfo {
string task_id = 1;
repeated string expected_output_keys = 2;
repeated string data_dependencies = 3;
string task_id = 1; /** The task ID. */
repeated string expected_output_keys = 2; /** The expected output IDs. A task have expected output IDs. */
repeated string data_dependencies = 3; /** The data dependencies IDs (inputs). A task have data dependencies. */
string payload_id = 4; /** Unique ID of the result that will be used as payload. Results are created implicitly. */
}

message CreationStatus {
Expand All @@ -41,11 +44,11 @@ message CreateTaskReply {
CreationStatusList creation_status_list = 1;
string error = 2;
}
string communication_token = 4;
string communication_token = 4; /** Communication token received by the worker during task processing */
}

message DataRequest {
string communication_token = 1;
string communication_token = 1; /** Communication token received by the worker during task processing */
string key = 2;
}

Expand All @@ -57,7 +60,7 @@ message DataReply {
string error = 3;
}
}
string communication_token = 1;
string communication_token = 1; /** Communication token received by the worker during task processing */
oneof type {
Init init = 2;
DataChunk data = 3;
Expand All @@ -70,13 +73,145 @@ message Result {
InitKeyedDataStream init = 1;
DataChunk data = 2;
}
string communication_token = 3;
string communication_token = 3; /** Communication token received by the worker during task processing */
}

message ResultReply {
string communication_token = 3;
string communication_token = 3; /** Communication token received by the worker during task processing */
oneof type {
Empty Ok = 1;
string Error = 2;
}
}

/*
* Request for creating results without data
*/
message CreateResultsMetaDataRequest {
/**
* A result to create.
*/
message ResultCreate {
string name = 1; /** The result name. Given by the client. */
}
repeated ResultCreate results = 1; /** The list of results to create. */
string session_id = 2; /** The session in which create results. */
string communication_token = 3; /** Communication token received by the worker during task processing */
}

/**
* Result metadata
*/
message ResultMetaData {
string session_id = 1; /** The session ID. */
string result_id = 2; /** The result ID. */
string name = 3; /** The result name. */
result_status.ResultStatus status = 4; /** The result status. */
google.protobuf.Timestamp created_at = 5; /** The result creation date. */
}

/*
* Response for creating results without data
*/
message CreateResultsMetaDataResponse {
repeated ResultMetaData results = 1; /** The list of metadata results that were created. */
string communication_token = 2; /** Communication token received by the worker during task processing */
}

/**
* Request to create tasks.
*/
message SubmitTasksRequest {
message TaskCreation {
repeated string expected_output_keys = 1; /** Unique ID of the results that will be produced by the task. Results are created using ResultsService. */
repeated string data_dependencies = 2; /** Unique ID of the results that will be used as datadependencies. Results are created using ResultsService. */
string payload_id = 3; /** Unique ID of the result that will be used as payload. Results are created using ResultsService. */
TaskOptions task_options = 4; /** Optionnal task options. */
}

string session_id = 1; /** The session ID. */
TaskOptions task_options = 2; /** The options for the tasks. Each task will have the same. Options are merged with the one from the session. */
repeated TaskCreation task_creations = 3; /** Task creation requests. */
string communication_token = 4; /** Communication token received by the worker during task processing */
}

/**
* Response to create tasks.
*
* expected_output_ids and data_dependencies must be created through ResultsService.
*
* Remark : this may have to be enriched to a better management of errors but
* will the client application be able to manage a missing data dependency or expected output ?
*/
message SubmitTasksResponse {
message TaskInfo {
string task_id = 1; /** The task ID. */
repeated string expected_output_ids = 2; /** The expected output IDs. A task has expected output IDs. */
repeated string data_dependencies = 3; /** The data dependencies IDs (inputs). A task has data dependencies. */
string payload_id = 4; /** Unique ID of the result that will be used as payload. Results are created implicitly. */
}

repeated TaskInfo task_infos = 1; /** List of task infos if submission successful, else throw gRPC exception. */
string communication_token = 2; /** Communication token received by the worker during task processing */
}

/*
* Request for creating results without data
*/
message CreateResultsRequest {
/**
* A result to create.
*/
message ResultCreate {
bytes data = 1; /** The actual data of the result. */
string name = 2; /** The result name. Given by the client. */
}
repeated ResultCreate results = 1; /** The results to create. */
string session_id = 2; /** The session in which create results. */
string communication_token = 3; /** Communication token received by the worker during task processing */
}

/*
* Response for creating results without data
*/
message CreateResultsResponse {
repeated ResultMetaData results = 1; /** The raw results that were created. */
string communication_token = 2; /** Communication token received by the worker during task processing */
}

/*
* Request for uploading results data through stream.
* Data must be sent in multiple chunks.
* Only one result can be uploaded.
*/
message UploadResultDataRequest {
/**
* The metadata to identify the result to update.
*/
message ResultIdentifier {
string session_id = 1; /** The session of the result. */
string result_id = 2; /** The ID of the result. */
}

/**
* The possible messages that constitute a UploadResultDataRequest
* They should be sent in the following order:
* - id
* - data_chunk (stream can have multiple data_chunk messages that represent data divided in several parts)
*
* Data chunk cannot exceed the size returned by the GetServiceConfiguration rpc method
*/
oneof type {
ResultIdentifier id = 1; /** The identifier of the result to which add data. */
bytes data_chunk = 2; /** A chunk of data. */
}
string communication_token = 4; /** Communication token received by the worker during task processing */
}

/*
* Response for uploading data with stream for result
*/
message UploadResultDataResponse {
string result_id = 1; /** The Id of the result to which data were added */
string communication_token = 2; /** Communication token received by the worker during task processing */
}
21 changes: 21 additions & 0 deletions Protos/V1/agent_service.proto
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,27 @@ import "agent_common.proto";
option csharp_namespace = "ArmoniK.Api.gRPC.V1.Agent";

service Agent {
/**
* Create the metadata of multiple results at once
* Data have to be uploaded separately
*/
rpc CreateResultsMetaData(CreateResultsMetaDataRequest) returns (CreateResultsMetaDataResponse) {}

/**
* Create one result with data included in the request
*/
rpc CreateResults(CreateResultsRequest) returns (CreateResultsResponse) {}

/**
* Upload data for result with stream
*/
rpc UploadResultData(stream UploadResultDataRequest) returns (UploadResultDataResponse) {}

/**
* Create tasks metadata and submit task for processing.
*/
rpc SubmitTasks(SubmitTasksRequest) returns (SubmitTasksResponse) {}

rpc CreateTask(stream CreateTaskRequest) returns (CreateTaskReply);
rpc GetResourceData(DataRequest) returns (stream DataReply);
rpc GetCommonData(DataRequest) returns (stream DataReply);
Expand Down
11 changes: 6 additions & 5 deletions Protos/V1/objects.proto
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,10 @@ message Output {
}

message TaskRequest {
repeated string expected_output_keys = 1;
repeated string data_dependencies = 2;
bytes payload = 3;
repeated string expected_output_keys = 1; /** Given names to the expected outputs that will be created implicitly. IDs are returned after task creation */
repeated string data_dependencies = 2; /** IDs of the results that will be used as data dependency. */
bytes payload = 3; /** Content of the payload for the task. */
string payload_name = 4; /** Name that will be associated to the result created for the payload. Optionnal */
}

message InitKeyedDataStream {
Expand All @@ -61,8 +62,8 @@ message DataChunk {
}

message TaskRequestHeader {
repeated string expected_output_keys = 1;
repeated string data_dependencies = 2;
repeated string expected_output_keys = 1; /** Given names to the expected outputs that will be created implicitly. IDs are returned after task creation */
repeated string data_dependencies = 2; /** IDs of the results that will be used as data dependency. */
}

message InitTaskRequest {
Expand Down
Loading

0 comments on commit 46cab83

Please sign in to comment.