From bba6292a29076f99ff160ad813f1148e805602e2 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 15 Apr 2025 16:52:12 +0200 Subject: [PATCH 01/22] core: rename type to scope, #TASK-7610 --- .../analysis/tools/WorkflowExecutorTest.java | 4 ++-- .../executors/WorkflowsCommandExecutor.java | 4 ++-- .../main/options/WorkflowsCommandOptions.java | 8 ++++---- .../catalog/db/api/WorkflowDBAdaptor.java | 2 +- .../db/mongodb/WorkflowMongoDBAdaptor.java | 4 ++-- .../catalog/managers/WorkflowManager.java | 2 +- .../catalog/managers/WorkflowManagerTest.java | 14 ++++++------- .../opencga/core/api/FieldConstants.java | 3 ++- .../core/models/workflow/Workflow.java | 20 +++++++++---------- .../models/workflow/WorkflowCreateParams.java | 20 +++++++++---------- .../models/workflow/WorkflowUpdateParams.java | 18 ++++++++--------- 11 files changed, 50 insertions(+), 49 deletions(-) diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/WorkflowExecutorTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/WorkflowExecutorTest.java index 3d913f18e18..a1f9e67599d 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/WorkflowExecutorTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/WorkflowExecutorTest.java @@ -87,7 +87,7 @@ public void nextflowDockerTest() throws ToolException, CatalogException, IOExcep StorageConfiguration storageConfiguration = StorageConfiguration.load(inputStream, "yml"); WorkflowCreateParams workflow = new WorkflowCreateParams() .setId("workflow") - .setType(Workflow.Type.OTHER) + .setScope(Workflow.Scope.OTHER) .setVariables(Arrays.asList( new WorkflowVariable() .setId("input") @@ -147,7 +147,7 @@ private WorkflowCreateParams createDummyWorkflow(String pipelineId) throws IOExc String content = IOUtils.toString(inputStream, "UTF-8"); return new WorkflowCreateParams() .setId("workflow") - .setType(Workflow.Type.OTHER) + .setScope(Workflow.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", content, true))); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java index 967a6e6fcd7..61c280f72db 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java @@ -164,7 +164,7 @@ private RestResponse create() throws Exception { putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); putNestedIfNotNull(beanParams, "manager.id", commandOptions.managerId, true); putNestedIfNotEmpty(beanParams, "manager.version", commandOptions.managerVersion, true); - putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotNull(beanParams, "scope", commandOptions.scope, true); putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); putNestedIfNotNull(beanParams, "draft", commandOptions.draft, true); putNestedIfNotEmpty(beanParams, "repository.id", commandOptions.repositoryId, true); @@ -350,7 +350,7 @@ private RestResponse update() throws Exception { putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); putNestedIfNotNull(beanParams, "manager.id", commandOptions.managerId, true); putNestedIfNotEmpty(beanParams, "manager.version", commandOptions.managerVersion, true); - putNestedIfNotNull(beanParams, "type", commandOptions.type, true); + putNestedIfNotNull(beanParams, "scope", commandOptions.scope, true); putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); putNestedIfNotNull(beanParams, "draft", commandOptions.draft, true); putNestedIfNotEmpty(beanParams, "repository.id", commandOptions.repositoryId, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java index d2f42a95541..05b8fe9aef9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java @@ -130,8 +130,8 @@ public class CreateCommandOptions { @Parameter(names = {"--manager-version"}, description = "Workflow system id. Valid values: NEXTFLOW.", required = false, arity = 1) public String managerVersion; - @Parameter(names = {"--type"}, description = "Workflow type. Valid values: NEXTFLOW.", required = false, arity = 1) - public String type; + @Parameter(names = {"--scope"}, description = "Workflow scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) + public String scope; @Parameter(names = {"--tags"}, description = "List of tags.", required = false, arity = 1) public String tags; @@ -404,8 +404,8 @@ public class UpdateCommandOptions { @Parameter(names = {"--manager-version"}, description = "Workflow system id. Valid values: NEXTFLOW.", required = false, arity = 1) public String managerVersion; - @Parameter(names = {"--type"}, description = "Workflow type. Valid values: NEXTFLOW.", required = false, arity = 1) - public String type; + @Parameter(names = {"--scope"}, description = "Workflow scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) + public String scope; @Parameter(names = {"--tags"}, description = "List of tags.", required = false, arity = 1) public String tags; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/WorkflowDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/WorkflowDBAdaptor.java index 286756af2b3..f22fe3223bb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/WorkflowDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/WorkflowDBAdaptor.java @@ -23,7 +23,7 @@ enum QueryParams implements QueryParam { NAME("name", TEXT, ""), DESCRIPTION("description", TEXT, ""), DRAFT("draft", BOOLEAN, ""), - TYPE("type", TEXT, ""), + SCOPE("scope", TEXT, ""), TAGS("tags", TEXT_ARRAY, ""), COMMAND_LINE("commandLine", TEXT, ""), MANAGER("manager", OBJECT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/WorkflowMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/WorkflowMongoDBAdaptor.java index 98dc23620e4..7817bcdc8b1 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/WorkflowMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/WorkflowMongoDBAdaptor.java @@ -332,7 +332,7 @@ private UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, QueryO filterBooleanParams(parameters, document.getSet(), acceptedBooleanParams); final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.DESCRIPTION.key(), QueryParams.COMMAND_LINE.key(), - QueryParams.TYPE.key()}; + QueryParams.SCOPE.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); final String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key()}; @@ -573,7 +573,7 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) case VERSION: case INTERNAL_REGISTRATION_USER_ID: case MANAGER_ID: - case TYPE: + case SCOPE: case DRAFT: addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); break; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java index c81d6b98cd2..5740c4a86b0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java @@ -911,7 +911,7 @@ private void validateNewWorkflow(Workflow workflow, String userId) throws Catalo if (workflow.getManager().getId() == null) { workflow.getManager().setId(WorkflowSystem.SystemId.NEXTFLOW); } - workflow.setType(ParamUtils.defaultObject(workflow.getType(), Workflow.Type.OTHER)); + workflow.setScope(ParamUtils.defaultObject(workflow.getScope(), Workflow.Scope.OTHER)); workflow.setTags(workflow.getTags() != null ? workflow.getTags() : Collections.emptyList()); workflow.setScripts(workflow.getScripts() != null ? workflow.getScripts() : Collections.emptyList()); boolean main = false; diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java index 383160b4dca..e45c502592c 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java @@ -60,7 +60,7 @@ public void importWorkflow() throws CatalogException { public void createWorkflowTest() throws CatalogException { Workflow workflow = new Workflow() .setId("workflow") - .setType(Workflow.Type.OTHER) + .setScope(Workflow.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); OpenCGAResult result = workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); @@ -105,13 +105,13 @@ public void createWorkflowTest() throws CatalogException { public void workflowSearchTest() throws CatalogException { Workflow workflow = new Workflow() .setId("workflow") - .setType(Workflow.Type.OTHER) + .setScope(Workflow.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken); workflow = new Workflow() .setId("workflow2") - .setType(Workflow.Type.OTHER) + .setScope(Workflow.Scope.OTHER) .setDraft(true) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken); @@ -136,13 +136,13 @@ public void workflowSearchTest() throws CatalogException { public void updateWorkflowTest() throws CatalogException { Workflow workflow = new Workflow() .setId("workflow") - .setType(Workflow.Type.OTHER) + .setScope(Workflow.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken); WorkflowUpdateParams updateParams = new WorkflowUpdateParams() .setName("newName") - .setType(Workflow.Type.OTHER) + .setScope(Workflow.Scope.OTHER) .setDraft(true) .setCreationDate("20240101000000") .setModificationDate("20240201000000") @@ -162,7 +162,7 @@ public void updateWorkflowTest() throws CatalogException { public void deleteWorkflowTest() throws CatalogException { Workflow workflow = new Workflow() .setId("workflow") - .setType(Workflow.Type.OTHER) + .setScope(Workflow.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); workflowManager.create(studyFqn, workflow, QueryOptions.empty(), ownerToken); @@ -178,7 +178,7 @@ public void deleteWorkflowTest() throws CatalogException { public void updateWorkflowAclTest() throws CatalogException { Workflow workflow = new Workflow() .setId("workflow") - .setType(Workflow.Type.OTHER) + .setScope(Workflow.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); workflowManager.create(studyFqn, workflow, QueryOptions.empty(), ownerToken); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 8b2843a67b1..82f64bb628a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -338,7 +338,8 @@ public class FieldConstants { // Workflow public static final String WORKFLOW_ID_DESCRIPTION = "Workflow ID."; public static final String WORKFLOW_NAME_DESCRIPTION = "Workflow name."; - public static final String WORKFLOW_TYPE_DESCRIPTION = "Workflow type. Valid values: NEXTFLOW."; + public static final String WORKFLOW_SCOPE_DESCRIPTION = "Workflow scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, " + + "CLINICAL_INTERPRETATION_ANALYSIS or OTHER."; public static final String WORKFLOW_MANAGER_DESCRIPTION = "Workflow system corresponding to the workflow."; public static final String WORKFLOW_DRAFT_DESCRIPTION = "Flag indicating whether the workflow is still a draft or not."; public static final String WORKFLOW_REPOSITORY_DESCRIPTION = "Workflow repository image to execute. If any, providing a list of" diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/Workflow.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/Workflow.java index f10445b374b..8d4e85cd38d 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/Workflow.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/Workflow.java @@ -26,8 +26,8 @@ public class Workflow extends PrivateStudyUid { @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) private String description; - @DataField(id = "type", description = FieldConstants.WORKFLOW_TYPE_DESCRIPTION) - private Type type; + @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) + private Scope scope; @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) private WorkflowSystem manager; @@ -68,7 +68,7 @@ public class Workflow extends PrivateStudyUid { @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) private Map attributes; - public enum Type { + public enum Scope { SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS, @@ -78,7 +78,7 @@ public enum Type { public Workflow() { } - public Workflow(String id, String name, String description, Type type, WorkflowSystem manager, List tags, + public Workflow(String id, String name, String description, Scope scope, WorkflowSystem manager, List tags, List variables, MinimumRequirements minimumRequirements, boolean draft, WorkflowRepository repository, List scripts, WorkflowInternal internal, String creationDate, String modificationDate, Map attributes) { @@ -86,7 +86,7 @@ public Workflow(String id, String name, String description, Type type, WorkflowS this.name = name; this.description = description; this.draft = draft; - this.type = type; + this.scope = scope; this.manager = manager; this.repository = repository; this.scripts = scripts; @@ -106,7 +106,7 @@ public String toString() { sb.append(", uuid='").append(uuid).append('\''); sb.append(", name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); - sb.append(", type=").append(type); + sb.append(", scope=").append(scope); sb.append(", manager=").append(manager); sb.append(", tags=").append(tags); sb.append(", variables=").append(variables); @@ -194,12 +194,12 @@ public Workflow setRelease(int release) { return this; } - public Type getType() { - return type; + public Scope getScope() { + return scope; } - public Workflow setType(Type type) { - this.type = type; + public Workflow setScope(Scope scope) { + this.scope = scope; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowCreateParams.java index 93e3c84a46b..1d7971fd957 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowCreateParams.java @@ -22,8 +22,8 @@ public class WorkflowCreateParams { @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) private WorkflowSystem manager; - @DataField(id = "type", description = FieldConstants.WORKFLOW_TYPE_DESCRIPTION) - private Workflow.Type type; + @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) + private Workflow.Scope scope; @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) private List tags; @@ -55,7 +55,7 @@ public class WorkflowCreateParams { public WorkflowCreateParams() { } - public WorkflowCreateParams(String id, String name, String description, WorkflowSystem manager, Workflow.Type type, List tags, + public WorkflowCreateParams(String id, String name, String description, WorkflowSystem manager, Workflow.Scope scope, List tags, boolean draft, WorkflowRepository repository, List variables, MinimumRequirements minimumRequirements, List scripts, String creationDate, String modificationDate, Map attributes) { @@ -63,7 +63,7 @@ public WorkflowCreateParams(String id, String name, String description, Workflow this.name = name; this.description = description; this.manager = manager; - this.type = type; + this.scope = scope; this.tags = tags; this.draft = draft; this.repository = repository; @@ -82,7 +82,7 @@ public String toString() { sb.append(", name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); sb.append(", manager=").append(manager); - sb.append(", type=").append(type); + sb.append(", scope=").append(scope); sb.append(", tags=").append(tags); sb.append(", draft=").append(draft); sb.append(", repository=").append(repository); @@ -97,7 +97,7 @@ public String toString() { } public Workflow toWorkflow() { - return new Workflow(id, name, description, type, manager, tags, variables, minimumRequirements, draft, repository, scripts, + return new Workflow(id, name, description, scope, manager, tags, variables, minimumRequirements, draft, repository, scripts, new WorkflowInternal(), creationDate, modificationDate, attributes); } @@ -137,12 +137,12 @@ public WorkflowCreateParams setManager(WorkflowSystem manager) { return this; } - public Workflow.Type getType() { - return type; + public Workflow.Scope getScope() { + return scope; } - public WorkflowCreateParams setType(Workflow.Type type) { - this.type = type; + public WorkflowCreateParams setScope(Workflow.Scope scope) { + this.scope = scope; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowUpdateParams.java index f474a8e4ad8..5fe8db106cc 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowUpdateParams.java @@ -18,8 +18,8 @@ public class WorkflowUpdateParams { @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) private WorkflowSystem manager; - @DataField(id = "type", description = FieldConstants.WORKFLOW_TYPE_DESCRIPTION) - private Workflow.Type type; + @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) + private Workflow.Scope scope; @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) private List tags; @@ -51,14 +51,14 @@ public class WorkflowUpdateParams { public WorkflowUpdateParams() { } - public WorkflowUpdateParams(String name, String description, WorkflowSystem manager, Workflow.Type type, List tags, + public WorkflowUpdateParams(String name, String description, WorkflowSystem manager, Workflow.Scope scope, List tags, boolean draft, WorkflowRepository repository, List scripts, List variables, MinimumRequirements minimumRequirements, String creationDate, String modificationDate, Map attributes) { this.name = name; this.description = description; this.manager = manager; - this.type = type; + this.scope = scope; this.tags = tags; this.draft = draft; this.repository = repository; @@ -76,7 +76,7 @@ public String toString() { sb.append("name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); sb.append(", manager=").append(manager); - sb.append(", type=").append(type); + sb.append(", scope=").append(scope); sb.append(", tags=").append(tags); sb.append(", draft=").append(draft); sb.append(", repository=").append(repository); @@ -117,12 +117,12 @@ public WorkflowUpdateParams setManager(WorkflowSystem manager) { return this; } - public Workflow.Type getType() { - return type; + public Workflow.Scope getScope() { + return scope; } - public WorkflowUpdateParams setType(Workflow.Type type) { - this.type = type; + public WorkflowUpdateParams setScope(Workflow.Scope scope) { + this.scope = scope; return this; } From 32393504b2183ae285ac6b05be2a600430227b06 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 15 Apr 2025 17:11:54 +0200 Subject: [PATCH 02/22] core: rename Workflow for ExternalTool, #TASK-7610 --- .../analysis/workflow/NextFlowExecutor.java | 108 +++--- ...est.java => ExternalToolExecutorTest.java} | 28 +- .../executors/WorkflowCommandExecutor.java | 2 +- .../executors/WorkflowsCommandExecutor.java | 80 ++-- .../main/options/WorkflowsCommandOptions.java | 4 +- .../authorization/AuthorizationManager.java | 4 +- .../CatalogAuthorizationManager.java | 20 +- .../opencga/catalog/db/DBAdaptorFactory.java | 2 +- ...daptor.java => ExternalToolDBAdaptor.java} | 6 +- ...r.java => ExternalToolMongoDBAdaptor.java} | 129 ++++--- .../db/mongodb/MongoDBAdaptorFactory.java | 2 +- .../OrganizationMongoDBAdaptorFactory.java | 6 +- .../mongodb/converters/WorkflowConverter.java | 8 +- .../catalog/managers/AbstractManager.java | 2 +- .../catalog/managers/CatalogManager.java | 8 +- ...wManager.java => ExternalToolManager.java} | 363 +++++++++--------- ...Test.java => ExternalToolManagerTest.java} | 120 +++--- .../client/rest/clients/WorkflowClient.java | 48 +-- .../opencga/core/common/JacksonUtils.java | 6 +- .../opencga/core/config/Configuration.java | 2 +- .../ExternalTool.java} | 68 ++-- .../ExternalToolAclEntryList.java | 6 + .../ExternalToolAclUpdateParams.java | 44 +++ .../ExternalToolCreateParams.java} | 56 +-- .../ExternalToolImportParams.java} | 12 +- .../ExternalToolInternal.java} | 10 +- .../externalTool/ExternalToolPermissions.java | 30 ++ .../ExternalToolUpdateParams.java} | 48 +-- .../ExternalToolVariable.java} | 24 +- .../NextFlowRunParams.java | 2 +- .../WorkflowRepository.java | 2 +- .../WorkflowRepositoryParams.java | 2 +- .../WorkflowScript.java | 2 +- .../WorkflowSystem.java | 2 +- .../core/models/study/StudyPermissions.java | 8 +- .../models/workflow/WorkflowAclEntryList.java | 6 - .../workflow/WorkflowAclUpdateParams.java | 44 --- .../models/workflow/WorkflowPermissions.java | 30 -- .../opencga/server/rest/JobWSServer.java | 2 +- .../opencga/server/rest/WorkflowWSServer.java | 50 +-- 40 files changed, 705 insertions(+), 691 deletions(-) rename opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/{WorkflowExecutorTest.java => ExternalToolExecutorTest.java} (89%) rename opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/{WorkflowDBAdaptor.java => ExternalToolDBAdaptor.java} (91%) rename opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/{WorkflowMongoDBAdaptor.java => ExternalToolMongoDBAdaptor.java} (81%) rename opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/{WorkflowManager.java => ExternalToolManager.java} (74%) rename opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/{WorkflowManagerTest.java => ExternalToolManagerTest.java} (52%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow/Workflow.java => externalTool/ExternalTool.java} (77%) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolAclEntryList.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolAclUpdateParams.java rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow/WorkflowCreateParams.java => externalTool/ExternalToolCreateParams.java} (71%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow/WorkflowImportParams.java => externalTool/ExternalToolImportParams.java} (66%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow/WorkflowInternal.java => externalTool/ExternalToolInternal.java} (72%) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolPermissions.java rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow/WorkflowUpdateParams.java => externalTool/ExternalToolUpdateParams.java} (73%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow/WorkflowVariable.java => externalTool/ExternalToolVariable.java} (72%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow => externalTool}/NextFlowRunParams.java (96%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow => externalTool}/WorkflowRepository.java (96%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow => externalTool}/WorkflowRepositoryParams.java (95%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow => externalTool}/WorkflowScript.java (96%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/{workflow => externalTool}/WorkflowSystem.java (95%) delete mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowAclEntryList.java delete mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowAclUpdateParams.java delete mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowPermissions.java diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java index 5ce5cd06049..5fd44aa1b96 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java @@ -10,7 +10,7 @@ import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.tools.OpenCgaDockerToolScopeStudy; -import org.opencb.opencga.catalog.db.api.WorkflowDBAdaptor; +import org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor; import org.opencb.opencga.catalog.utils.InputFileUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.GitRepositoryState; @@ -18,11 +18,11 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.externalTool.ExternalTool; import org.opencb.opencga.core.models.job.ToolInfoExecutor; -import org.opencb.opencga.core.models.workflow.NextFlowRunParams; -import org.opencb.opencga.core.models.workflow.Workflow; -import org.opencb.opencga.core.models.workflow.WorkflowScript; -import org.opencb.opencga.core.models.workflow.WorkflowVariable; +import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; +import org.opencb.opencga.core.models.externalTool.WorkflowScript; +import org.opencb.opencga.core.models.externalTool.ExternalToolVariable; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.ToolDependency; import org.opencb.opencga.core.tools.annotations.Tool; @@ -53,7 +53,7 @@ public class NextFlowExecutor extends OpenCgaDockerToolScopeStudy { @ToolParams protected NextFlowRunParams nextflowParams = new NextFlowRunParams(); - private Workflow workflow; + private ExternalTool externalTool; private String cliParams; private String outDirPath; @@ -74,9 +74,9 @@ protected void check() throws Exception { InputFileUtils inputFileUtils = new InputFileUtils(catalogManager); - OpenCGAResult result; + OpenCGAResult result; if (nextflowParams.getVersion() != null) { - Query query = new Query(WorkflowDBAdaptor.QueryParams.VERSION.key(), nextflowParams.getVersion()); + Query query = new Query(ExternalToolDBAdaptor.QueryParams.VERSION.key(), nextflowParams.getVersion()); result = catalogManager.getWorkflowManager().get(study, Collections.singletonList(nextflowParams.getId()), query, QueryOptions.empty(), false, token); } else { @@ -85,9 +85,9 @@ protected void check() throws Exception { if (result.getNumResults() == 0) { throw new ToolException("Workflow '" + nextflowParams.getId() + "' not found"); } - workflow = result.first(); + externalTool = result.first(); - if (workflow == null) { + if (externalTool == null) { throw new ToolException("Workflow '" + nextflowParams.getId() + "' is null"); } @@ -95,9 +95,9 @@ protected void check() throws Exception { ephimeralDirPath = getScratchDir().toAbsolutePath().toString(); Set mandatoryParams = new HashSet<>(); - Map variableMap = new HashMap<>(); - if (CollectionUtils.isNotEmpty(workflow.getVariables())) { - for (WorkflowVariable variable : workflow.getVariables()) { + Map variableMap = new HashMap<>(); + if (CollectionUtils.isNotEmpty(externalTool.getVariables())) { + for (ExternalToolVariable variable : externalTool.getVariables()) { String variableId = removePrefix(variable.getId()); variableMap.put(variableId, variable); if (variable.isRequired()) { @@ -106,26 +106,26 @@ protected void check() throws Exception { } } - if (StringUtils.isEmpty(workflow.getManager().getVersion())) { - workflow.getManager().setVersion(ParamConstants.DEFAULT_MIN_NEXTFLOW_VERSION); + if (StringUtils.isEmpty(externalTool.getManager().getVersion())) { + externalTool.getManager().setVersion(ParamConstants.DEFAULT_MIN_NEXTFLOW_VERSION); } // Update job tags and attributes - ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(workflow.getManager().getId().name().toLowerCase(), - workflow.getManager().getVersion()); + ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(externalTool.getManager().getId().name().toLowerCase(), + externalTool.getManager().getVersion()); Set tags = new HashSet<>(); tags.add(ID); - tags.add(workflow.getManager().getId().name()); - tags.add(workflow.getManager().getId() + ":" + workflow.getManager().getVersion()); - tags.add(workflow.getId()); - if (CollectionUtils.isNotEmpty(workflow.getTags())) { - tags.addAll(workflow.getTags()); + tags.add(externalTool.getManager().getId().name()); + tags.add(externalTool.getManager().getId() + ":" + externalTool.getManager().getVersion()); + tags.add(externalTool.getId()); + if (CollectionUtils.isNotEmpty(externalTool.getTags())) { + tags.addAll(externalTool.getTags()); } List dependencyList = new ArrayList<>(4); dependencyList.add(new ToolDependency("opencb/opencga-workflow", GitRepositoryState.getInstance().getBuildVersion() + "-hdp3.1")); - dependencyList.add(new ToolDependency("nextflow", workflow.getManager().getVersion())); - dependencyList.add(new ToolDependency(workflow.getId(), String.valueOf(workflow.getVersion()))); - if (workflow.getRepository() != null && StringUtils.isNotEmpty(workflow.getRepository().getId())) { - dependencyList.add(new ToolDependency(workflow.getRepository().getId(), workflow.getRepository().getVersion())); + dependencyList.add(new ToolDependency("nextflow", externalTool.getManager().getVersion())); + dependencyList.add(new ToolDependency(externalTool.getId(), String.valueOf(externalTool.getVersion()))); + if (externalTool.getRepository() != null && StringUtils.isNotEmpty(externalTool.getRepository().getId())) { + dependencyList.add(new ToolDependency(externalTool.getRepository().getId(), externalTool.getRepository().getVersion())); } addDependencies(dependencyList); updateJobInformation(new ArrayList<>(tags), toolInfoExecutor); @@ -137,7 +137,7 @@ protected void check() throws Exception { // Remove from the mandatoryParams set mandatoryParams.remove(variableId); - WorkflowVariable workflowVariable = variableMap.get(variableId); + ExternalToolVariable externalToolVariable = variableMap.get(variableId); if (entry.getKey().startsWith("-")) { cliParamsBuilder.append(entry.getKey()).append(" "); @@ -145,17 +145,17 @@ protected void check() throws Exception { cliParamsBuilder.append("--").append(entry.getKey()).append(" "); } if (StringUtils.isNotEmpty(entry.getValue())) { - if ((workflowVariable != null && workflowVariable.isOutput()) || inputFileUtils.isDynamicOutputFolder(entry.getValue())) { + if ((externalToolVariable != null && externalToolVariable.isOutput()) || inputFileUtils.isDynamicOutputFolder(entry.getValue())) { processOutputCli(entry.getValue(), inputFileUtils, cliParamsBuilder); } else if (!inputFileUtils.isFlag(entry.getValue())) { processInputCli(entry.getValue(), inputFileUtils, cliParamsBuilder); } - } else if (workflowVariable != null) { - if (StringUtils.isNotEmpty(workflowVariable.getDefaultValue())) { - cliParamsBuilder.append(workflowVariable.getDefaultValue()).append(" "); - } else if (workflowVariable.isOutput()) { + } else if (externalToolVariable != null) { + if (StringUtils.isNotEmpty(externalToolVariable.getDefaultValue())) { + cliParamsBuilder.append(externalToolVariable.getDefaultValue()).append(" "); + } else if (externalToolVariable.isOutput()) { processOutputCli("", inputFileUtils, cliParamsBuilder); - } else if (workflowVariable.isRequired() && workflowVariable.getType() != WorkflowVariable.WorkflowVariableType.FLAG) { + } else if (externalToolVariable.isRequired() && externalToolVariable.getType() != ExternalToolVariable.WorkflowVariableType.FLAG) { throw new ToolException("Missing value for mandatory parameter: '" + variableId + "'."); } } @@ -164,23 +164,23 @@ protected void check() throws Exception { for (String mandatoryParam : mandatoryParams) { logger.info("Processing missing mandatory param: '{}'", mandatoryParam); - WorkflowVariable workflowVariable = variableMap.get(mandatoryParam); + ExternalToolVariable externalToolVariable = variableMap.get(mandatoryParam); - if (workflowVariable.getId().startsWith("-")) { - cliParamsBuilder.append(workflowVariable.getId()).append(" "); + if (externalToolVariable.getId().startsWith("-")) { + cliParamsBuilder.append(externalToolVariable.getId()).append(" "); } else { - cliParamsBuilder.append("--").append(workflowVariable.getId()).append(" "); + cliParamsBuilder.append("--").append(externalToolVariable.getId()).append(" "); } - if (StringUtils.isNotEmpty(workflowVariable.getDefaultValue())) { - if (workflowVariable.isOutput()) { - processOutputCli(workflowVariable.getDefaultValue(), inputFileUtils, cliParamsBuilder); + if (StringUtils.isNotEmpty(externalToolVariable.getDefaultValue())) { + if (externalToolVariable.isOutput()) { + processOutputCli(externalToolVariable.getDefaultValue(), inputFileUtils, cliParamsBuilder); } else { - processInputCli(workflowVariable.getDefaultValue(), inputFileUtils, cliParamsBuilder); + processInputCli(externalToolVariable.getDefaultValue(), inputFileUtils, cliParamsBuilder); } - } else if (workflowVariable.isOutput()) { + } else if (externalToolVariable.isOutput()) { processOutputCli("", inputFileUtils, cliParamsBuilder); - } else if (workflowVariable.getType() != WorkflowVariable.WorkflowVariableType.FLAG) { + } else if (externalToolVariable.getType() != ExternalToolVariable.WorkflowVariableType.FLAG) { throw new ToolException("Missing mandatory parameter: '" + mandatoryParam + "'."); } } @@ -190,7 +190,7 @@ protected void check() throws Exception { @Override protected void run() throws Exception { - for (WorkflowScript script : workflow.getScripts()) { + for (WorkflowScript script : externalTool.getScripts()) { // Write script files Path path = temporalInputDir.resolve(script.getFileName()); Files.write(path, script.getContent().getBytes()); @@ -216,21 +216,21 @@ protected void run() throws Exception { String dockerImage = "opencb/opencga-workflow:" + GitRepositoryState.getInstance().getBuildVersion() + "-hdp3.1"; StringBuilder stringBuilder = new StringBuilder() .append("bash -c \"NXF_VER=") - .append(workflow.getManager().getVersion()) + .append(externalTool.getManager().getVersion()) .append(" nextflow -c ") .append(nextflowConfigPath) .append(" run "); - if (workflow.getRepository() != null && StringUtils.isNotEmpty(workflow.getRepository().getId())) { + if (externalTool.getRepository() != null && StringUtils.isNotEmpty(externalTool.getRepository().getId())) { // stringBuilder.append(workflow.getRepository().getImage()).append(" -with-docker"); - stringBuilder.append(workflow.getRepository().getId()); + stringBuilder.append(externalTool.getRepository().getId()); // Add the repository version if we have it and the user is not specifying it - if (StringUtils.isNotEmpty(workflow.getRepository().getVersion()) && !cliParams.contains("-r ")) { + if (StringUtils.isNotEmpty(externalTool.getRepository().getVersion()) && !cliParams.contains("-r ")) { stringBuilder .append(" -r ") - .append(workflow.getRepository().getVersion()); + .append(externalTool.getRepository().getVersion()); } } else { - for (WorkflowScript script : workflow.getScripts()) { + for (WorkflowScript script : externalTool.getScripts()) { if (script.isMain()) { stringBuilder.append(temporalInputDir.resolve(script.getFileName())); break; @@ -299,16 +299,16 @@ private void writeTraceConfig(BufferedWriter writer) throws IOException { } private void writeResourceLimitsConfig(BufferedWriter writer) throws IOException { - if (workflow.getMinimumRequirements() == null || StringUtils.isEmpty(workflow.getMinimumRequirements().getMemory()) || - StringUtils.isEmpty(workflow.getMinimumRequirements().getCpu())) { + if (externalTool.getMinimumRequirements() == null || StringUtils.isEmpty(externalTool.getMinimumRequirements().getMemory()) || + StringUtils.isEmpty(externalTool.getMinimumRequirements().getCpu())) { return; } writer.newLine(); writer.write("process {\n"); writer.write(" resourceLimits = [\n"); - writer.write(" cpus: " + workflow.getMinimumRequirements().getCpu() + ",\n"); - writer.write(" memory: " + workflow.getMinimumRequirements().getMemory() + "\n"); + writer.write(" cpus: " + externalTool.getMinimumRequirements().getCpu() + ",\n"); + writer.write(" memory: " + externalTool.getMinimumRequirements().getMemory() + "\n"); writer.write(" ]\n"); writer.write("}\n"); } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/WorkflowExecutorTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java similarity index 89% rename from opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/WorkflowExecutorTest.java rename to opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java index a1f9e67599d..832f0bc3fea 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/WorkflowExecutorTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java @@ -15,7 +15,7 @@ import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileCreateParams; -import org.opencb.opencga.core.models.workflow.*; +import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.storage.core.StorageEngineFactory; import java.io.IOException; @@ -28,7 +28,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -public class WorkflowExecutorTest extends AbstractManagerTest { +public class ExternalToolExecutorTest extends AbstractManagerTest { @Ignore @Test @@ -36,7 +36,7 @@ public void nextflowScriptTest() throws ToolException, CatalogException, IOExcep InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml"); StorageConfiguration storageConfiguration = StorageConfiguration.load(inputStream, "yml"); - WorkflowCreateParams workflow = createDummyWorkflow(); + ExternalToolCreateParams workflow = createDummyWorkflow(); catalogManager.getWorkflowManager().create(studyFqn, workflow.toWorkflow(), QueryOptions.empty(), ownerToken); Path outDir = Paths.get(catalogManagerResource.createTmpOutdir("_nextflow")); @@ -61,7 +61,7 @@ public void nextflowScriptWithFilesTest() throws ToolException, CatalogException catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setPath("myfile.txt").setContent("hello world").setType(File.Type.FILE), false, ownerToken); - WorkflowCreateParams workflow = createDummyWorkflow("pipeline_cat_file.nf"); + ExternalToolCreateParams workflow = createDummyWorkflow("pipeline_cat_file.nf"); catalogManager.getWorkflowManager().create(studyFqn, workflow.toWorkflow(), QueryOptions.empty(), ownerToken); Path outDir = Paths.get(catalogManagerResource.createTmpOutdir("_nextflow")); @@ -85,22 +85,22 @@ public void nextflowScriptWithFilesTest() throws ToolException, CatalogException public void nextflowDockerTest() throws ToolException, CatalogException, IOException { InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml"); StorageConfiguration storageConfiguration = StorageConfiguration.load(inputStream, "yml"); - WorkflowCreateParams workflow = new WorkflowCreateParams() + ExternalToolCreateParams workflow = new ExternalToolCreateParams() .setId("workflow") - .setScope(Workflow.Scope.OTHER) + .setScope(ExternalTool.Scope.OTHER) .setVariables(Arrays.asList( - new WorkflowVariable() + new ExternalToolVariable() .setId("input") .setRequired(true), - new WorkflowVariable() + new ExternalToolVariable() .setId("outdir") .setOutput(true) .setRequired(true), - new WorkflowVariable() + new ExternalToolVariable() .setId("genome") .setRequired(true) .setDefaultValue("GRCh37"), - new WorkflowVariable() + new ExternalToolVariable() .setId("-profile") .setRequired(true) .setDefaultValue("docker") @@ -138,16 +138,16 @@ public void nextflowDockerTest() throws ToolException, CatalogException, IOExcep System.out.println(stopWatch.getTime(TimeUnit.MILLISECONDS)); } - private WorkflowCreateParams createDummyWorkflow() throws IOException { + private ExternalToolCreateParams createDummyWorkflow() throws IOException { return createDummyWorkflow("pipeline.nf"); } - private WorkflowCreateParams createDummyWorkflow(String pipelineId) throws IOException { + private ExternalToolCreateParams createDummyWorkflow(String pipelineId) throws IOException { InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream("nextflow/" + pipelineId); String content = IOUtils.toString(inputStream, "UTF-8"); - return new WorkflowCreateParams() + return new ExternalToolCreateParams() .setId("workflow") - .setScope(Workflow.Scope.OTHER) + .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", content, true))); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/WorkflowCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/WorkflowCommandExecutor.java index 12898b57b57..a9b39a88729 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/WorkflowCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/WorkflowCommandExecutor.java @@ -4,7 +4,7 @@ import org.opencb.opencga.analysis.workflow.NextFlowExecutor; import org.opencb.opencga.app.cli.internal.options.WorkflowCommandOptions; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.workflow.NextFlowRunParams; +import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java index 61c280f72db..0246f81105a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java @@ -13,17 +13,17 @@ import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ClientException; +import org.opencb.opencga.core.models.externalTool.ExternalTool; +import org.opencb.opencga.core.models.externalTool.ExternalToolAclEntryList; +import org.opencb.opencga.core.models.externalTool.ExternalToolAclUpdateParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolCreateParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolUpdateParams; +import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; +import org.opencb.opencga.core.models.externalTool.WorkflowRepository; +import org.opencb.opencga.core.models.externalTool.WorkflowRepositoryParams; +import org.opencb.opencga.core.models.externalTool.WorkflowSystem; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.job.MinimumRequirements; -import org.opencb.opencga.core.models.workflow.NextFlowRunParams; -import org.opencb.opencga.core.models.workflow.Workflow; -import org.opencb.opencga.core.models.workflow.WorkflowAclEntryList; -import org.opencb.opencga.core.models.workflow.WorkflowAclUpdateParams; -import org.opencb.opencga.core.models.workflow.WorkflowCreateParams; -import org.opencb.opencga.core.models.workflow.WorkflowRepository; -import org.opencb.opencga.core.models.workflow.WorkflowRepositoryParams; -import org.opencb.opencga.core.models.workflow.WorkflowSystem; -import org.opencb.opencga.core.models.workflow.WorkflowUpdateParams; import org.opencb.opencga.core.response.QueryType; import org.opencb.opencga.core.response.RestResponse; @@ -100,7 +100,7 @@ public void execute() throws Exception { } - private RestResponse updateAcl() throws Exception { + private RestResponse updateAcl() throws Exception { logger.debug("Executing updateAcl in Workflows command line"); WorkflowsCommandOptions.UpdateAclCommandOptions commandOptions = workflowsCommandOptions.updateAclCommandOptions; @@ -112,28 +112,28 @@ private RestResponse updateAcl() throws Exception { } - WorkflowAclUpdateParams workflowAclUpdateParams = null; + ExternalToolAclUpdateParams externalToolAclUpdateParams = null; if (commandOptions.jsonDataModel) { - RestResponse res = new RestResponse<>(); + RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/workflows/acl/{members}/update")); return res; } else if (commandOptions.jsonFile != null) { - workflowAclUpdateParams = JacksonUtils.getDefaultObjectMapper() - .readValue(new java.io.File(commandOptions.jsonFile), WorkflowAclUpdateParams.class); + externalToolAclUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), ExternalToolAclUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotNull(beanParams, "workflowIds", commandOptions.workflowIds, true); + putNestedIfNotNull(beanParams, "externalToolIds", commandOptions.externalToolIds, true); putNestedIfNotNull(beanParams, "permissions", commandOptions.permissions, true); - workflowAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + externalToolAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) - .readValue(beanParams.toJson(), WorkflowAclUpdateParams.class); + .readValue(beanParams.toJson(), ExternalToolAclUpdateParams.class); } - return openCGAClient.getWorkflowClient().updateAcl(commandOptions.members, commandOptions.action, workflowAclUpdateParams, queryParams); + return openCGAClient.getWorkflowClient().updateAcl(commandOptions.members, commandOptions.action, externalToolAclUpdateParams, queryParams); } - private RestResponse create() throws Exception { + private RestResponse create() throws Exception { logger.debug("Executing create in Workflows command line"); WorkflowsCommandOptions.CreateCommandOptions commandOptions = workflowsCommandOptions.createCommandOptions; @@ -148,15 +148,15 @@ private RestResponse create() throws Exception { } - WorkflowCreateParams workflowCreateParams = null; + ExternalToolCreateParams externalToolCreateParams = null; if (commandOptions.jsonDataModel) { - RestResponse res = new RestResponse<>(); + RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/workflows/create")); return res; } else if (commandOptions.jsonFile != null) { - workflowCreateParams = JacksonUtils.getDefaultObjectMapper() - .readValue(new java.io.File(commandOptions.jsonFile), WorkflowCreateParams.class); + externalToolCreateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), ExternalToolCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); @@ -178,11 +178,11 @@ private RestResponse create() throws Exception { putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); - workflowCreateParams = JacksonUtils.getDefaultObjectMapper().copy() + externalToolCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) - .readValue(beanParams.toJson(), WorkflowCreateParams.class); + .readValue(beanParams.toJson(), ExternalToolCreateParams.class); } - return openCGAClient.getWorkflowClient().create(workflowCreateParams, queryParams); + return openCGAClient.getWorkflowClient().create(externalToolCreateParams, queryParams); } private RestResponse distinct() throws Exception { @@ -213,7 +213,7 @@ private RestResponse distinct() throws Exception { return openCGAClient.getWorkflowClient().distinct(commandOptions.field, queryParams); } - private RestResponse importWorkflow() throws Exception { + private RestResponse importWorkflow() throws Exception { logger.debug("Executing importWorkflow in Workflows command line"); WorkflowsCommandOptions.ImportCommandOptions commandOptions = workflowsCommandOptions.importCommandOptions; @@ -227,7 +227,7 @@ private RestResponse importWorkflow() throws Exception { WorkflowRepositoryParams workflowRepositoryParams = null; if (commandOptions.jsonDataModel) { - RestResponse res = new RestResponse<>(); + RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/workflows/import")); return res; @@ -287,7 +287,7 @@ private RestResponse run() throws Exception { return openCGAClient.getWorkflowClient().run(nextFlowRunParams, queryParams); } - private RestResponse search() throws Exception { + private RestResponse search() throws Exception { logger.debug("Executing search in Workflows command line"); WorkflowsCommandOptions.SearchCommandOptions commandOptions = workflowsCommandOptions.searchCommandOptions; @@ -320,7 +320,7 @@ private RestResponse search() throws Exception { return openCGAClient.getWorkflowClient().search(queryParams); } - private RestResponse update() throws Exception { + private RestResponse update() throws Exception { logger.debug("Executing update in Workflows command line"); WorkflowsCommandOptions.UpdateCommandOptions commandOptions = workflowsCommandOptions.updateCommandOptions; @@ -335,15 +335,15 @@ private RestResponse update() throws Exception { } - WorkflowUpdateParams workflowUpdateParams = null; + ExternalToolUpdateParams externalToolUpdateParams = null; if (commandOptions.jsonDataModel) { - RestResponse res = new RestResponse<>(); + RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/workflows/{workflowId}/update")); return res; } else if (commandOptions.jsonFile != null) { - workflowUpdateParams = JacksonUtils.getDefaultObjectMapper() - .readValue(new java.io.File(commandOptions.jsonFile), WorkflowUpdateParams.class); + externalToolUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), ExternalToolUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); @@ -364,14 +364,14 @@ private RestResponse update() throws Exception { putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); - workflowUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + externalToolUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) - .readValue(beanParams.toJson(), WorkflowUpdateParams.class); + .readValue(beanParams.toJson(), ExternalToolUpdateParams.class); } - return openCGAClient.getWorkflowClient().update(commandOptions.workflowId, workflowUpdateParams, queryParams); + return openCGAClient.getWorkflowClient().update(commandOptions.workflowId, externalToolUpdateParams, queryParams); } - private RestResponse acl() throws Exception { + private RestResponse acl() throws Exception { logger.debug("Executing acl in Workflows command line"); WorkflowsCommandOptions.AclCommandOptions commandOptions = workflowsCommandOptions.aclCommandOptions; @@ -387,7 +387,7 @@ private RestResponse acl() throws Exception { return openCGAClient.getWorkflowClient().acl(commandOptions.workflows, queryParams); } - private RestResponse delete() throws Exception { + private RestResponse delete() throws Exception { logger.debug("Executing delete in Workflows command line"); WorkflowsCommandOptions.DeleteCommandOptions commandOptions = workflowsCommandOptions.deleteCommandOptions; @@ -401,7 +401,7 @@ private RestResponse delete() throws Exception { return openCGAClient.getWorkflowClient().delete(commandOptions.workflows, queryParams); } - private RestResponse info() throws Exception { + private RestResponse info() throws Exception { logger.debug("Executing info in Workflows command line"); WorkflowsCommandOptions.InfoCommandOptions commandOptions = workflowsCommandOptions.infoCommandOptions; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java index 05b8fe9aef9..a46e29833a3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java @@ -83,8 +83,8 @@ public class UpdateAclCommandOptions { @Parameter(names = {"--action"}, description = "Action to be performed [ADD, SET, REMOVE or RESET].", required = true, arity = 1) public String action = "ADD"; - @Parameter(names = {"--workflow-ids"}, description = "The body web service workflowIds parameter", required = false, arity = 1) - public String workflowIds; + @Parameter(names = {"--external-tool-ids"}, description = "The body web service externalToolIds parameter", required = false, arity = 1) + public String externalToolIds; @Parameter(names = {"--permissions"}, description = "The body web service permissions parameter", required = false, arity = 1) public String permissions; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationManager.java index 722207acce5..3f1dbeff05e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/AuthorizationManager.java @@ -35,7 +35,7 @@ import org.opencb.opencga.core.models.study.PermissionRule; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; -import org.opencb.opencga.core.models.workflow.WorkflowPermissions; +import org.opencb.opencga.core.models.externalTool.ExternalToolPermissions; import org.opencb.opencga.core.response.OpenCGAResult; import javax.annotation.Nullable; @@ -226,7 +226,7 @@ void checkFamilyPermission(String organizationId, long studyId, long familyId, S void checkClinicalAnalysisPermission(String organizationId, long studyId, long analysisId, String userId, ClinicalAnalysisPermissions permission) throws CatalogException; - void checkWorkflowPermission(String organizationId, long studyUid, long workflowUid, String userId, WorkflowPermissions permission) + void checkWorkflowPermission(String organizationId, long studyUid, long workflowUid, String userId, ExternalToolPermissions permission) throws CatalogException; default List getEffectivePermissions(String organizationId, long studyUid, String resourceId, Enums.Resource resource) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java index 2232c3bd0e2..bb764152309 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java @@ -46,7 +46,7 @@ import org.opencb.opencga.core.models.study.PermissionRule; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; -import org.opencb.opencga.core.models.workflow.WorkflowPermissions; +import org.opencb.opencga.core.models.externalTool.ExternalToolPermissions; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -446,10 +446,10 @@ public void checkClinicalAnalysisPermission(String organizationId, long studyId, @Override public void checkWorkflowPermission(String organizationId, long studyUid, long workflowUid, String userId, - WorkflowPermissions permission) throws CatalogException { + ExternalToolPermissions permission) throws CatalogException { Query query = new Query() - .append(WorkflowDBAdaptor.QueryParams.UID.key(), workflowUid) - .append(WorkflowDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) + .append(ExternalToolDBAdaptor.QueryParams.UID.key(), workflowUid) + .append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(ParamConstants.ACL_PARAM, userId + ":" + permission.name()); if (checkUserPermission(organizationId, userId, query, dbAdaptorFactory.getWorkflowDBAdaptor(organizationId))) { @@ -757,8 +757,8 @@ private void setDependentPermissions(List aclParams) throws Ca case WORKFLOW: allPermissions.addAll(aclParam.getPermissions() .stream() - .map(WorkflowPermissions::valueOf) - .map(WorkflowPermissions::getDependentPermissions) + .map(ExternalToolPermissions::valueOf) + .map(ExternalToolPermissions::getDependentPermissions) .flatMap(List::stream) .collect(Collectors.toSet()) .stream().map(Enum::name) @@ -882,8 +882,8 @@ private void setImplicitPermissions(List aclParams) throws Cat case WORKFLOW: allPermissions.addAll(aclParam.getPermissions() .stream() - .map(WorkflowPermissions::valueOf) - .map(WorkflowPermissions::getImplicitPermissions) + .map(ExternalToolPermissions::valueOf) + .map(ExternalToolPermissions::getImplicitPermissions) .flatMap(List::stream) .collect(Collectors.toSet()) .stream().map(Enum::name) @@ -1005,8 +1005,8 @@ private List getImplicitPermissions(List permissions, Enums.Reso case WORKFLOW: allPermissions.addAll(permissions .stream() - .map(WorkflowPermissions::valueOf) - .map(WorkflowPermissions::getImplicitPermissions) + .map(ExternalToolPermissions::valueOf) + .map(ExternalToolPermissions::getImplicitPermissions) .flatMap(List::stream) .collect(Collectors.toSet()) .stream().map(Enum::name) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java index 8b2fc5b102b..aeaaed941ad 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/DBAdaptorFactory.java @@ -115,5 +115,5 @@ default String getCatalogDatabase(String prefix, String organization) { InterpretationDBAdaptor getInterpretationDBAdaptor(String organization) throws CatalogDBException; - WorkflowDBAdaptor getWorkflowDBAdaptor(String organization) throws CatalogDBException; + ExternalToolDBAdaptor getWorkflowDBAdaptor(String organization) throws CatalogDBException; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/WorkflowDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java similarity index 91% rename from opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/WorkflowDBAdaptor.java rename to opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java index f22fe3223bb..00c6c28cc90 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/WorkflowDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java @@ -4,16 +4,16 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryParam; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.core.models.workflow.Workflow; +import org.opencb.opencga.core.models.externalTool.ExternalTool; import org.opencb.opencga.core.response.OpenCGAResult; import java.util.Map; import static org.opencb.commons.datastore.core.QueryParam.Type.*; -public interface WorkflowDBAdaptor extends CoreDBAdaptor { +public interface ExternalToolDBAdaptor extends CoreDBAdaptor { - OpenCGAResult insert(long studyUid, Workflow workflow, QueryOptions options) + OpenCGAResult insert(long studyUid, ExternalTool externalTool, QueryOptions options) throws CatalogException; enum QueryParams implements QueryParam { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/WorkflowMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java similarity index 81% rename from opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/WorkflowMongoDBAdaptor.java rename to opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java index 7817bcdc8b1..aab9cd5c7b4 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/WorkflowMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java @@ -10,7 +10,7 @@ import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDBIterator; import org.opencb.opencga.catalog.db.api.DBIterator; -import org.opencb.opencga.catalog.db.api.WorkflowDBAdaptor; +import org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor; import org.opencb.opencga.catalog.db.mongodb.converters.WorkflowConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.WorkflowCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; @@ -22,8 +22,8 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.workflow.Workflow; -import org.opencb.opencga.core.models.workflow.WorkflowPermissions; +import org.opencb.opencga.core.models.externalTool.ExternalTool; +import org.opencb.opencga.core.models.externalTool.ExternalToolPermissions; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.LoggerFactory; @@ -36,7 +36,7 @@ import static org.opencb.opencga.catalog.db.mongodb.AuthorizationMongoDBUtils.getQueryForAuthorisedEntries; import static org.opencb.opencga.catalog.db.mongodb.MongoDBUtils.*; -public class WorkflowMongoDBAdaptor extends CatalogMongoDBAdaptor implements WorkflowDBAdaptor { +public class ExternalToolMongoDBAdaptor extends CatalogMongoDBAdaptor implements ExternalToolDBAdaptor { private final MongoDBCollection workflowCollection; private final MongoDBCollection archiveWorkflowCollection; @@ -44,10 +44,10 @@ public class WorkflowMongoDBAdaptor extends CatalogMongoDBAdaptor implements Wor private final SnapshotVersionedMongoDBAdaptor versionedMongoDBAdaptor; private final WorkflowConverter workflowConverter; - public WorkflowMongoDBAdaptor(MongoDBCollection workflowCollection, MongoDBCollection archiveWorkflowCollection, - MongoDBCollection deleteWorkflowCollection, Configuration configuration, - OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { - super(configuration, LoggerFactory.getLogger(WorkflowMongoDBAdaptor.class)); + public ExternalToolMongoDBAdaptor(MongoDBCollection workflowCollection, MongoDBCollection archiveWorkflowCollection, + MongoDBCollection deleteWorkflowCollection, Configuration configuration, + OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { + super(configuration, LoggerFactory.getLogger(ExternalToolMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.workflowCollection = workflowCollection; this.archiveWorkflowCollection = archiveWorkflowCollection; @@ -65,66 +65,66 @@ public MongoDBCollection getArchiveCollection() { return archiveWorkflowCollection; } - Workflow insert(ClientSession clientSession, long studyUid, Workflow workflow) throws CatalogDBException { + ExternalTool insert(ClientSession clientSession, long studyUid, ExternalTool externalTool) throws CatalogDBException { dbAdaptorFactory.getCatalogStudyDBAdaptor().checkId(studyUid); - if (StringUtils.isEmpty(workflow.getId())) { + if (StringUtils.isEmpty(externalTool.getId())) { throw new CatalogDBException("Missing workflow id"); } // Check the workflow does not exist Bson bson = Filters.and( - Filters.eq(QueryParams.ID.key(), workflow.getId()), + Filters.eq(QueryParams.ID.key(), externalTool.getId()), Filters.eq(QueryParams.STUDY_UID.key(), studyUid) ); DataResult count = workflowCollection.count(clientSession, bson); if (count.getNumMatches() > 0) { - throw new CatalogDBException("Workflow { id: '" + workflow.getId() + "'} already exists."); + throw new CatalogDBException("Workflow { id: '" + externalTool.getId() + "'} already exists."); } long uid = getNewUid(clientSession); - workflow.setUid(uid); - workflow.setStudyUid(studyUid); - workflow.setRelease(dbAdaptorFactory.getCatalogStudyDBAdaptor().getCurrentRelease(clientSession, studyUid)); + externalTool.setUid(uid); + externalTool.setStudyUid(studyUid); + externalTool.setRelease(dbAdaptorFactory.getCatalogStudyDBAdaptor().getCurrentRelease(clientSession, studyUid)); - Document workflowObject = workflowConverter.convertToStorageType(workflow); + Document workflowObject = workflowConverter.convertToStorageType(externalTool); - workflowObject.put(PRIVATE_CREATION_DATE, - StringUtils.isNotEmpty(workflow.getCreationDate()) ? TimeUtils.toDate(workflow.getCreationDate()) : TimeUtils.getDate()); - workflowObject.put(PRIVATE_MODIFICATION_DATE, StringUtils.isNotEmpty(workflow.getModificationDate()) - ? TimeUtils.toDate(workflow.getModificationDate()) : TimeUtils.getDate()); + workflowObject.put(PRIVATE_CREATION_DATE, StringUtils.isNotEmpty(externalTool.getCreationDate()) + ? TimeUtils.toDate(externalTool.getCreationDate()) : TimeUtils.getDate()); + workflowObject.put(PRIVATE_MODIFICATION_DATE, StringUtils.isNotEmpty(externalTool.getModificationDate()) + ? TimeUtils.toDate(externalTool.getModificationDate()) : TimeUtils.getDate()); - logger.debug("Inserting workflow '{}' ({})...", workflow.getId(), workflow.getUid()); + logger.debug("Inserting workflow '{}' ({})...", externalTool.getId(), externalTool.getUid()); versionedMongoDBAdaptor.insert(clientSession, workflowObject); - logger.debug("Workflow '{}' successfully inserted", workflow.getId()); + logger.debug("Workflow '{}' successfully inserted", externalTool.getId()); - return workflow; + return externalTool; } @Override - public OpenCGAResult insert(long studyUid, Workflow workflow, QueryOptions options) throws CatalogException { + public OpenCGAResult insert(long studyUid, ExternalTool externalTool, QueryOptions options) throws CatalogException { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); - logger.debug("Starting workflow insert transaction for workflow id '{}'", workflow.getId()); + logger.debug("Starting workflow insert transaction for workflow id '{}'", externalTool.getId()); - insert(clientSession, studyUid, workflow); + insert(clientSession, studyUid, externalTool); return endWrite(tmpStartTime, 1, 1, 0, 0, null); - }, e -> logger.error("Could not create workflow {}: {}", workflow.getId(), e.getMessage())); + }, e -> logger.error("Could not create workflow {}: {}", externalTool.getId(), e.getMessage())); } @Override - public OpenCGAResult get(long studyUid, Query query, QueryOptions options, String user) + public OpenCGAResult get(long studyUid, Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { long startTime = startQuery(); - try (DBIterator dbIterator = iterator(studyUid, query, options, user)) { + try (DBIterator dbIterator = iterator(studyUid, query, options, user)) { return endQuery(startTime, dbIterator); } } @Override - public OpenCGAResult get(Query query, QueryOptions options) + public OpenCGAResult get(Query query, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long startTime = startQuery(); - try (DBIterator dbIterator = iterator(query, options)) { + try (DBIterator dbIterator = iterator(query, options)) { return endQuery(startTime, dbIterator); } } @@ -148,7 +148,7 @@ OpenCGAResult nativeGet(Query query, QueryOptions options) } @Override - public DBIterator iterator(long studyUid, Query query, QueryOptions options, String user) + public DBIterator iterator(long studyUid, Query query, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { query.put(PRIVATE_STUDY_UID, studyUid); MongoDBIterator mongoCursor = getMongoCursor(null, query, options, user); @@ -156,7 +156,7 @@ public DBIterator iterator(long studyUid, Query query, QueryOptions op } @Override - public DBIterator iterator(Query query, QueryOptions options) + public DBIterator iterator(Query query, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { MongoDBIterator mongoCursor = getMongoCursor(null, query, options, null); return new WorkflowCatalogMongoDBIterator<>(mongoCursor, null, workflowConverter, null, options); @@ -197,30 +197,30 @@ public OpenCGAResult count(Query query) } @Override - public OpenCGAResult groupBy(Query query, List fields, QueryOptions options, String user) + public OpenCGAResult groupBy(Query query, List fields, QueryOptions options, String user) throws CatalogDBException, CatalogAuthorizationException, CatalogParameterException { return null; } @Override - public OpenCGAResult groupBy(Query query, String field, QueryOptions options) + public OpenCGAResult groupBy(Query query, String field, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Bson bsonQuery = parseQuery(query); - return groupBy(workflowCollection, bsonQuery, field, WorkflowDBAdaptor.QueryParams.ID.key(), options); + return groupBy(workflowCollection, bsonQuery, field, ExternalToolDBAdaptor.QueryParams.ID.key(), options); } @Override - public OpenCGAResult groupBy(Query query, List fields, QueryOptions options) + public OpenCGAResult groupBy(Query query, List fields, QueryOptions options) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Bson bsonQuery = parseQuery(query); - return groupBy(workflowCollection, bsonQuery, fields, WorkflowDBAdaptor.QueryParams.ID.key(), options); + return groupBy(workflowCollection, bsonQuery, fields, ExternalToolDBAdaptor.QueryParams.ID.key(), options); } @Override public OpenCGAResult distinct(long studyUid, String field, Query query, String userId) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Query finalQuery = query != null ? new Query(query) : new Query(); - finalQuery.put(WorkflowDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); + finalQuery.put(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); Bson bson = parseQuery(finalQuery, userId); return new OpenCGAResult<>(workflowCollection.distinct(field, bson)); } @@ -230,7 +230,7 @@ public OpenCGAResult distinct(long studyUid, List fields, Query query throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { StopWatch stopWatch = StopWatch.createStarted(); Query finalQuery = query != null ? new Query(query) : new Query(); - finalQuery.put(WorkflowDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); + finalQuery.put(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); Bson bson = parseQuery(finalQuery, userId); Set results = new LinkedHashSet<>(); @@ -250,19 +250,19 @@ public OpenCGAResult facet(long studyUid, Query query, String facet, } @Override - public OpenCGAResult stats(Query query) { + public OpenCGAResult stats(Query query) { return null; } @Override - public OpenCGAResult update(long uid, ObjectMap parameters, QueryOptions queryOptions) + public OpenCGAResult update(long uid, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { Query query = new Query(QueryParams.UID.key(), uid); return update(query, parameters, queryOptions); } @Override - public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions queryOptions) + public OpenCGAResult update(Query query, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { return runTransaction(clientSession -> privateUpdate(clientSession, query, parameters, queryOptions)); @@ -272,7 +272,7 @@ public OpenCGAResult update(Query query, ObjectMap parameters, QueryOp } } - OpenCGAResult privateUpdate(ClientSession clientSession, Query query, ObjectMap parameters, QueryOptions queryOptions) + OpenCGAResult privateUpdate(ClientSession clientSession, Query query, ObjectMap parameters, QueryOptions queryOptions) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long tmpStartTime = startQuery(); Bson bsonQuery = parseQuery(query); @@ -381,27 +381,28 @@ private UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, QueryO } @Override - public OpenCGAResult delete(Workflow workflow) + public OpenCGAResult delete(ExternalTool externalTool) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try { Query query = new Query() - .append(QueryParams.UID.key(), workflow.getUid()) - .append(QueryParams.STUDY_UID.key(), workflow.getStudyUid()); + .append(QueryParams.UID.key(), externalTool.getUid()) + .append(QueryParams.STUDY_UID.key(), externalTool.getStudyUid()); OpenCGAResult result = nativeGet(query, new QueryOptions()); if (result.getNumResults() == 0) { - throw new CatalogDBException("Could not find workflow " + workflow.getId() + " with uid " + workflow.getUid()); + throw new CatalogDBException("Could not find workflow " + externalTool.getId() + " with uid " + externalTool.getUid()); } return runTransaction(clientSession -> privateDelete(clientSession, result.first())); } catch (CatalogException e) { - logger.error("Could not delete workflow {}: {}", workflow.getId(), e.getMessage(), e); - throw new CatalogDBException("Could not delete workflow " + workflow.getId() + ": " + e.getMessage(), e); + logger.error("Could not delete workflow {}: {}", externalTool.getId(), e.getMessage(), e); + throw new CatalogDBException("Could not delete workflow " + externalTool.getId() + ": " + e.getMessage(), e); } } @Override - public OpenCGAResult delete(Query query) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + public OpenCGAResult delete(Query query) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { try (DBIterator iterator = nativeIterator(query, new QueryOptions())) { - OpenCGAResult result = OpenCGAResult.empty(Workflow.class); + OpenCGAResult result = OpenCGAResult.empty(ExternalTool.class); while (iterator.hasNext()) { Document workflow = iterator.next(); String workflowId = workflow.getString(QueryParams.ID.key()); @@ -418,7 +419,7 @@ public OpenCGAResult delete(Query query) throws CatalogDBException, Ca } } - OpenCGAResult privateDelete(ClientSession clientSession, Document workflowDocument) + OpenCGAResult privateDelete(ClientSession clientSession, Document workflowDocument) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { long tmpStartTime = startQuery(); @@ -438,17 +439,17 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document work } @Override - public OpenCGAResult restore(long id, QueryOptions queryOptions) throws CatalogDBException { + public OpenCGAResult restore(long id, QueryOptions queryOptions) throws CatalogDBException { return null; } @Override - public OpenCGAResult restore(Query query, QueryOptions queryOptions) throws CatalogDBException { + public OpenCGAResult restore(Query query, QueryOptions queryOptions) throws CatalogDBException { return null; } @Override - public OpenCGAResult rank(Query query, String field, int numResults, boolean asc) + public OpenCGAResult rank(Query query, String field, int numResults, boolean asc) throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { return null; } @@ -506,7 +507,7 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.WORKFLOW, user, simplifyPermissions)); } else { - andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, WorkflowPermissions.VIEW.name(), + andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, ExternalToolPermissions.VIEW.name(), Enums.Resource.WORKFLOW, simplifyPermissions)); } @@ -516,18 +517,18 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) Query queryCopy = new Query(query); // queryCopy.remove(WorkflowDBAdaptor.QueryParams.DELETED.key()); - if ("all".equalsIgnoreCase(queryCopy.getString(WorkflowDBAdaptor.QueryParams.VERSION.key()))) { + if ("all".equalsIgnoreCase(queryCopy.getString(ExternalToolDBAdaptor.QueryParams.VERSION.key()))) { queryCopy.put(Constants.ALL_VERSIONS, true); - queryCopy.remove(WorkflowDBAdaptor.QueryParams.VERSION.key()); + queryCopy.remove(ExternalToolDBAdaptor.QueryParams.VERSION.key()); } boolean uidVersionQueryFlag = versionedMongoDBAdaptor.generateUidVersionQuery(queryCopy, andBsonList); for (Map.Entry entry : queryCopy.entrySet()) { String key = entry.getKey().split("\\.")[0]; - WorkflowDBAdaptor.QueryParams queryParam = WorkflowDBAdaptor.QueryParams.getParam(entry.getKey()) != null - ? WorkflowDBAdaptor.QueryParams.getParam(entry.getKey()) - : WorkflowDBAdaptor.QueryParams.getParam(key); + ExternalToolDBAdaptor.QueryParams queryParam = ExternalToolDBAdaptor.QueryParams.getParam(entry.getKey()) != null + ? ExternalToolDBAdaptor.QueryParams.getParam(entry.getKey()) + : ExternalToolDBAdaptor.QueryParams.getParam(key); if (queryParam == null) { if (Constants.ALL_VERSIONS.equals(entry.getKey())) { continue; @@ -591,8 +592,8 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) // If the user doesn't look for a concrete version... if (!uidVersionQueryFlag && !queryCopy.getBoolean(Constants.ALL_VERSIONS) - && !queryCopy.containsKey(WorkflowDBAdaptor.QueryParams.VERSION.key()) - && queryCopy.containsKey(WorkflowDBAdaptor.QueryParams.SNAPSHOT.key())) { + && !queryCopy.containsKey(ExternalToolDBAdaptor.QueryParams.VERSION.key()) + && queryCopy.containsKey(ExternalToolDBAdaptor.QueryParams.SNAPSHOT.key())) { // If the user looks for anything from some release, we will try to find the latest from the release (snapshot) andBsonList.add(Filters.eq(LAST_OF_RELEASE, true)); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index 019f2adb355..42e86c87ddb 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -405,7 +405,7 @@ public InterpretationDBAdaptor getInterpretationDBAdaptor(String organizationId) } @Override - public WorkflowDBAdaptor getWorkflowDBAdaptor(String organization) throws CatalogDBException { + public ExternalToolDBAdaptor getWorkflowDBAdaptor(String organization) throws CatalogDBException { return getOrganizationMongoDBAdaptorFactory(organization).getWorkflowDBAdaptor(); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java index 71741f8ca0e..3b0aaa63edf 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java @@ -140,7 +140,7 @@ public class OrganizationMongoDBAdaptorFactory { private final PanelMongoDBAdaptor panelDBAdaptor; private final ClinicalAnalysisMongoDBAdaptor clinicalDBAdaptor; private final InterpretationMongoDBAdaptor interpretationDBAdaptor; - private final WorkflowMongoDBAdaptor workflowDBAdaptor; + private final ExternalToolMongoDBAdaptor workflowDBAdaptor; private final AuditMongoDBAdaptor auditDBAdaptor; // private final MetaMongoDBAdaptor metaDBAdaptor; private final MigrationMongoDBAdaptor migrationDBAdaptor; @@ -230,7 +230,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa configuration, this); interpretationDBAdaptor = new InterpretationMongoDBAdaptor(interpretationCollection, interpretationArchivedCollection, deletedInterpretationCollection, configuration, this); - workflowDBAdaptor = new WorkflowMongoDBAdaptor(workflowCollection, workflowArchivedCollection, deletedWorkflowCollection, + workflowDBAdaptor = new ExternalToolMongoDBAdaptor(workflowCollection, workflowArchivedCollection, deletedWorkflowCollection, configuration, this); // metaDBAdaptor = new MetaMongoDBAdaptor(metaCollection, configuration, this); migrationDBAdaptor = new MigrationMongoDBAdaptor(migrationCollection, configuration, this); @@ -499,7 +499,7 @@ public InterpretationMongoDBAdaptor getInterpretationDBAdaptor() { return interpretationDBAdaptor; } - public WorkflowMongoDBAdaptor getWorkflowDBAdaptor() { + public ExternalToolMongoDBAdaptor getWorkflowDBAdaptor() { return workflowDBAdaptor; } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/WorkflowConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/WorkflowConverter.java index 182385965bf..0a4464dd0d6 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/WorkflowConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/WorkflowConverter.java @@ -1,16 +1,16 @@ package org.opencb.opencga.catalog.db.mongodb.converters; import org.bson.Document; -import org.opencb.opencga.core.models.workflow.Workflow; +import org.opencb.opencga.core.models.externalTool.ExternalTool; -public class WorkflowConverter extends OpenCgaMongoConverter { +public class WorkflowConverter extends OpenCgaMongoConverter { public WorkflowConverter() { - super(Workflow.class); + super(ExternalTool.class); } @Override - public Document convertToStorageType(Workflow object) { + public Document convertToStorageType(ExternalTool object) { Document document = super.convertToStorageType(object); // Keep long values in uids document.put("uid", object.getUid()); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java index 29488aa0b14..42366e53bdc 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/AbstractManager.java @@ -142,7 +142,7 @@ protected InterpretationDBAdaptor getInterpretationDBAdaptor(String organization return getCatalogDBAdaptorFactory().getInterpretationDBAdaptor(organization); } - protected WorkflowDBAdaptor getWorkflowDBAdaptor(String organization) throws CatalogDBException { + protected ExternalToolDBAdaptor getWorkflowDBAdaptor(String organization) throws CatalogDBException { return catalogDBAdaptorFactory.getWorkflowDBAdaptor(organization); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index f832c848e68..876cd7c023d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -88,7 +88,7 @@ public class CatalogManager implements AutoCloseable { private ClinicalAnalysisManager clinicalAnalysisManager; private InterpretationManager interpretationManager; private PanelManager panelManager; - private WorkflowManager workflowManager; + private ExternalToolManager externalToolManager; private AuditManager auditManager; private AuthorizationManager authorizationManager; @@ -164,7 +164,7 @@ private void configureManagers(Configuration configuration) throws CatalogExcept clinicalAnalysisManager = new ClinicalAnalysisManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, configuration); interpretationManager = new InterpretationManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, configuration); - workflowManager = new WorkflowManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, ioManagerFactory, + externalToolManager = new ExternalToolManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, ioManagerFactory, catalogIOManager, configuration); } @@ -458,7 +458,7 @@ public MigrationManager getMigrationManager() { return migrationManager; } - public WorkflowManager getWorkflowManager() { - return workflowManager; + public ExternalToolManager getWorkflowManager() { + return externalToolManager; } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java similarity index 74% rename from opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java rename to opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java index 5740c4a86b0..1651b9f40c9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java @@ -9,7 +9,7 @@ import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; import org.opencb.opencga.catalog.db.api.DBIterator; -import org.opencb.opencga.catalog.db.api.WorkflowDBAdaptor; +import org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; @@ -35,7 +35,7 @@ import org.opencb.opencga.core.models.job.ToolInfo; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; -import org.opencb.opencga.core.models.workflow.*; +import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,10 +51,10 @@ import java.util.stream.Collectors; import static org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager.checkPermissions; -import static org.opencb.opencga.catalog.db.api.WorkflowDBAdaptor.QueryParams.*; +import static org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor.QueryParams.*; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; -public class WorkflowManager extends ResourceManager { +public class ExternalToolManager extends ResourceManager { public static final QueryOptions INCLUDE_WORKFLOW_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(ID.key(), UID.key(), UUID.key(), VERSION.key(), DESCRIPTION.key(), STUDY_UID.key(), MANAGER.key())); @@ -70,14 +70,14 @@ public class WorkflowManager extends ResourceManager { private final Logger logger; - WorkflowManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, - DBAdaptorFactory catalogDBAdaptorFactory, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager, - Configuration configuration) { + ExternalToolManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, + DBAdaptorFactory catalogDBAdaptorFactory, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager, + Configuration configuration) { super(authorizationManager, auditManager, catalogManager, catalogDBAdaptorFactory, configuration); this.catalogIOManager = catalogIOManager; this.ioManagerFactory = ioManagerFactory; - this.logger = LoggerFactory.getLogger(WorkflowManager.class); + this.logger = LoggerFactory.getLogger(ExternalToolManager.class); } @@ -87,8 +87,9 @@ Enums.Resource getEntity() { } @Override - InternalGetDataResult internalGet(String organizationId, long studyUid, List workflowIdList, @Nullable Query query, - QueryOptions options, String user, boolean ignoreException) throws CatalogException { + InternalGetDataResult internalGet(String organizationId, long studyUid, List workflowIdList, + @Nullable Query query, QueryOptions options, String user, boolean ignoreException) + throws CatalogException { if (ListUtils.isEmpty(workflowIdList)) { throw new CatalogException("Missing workflow entries."); } @@ -105,24 +106,24 @@ InternalGetDataResult internalGet(String organizationId, long studyUid throw new CatalogException("Only one workflow allowed when requesting multiple versions"); } - WorkflowDBAdaptor.QueryParams idQueryParam = getFieldFilter(uniqueList); + ExternalToolDBAdaptor.QueryParams idQueryParam = getFieldFilter(uniqueList); queryCopy.put(idQueryParam.key(), uniqueList); // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult workflowResult = getWorkflowDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult workflowResult = getWorkflowDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); - Function workflowStringFunction = Workflow::getId; + Function workflowStringFunction = ExternalTool::getId; if (idQueryParam.equals(UUID)) { - workflowStringFunction = Workflow::getUuid; + workflowStringFunction = ExternalTool::getUuid; } if (ignoreException || workflowResult.getNumResults() >= uniqueList.size()) { return keepOriginalOrder(uniqueList, workflowStringFunction, workflowResult, ignoreException, versioned); } // Query without adding the user check - OpenCGAResult resultsNoCheck = getWorkflowDBAdaptor(organizationId).get(queryCopy, queryOptions); + OpenCGAResult resultsNoCheck = getWorkflowDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == workflowResult.getNumResults()) { throw CatalogException.notFound("workflows", getMissingFields(uniqueList, workflowResult.getResults(), workflowStringFunction)); @@ -132,7 +133,8 @@ InternalGetDataResult internalGet(String organizationId, long studyUid } @Override - public OpenCGAResult create(String studyStr, Workflow workflow, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult create(String studyStr, ExternalTool externalTool, QueryOptions options, String token) + throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); @@ -140,7 +142,7 @@ public OpenCGAResult create(String studyStr, Workflow workflow, QueryO ObjectMap auditParams = new ObjectMap() .append("study", studyStr) - .append("workflow", workflow) + .append("workflow", externalTool) .append("options", options) .append("token", token); @@ -158,24 +160,24 @@ public OpenCGAResult create(String studyStr, Workflow workflow, QueryO StudyPermissions.Permissions.WRITE_WORKFLOWS); // 2. Validate the workflow parameters - validateNewWorkflow(workflow, userId); + validateNewWorkflow(externalTool, userId); // 3. We insert the workflow - OpenCGAResult insert = getWorkflowDBAdaptor(organizationId).insert(study.getUid(), workflow, options); + OpenCGAResult insert = getWorkflowDBAdaptor(organizationId).insert(study.getUid(), externalTool, options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created workflow Query query = new Query() .append(STUDY_UID.key(), study.getUid()) - .append(UID.key(), workflow.getUid()); - OpenCGAResult result = getWorkflowDBAdaptor(organizationId).get(query, options); + .append(UID.key(), externalTool.getUid()); + OpenCGAResult result = getWorkflowDBAdaptor(organizationId).get(query, options); insert.setResults(result.getResults()); } - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, workflow.getId(), workflow.getUuid(), studyId, + auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), externalTool.getUuid(), studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; } catch (CatalogException e) { - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, workflow.getId(), "", studyId, studyUuid, auditParams, - new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), "", studyId, studyUuid, + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } } @@ -194,15 +196,16 @@ public OpenCGAResult submit(String studyStr, NextFlowRunParams runParams, S if (runParams.getVersion() != null) { query.append(VERSION.key(), runParams.getVersion()); } - Workflow workflow = internalGet(organizationId, study.getUid(), runParams.getId(), query, QueryOptions.empty(), userId).first(); + ExternalTool externalTool = internalGet(organizationId, study.getUid(), runParams.getId(), query, QueryOptions.empty(), userId) + .first(); ToolInfo toolInfo = new ToolInfo() - .setId(workflow.getId()) + .setId(externalTool.getId()) // Set workflow minimum requirements - .setMinimumRequirements(workflow.getMinimumRequirements() != null - ? workflow.getMinimumRequirements() + .setMinimumRequirements(externalTool.getMinimumRequirements() != null + ? externalTool.getMinimumRequirements() : configuration.getAnalysis().getWorkflow().getMinRequirements()) - .setDescription(workflow.getDescription()); + .setDescription(externalTool.getDescription()); Map paramsMap = runParams.toParams(); paramsMap.putIfAbsent(ParamConstants.STUDY_PARAM, study.getFqn()); @@ -230,8 +233,8 @@ public OpenCGAResult submit(String studyStr, NextFlowRunParams runParams, S jobDependsOn, jobTags, null, jobScheduledStartTime, dryRun, Collections.emptyMap(), token); } - public OpenCGAResult importWorkflow(String studyStr, WorkflowRepositoryParams repository, QueryOptions options, String token) - throws CatalogException { + public OpenCGAResult importWorkflow(String studyStr, WorkflowRepositoryParams repository, QueryOptions options, + String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); @@ -257,24 +260,24 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepositor StudyPermissions.Permissions.WRITE_WORKFLOWS); // 2. Download workflow - Workflow workflow = downloadWorkflow(repository); - validateNewWorkflow(workflow, userId); - workflowId = workflow.getId(); + ExternalTool externalTool = downloadWorkflow(repository); + validateNewWorkflow(externalTool, userId); + workflowId = externalTool.getId(); - OpenCGAResult result; + OpenCGAResult result; Query query = new Query() .append(STUDY_UID.key(), study.getUid()) - .append(ID.key(), workflow.getId()); - OpenCGAResult tmpResult = getWorkflowDBAdaptor(organizationId).get(query, INCLUDE_WORKFLOW_IDS); + .append(ID.key(), externalTool.getId()); + OpenCGAResult tmpResult = getWorkflowDBAdaptor(organizationId).get(query, INCLUDE_WORKFLOW_IDS); if (tmpResult.getNumResults() > 0) { logger.warn("Workflow '" + workflowId + "' already exists. Updating with the latest workflow information."); try { // Set workflow uid just in case users want to get the final result - workflow.setUid(tmpResult.first().getUid()); + externalTool.setUid(tmpResult.first().getUid()); // Create update map removing the id to avoid the dbAdaptor exception - ObjectMap updateMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(workflow)); + ObjectMap updateMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(externalTool)); updateMap.remove("id"); result = getWorkflowDBAdaptor(organizationId).update(tmpResult.first().getUid(), updateMap, QueryOptions.empty()); } catch (JsonProcessingException e) { @@ -283,18 +286,18 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepositor } } else { // 3. We insert the workflow - result = getWorkflowDBAdaptor(organizationId).insert(study.getUid(), workflow, options); + result = getWorkflowDBAdaptor(organizationId).insert(study.getUid(), externalTool, options); } if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch created workflow query = new Query() .append(STUDY_UID.key(), study.getUid()) - .append(UID.key(), workflow.getUid()); - OpenCGAResult tmpTmpResult = getWorkflowDBAdaptor(organizationId).get(query, options); + .append(UID.key(), externalTool.getUid()); + OpenCGAResult tmpTmpResult = getWorkflowDBAdaptor(organizationId).get(query, options); result.setResults(tmpTmpResult.getResults()); } - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, workflow.getId(), workflow.getUuid(), studyId, + auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), externalTool.getUuid(), studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { @@ -304,27 +307,27 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepositor } } - private Workflow downloadWorkflow(WorkflowRepositoryParams repository) throws CatalogException { + private ExternalTool downloadWorkflow(WorkflowRepositoryParams repository) throws CatalogException { ParamUtils.checkObj(repository, "Workflow repository parameters"); if (StringUtils.isEmpty(repository.getId())) { throw new CatalogParameterException("Missing 'id' field in workflow import parameters"); } String workflowId = repository.getId().replace("/", "."); - Workflow workflow = new Workflow("", "", "", null, new WorkflowSystem(WorkflowSystem.SystemId.NEXTFLOW, ""), new LinkedList<>(), - new LinkedList<>(), new MinimumRequirements(), false, repository.toWorkflowRepository(), new LinkedList<>(), - new WorkflowInternal(), TimeUtils.getTime(), TimeUtils.getTime(), new HashMap<>()); + ExternalTool externalTool = new ExternalTool("", "", "", null, new WorkflowSystem(WorkflowSystem.SystemId.NEXTFLOW, ""), + new LinkedList<>(), new LinkedList<>(), new MinimumRequirements(), false, repository.toWorkflowRepository(), + new LinkedList<>(), new ExternalToolInternal(), TimeUtils.getTime(), TimeUtils.getTime(), new HashMap<>()); try { - processNextflowConfig(workflow, repository); - processMemoryRequirements(workflow, repository); + processNextflowConfig(externalTool, repository); + processMemoryRequirements(externalTool, repository); } catch (CatalogException e) { throw new CatalogException("Could not process repository information from workflow '" + workflowId + "'.", e); } - return workflow; + return externalTool; } - private void processMemoryRequirements(Workflow workflow, WorkflowRepositoryParams repository) throws CatalogException { + private void processMemoryRequirements(ExternalTool externalTool, WorkflowRepositoryParams repository) throws CatalogException { String urlStr; if (StringUtils.isEmpty(repository.getVersion())) { @@ -369,10 +372,10 @@ private void processMemoryRequirements(Workflow workflow, WorkflowRepositoryPara } } if (cpus > 0 && memory != null) { - workflow.getMinimumRequirements().setCpu(String.valueOf(cpus)); - workflow.getMinimumRequirements().setMemory(memory); + externalTool.getMinimumRequirements().setCpu(String.valueOf(cpus)); + externalTool.getMinimumRequirements().setMemory(memory); } else { - logger.warn("Could not find the minimum requirements for the workflow " + workflow.getId()); + logger.warn("Could not find the minimum requirements for the workflow " + externalTool.getId()); } in.close(); } catch (Exception e) { @@ -380,7 +383,7 @@ private void processMemoryRequirements(Workflow workflow, WorkflowRepositoryPara } } - private void processNextflowConfig(Workflow workflow, WorkflowRepositoryParams repository) throws CatalogException { + private void processNextflowConfig(ExternalTool externalTool, WorkflowRepositoryParams repository) throws CatalogException { String urlStr; if (StringUtils.isEmpty(repository.getVersion())) { urlStr = "https://raw.githubusercontent.com/" + repository.getId() + "/refs/heads/master/nextflow.config"; @@ -406,7 +409,7 @@ private void processNextflowConfig(Workflow workflow, WorkflowRepositoryParams r manifestBracketClose = null; } else { // Process manifest line - fillWithWorkflowManifest(workflow, inputLine); + fillWithWorkflowManifest(externalTool, inputLine); } } else if (profilesBracketClose != null) { if (gitpodBracketClose != null) { @@ -414,7 +417,7 @@ private void processNextflowConfig(Workflow workflow, WorkflowRepositoryParams r gitpodBracketClose = null; } else { // Process gitpod line - fillWithGitpodManifest(workflow, inputLine); + fillWithGitpodManifest(externalTool, inputLine); } } else if (inputLine.trim().startsWith("gitpod {")) { int position = inputLine.indexOf("gitpod {"); @@ -437,7 +440,7 @@ private void processNextflowConfig(Workflow workflow, WorkflowRepositoryParams r } } - private void fillWithWorkflowManifest(Workflow workflow, String rawline) { + private void fillWithWorkflowManifest(ExternalTool externalTool, String rawline) { String[] split = rawline.split("= "); if (split.length != 2) { return; @@ -447,32 +450,32 @@ private void fillWithWorkflowManifest(Workflow workflow, String rawline) { String value = split[1].replace("\"", "").replace("'", "").trim(); switch (key) { case "name": - workflow.setId(value.replace("/", ".")); - workflow.setName(value.replace("/", " ")); - workflow.getRepository().setId(value); + externalTool.setId(value.replace("/", ".")); + externalTool.setName(value.replace("/", " ")); + externalTool.getRepository().setId(value); break; case "author": - workflow.getRepository().setAuthor(value); + externalTool.getRepository().setAuthor(value); break; case "description": - workflow.setDescription(value); - workflow.getRepository().setDescription(value); + externalTool.setDescription(value); + externalTool.getRepository().setDescription(value); break; case "version": String version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); - workflow.getRepository().setVersion(version); + externalTool.getRepository().setVersion(version); break; case "nextflowVersion": // Nextflow version must start with a number version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); - workflow.getManager().setVersion(version); + externalTool.getManager().setVersion(version); break; default: break; } } - private void fillWithGitpodManifest(Workflow workflow, String rawline) { + private void fillWithGitpodManifest(ExternalTool externalTool, String rawline) { String[] split = rawline.split("="); if (split.length != 2) { return; @@ -481,10 +484,10 @@ private void fillWithGitpodManifest(Workflow workflow, String rawline) { String value = split[1].replace("\"", "").replace("'", "").trim(); switch (key) { case "executor.cpus": - workflow.getMinimumRequirements().setCpu(value); + externalTool.getMinimumRequirements().setCpu(value); break; case "executor.memory": - workflow.getMinimumRequirements().setMemory(value); + externalTool.getMinimumRequirements().setMemory(value); break; default: break; @@ -492,8 +495,8 @@ private void fillWithGitpodManifest(Workflow workflow, String rawline) { } - public OpenCGAResult update(String studyStr, String workflowId, WorkflowUpdateParams updateParams, QueryOptions options, - String token) throws CatalogException { + public OpenCGAResult update(String studyStr, String workflowId, ExternalToolUpdateParams updateParams, + QueryOptions options, String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); @@ -517,14 +520,14 @@ public OpenCGAResult update(String studyStr, String workflowId, Workfl studyId = study.getId(); studyUuid = study.getUuid(); - Workflow workflow = internalGet(organizationId, study.getUid(), Collections.singletonList(workflowId), null, + ExternalTool externalTool = internalGet(organizationId, study.getUid(), Collections.singletonList(workflowId), null, INCLUDE_WORKFLOW_IDS, userId, false).first(); - id = workflow.getId(); - uuid = workflow.getUuid(); + id = externalTool.getId(); + uuid = externalTool.getUuid(); // Check permission - authorizationManager.checkWorkflowPermission(organizationId, study.getUid(), workflow.getUid(), userId, - WorkflowPermissions.WRITE); + authorizationManager.checkWorkflowPermission(organizationId, study.getUid(), externalTool.getUid(), userId, + ExternalToolPermissions.WRITE); if (updateParams == null) { throw new CatalogException("Missing parameters to update the workflow."); @@ -547,12 +550,12 @@ public OpenCGAResult update(String studyStr, String workflowId, Workfl } // 2. Update workflow object - OpenCGAResult insert = getWorkflowDBAdaptor(organizationId).update(workflow.getUid(), updateMap, options); + OpenCGAResult insert = getWorkflowDBAdaptor(organizationId).update(externalTool.getUid(), updateMap, options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated workflow Query query = new Query() - .append(UID.key(), workflow.getUid()); - OpenCGAResult result = getWorkflowDBAdaptor(organizationId).get(query, options); + .append(UID.key(), externalTool.getUid()); + OpenCGAResult result = getWorkflowDBAdaptor(organizationId).get(query, options); insert.setResults(result.getResults()); } auditManager.auditUpdate(organizationId, userId, Enums.Resource.WORKFLOW, id, uuid, studyId, studyUuid, auditParams, @@ -566,7 +569,7 @@ public OpenCGAResult update(String studyStr, String workflowId, Workfl } @Override - public DBIterator iterator(String studyStr, Query query, QueryOptions options, String token) throws CatalogException { + public DBIterator iterator(String studyStr, Query query, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); @@ -578,7 +581,7 @@ public DBIterator iterator(String studyStr, Query query, QueryOptions Query finalQuery = new Query(query); fixQueryObject(finalQuery); - finalQuery.append(WorkflowDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); + finalQuery.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); return getWorkflowDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, options, userId); } @@ -596,13 +599,13 @@ public OpenCGAResult facet(String studyStr, Query query, String face Query finalQuery = new Query(query); fixQueryObject(finalQuery); - finalQuery.append(WorkflowDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); + finalQuery.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); return getWorkflowDBAdaptor(organizationId).facet(study.getUid(), finalQuery, facet, userId); } @Override - public OpenCGAResult search(String studyStr, Query query, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult search(String studyStr, Query query, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); options = ParamUtils.defaultObject(options, QueryOptions::new); @@ -624,9 +627,9 @@ public OpenCGAResult search(String studyStr, Query query, QueryOptions studyUuid = study.getUuid(); fixQueryObject(query); - query.append(WorkflowDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); + query.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - OpenCGAResult queryResult = getWorkflowDBAdaptor(organizationId).get(study.getUid(), query, options, userId); + OpenCGAResult queryResult = getWorkflowDBAdaptor(organizationId).get(study.getUid(), query, options, userId); auditManager.auditSearch(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); @@ -658,7 +661,7 @@ public OpenCGAResult distinct(String studyStr, List fields, Query que try { fixQueryObject(query); - query.append(WorkflowDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); + query.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); OpenCGAResult result = getWorkflowDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); auditManager.auditDistinct(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, @@ -673,7 +676,7 @@ public OpenCGAResult distinct(String studyStr, List fields, Query que } @Override - public OpenCGAResult count(String studyStr, Query query, String token) throws CatalogException { + public OpenCGAResult count(String studyStr, Query query, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); String organizationId = studyFqn.getOrganizationId(); @@ -690,7 +693,7 @@ public OpenCGAResult count(String studyStr, Query query, String token) try { fixQueryObject(query); - query.append(WorkflowDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); + query.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); OpenCGAResult queryResultAux = getWorkflowDBAdaptor(organizationId).count(query, userId); auditManager.auditCount(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, @@ -706,11 +709,12 @@ public OpenCGAResult count(String studyStr, Query query, String token) } @Override - public OpenCGAResult delete(String studyStr, List ids, QueryOptions options, String token) throws CatalogException { + public OpenCGAResult delete(String studyStr, List ids, QueryOptions options, String token) + throws CatalogException { return delete(studyStr, ids, options, false, token); } - public OpenCGAResult delete(String studyStr, List ids, ObjectMap params, boolean ignoreException, String token) + public OpenCGAResult delete(String studyStr, List ids, ObjectMap params, boolean ignoreException, String token) throws CatalogException { if (ids == null || ListUtils.isEmpty(ids)) { throw new CatalogException("Missing list of workflow ids"); @@ -743,33 +747,34 @@ public OpenCGAResult delete(String studyStr, List ids, ObjectM } auditManager.initAuditBatch(operationId); - OpenCGAResult result = OpenCGAResult.empty(Workflow.class); + OpenCGAResult result = OpenCGAResult.empty(ExternalTool.class); for (String id : ids) { String workflowId = id; String workflowUuid = ""; try { - OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_WORKFLOW_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_WORKFLOW_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Workflow '" + id + "' not found"); } - Workflow workflow = internalResult.first(); + ExternalTool externalTool = internalResult.first(); // We set the proper values for the audit - workflowId = workflow.getId(); - workflowUuid = workflow.getUuid(); + workflowId = externalTool.getId(); + workflowUuid = externalTool.getUuid(); if (checkPermissions) { - authorizationManager.checkWorkflowPermission(organizationId, study.getUid(), workflow.getUid(), userId, - WorkflowPermissions.DELETE); + authorizationManager.checkWorkflowPermission(organizationId, study.getUid(), externalTool.getUid(), userId, + ExternalToolPermissions.DELETE); } // Check if the workflow can be deleted - checkWorkflowCanBeDeleted(organizationId, study.getUid(), workflow, params.getBoolean(Constants.FORCE, false)); + checkWorkflowCanBeDeleted(organizationId, study.getUid(), externalTool, params.getBoolean(Constants.FORCE, false)); - result.append(getWorkflowDBAdaptor(organizationId).delete(workflow)); + result.append(getWorkflowDBAdaptor(organizationId).delete(externalTool)); - auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.WORKFLOW, workflow.getId(), workflow.getUuid(), - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), + externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { String errorMsg = "Cannot delete workflow " + workflowId + ": " + e.getMessage(); @@ -787,7 +792,7 @@ public OpenCGAResult delete(String studyStr, List ids, ObjectM return endResult(result, ignoreException); } - private void checkWorkflowCanBeDeleted(String organizationId, long uid, Workflow workflow, boolean force) { + private void checkWorkflowCanBeDeleted(String organizationId, long uid, ExternalTool externalTool, boolean force) { return; } @@ -822,10 +827,10 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool boolean checkPermissions; // We try to get an iterator containing all the workflows to be deleted - DBIterator iterator; + DBIterator iterator; try { fixQueryObject(finalQuery); - finalQuery.append(WorkflowDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); + finalQuery.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); iterator = getWorkflowDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_WORKFLOW_IDS, userId); @@ -840,32 +845,32 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool auditManager.initAuditBatch(operationUuid); while (iterator.hasNext()) { - Workflow workflow = iterator.next(); + ExternalTool externalTool = iterator.next(); try { if (checkPermissions) { - authorizationManager.checkWorkflowPermission(organizationId, study.getUid(), workflow.getUid(), userId, - WorkflowPermissions.DELETE); + authorizationManager.checkWorkflowPermission(organizationId, study.getUid(), externalTool.getUid(), userId, + ExternalToolPermissions.DELETE); } // Check if the workflow can be deleted - checkWorkflowCanBeDeleted(organizationId, study.getUid(), workflow, params.getBoolean(Constants.FORCE, false)); + checkWorkflowCanBeDeleted(organizationId, study.getUid(), externalTool, params.getBoolean(Constants.FORCE, false)); - result.append(getWorkflowDBAdaptor(organizationId).delete(workflow)); + result.append(getWorkflowDBAdaptor(organizationId).delete(externalTool)); - auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, workflow.getId(), - workflow.getUuid(), study.getId(), study.getUuid(), auditParams, + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, externalTool.getId(), + externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { - String errorMsg = "Cannot delete workflow " + workflow.getId() + ": " + e.getMessage(); + String errorMsg = "Cannot delete workflow " + externalTool.getId() + ": " + e.getMessage(); - Event event = new Event(Event.Type.ERROR, workflow.getId(), e.getMessage()); + Event event = new Event(Event.Type.ERROR, externalTool.getId(), e.getMessage()); result.getEvents().add(event); result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, workflow.getId(), - workflow.getUuid(), study.getId(), study.getUuid(), auditParams, + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, externalTool.getId(), + externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } @@ -886,10 +891,10 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List idList) throws CatalogException { - WorkflowDBAdaptor.QueryParams idQueryParam = null; + private ExternalToolDBAdaptor.QueryParams getFieldFilter(List idList) throws CatalogException { + ExternalToolDBAdaptor.QueryParams idQueryParam = null; for (String entry : idList) { - WorkflowDBAdaptor.QueryParams param = ID; + ExternalToolDBAdaptor.QueryParams param = ID; if (UuidUtils.isOpenCgaUuid(entry)) { param = UUID; } @@ -903,19 +908,19 @@ private WorkflowDBAdaptor.QueryParams getFieldFilter(List idList) throws return idQueryParam; } - private void validateNewWorkflow(Workflow workflow, String userId) throws CatalogParameterException { - ParamUtils.checkIdentifier(workflow.getId(), ID.key()); - if (workflow.getManager() == null) { - workflow.setManager(new WorkflowSystem()); + private void validateNewWorkflow(ExternalTool externalTool, String userId) throws CatalogParameterException { + ParamUtils.checkIdentifier(externalTool.getId(), ID.key()); + if (externalTool.getManager() == null) { + externalTool.setManager(new WorkflowSystem()); } - if (workflow.getManager().getId() == null) { - workflow.getManager().setId(WorkflowSystem.SystemId.NEXTFLOW); + if (externalTool.getManager().getId() == null) { + externalTool.getManager().setId(WorkflowSystem.SystemId.NEXTFLOW); } - workflow.setScope(ParamUtils.defaultObject(workflow.getScope(), Workflow.Scope.OTHER)); - workflow.setTags(workflow.getTags() != null ? workflow.getTags() : Collections.emptyList()); - workflow.setScripts(workflow.getScripts() != null ? workflow.getScripts() : Collections.emptyList()); + externalTool.setScope(ParamUtils.defaultObject(externalTool.getScope(), ExternalTool.Scope.OTHER)); + externalTool.setTags(externalTool.getTags() != null ? externalTool.getTags() : Collections.emptyList()); + externalTool.setScripts(externalTool.getScripts() != null ? externalTool.getScripts() : Collections.emptyList()); boolean main = false; - for (WorkflowScript script : workflow.getScripts()) { + for (WorkflowScript script : externalTool.getScripts()) { ParamUtils.checkIdentifier(script.getFileName(), SCRIPTS.key() + ".id"); ParamUtils.checkParameter(script.getContent(), SCRIPTS.key() + ".content"); if (script.isMain()) { @@ -925,38 +930,38 @@ private void validateNewWorkflow(Workflow workflow, String userId) throws Catalo main = script.isMain(); } } - if (CollectionUtils.isNotEmpty(workflow.getScripts()) && !main) { + if (CollectionUtils.isNotEmpty(externalTool.getScripts()) && !main) { throw new CatalogParameterException("No main script found."); } - workflow.setRepository(workflow.getRepository() != null ? workflow.getRepository() : new WorkflowRepository("")); - if (StringUtils.isEmpty(workflow.getRepository().getId()) && CollectionUtils.isEmpty(workflow.getScripts())) { + externalTool.setRepository(externalTool.getRepository() != null ? externalTool.getRepository() : new WorkflowRepository("")); + if (StringUtils.isEmpty(externalTool.getRepository().getId()) && CollectionUtils.isEmpty(externalTool.getScripts())) { throw new CatalogParameterException("No repository image or scripts found."); } - if (StringUtils.isNotEmpty(workflow.getRepository().getId()) && CollectionUtils.isNotEmpty(workflow.getScripts())) { + if (StringUtils.isNotEmpty(externalTool.getRepository().getId()) && CollectionUtils.isNotEmpty(externalTool.getScripts())) { throw new CatalogParameterException("Both repository image and scripts found. Please, either add scripts or a repository" + " image."); } - workflow.setName(ParamUtils.defaultString(workflow.getName(), workflow.getId())); - workflow.setDescription(ParamUtils.defaultString(workflow.getDescription(), "")); - workflow.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.WORKFLOW)); - workflow.setVersion(1); - workflow.setCreationDate(ParamUtils.checkDateOrGetCurrentDate(workflow.getCreationDate(), CREATION_DATE.key())); - workflow.setModificationDate(ParamUtils.checkDateOrGetCurrentDate(workflow.getModificationDate(), MODIFICATION_DATE.key())); - workflow.setAttributes(ParamUtils.defaultObject(workflow.getAttributes(), Collections.emptyMap())); - workflow.setInternal(new WorkflowInternal(new InternalStatus(InternalStatus.READY), TimeUtils.getTime(), TimeUtils.getTime(), - userId)); + externalTool.setName(ParamUtils.defaultString(externalTool.getName(), externalTool.getId())); + externalTool.setDescription(ParamUtils.defaultString(externalTool.getDescription(), "")); + externalTool.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.WORKFLOW)); + externalTool.setVersion(1); + externalTool.setCreationDate(ParamUtils.checkDateOrGetCurrentDate(externalTool.getCreationDate(), CREATION_DATE.key())); + externalTool.setModificationDate(ParamUtils.checkDateOrGetCurrentDate(externalTool.getModificationDate(), MODIFICATION_DATE.key())); + externalTool.setAttributes(ParamUtils.defaultObject(externalTool.getAttributes(), Collections.emptyMap())); + externalTool.setInternal(new ExternalToolInternal(new InternalStatus(InternalStatus.READY), TimeUtils.getTime(), + TimeUtils.getTime(), userId)); } // ************************** ACLs ******************************** // - public OpenCGAResult> getAcls(String studyId, List workflowList, String member, - boolean ignoreException, String token) throws CatalogException { + public OpenCGAResult> getAcls(String studyId, List workflowList, String member, + boolean ignoreException, String token) throws CatalogException { return getAcls(studyId, workflowList, StringUtils.isNotEmpty(member) ? Collections.singletonList(member) : Collections.emptyList(), ignoreException, token); } - public OpenCGAResult> getAcls(String studyStr, List workflowList, List members, - boolean ignoreException, String token) throws CatalogException { + public OpenCGAResult> getAcls(String studyStr, List workflowList, List members, + boolean ignoreException, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); String organizationId = studyFqn.getOrganizationId(); @@ -971,37 +976,37 @@ public OpenCGAResult> getAcls(String studyStr, .append("ignoreException", ignoreException) .append("token", token); - OpenCGAResult> workflowAcls = OpenCGAResult.empty(); + OpenCGAResult> workflowAcls = OpenCGAResult.empty(); Map missingMap = new HashMap<>(); try { auditManager.initAuditBatch(operationId); - InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), workflowList, INCLUDE_WORKFLOW_IDS, - userId, ignoreException); + InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), workflowList, + INCLUDE_WORKFLOW_IDS, userId, ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() .collect(Collectors.toMap(InternalGetDataResult.Missing::getId, Function.identity())); } - List workflowUids = queryResult.getResults().stream().map(Workflow::getUid).collect(Collectors.toList()); + List workflowUids = queryResult.getResults().stream().map(ExternalTool::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, members, Enums.Resource.WORKFLOW, - WorkflowPermissions.class, userId); + ExternalToolPermissions.class, userId); } else { workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, Enums.Resource.WORKFLOW, - WorkflowPermissions.class, userId); + ExternalToolPermissions.class, userId); } // Include non-existing samples to the result list - List> resultList = new ArrayList<>(workflowList.size()); + List> resultList = new ArrayList<>(workflowList.size()); List eventList = new ArrayList<>(missingMap.size()); int counter = 0; for (String workflowId : workflowList) { if (!missingMap.containsKey(workflowId)) { - Workflow workflow = queryResult.getResults().get(counter); + ExternalTool externalTool = queryResult.getResults().get(counter); resultList.add(workflowAcls.getResults().get(counter)); auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.WORKFLOW, - workflow.getId(), workflow.getUuid(), study.getId(), study.getUuid(), auditParams, + externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { @@ -1038,8 +1043,9 @@ public OpenCGAResult> getAcls(String studyStr, return workflowAcls; } - public OpenCGAResult> updateAcl(String studyStr, String memberList, WorkflowAclUpdateParams params, - ParamUtils.AclAction action, String token) throws CatalogException { + public OpenCGAResult> updateAcl(String studyStr, String memberList, + ExternalToolAclUpdateParams params, ParamUtils.AclAction action, + String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); String organizationId = studyFqn.getOrganizationId(); @@ -1055,23 +1061,23 @@ public OpenCGAResult> updateAcl(String studySt String operationId = UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.AUDIT); List members; - List workflowList; + List externalToolList; try { auditManager.initAuditBatch(operationId); ParamUtils.checkObj(params, "WorkflowAclUpdateParams"); - if (CollectionUtils.isEmpty(params.getWorkflowIds())) { + if (CollectionUtils.isEmpty(params.getExternalToolIds())) { throw new CatalogException("Update ACL: No workflows provided to be updated."); } if (action == null) { throw new CatalogException("Invalid action found. Please choose a valid action to be performed."); } if (CollectionUtils.isNotEmpty(params.getPermissions())) { - checkPermissions(params.getPermissions(), WorkflowPermissions::valueOf); + checkPermissions(params.getPermissions(), ExternalToolPermissions::valueOf); } - workflowList = internalGet(organizationId, study.getUid(), params.getWorkflowIds(), INCLUDE_WORKFLOW_IDS, userId, false) + externalToolList = internalGet(organizationId, study.getUid(), params.getExternalToolIds(), INCLUDE_WORKFLOW_IDS, userId, false) .getResults(); authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); @@ -1084,8 +1090,8 @@ public OpenCGAResult> updateAcl(String studySt checkMembers(organizationId, study.getUid(), members); authorizationManager.checkNotAssigningPermissionsToAdminsGroup(members); } catch (CatalogException e) { - if (CollectionUtils.isNotEmpty(params.getWorkflowIds())) { - for (String workflowId : params.getWorkflowIds()) { + if (CollectionUtils.isNotEmpty(params.getExternalToolIds())) { + for (String workflowId : params.getExternalToolIds()) { auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, workflowId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); @@ -1095,17 +1101,17 @@ public OpenCGAResult> updateAcl(String studySt throw e; } - OpenCGAResult> aclResultList = OpenCGAResult.empty(); + OpenCGAResult> aclResultList = OpenCGAResult.empty(); int numProcessed = 0; do { - List batchWorkflowList = new ArrayList<>(); - while (numProcessed < Math.min(numProcessed + BATCH_OPERATION_SIZE, workflowList.size())) { - batchWorkflowList.add(workflowList.get(numProcessed)); + List batchExternalToolList = new ArrayList<>(); + while (numProcessed < Math.min(numProcessed + BATCH_OPERATION_SIZE, externalToolList.size())) { + batchExternalToolList.add(externalToolList.get(numProcessed)); numProcessed += 1; } - List workflowUids = batchWorkflowList.stream().map(Workflow::getUid).collect(Collectors.toList()); - List workflowIds = batchWorkflowList.stream().map(Workflow::getId).collect(Collectors.toList()); + List workflowUids = batchExternalToolList.stream().map(ExternalTool::getUid).collect(Collectors.toList()); + List workflowIds = batchExternalToolList.stream().map(ExternalTool::getId).collect(Collectors.toList()); List aclParamsList = new ArrayList<>(); AuthorizationManager.CatalogAclParams.addToList(workflowUids, params.getPermissions(), Enums.Resource.WORKFLOW, aclParamsList); @@ -1130,39 +1136,40 @@ public OpenCGAResult> updateAcl(String studySt throw new CatalogException("Unexpected error occurred. No valid action found."); } - OpenCGAResult> queryResults = authorizationManager.getAcls(organizationId, study.getUid(), - workflowUids, members, Enums.Resource.WORKFLOW, WorkflowPermissions.class); + OpenCGAResult> queryResults = authorizationManager.getAcls(organizationId, + study.getUid(), + workflowUids, members, Enums.Resource.WORKFLOW, ExternalToolPermissions.class); for (int i = 0; i < queryResults.getResults().size(); i++) { queryResults.getResults().get(i).setId(workflowIds.get(i)); } aclResultList.append(queryResults); - for (Workflow workflow : batchWorkflowList) { + for (ExternalTool externalTool : batchExternalToolList) { auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, - workflow.getId(), workflow.getUuid(), study.getId(), study.getUuid(), auditParams, + externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } } catch (CatalogException e) { // Process current batch - for (Workflow workflow : batchWorkflowList) { + for (ExternalTool externalTool : batchExternalToolList) { auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, - workflow.getId(), workflow.getUuid(), study.getId(), study.getUuid(), auditParams, + externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } // Process remaining unprocessed batches - while (numProcessed < workflowList.size()) { - Workflow workflow = workflowList.get(numProcessed); + while (numProcessed < externalToolList.size()) { + ExternalTool externalTool = externalToolList.get(numProcessed); auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, - workflow.getId(), workflow.getUuid(), study.getId(), study.getUuid(), auditParams, + externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } auditManager.finishAuditBatch(organizationId, operationId); throw e; } - } while (numProcessed < workflowList.size()); + } while (numProcessed < externalToolList.size()); auditManager.finishAuditBatch(organizationId, operationId); return aclResultList; diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java similarity index 52% rename from opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java rename to opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java index e45c502592c..6032bfb6f50 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java @@ -5,11 +5,11 @@ import org.junit.experimental.categories.Category; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.db.api.WorkflowDBAdaptor; +import org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.core.models.workflow.*; +import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -19,114 +19,114 @@ import static org.junit.Assert.*; @Category(MediumTests.class) -public class WorkflowManagerTest extends AbstractManagerTest { +public class ExternalToolManagerTest extends AbstractManagerTest { - private WorkflowManager workflowManager; + private ExternalToolManager externalToolManager; @Before public void setUp() throws Exception { super.setUp(); - workflowManager = catalogManager.getWorkflowManager(); + externalToolManager = catalogManager.getWorkflowManager(); } @Test public void importWorkflow() throws CatalogException { WorkflowRepositoryParams params = new WorkflowRepositoryParams("nf-core/rnaseq"); - OpenCGAResult result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + OpenCGAResult result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); // Update imported workflow - result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(2, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/proteinfold"); - result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/methylseq"); - result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/pacvar"); - result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); } @Test public void createWorkflowTest() throws CatalogException { - Workflow workflow = new Workflow() + ExternalTool externalTool = new ExternalTool() .setId("workflow") - .setScope(Workflow.Scope.OTHER) + .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - OpenCGAResult result = workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken); + OpenCGAResult result = externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertNotNull(result.first()); - assertEquals(workflow.getId(), result.first().getId()); + assertEquals(externalTool.getId(), result.first().getId()); // Add repository to workflow - workflow.setId("workflow2"); - workflow.setRepository(new WorkflowRepository("blabla")); + externalTool.setId("workflow2"); + externalTool.setRepository(new WorkflowRepository("blabla")); CatalogException catalogException = assertThrows(CatalogException.class, - () -> workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken)); + () -> externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("repository")); // Remove script from workflow - workflow.setScripts(Collections.emptyList()); - result = workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken); + externalTool.setScripts(Collections.emptyList()); + result = externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertNotNull(result.first()); - assertEquals(workflow.getId(), result.first().getId()); + assertEquals(externalTool.getId(), result.first().getId()); // Remove script and add two scripts with 2 mains - workflow.setId("workflow3"); - workflow.setRepository(null); - workflow.setScripts(Arrays.asList( + externalTool.setId("workflow3"); + externalTool.setRepository(null); + externalTool.setScripts(Arrays.asList( new WorkflowScript("script1", "echo 'Hello'", true), new WorkflowScript("script2", "echo 'World'", true) )); catalogException = assertThrows(CatalogException.class, - () -> workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken)); + () -> externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main")); // Add one single script without main - workflow.setScripts(Collections.singletonList( + externalTool.setScripts(Collections.singletonList( new WorkflowScript("script1", "echo 'Hello'", false) )); catalogException = assertThrows(CatalogException.class, - () -> workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken)); + () -> externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main")); } @Test public void workflowSearchTest() throws CatalogException { - Workflow workflow = new Workflow() + ExternalTool externalTool = new ExternalTool() .setId("workflow") - .setScope(Workflow.Scope.OTHER) + .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken); + externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); - workflow = new Workflow() + externalTool = new ExternalTool() .setId("workflow2") - .setScope(Workflow.Scope.OTHER) + .setScope(ExternalTool.Scope.OTHER) .setDraft(true) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken); + externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); - OpenCGAResult search = workflowManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); + OpenCGAResult search = externalToolManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); assertEquals(2, search.getNumResults()); - Query query = new Query(WorkflowDBAdaptor.QueryParams.DRAFT.key(), true); - search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); + Query query = new Query(ExternalToolDBAdaptor.QueryParams.DRAFT.key(), true); + search = externalToolManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("workflow2", search.first().getId()); assertTrue(search.first().isDraft()); - query = new Query(WorkflowDBAdaptor.QueryParams.DRAFT.key(), false); - search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); + query = new Query(ExternalToolDBAdaptor.QueryParams.DRAFT.key(), false); + search = externalToolManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("workflow", search.first().getId()); assertFalse(search.first().isDraft()); @@ -134,61 +134,61 @@ public void workflowSearchTest() throws CatalogException { @Test public void updateWorkflowTest() throws CatalogException { - Workflow workflow = new Workflow() + ExternalTool externalTool = new ExternalTool() .setId("workflow") - .setScope(Workflow.Scope.OTHER) + .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken); + externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); - WorkflowUpdateParams updateParams = new WorkflowUpdateParams() + ExternalToolUpdateParams updateParams = new ExternalToolUpdateParams() .setName("newName") - .setScope(Workflow.Scope.OTHER) + .setScope(ExternalTool.Scope.OTHER) .setDraft(true) .setCreationDate("20240101000000") .setModificationDate("20240201000000") .setDescription("description"); - OpenCGAResult update = workflowManager.update(studyFqn, workflow.getId(), updateParams, INCLUDE_RESULT, ownerToken); + OpenCGAResult update = externalToolManager.update(studyFqn, externalTool.getId(), updateParams, INCLUDE_RESULT, ownerToken); assertEquals(1, update.getNumUpdated()); - Workflow updatedWorkflow = update.first(); - assertEquals(updateParams.getName(), updatedWorkflow.getName()); - assertEquals(updateParams.isDraft(), updatedWorkflow.isDraft()); - assertEquals(updateParams.getCreationDate(), updatedWorkflow.getCreationDate()); - assertEquals(updateParams.getModificationDate(), updatedWorkflow.getModificationDate()); - assertEquals(updateParams.getDescription(), updatedWorkflow.getDescription()); + ExternalTool updatedExternalTool = update.first(); + assertEquals(updateParams.getName(), updatedExternalTool.getName()); + assertEquals(updateParams.isDraft(), updatedExternalTool.isDraft()); + assertEquals(updateParams.getCreationDate(), updatedExternalTool.getCreationDate()); + assertEquals(updateParams.getModificationDate(), updatedExternalTool.getModificationDate()); + assertEquals(updateParams.getDescription(), updatedExternalTool.getDescription()); } @Test public void deleteWorkflowTest() throws CatalogException { - Workflow workflow = new Workflow() + ExternalTool externalTool = new ExternalTool() .setId("workflow") - .setScope(Workflow.Scope.OTHER) + .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - workflowManager.create(studyFqn, workflow, QueryOptions.empty(), ownerToken); + externalToolManager.create(studyFqn, externalTool, QueryOptions.empty(), ownerToken); - OpenCGAResult result = workflowManager.delete(studyFqn, workflow.getId(), QueryOptions.empty(), ownerToken); + OpenCGAResult result = externalToolManager.delete(studyFqn, externalTool.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumDeleted()); CatalogException exception = assertThrows(CatalogException.class, - () -> workflowManager.get(studyFqn, workflow.getId(), QueryOptions.empty(), ownerToken)); + () -> externalToolManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), ownerToken)); assertTrue(exception.getMessage().contains("not found")); } @Test public void updateWorkflowAclTest() throws CatalogException { - Workflow workflow = new Workflow() + ExternalTool externalTool = new ExternalTool() .setId("workflow") - .setScope(Workflow.Scope.OTHER) + .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - workflowManager.create(studyFqn, workflow, QueryOptions.empty(), ownerToken); + externalToolManager.create(studyFqn, externalTool, QueryOptions.empty(), ownerToken); CatalogAuthorizationException catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, - () -> workflowManager.get(studyFqn, workflow.getId(), QueryOptions.empty(), noAccessToken1)); + () -> externalToolManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), noAccessToken1)); assertTrue(catalogAuthorizationException.getMessage().contains("denied")); - workflowManager.updateAcl(studyFqn, noAccessUserId1, new WorkflowAclUpdateParams(Collections.singletonList(workflow.getId()), Collections.singletonList("VIEW")), + externalToolManager.updateAcl(studyFqn, noAccessUserId1, new ExternalToolAclUpdateParams(Collections.singletonList(externalTool.getId()), Collections.singletonList("VIEW")), ParamUtils.AclAction.ADD, ownerToken); - OpenCGAResult result = workflowManager.get(studyFqn, workflow.getId(), QueryOptions.empty(), noAccessToken1); + OpenCGAResult result = externalToolManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), noAccessToken1); assertEquals(1, result.getNumResults()); } diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java index 659c1ece017..d8ec5dc79a5 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java @@ -22,14 +22,14 @@ import org.opencb.opencga.core.client.ParentClient; import org.opencb.opencga.core.config.client.ClientConfiguration; import org.opencb.opencga.core.exceptions.ClientException; +import org.opencb.opencga.core.models.externalTool.ExternalTool; +import org.opencb.opencga.core.models.externalTool.ExternalToolAclEntryList; +import org.opencb.opencga.core.models.externalTool.ExternalToolAclUpdateParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolCreateParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolUpdateParams; +import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; +import org.opencb.opencga.core.models.externalTool.WorkflowRepositoryParams; import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.workflow.NextFlowRunParams; -import org.opencb.opencga.core.models.workflow.Workflow; -import org.opencb.opencga.core.models.workflow.WorkflowAclEntryList; -import org.opencb.opencga.core.models.workflow.WorkflowAclUpdateParams; -import org.opencb.opencga.core.models.workflow.WorkflowCreateParams; -import org.opencb.opencga.core.models.workflow.WorkflowRepositoryParams; -import org.opencb.opencga.core.models.workflow.WorkflowUpdateParams; import org.opencb.opencga.core.response.RestResponse; @@ -63,12 +63,12 @@ public WorkflowClient(String token, ClientConfiguration configuration) { * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse updateAcl(String members, String action, WorkflowAclUpdateParams data, ObjectMap params) - throws ClientException { + public RestResponse updateAcl(String members, String action, ExternalToolAclUpdateParams data, ObjectMap + params) throws ClientException { params = params != null ? params : new ObjectMap(); params.putIfNotNull("action", action); params.put("body", data); - return execute("workflows", null, "acl", members, "update", params, POST, WorkflowAclEntryList.class); + return execute("workflows", null, "acl", members, "update", params, POST, ExternalToolAclEntryList.class); } /** @@ -82,10 +82,10 @@ public RestResponse updateAcl(String members, String actio * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse create(WorkflowCreateParams data, ObjectMap params) throws ClientException { + public RestResponse create(ExternalToolCreateParams data, ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); params.put("body", data); - return execute("workflows", null, null, null, "create", params, POST, Workflow.class); + return execute("workflows", null, null, null, "create", params, POST, ExternalTool.class); } /** @@ -128,10 +128,10 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse importWorkflow(WorkflowRepositoryParams data, ObjectMap params) throws ClientException { + public RestResponse importWorkflow(WorkflowRepositoryParams data, ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); params.put("body", data); - return execute("workflows", null, null, null, "import", params, POST, Workflow.class); + return execute("workflows", null, null, null, "import", params, POST, ExternalTool.class); } /** @@ -186,9 +186,9 @@ public RestResponse run(NextFlowRunParams data, ObjectMap params) throws Cl * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse search(ObjectMap params) throws ClientException { + public RestResponse search(ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); - return execute("workflows", null, null, null, "search", params, GET, Workflow.class); + return execute("workflows", null, null, null, "search", params, GET, ExternalTool.class); } /** @@ -203,10 +203,10 @@ public RestResponse search(ObjectMap params) throws ClientException { * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse update(String workflowId, WorkflowUpdateParams data, ObjectMap params) throws ClientException { + public RestResponse update(String workflowId, ExternalToolUpdateParams data, ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); params.put("body", data); - return execute("workflows", workflowId, null, null, "update", params, POST, Workflow.class); + return execute("workflows", workflowId, null, null, "update", params, POST, ExternalTool.class); } /** @@ -220,9 +220,9 @@ public RestResponse update(String workflowId, WorkflowUpdateParams dat * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse acl(String workflows, ObjectMap params) throws ClientException { + public RestResponse acl(String workflows, ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); - return execute("workflows", workflows, null, null, "acl", params, GET, WorkflowAclEntryList.class); + return execute("workflows", workflows, null, null, "acl", params, GET, ExternalToolAclEntryList.class); } /** @@ -233,9 +233,9 @@ public RestResponse acl(String workflows, ObjectMap params * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse delete(String workflows, ObjectMap params) throws ClientException { + public RestResponse delete(String workflows, ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); - return execute("workflows", workflows, null, null, "delete", params, DELETE, Workflow.class); + return execute("workflows", workflows, null, null, "delete", params, DELETE, ExternalTool.class); } /** @@ -251,8 +251,8 @@ public RestResponse delete(String workflows, ObjectMap params) throws * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse info(String workflows, ObjectMap params) throws ClientException { + public RestResponse info(String workflows, ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); - return execute("workflows", workflows, null, null, "info", params, GET, Workflow.class); + return execute("workflows", workflows, null, null, "info", params, GET, ExternalTool.class); } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/common/JacksonUtils.java b/opencga-core/src/main/java/org/opencb/opencga/core/common/JacksonUtils.java index 187aa5c606d..3cc756c11c6 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/common/JacksonUtils.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/common/JacksonUtils.java @@ -49,7 +49,7 @@ import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.VariableSet; -import org.opencb.opencga.core.models.workflow.Workflow; +import org.opencb.opencga.core.models.externalTool.ExternalTool; import javax.ws.rs.ext.ContextResolver; import java.io.IOException; @@ -114,7 +114,7 @@ private static ObjectMapper generateUpdateObjectMapper(JsonFactory jf) { objectMapper.addMixIn(VariableSet.class, PrivateUidMixin.class); objectMapper.addMixIn(ClinicalAnalysis.class, PrivateUidMixin.class); objectMapper.addMixIn(Interpretation.class, PrivateUidMixin.class); - objectMapper.addMixIn(Workflow.class, PrivateUidMixin.class); + objectMapper.addMixIn(ExternalTool.class, PrivateUidMixin.class); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); @@ -149,7 +149,7 @@ private static ObjectMapper generateOpenCGAObjectMapper(JsonFactory jf) { objectMapper.addMixIn(VariableSet.class, PrivateUidMixin.class); objectMapper.addMixIn(ClinicalAnalysis.class, PrivateUidMixin.class); objectMapper.addMixIn(Interpretation.class, PrivateUidMixin.class); - objectMapper.addMixIn(Workflow.class, PrivateUidMixin.class); + objectMapper.addMixIn(ExternalTool.class, PrivateUidMixin.class); objectMapper.addMixIn(FederationServerParams.class, FederationServerParamsMixin.class); objectMapper.addMixIn(FederationClientParams.class, FederationClientParamsMixin.class); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java index 1366b121a8d..3e1e76acf44 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java @@ -22,7 +22,7 @@ import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.models.job.MinimumRequirements; -import org.opencb.opencga.core.models.workflow.WorkflowSystem; +import org.opencb.opencga.core.models.externalTool.WorkflowSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/Workflow.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java similarity index 77% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/Workflow.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java index 8d4e85cd38d..2ebc9ea82fc 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/Workflow.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; import org.opencb.commons.annotations.DataClass; import org.opencb.commons.annotations.DataField; @@ -9,8 +9,8 @@ import java.util.List; import java.util.Map; -@DataClass(id = "Workflow", since = "3.3.0", description = "Job data model hosts information about any job.") -public class Workflow extends PrivateStudyUid { +@DataClass(id = "ExternalTool", since = "3.3.0", description = "External Tool.") +public class ExternalTool extends PrivateStudyUid { @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, description = FieldConstants.WORKFLOW_ID_DESCRIPTION) @@ -36,7 +36,7 @@ public class Workflow extends PrivateStudyUid { private List tags; @DataField(id = "variables", description = FieldConstants.WORKFLOW_VARIABLES_DESCRIPTION) - private List variables; + private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) private MinimumRequirements minimumRequirements; @@ -57,7 +57,7 @@ public class Workflow extends PrivateStudyUid { private int release; @DataField(id = "internal", description = FieldConstants.WORKFLOW_INTERNAL_DESCRIPTION) - private WorkflowInternal internal; + private ExternalToolInternal internal; @DataField(id = "creationDate", indexed = true, description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) private String creationDate; @@ -68,6 +68,12 @@ public class Workflow extends PrivateStudyUid { @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) private Map attributes; + public enum Type { + TOOL, + VARIANT_WALKER, + WORKFLOW + } + public enum Scope { SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, @@ -75,13 +81,13 @@ public enum Scope { OTHER } - public Workflow() { + public ExternalTool() { } - public Workflow(String id, String name, String description, Scope scope, WorkflowSystem manager, List tags, - List variables, MinimumRequirements minimumRequirements, boolean draft, - WorkflowRepository repository, List scripts, WorkflowInternal internal, String creationDate, - String modificationDate, Map attributes) { + public ExternalTool(String id, String name, String description, Scope scope, WorkflowSystem manager, List tags, + List variables, MinimumRequirements minimumRequirements, boolean draft, + WorkflowRepository repository, List scripts, ExternalToolInternal internal, String creationDate, + String modificationDate, Map attributes) { this.id = id; this.name = name; this.description = description; @@ -129,13 +135,13 @@ public String getId() { } @Override - public Workflow setId(String id) { + public ExternalTool setId(String id) { this.id = id; return this; } @Override - public Workflow setUid(long uid) { + public ExternalTool setUid(long uid) { super.setUid(uid); return this; } @@ -144,7 +150,7 @@ public String getUuid() { return uuid; } - public Workflow setUuid(String uuid) { + public ExternalTool setUuid(String uuid) { this.uuid = uuid; return this; } @@ -153,7 +159,7 @@ public String getName() { return name; } - public Workflow setName(String name) { + public ExternalTool setName(String name) { this.name = name; return this; } @@ -162,7 +168,7 @@ public String getDescription() { return description; } - public Workflow setDescription(String description) { + public ExternalTool setDescription(String description) { this.description = description; return this; } @@ -171,7 +177,7 @@ public boolean isDraft() { return draft; } - public Workflow setDraft(boolean draft) { + public ExternalTool setDraft(boolean draft) { this.draft = draft; return this; } @@ -180,7 +186,7 @@ public int getVersion() { return version; } - public Workflow setVersion(int version) { + public ExternalTool setVersion(int version) { this.version = version; return this; } @@ -189,7 +195,7 @@ public int getRelease() { return release; } - public Workflow setRelease(int release) { + public ExternalTool setRelease(int release) { this.release = release; return this; } @@ -198,7 +204,7 @@ public Scope getScope() { return scope; } - public Workflow setScope(Scope scope) { + public ExternalTool setScope(Scope scope) { this.scope = scope; return this; } @@ -207,7 +213,7 @@ public WorkflowSystem getManager() { return manager; } - public Workflow setManager(WorkflowSystem manager) { + public ExternalTool setManager(WorkflowSystem manager) { this.manager = manager; return this; } @@ -216,7 +222,7 @@ public WorkflowRepository getRepository() { return repository; } - public Workflow setRepository(WorkflowRepository repository) { + public ExternalTool setRepository(WorkflowRepository repository) { this.repository = repository; return this; } @@ -225,7 +231,7 @@ public List getScripts() { return scripts; } - public Workflow setScripts(List scripts) { + public ExternalTool setScripts(List scripts) { this.scripts = scripts; return this; } @@ -234,16 +240,16 @@ public List getTags() { return tags; } - public Workflow setTags(List tags) { + public ExternalTool setTags(List tags) { this.tags = tags; return this; } - public List getVariables() { + public List getVariables() { return variables; } - public Workflow setVariables(List variables) { + public ExternalTool setVariables(List variables) { this.variables = variables; return this; } @@ -252,16 +258,16 @@ public MinimumRequirements getMinimumRequirements() { return minimumRequirements; } - public Workflow setMinimumRequirements(MinimumRequirements minimumRequirements) { + public ExternalTool setMinimumRequirements(MinimumRequirements minimumRequirements) { this.minimumRequirements = minimumRequirements; return this; } - public WorkflowInternal getInternal() { + public ExternalToolInternal getInternal() { return internal; } - public Workflow setInternal(WorkflowInternal internal) { + public ExternalTool setInternal(ExternalToolInternal internal) { this.internal = internal; return this; } @@ -270,7 +276,7 @@ public String getCreationDate() { return creationDate; } - public Workflow setCreationDate(String creationDate) { + public ExternalTool setCreationDate(String creationDate) { this.creationDate = creationDate; return this; } @@ -279,7 +285,7 @@ public String getModificationDate() { return modificationDate; } - public Workflow setModificationDate(String modificationDate) { + public ExternalTool setModificationDate(String modificationDate) { this.modificationDate = modificationDate; return this; } @@ -288,7 +294,7 @@ public Map getAttributes() { return attributes; } - public Workflow setAttributes(Map attributes) { + public ExternalTool setAttributes(Map attributes) { this.attributes = attributes; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolAclEntryList.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolAclEntryList.java new file mode 100644 index 00000000000..ff8b4220e06 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolAclEntryList.java @@ -0,0 +1,6 @@ +package org.opencb.opencga.core.models.externalTool; + +import org.opencb.opencga.core.models.AclEntryList; + +public class ExternalToolAclEntryList extends AclEntryList { +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolAclUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolAclUpdateParams.java new file mode 100644 index 00000000000..e021449a347 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolAclUpdateParams.java @@ -0,0 +1,44 @@ +package org.opencb.opencga.core.models.externalTool; + +import java.util.List; + +public class ExternalToolAclUpdateParams { + + private List externalToolIds; + private List permissions; + + public ExternalToolAclUpdateParams() { + } + + public ExternalToolAclUpdateParams(List externalToolIds, List permissions) { + this.externalToolIds = externalToolIds; + this.permissions = permissions; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("WorkflowAclUpdateParams{"); + sb.append("externalToolIds=").append(externalToolIds); + sb.append(", permissions=").append(permissions); + sb.append('}'); + return sb.toString(); + } + + public List getExternalToolIds() { + return externalToolIds; + } + + public ExternalToolAclUpdateParams setExternalToolIds(List externalToolIds) { + this.externalToolIds = externalToolIds; + return this; + } + + public List getPermissions() { + return permissions; + } + + public ExternalToolAclUpdateParams setPermissions(List permissions) { + this.permissions = permissions; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java similarity index 71% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowCreateParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java index 1d7971fd957..88ed58027d9 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; @@ -7,7 +7,7 @@ import java.util.List; import java.util.Map; -public class WorkflowCreateParams { +public class ExternalToolCreateParams { @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, description = FieldConstants.WORKFLOW_ID_DESCRIPTION) @@ -23,7 +23,7 @@ public class WorkflowCreateParams { private WorkflowSystem manager; @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) - private Workflow.Scope scope; + private ExternalTool.Scope scope; @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) private List tags; @@ -35,7 +35,7 @@ public class WorkflowCreateParams { private WorkflowRepository repository; @DataField(id = "variables", description = FieldConstants.WORKFLOW_VARIABLES_DESCRIPTION) - private List variables; + private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) private MinimumRequirements minimumRequirements; @@ -52,13 +52,13 @@ public class WorkflowCreateParams { @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) private Map attributes; - public WorkflowCreateParams() { + public ExternalToolCreateParams() { } - public WorkflowCreateParams(String id, String name, String description, WorkflowSystem manager, Workflow.Scope scope, List tags, - boolean draft, WorkflowRepository repository, List variables, - MinimumRequirements minimumRequirements, List scripts, String creationDate, - String modificationDate, Map attributes) { + public ExternalToolCreateParams(String id, String name, String description, WorkflowSystem manager, ExternalTool.Scope scope, List tags, + boolean draft, WorkflowRepository repository, List variables, + MinimumRequirements minimumRequirements, List scripts, String creationDate, + String modificationDate, Map attributes) { this.id = id; this.name = name; this.description = description; @@ -96,16 +96,16 @@ public String toString() { return sb.toString(); } - public Workflow toWorkflow() { - return new Workflow(id, name, description, scope, manager, tags, variables, minimumRequirements, draft, repository, scripts, - new WorkflowInternal(), creationDate, modificationDate, attributes); + public ExternalTool toWorkflow() { + return new ExternalTool(id, name, description, scope, manager, tags, variables, minimumRequirements, draft, repository, scripts, + new ExternalToolInternal(), creationDate, modificationDate, attributes); } public String getId() { return id; } - public WorkflowCreateParams setId(String id) { + public ExternalToolCreateParams setId(String id) { this.id = id; return this; } @@ -114,7 +114,7 @@ public String getName() { return name; } - public WorkflowCreateParams setName(String name) { + public ExternalToolCreateParams setName(String name) { this.name = name; return this; } @@ -123,7 +123,7 @@ public String getDescription() { return description; } - public WorkflowCreateParams setDescription(String description) { + public ExternalToolCreateParams setDescription(String description) { this.description = description; return this; } @@ -132,16 +132,16 @@ public WorkflowSystem getManager() { return manager; } - public WorkflowCreateParams setManager(WorkflowSystem manager) { + public ExternalToolCreateParams setManager(WorkflowSystem manager) { this.manager = manager; return this; } - public Workflow.Scope getScope() { + public ExternalTool.Scope getScope() { return scope; } - public WorkflowCreateParams setScope(Workflow.Scope scope) { + public ExternalToolCreateParams setScope(ExternalTool.Scope scope) { this.scope = scope; return this; } @@ -150,7 +150,7 @@ public List getTags() { return tags; } - public WorkflowCreateParams setTags(List tags) { + public ExternalToolCreateParams setTags(List tags) { this.tags = tags; return this; } @@ -159,7 +159,7 @@ public boolean isDraft() { return draft; } - public WorkflowCreateParams setDraft(boolean draft) { + public ExternalToolCreateParams setDraft(boolean draft) { this.draft = draft; return this; } @@ -168,16 +168,16 @@ public WorkflowRepository getRepository() { return repository; } - public WorkflowCreateParams setRepository(WorkflowRepository repository) { + public ExternalToolCreateParams setRepository(WorkflowRepository repository) { this.repository = repository; return this; } - public List getVariables() { + public List getVariables() { return variables; } - public WorkflowCreateParams setVariables(List variables) { + public ExternalToolCreateParams setVariables(List variables) { this.variables = variables; return this; } @@ -186,7 +186,7 @@ public MinimumRequirements getMinimumRequirements() { return minimumRequirements; } - public WorkflowCreateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { + public ExternalToolCreateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { this.minimumRequirements = minimumRequirements; return this; } @@ -195,7 +195,7 @@ public List getScripts() { return scripts; } - public WorkflowCreateParams setScripts(List scripts) { + public ExternalToolCreateParams setScripts(List scripts) { this.scripts = scripts; return this; } @@ -204,7 +204,7 @@ public String getCreationDate() { return creationDate; } - public WorkflowCreateParams setCreationDate(String creationDate) { + public ExternalToolCreateParams setCreationDate(String creationDate) { this.creationDate = creationDate; return this; } @@ -213,7 +213,7 @@ public String getModificationDate() { return modificationDate; } - public WorkflowCreateParams setModificationDate(String modificationDate) { + public ExternalToolCreateParams setModificationDate(String modificationDate) { this.modificationDate = modificationDate; return this; } @@ -222,7 +222,7 @@ public Map getAttributes() { return attributes; } - public WorkflowCreateParams setAttributes(Map attributes) { + public ExternalToolCreateParams setAttributes(Map attributes) { this.attributes = attributes; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowImportParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolImportParams.java similarity index 66% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowImportParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolImportParams.java index 310e06998aa..0dabad4b948 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowImportParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolImportParams.java @@ -1,14 +1,14 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; -public class WorkflowImportParams { +public class ExternalToolImportParams { private String source; private String id; - public WorkflowImportParams() { + public ExternalToolImportParams() { } - public WorkflowImportParams(String source, String id) { + public ExternalToolImportParams(String source, String id) { this.source = source; this.id = id; } @@ -26,7 +26,7 @@ public String getSource() { return source; } - public WorkflowImportParams setSource(String source) { + public ExternalToolImportParams setSource(String source) { this.source = source; return this; } @@ -35,7 +35,7 @@ public String getId() { return id; } - public WorkflowImportParams setId(String id) { + public ExternalToolImportParams setId(String id) { this.id = id; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolInternal.java similarity index 72% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowInternal.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolInternal.java index 10f58513e53..cf6308c5fa6 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolInternal.java @@ -1,16 +1,16 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; import org.opencb.opencga.core.models.common.Internal; import org.opencb.opencga.core.models.common.InternalStatus; -public class WorkflowInternal extends Internal { +public class ExternalToolInternal extends Internal { private String registrationUserId; - public WorkflowInternal() { + public ExternalToolInternal() { } - public WorkflowInternal(InternalStatus status, String registrationDate, String lastModified, String registrationUserId) { + public ExternalToolInternal(InternalStatus status, String registrationDate, String lastModified, String registrationUserId) { super(status, registrationDate, lastModified); this.registrationUserId = registrationUserId; } @@ -30,7 +30,7 @@ public String getRegistrationUserId() { return registrationUserId; } - public WorkflowInternal setRegistrationUserId(String registrationUserId) { + public ExternalToolInternal setRegistrationUserId(String registrationUserId) { this.registrationUserId = registrationUserId; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolPermissions.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolPermissions.java new file mode 100644 index 00000000000..99e12599f93 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolPermissions.java @@ -0,0 +1,30 @@ +package org.opencb.opencga.core.models.externalTool; + +import java.util.*; + +public enum ExternalToolPermissions { + NONE(Collections.emptyList()), + VIEW(Collections.emptyList()), + WRITE(Collections.singletonList(VIEW)), + DELETE(Arrays.asList(VIEW, WRITE)); + + private final List implicitPermissions; + + ExternalToolPermissions(List implicitPermissions) { + this.implicitPermissions = implicitPermissions; + } + + public List getImplicitPermissions() { + return implicitPermissions; + } + + public List getDependentPermissions() { + List dependentPermissions = new LinkedList<>(); + for (ExternalToolPermissions permission : EnumSet.complementOf(EnumSet.of(this))) { + if (permission.getImplicitPermissions().contains(this)) { + dependentPermissions.add(permission); + } + } + return dependentPermissions; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java similarity index 73% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowUpdateParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java index 5fe8db106cc..48e9bc6361a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; @@ -7,7 +7,7 @@ import java.util.List; import java.util.Map; -public class WorkflowUpdateParams { +public class ExternalToolUpdateParams { @DataField(id = "name", description = FieldConstants.WORKFLOW_NAME_DESCRIPTION) private String name; @@ -19,7 +19,7 @@ public class WorkflowUpdateParams { private WorkflowSystem manager; @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) - private Workflow.Scope scope; + private ExternalTool.Scope scope; @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) private List tags; @@ -34,7 +34,7 @@ public class WorkflowUpdateParams { private List scripts; @DataField(id = "variables", description = FieldConstants.WORKFLOW_VARIABLES_DESCRIPTION) - private List variables; + private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) private MinimumRequirements minimumRequirements; @@ -48,13 +48,13 @@ public class WorkflowUpdateParams { @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) private Map attributes; - public WorkflowUpdateParams() { + public ExternalToolUpdateParams() { } - public WorkflowUpdateParams(String name, String description, WorkflowSystem manager, Workflow.Scope scope, List tags, - boolean draft, WorkflowRepository repository, List scripts, - List variables, MinimumRequirements minimumRequirements, String creationDate, - String modificationDate, Map attributes) { + public ExternalToolUpdateParams(String name, String description, WorkflowSystem manager, ExternalTool.Scope scope, List tags, + boolean draft, WorkflowRepository repository, List scripts, + List variables, MinimumRequirements minimumRequirements, String creationDate, + String modificationDate, Map attributes) { this.name = name; this.description = description; this.manager = manager; @@ -94,7 +94,7 @@ public String getName() { return name; } - public WorkflowUpdateParams setName(String name) { + public ExternalToolUpdateParams setName(String name) { this.name = name; return this; } @@ -103,7 +103,7 @@ public String getDescription() { return description; } - public WorkflowUpdateParams setDescription(String description) { + public ExternalToolUpdateParams setDescription(String description) { this.description = description; return this; } @@ -112,16 +112,16 @@ public WorkflowSystem getManager() { return manager; } - public WorkflowUpdateParams setManager(WorkflowSystem manager) { + public ExternalToolUpdateParams setManager(WorkflowSystem manager) { this.manager = manager; return this; } - public Workflow.Scope getScope() { + public ExternalTool.Scope getScope() { return scope; } - public WorkflowUpdateParams setScope(Workflow.Scope scope) { + public ExternalToolUpdateParams setScope(ExternalTool.Scope scope) { this.scope = scope; return this; } @@ -130,7 +130,7 @@ public List getTags() { return tags; } - public WorkflowUpdateParams setTags(List tags) { + public ExternalToolUpdateParams setTags(List tags) { this.tags = tags; return this; } @@ -139,7 +139,7 @@ public boolean isDraft() { return draft; } - public WorkflowUpdateParams setDraft(boolean draft) { + public ExternalToolUpdateParams setDraft(boolean draft) { this.draft = draft; return this; } @@ -148,7 +148,7 @@ public WorkflowRepository getRepository() { return repository; } - public WorkflowUpdateParams setRepository(WorkflowRepository repository) { + public ExternalToolUpdateParams setRepository(WorkflowRepository repository) { this.repository = repository; return this; } @@ -157,16 +157,16 @@ public List getScripts() { return scripts; } - public WorkflowUpdateParams setScripts(List scripts) { + public ExternalToolUpdateParams setScripts(List scripts) { this.scripts = scripts; return this; } - public List getVariables() { + public List getVariables() { return variables; } - public WorkflowUpdateParams setVariables(List variables) { + public ExternalToolUpdateParams setVariables(List variables) { this.variables = variables; return this; } @@ -175,7 +175,7 @@ public String getCreationDate() { return creationDate; } - public WorkflowUpdateParams setCreationDate(String creationDate) { + public ExternalToolUpdateParams setCreationDate(String creationDate) { this.creationDate = creationDate; return this; } @@ -184,7 +184,7 @@ public String getModificationDate() { return modificationDate; } - public WorkflowUpdateParams setModificationDate(String modificationDate) { + public ExternalToolUpdateParams setModificationDate(String modificationDate) { this.modificationDate = modificationDate; return this; } @@ -193,7 +193,7 @@ public MinimumRequirements getMinimumRequirements() { return minimumRequirements; } - public WorkflowUpdateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { + public ExternalToolUpdateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { this.minimumRequirements = minimumRequirements; return this; } @@ -202,7 +202,7 @@ public Map getAttributes() { return attributes; } - public WorkflowUpdateParams setAttributes(Map attributes) { + public ExternalToolUpdateParams setAttributes(Map attributes) { this.attributes = attributes; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowVariable.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolVariable.java similarity index 72% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowVariable.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolVariable.java index 678f4bf19de..13c71c3e002 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowVariable.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolVariable.java @@ -1,6 +1,6 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; -public class WorkflowVariable { +public class ExternalToolVariable { private String id; private String name; @@ -19,11 +19,11 @@ public enum WorkflowVariableType { FILE } - public WorkflowVariable() { + public ExternalToolVariable() { } - public WorkflowVariable(String id, String name, String description, WorkflowVariableType type, boolean required, String defaultValue, - boolean output) { + public ExternalToolVariable(String id, String name, String description, WorkflowVariableType type, boolean required, + String defaultValue, boolean output) { this.id = id; this.name = name; this.description = description; @@ -51,7 +51,7 @@ public String getId() { return id; } - public WorkflowVariable setId(String id) { + public ExternalToolVariable setId(String id) { this.id = id; return this; } @@ -60,7 +60,7 @@ public String getName() { return name; } - public WorkflowVariable setName(String name) { + public ExternalToolVariable setName(String name) { this.name = name; return this; } @@ -69,7 +69,7 @@ public String getDescription() { return description; } - public WorkflowVariable setDescription(String description) { + public ExternalToolVariable setDescription(String description) { this.description = description; return this; } @@ -78,7 +78,7 @@ public WorkflowVariableType getType() { return type; } - public WorkflowVariable setType(WorkflowVariableType type) { + public ExternalToolVariable setType(WorkflowVariableType type) { this.type = type; return this; } @@ -87,7 +87,7 @@ public boolean isRequired() { return required; } - public WorkflowVariable setRequired(boolean required) { + public ExternalToolVariable setRequired(boolean required) { this.required = required; return this; } @@ -96,7 +96,7 @@ public String getDefaultValue() { return defaultValue; } - public WorkflowVariable setDefaultValue(String defaultValue) { + public ExternalToolVariable setDefaultValue(String defaultValue) { this.defaultValue = defaultValue; return this; } @@ -105,7 +105,7 @@ public boolean isOutput() { return output; } - public WorkflowVariable setOutput(boolean output) { + public ExternalToolVariable setOutput(boolean output) { this.output = output; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/NextFlowRunParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/NextFlowRunParams.java similarity index 96% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/NextFlowRunParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/NextFlowRunParams.java index d35494041e7..c05043b9b59 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/NextFlowRunParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/NextFlowRunParams.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; import org.opencb.opencga.core.tools.ToolParams; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowRepository.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java similarity index 96% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowRepository.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java index ebed05781f2..06eb1cab136 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowRepository.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; public class WorkflowRepository { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowRepositoryParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepositoryParams.java similarity index 95% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowRepositoryParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepositoryParams.java index 275c712385d..0365490f761 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowRepositoryParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepositoryParams.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; public class WorkflowRepositoryParams { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowScript.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowScript.java similarity index 96% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowScript.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowScript.java index 7cafcfbd8d7..fbd37ebd0e0 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowScript.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowScript.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; import org.opencb.commons.annotations.DataField; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowSystem.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java similarity index 95% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowSystem.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java index dc57eeba66b..ab692be189a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowSystem.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.core.models.workflow; +package org.opencb.opencga.core.models.externalTool; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java index 857a4834f69..1ff5bad7314 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/StudyPermissions.java @@ -9,7 +9,7 @@ import org.opencb.opencga.core.models.job.JobPermissions; import org.opencb.opencga.core.models.panel.PanelPermissions; import org.opencb.opencga.core.models.sample.SamplePermissions; -import org.opencb.opencga.core.models.workflow.WorkflowPermissions; +import org.opencb.opencga.core.models.externalTool.ExternalToolPermissions; import java.util.*; @@ -117,9 +117,9 @@ public enum Permissions { ClinicalAnalysisPermissions.ADMIN.name(), CLINICAL_ANALYSIS), // WORKFLOWS - VIEW_WORKFLOWS(Collections.emptyList(), WorkflowPermissions.VIEW.name(), WORKFLOW), - WRITE_WORKFLOWS(Collections.singletonList(VIEW_WORKFLOWS), WorkflowPermissions.WRITE.name(), WORKFLOW), - DELETE_WORKFLOWS(Arrays.asList(VIEW_WORKFLOWS, WRITE_WORKFLOWS), WorkflowPermissions.DELETE.name(), WORKFLOW); + VIEW_WORKFLOWS(Collections.emptyList(), ExternalToolPermissions.VIEW.name(), WORKFLOW), + WRITE_WORKFLOWS(Collections.singletonList(VIEW_WORKFLOWS), ExternalToolPermissions.WRITE.name(), WORKFLOW), + DELETE_WORKFLOWS(Arrays.asList(VIEW_WORKFLOWS, WRITE_WORKFLOWS), ExternalToolPermissions.DELETE.name(), WORKFLOW); private final static Map map; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowAclEntryList.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowAclEntryList.java deleted file mode 100644 index b4234b0ee03..00000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowAclEntryList.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.opencb.opencga.core.models.workflow; - -import org.opencb.opencga.core.models.AclEntryList; - -public class WorkflowAclEntryList extends AclEntryList { -} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowAclUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowAclUpdateParams.java deleted file mode 100644 index 7f6cea398e2..00000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowAclUpdateParams.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.opencb.opencga.core.models.workflow; - -import java.util.List; - -public class WorkflowAclUpdateParams { - - private List workflowIds; - private List permissions; - - public WorkflowAclUpdateParams() { - } - - public WorkflowAclUpdateParams(List workflowIds, List permissions) { - this.workflowIds = workflowIds; - this.permissions = permissions; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("WorkflowAclUpdateParams{"); - sb.append("workflowIds=").append(workflowIds); - sb.append(", permissions=").append(permissions); - sb.append('}'); - return sb.toString(); - } - - public List getWorkflowIds() { - return workflowIds; - } - - public WorkflowAclUpdateParams setWorkflowIds(List workflowIds) { - this.workflowIds = workflowIds; - return this; - } - - public List getPermissions() { - return permissions; - } - - public WorkflowAclUpdateParams setPermissions(List permissions) { - this.permissions = permissions; - return this; - } -} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowPermissions.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowPermissions.java deleted file mode 100644 index 3f14ccf1d94..00000000000 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/workflow/WorkflowPermissions.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.opencb.opencga.core.models.workflow; - -import java.util.*; - -public enum WorkflowPermissions { - NONE(Collections.emptyList()), - VIEW(Collections.emptyList()), - WRITE(Collections.singletonList(VIEW)), - DELETE(Arrays.asList(VIEW, WRITE)); - - private final List implicitPermissions; - - WorkflowPermissions(List implicitPermissions) { - this.implicitPermissions = implicitPermissions; - } - - public List getImplicitPermissions() { - return implicitPermissions; - } - - public List getDependentPermissions() { - List dependentPermissions = new LinkedList<>(); - for (WorkflowPermissions permission : EnumSet.complementOf(EnumSet.of(this))) { - if (permission.getImplicitPermissions().contains(this)) { - dependentPermissions.add(permission); - } - } - return dependentPermissions; - } -} diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java index 0edeed7373b..efa484e89c2 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java @@ -32,7 +32,7 @@ import org.opencb.opencga.core.models.AclParams; import org.opencb.opencga.core.models.file.FileContent; import org.opencb.opencga.core.models.job.*; -import org.opencb.opencga.core.models.workflow.NextFlowRunParams; +import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.*; diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java index 3c34dda26b4..695942090ad 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java @@ -3,13 +3,13 @@ import org.apache.commons.lang3.ObjectUtils; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.managers.WorkflowManager; +import org.opencb.opencga.catalog.managers.ExternalToolManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.job.Job; -import org.opencb.opencga.core.models.workflow.*; +import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.tools.annotations.*; import javax.servlet.http.HttpServletRequest; @@ -25,17 +25,17 @@ @Api(value = "Workflows", description = "Methods for working with 'workflows' endpoint") public class WorkflowWSServer extends OpenCGAWSServer { - private WorkflowManager workflowManager; + private ExternalToolManager externalToolManager; public WorkflowWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest, @Context HttpHeaders httpHeaders) throws IOException, VersionException { super(uriInfo, httpServletRequest, httpHeaders); - workflowManager = catalogManager.getWorkflowManager(); + externalToolManager = catalogManager.getWorkflowManager(); } @GET @Path("/{workflows}/info") - @ApiOperation(value = "Get workflow information", response = Workflow.class) + @ApiOperation(value = "Get workflow information", response = ExternalTool.class) @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, format = "", example = "name,attributes", dataType = "string", paramType = "query"), @@ -51,7 +51,7 @@ public Response workflowInfo( query.remove(ParamConstants.STUDY_PARAM); List workflowList = getIdList(workflowStr); - DataResult workflowDataResult = workflowManager.get(studyStr, workflowList, query, queryOptions, true, token); + DataResult workflowDataResult = externalToolManager.get(studyStr, workflowList, query, queryOptions, true, token); return createOkResponse(workflowDataResult); } catch (Exception e) { return createErrorResponse(e); @@ -60,7 +60,7 @@ public Response workflowInfo( @POST @Path("/create") - @ApiOperation(value = "Create a workflow", response = Workflow.class, notes = "Create a workflow.") + @ApiOperation(value = "Create a workflow", response = ExternalTool.class, notes = "Create a workflow.") @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), @@ -70,13 +70,13 @@ public Response workflowInfo( public Response createWorkflowPOST( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, - @ApiParam(value = "JSON containing workflow information", required = true) WorkflowCreateParams params) { + @ApiParam(value = "JSON containing workflow information", required = true) ExternalToolCreateParams params) { try { - params = ObjectUtils.defaultIfNull(params, new WorkflowCreateParams()); + params = ObjectUtils.defaultIfNull(params, new ExternalToolCreateParams()); - Workflow workflow = params.toWorkflow(); + ExternalTool externalTool = params.toWorkflow(); - return createOkResponse(workflowManager.create(studyStr, workflow, queryOptions, token)); + return createOkResponse(externalToolManager.create(studyStr, externalTool, queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -103,7 +103,7 @@ public Response updateByPost( @POST @Path("/import") @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation(value = "Import a workflow", response = Workflow.class) + @ApiOperation(value = "Import a workflow", response = ExternalTool.class) public Response importWorkflow( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "Repository parameters", required = true) WorkflowRepositoryParams params) { @@ -112,7 +112,7 @@ public Response importWorkflow( @GET @Path("/search") - @ApiOperation(value = "Workflow search method", response = Workflow.class) + @ApiOperation(value = "Workflow search method", response = ExternalTool.class) @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, example = "name,attributes", dataType = "string", paramType = "query"), @@ -143,7 +143,7 @@ public Response search( ) { try { query.remove(ParamConstants.STUDY_PARAM); - return createOkResponse(workflowManager.search(studyStr, query, queryOptions, token)); + return createOkResponse(externalToolManager.search(studyStr, query, queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -173,7 +173,7 @@ public Response distinct( query.remove(ParamConstants.STUDY_PARAM); query.remove(ParamConstants.DISTINCT_FIELD_PARAM); List fields = split(field, ParamConstants.DISTINCT_FIELD_PARAM, true); - return createOkResponse(workflowManager.distinct(studyStr, fields, query, token)); + return createOkResponse(externalToolManager.distinct(studyStr, fields, query, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -182,7 +182,7 @@ public Response distinct( @POST @Path("/{workflowId}/update") @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation(value = "Update some workflow attributes", response = Workflow.class) + @ApiOperation(value = "Update some workflow attributes", response = ExternalTool.class) @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), @@ -194,12 +194,12 @@ public Response updateByPost( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, // @ApiParam(value = ParamConstants.WORKFLOW_SCRIPTS_ACTION_DESCRIPTION, allowableValues = "ADD,SET,REMOVE", defaultValue = "ADD") @QueryParam(ParamConstants.WORKFLOW_SCRIPTS_ACTION_PARAM) ParamUtils.BasicUpdateAction workflowScriptsAction, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, - @ApiParam(value = "body") WorkflowUpdateParams parameters) { + @ApiParam(value = "body") ExternalToolUpdateParams parameters) { try { // Map actionMap = new HashMap<>(); // queryOptions.put(Constants.ACTIONS, actionMap); - return createOkResponse(workflowManager.update(studyStr, workflowId, parameters, queryOptions, token), "Workflow update success"); + return createOkResponse(externalToolManager.update(studyStr, workflowId, parameters, queryOptions, token), "Workflow update success"); } catch (Exception e) { return createErrorResponse(e); } @@ -207,12 +207,12 @@ public Response updateByPost( @DELETE @Path("/{workflows}/delete") - @ApiOperation(value = "Delete workflows", response = Workflow.class) + @ApiOperation(value = "Delete workflows", response = ExternalTool.class) public Response delete( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.WORKFLOWS_DESCRIPTION) @PathParam("workflows") String workflows) { try { - return createOkResponse(workflowManager.delete(studyStr, getIdList(workflows), queryOptions, token)); + return createOkResponse(externalToolManager.delete(studyStr, getIdList(workflows), queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -221,14 +221,14 @@ public Response delete( @GET @Path("/{workflows}/acl") @ApiOperation(value = "Returns the acl of the workflows. If member is provided, it will only return the acl for the member.", - response = WorkflowAclEntryList.class) + response = ExternalToolAclEntryList.class) public Response getAcls(@ApiParam(value = ParamConstants.WORKFLOWS_DESCRIPTION, required = true) @PathParam("workflows") String workflowIdsStr, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "User or group id") @QueryParam("member") String member, @ApiParam(value = ParamConstants.SILENT_DESCRIPTION, defaultValue = "false") @QueryParam(Constants.SILENT) boolean silent) { try { List idList = getIdList(workflowIdsStr); - return createOkResponse(workflowManager.getAcls(studyStr, idList, member, silent, token)); + return createOkResponse(externalToolManager.getAcls(studyStr, idList, member, silent, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -236,15 +236,15 @@ public Response getAcls(@ApiParam(value = ParamConstants.WORKFLOWS_DESCRIPTION, @POST @Path("/acl/{members}/update") - @ApiOperation(value = "Update the set of workflow permissions granted for the member", response = WorkflowAclEntryList.class) + @ApiOperation(value = "Update the set of workflow permissions granted for the member", response = ExternalToolAclEntryList.class) public Response updateAcl( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "Comma separated list of user or group ids", required = true) @PathParam("members") String memberIds, @ApiParam(value = ParamConstants.ACL_ACTION_DESCRIPTION, required = true, defaultValue = "ADD") @QueryParam(ParamConstants.ACL_ACTION_PARAM) ParamUtils.AclAction action, - @ApiParam(value = "JSON containing the parameters to update the permissions.", required = true) WorkflowAclUpdateParams params) { + @ApiParam(value = "JSON containing the parameters to update the permissions.", required = true) ExternalToolAclUpdateParams params) { try { - return createOkResponse(workflowManager.updateAcl(studyStr, memberIds, params, action, token)); + return createOkResponse(externalToolManager.updateAcl(studyStr, memberIds, params, action, token)); } catch (Exception e) { return createErrorResponse(e); } From faf381879f77be13709bbdf2cd1a79717c498e3a Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 16 Apr 2025 15:59:11 +0200 Subject: [PATCH 03/22] wip --- .../analysis/workflow/NextFlowExecutor.java | 41 ++-- .../CatalogAuthorizationManager.java | 6 +- .../catalog/db/api/ExternalToolDBAdaptor.java | 1 + .../mongodb/AuthorizationMongoDBAdaptor.java | 12 +- .../db/mongodb/AuthorizationMongoDBUtils.java | 2 +- .../mongodb/ExternalToolMongoDBAdaptor.java | 4 +- .../catalog/managers/CatalogManager.java | 8 +- ...lToolManager.java => WorkflowManager.java} | 126 ++++++----- ...agerTest.java => WorkflowManagerTest.java} | 54 ++--- .../opencga/core/api/FieldConstants.java | 39 ++-- .../opencga/core/models/common/Enums.java | 2 +- .../core/models/externalTool/Docker.java | 90 ++++++++ .../models/externalTool/ExternalTool.java | 139 +++++++----- .../ExternalToolCreateParams.java | 12 +- .../ExternalToolUpdateParams.java | 10 +- .../core/models/externalTool/Workflow.java | 65 ++++++ .../models/externalTool/WorkflowSystem.java | 2 +- .../workflow/WorkflowCreateParams.java | 211 ++++++++++++++++++ .../opencga/server/rest/WorkflowWSServer.java | 22 +- 19 files changed, 629 insertions(+), 217 deletions(-) rename opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/{ExternalToolManager.java => WorkflowManager.java} (91%) rename opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/{ExternalToolManagerTest.java => WorkflowManagerTest.java} (72%) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Docker.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Workflow.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowCreateParams.java diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java index 5fd44aa1b96..0b9489605b4 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java @@ -18,11 +18,8 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.externalTool.ExternalTool; +import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.models.job.ToolInfoExecutor; -import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; -import org.opencb.opencga.core.models.externalTool.WorkflowScript; -import org.opencb.opencga.core.models.externalTool.ExternalToolVariable; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.ToolDependency; import org.opencb.opencga.core.tools.annotations.Tool; @@ -44,7 +41,7 @@ import java.util.*; import java.util.stream.Stream; -@Tool(id = NextFlowExecutor.ID, resource = Enums.Resource.WORKFLOW, description = NextFlowExecutor.DESCRIPTION) +@Tool(id = NextFlowExecutor.ID, resource = Enums.Resource.EXTERNAL_TOOL, description = NextFlowExecutor.DESCRIPTION) public class NextFlowExecutor extends OpenCgaDockerToolScopeStudy { public final static String ID = "nextflow"; @@ -106,26 +103,28 @@ protected void check() throws Exception { } } - if (StringUtils.isEmpty(externalTool.getManager().getVersion())) { - externalTool.getManager().setVersion(ParamConstants.DEFAULT_MIN_NEXTFLOW_VERSION); + if (StringUtils.isEmpty(externalTool.getWorkflow().getManager().getVersion())) { + externalTool.getWorkflow().getManager().setVersion(ParamConstants.DEFAULT_MIN_NEXTFLOW_VERSION); } // Update job tags and attributes - ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(externalTool.getManager().getId().name().toLowerCase(), - externalTool.getManager().getVersion()); + ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(externalTool.getWorkflow().getManager().getId().name().toLowerCase(), + externalTool.getWorkflow().getManager().getVersion()); Set tags = new HashSet<>(); tags.add(ID); - tags.add(externalTool.getManager().getId().name()); - tags.add(externalTool.getManager().getId() + ":" + externalTool.getManager().getVersion()); + tags.add(externalTool.getWorkflow().getManager().getId().name()); + tags.add(externalTool.getWorkflow().getManager().getId() + ":" + externalTool.getWorkflow().getManager().getVersion()); tags.add(externalTool.getId()); if (CollectionUtils.isNotEmpty(externalTool.getTags())) { tags.addAll(externalTool.getTags()); } List dependencyList = new ArrayList<>(4); dependencyList.add(new ToolDependency("opencb/opencga-workflow", GitRepositoryState.getInstance().getBuildVersion() + "-hdp3.1")); - dependencyList.add(new ToolDependency("nextflow", externalTool.getManager().getVersion())); + dependencyList.add(new ToolDependency("nextflow", externalTool.getWorkflow().getManager().getVersion())); dependencyList.add(new ToolDependency(externalTool.getId(), String.valueOf(externalTool.getVersion()))); - if (externalTool.getRepository() != null && StringUtils.isNotEmpty(externalTool.getRepository().getId())) { - dependencyList.add(new ToolDependency(externalTool.getRepository().getId(), externalTool.getRepository().getVersion())); + if (externalTool.getWorkflow().getRepository() != null) { + if (StringUtils.isNotEmpty(externalTool.getWorkflow().getRepository().getId())) { + dependencyList.add(new ToolDependency(externalTool.getWorkflow().getRepository().getId(), externalTool.getWorkflow().getRepository().getVersion())); + } } addDependencies(dependencyList); updateJobInformation(new ArrayList<>(tags), toolInfoExecutor); @@ -190,7 +189,7 @@ protected void check() throws Exception { @Override protected void run() throws Exception { - for (WorkflowScript script : externalTool.getScripts()) { + for (WorkflowScript script : externalTool.getWorkflow().getScripts()) { // Write script files Path path = temporalInputDir.resolve(script.getFileName()); Files.write(path, script.getContent().getBytes()); @@ -216,21 +215,21 @@ protected void run() throws Exception { String dockerImage = "opencb/opencga-workflow:" + GitRepositoryState.getInstance().getBuildVersion() + "-hdp3.1"; StringBuilder stringBuilder = new StringBuilder() .append("bash -c \"NXF_VER=") - .append(externalTool.getManager().getVersion()) + .append(externalTool.getWorkflow().getManager().getVersion()) .append(" nextflow -c ") .append(nextflowConfigPath) .append(" run "); - if (externalTool.getRepository() != null && StringUtils.isNotEmpty(externalTool.getRepository().getId())) { + if (externalTool.getWorkflow().getRepository() != null && StringUtils.isNotEmpty(externalTool.getWorkflow().getRepository().getId())) { // stringBuilder.append(workflow.getRepository().getImage()).append(" -with-docker"); - stringBuilder.append(externalTool.getRepository().getId()); + stringBuilder.append(externalTool.getWorkflow().getRepository().getId()); // Add the repository version if we have it and the user is not specifying it - if (StringUtils.isNotEmpty(externalTool.getRepository().getVersion()) && !cliParams.contains("-r ")) { + if (StringUtils.isNotEmpty(externalTool.getWorkflow().getRepository().getVersion()) && !cliParams.contains("-r ")) { stringBuilder .append(" -r ") - .append(externalTool.getRepository().getVersion()); + .append(externalTool.getWorkflow().getRepository().getVersion()); } } else { - for (WorkflowScript script : externalTool.getScripts()) { + for (WorkflowScript script : externalTool.getWorkflow().getScripts()) { if (script.isMain()) { stringBuilder.append(temporalInputDir.resolve(script.getFileName())); break; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java index bb764152309..41f5624d38d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java @@ -754,7 +754,7 @@ private void setDependentPermissions(List aclParams) throws Ca .collect(Collectors.toSet()) ); break; - case WORKFLOW: + case EXTERNAL_TOOL: allPermissions.addAll(aclParam.getPermissions() .stream() .map(ExternalToolPermissions::valueOf) @@ -879,7 +879,7 @@ private void setImplicitPermissions(List aclParams) throws Cat .collect(Collectors.toSet()) ); break; - case WORKFLOW: + case EXTERNAL_TOOL: allPermissions.addAll(aclParam.getPermissions() .stream() .map(ExternalToolPermissions::valueOf) @@ -1002,7 +1002,7 @@ private List getImplicitPermissions(List permissions, Enums.Reso .collect(Collectors.toSet()) ); break; - case WORKFLOW: + case EXTERNAL_TOOL: allPermissions.addAll(permissions .stream() .map(ExternalToolPermissions::valueOf) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java index 00c6c28cc90..81d34a61e5a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java @@ -23,6 +23,7 @@ enum QueryParams implements QueryParam { NAME("name", TEXT, ""), DESCRIPTION("description", TEXT, ""), DRAFT("draft", BOOLEAN, ""), + TYPE("type", TEXT, ""), SCOPE("scope", TEXT, ""), TAGS("tags", TEXT_ARRAY, ""), COMMAND_LINE("commandLine", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java index d160c9f6d6b..947647a6dca 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java @@ -133,7 +133,7 @@ private void validateEntry(Enums.Resource entry) throws CatalogDBException { case FAMILY: case CLINICAL_ANALYSIS: case CLINICAL: - case WORKFLOW: + case EXTERNAL_TOOL: return; default: throw new CatalogDBException("Unexpected parameter received. " + entry + " has been received."); @@ -781,7 +781,7 @@ public OpenCGAResult resetMembersFromAllEntries(long studyId, List membe removePermissions(clientSession, studyId, members, Enums.Resource.DISEASE_PANEL); removePermissions(clientSession, studyId, members, Enums.Resource.FAMILY); removePermissions(clientSession, studyId, members, Enums.Resource.CLINICAL_ANALYSIS); - removePermissions(clientSession, studyId, members, Enums.Resource.WORKFLOW); + removePermissions(clientSession, studyId, members, Enums.Resource.EXTERNAL_TOOL); removeFromMembers(clientSession, Collections.singletonList(studyId), members, null, Enums.Resource.STUDY); return endWrite(tmpStartTime, -1, -1, null); @@ -1098,7 +1098,7 @@ private Bson parseQuery(Query query, Document rawQuery, Enums.Resource entry) th case CLINICAL_ANALYSIS: case CLINICAL: return dbAdaptorFactory.getClinicalAnalysisDBAdaptor().parseQuery(query, rawQuery); - case WORKFLOW: + case EXTERNAL_TOOL: return dbAdaptorFactory.getWorkflowDBAdaptor().parseQuery(query, rawQuery); default: throw new CatalogException("Unexpected parameter received. " + entry + " has been received."); @@ -1184,7 +1184,7 @@ private MongoDBCollection getMainCollection(Enums.Resource resource) throws Cata case CLINICAL_ANALYSIS: case CLINICAL: return dbAdaptorFactory.getClinicalAnalysisDBAdaptor().getCollection(); - case WORKFLOW: + case EXTERNAL_TOOL: return dbAdaptorFactory.getWorkflowDBAdaptor().getCollection(); default: throw new CatalogDBException("Unexpected resource '" + resource + "' parameter received."); @@ -1201,7 +1201,7 @@ private MongoDBCollection getArchiveCollection(Enums.Resource resource) throws C return dbAdaptorFactory.getCatalogPanelDBAdaptor().getPanelArchiveCollection(); case FAMILY: return dbAdaptorFactory.getCatalogFamilyDBAdaptor().getArchiveFamilyCollection(); - case WORKFLOW: + case EXTERNAL_TOOL: return dbAdaptorFactory.getWorkflowDBAdaptor().getArchiveCollection(); case CLINICAL: case CLINICAL_ANALYSIS: @@ -1214,6 +1214,6 @@ private MongoDBCollection getArchiveCollection(Enums.Resource resource) throws C private boolean hasArchiveCollection(Enums.Resource resource) { return resource == Enums.Resource.INDIVIDUAL || resource == Enums.Resource.SAMPLE || resource == Enums.Resource.DISEASE_PANEL || resource == Enums.Resource.FAMILY || resource == Enums.Resource.CLINICAL || resource == Enums.Resource.CLINICAL_ANALYSIS - || resource == Enums.Resource.WORKFLOW; + || resource == Enums.Resource.EXTERNAL_TOOL; } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java index 2218056e813..6913cbd465e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java @@ -191,7 +191,7 @@ private static int getPermissionType(Enums.Resource resource) throws CatalogPara return StudyPermissions.FAMILY; case CLINICAL_ANALYSIS: return StudyPermissions.CLINICAL_ANALYSIS; - case WORKFLOW: + case EXTERNAL_TOOL: return StudyPermissions.WORKFLOW; default: throw new CatalogParameterException("Unexpected resource '" + resource + "'."); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java index aab9cd5c7b4..7b7de5f79a3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java @@ -504,11 +504,11 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { - andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.WORKFLOW, user, + andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.EXTERNAL_TOOL, user, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, ExternalToolPermissions.VIEW.name(), - Enums.Resource.WORKFLOW, simplifyPermissions)); + Enums.Resource.EXTERNAL_TOOL, simplifyPermissions)); } query.remove(ParamConstants.ACL_PARAM); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 876cd7c023d..f832c848e68 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -88,7 +88,7 @@ public class CatalogManager implements AutoCloseable { private ClinicalAnalysisManager clinicalAnalysisManager; private InterpretationManager interpretationManager; private PanelManager panelManager; - private ExternalToolManager externalToolManager; + private WorkflowManager workflowManager; private AuditManager auditManager; private AuthorizationManager authorizationManager; @@ -164,7 +164,7 @@ private void configureManagers(Configuration configuration) throws CatalogExcept clinicalAnalysisManager = new ClinicalAnalysisManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, configuration); interpretationManager = new InterpretationManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, configuration); - externalToolManager = new ExternalToolManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, ioManagerFactory, + workflowManager = new WorkflowManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, ioManagerFactory, catalogIOManager, configuration); } @@ -458,7 +458,7 @@ public MigrationManager getMigrationManager() { return migrationManager; } - public ExternalToolManager getWorkflowManager() { - return externalToolManager; + public WorkflowManager getWorkflowManager() { + return workflowManager; } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java similarity index 91% rename from opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java rename to opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java index 1651b9f40c9..a5a8efb8c8b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.*; import org.opencb.commons.datastore.core.result.Error; @@ -29,6 +30,7 @@ import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.job.JobType; import org.opencb.opencga.core.models.job.MinimumRequirements; @@ -54,9 +56,9 @@ import static org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor.QueryParams.*; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; -public class ExternalToolManager extends ResourceManager { +public class WorkflowManager extends ResourceManager { - public static final QueryOptions INCLUDE_WORKFLOW_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(ID.key(), UID.key(), + public static final QueryOptions INCLUDE_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(ID.key(), UID.key(), UUID.key(), VERSION.key(), DESCRIPTION.key(), STUDY_UID.key(), MANAGER.key())); private final CatalogIOManager catalogIOManager; @@ -70,20 +72,20 @@ public class ExternalToolManager extends ResourceManager { private final Logger logger; - ExternalToolManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, - DBAdaptorFactory catalogDBAdaptorFactory, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager, - Configuration configuration) { + WorkflowManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, + DBAdaptorFactory catalogDBAdaptorFactory, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager, + Configuration configuration) { super(authorizationManager, auditManager, catalogManager, catalogDBAdaptorFactory, configuration); this.catalogIOManager = catalogIOManager; this.ioManagerFactory = ioManagerFactory; - this.logger = LoggerFactory.getLogger(ExternalToolManager.class); + this.logger = LoggerFactory.getLogger(WorkflowManager.class); } @Override Enums.Resource getEntity() { - return Enums.Resource.WORKFLOW; + return Enums.Resource.EXTERNAL_TOOL; } @Override @@ -99,6 +101,7 @@ InternalGetDataResult internalGet(String organizationId, long stud Query queryCopy = query == null ? new Query() : new Query(query); queryCopy.put(STUDY_UID.key(), studyUid); + queryCopy.put(TYPE.key(), ExternalTool.Type.WORKFLOW); boolean versioned = queryCopy.getBoolean(Constants.ALL_VERSIONS) || queryCopy.containsKey(VERSION.key()); @@ -135,6 +138,11 @@ InternalGetDataResult internalGet(String organizationId, long stud @Override public OpenCGAResult create(String studyStr, ExternalTool externalTool, QueryOptions options, String token) throws CatalogException { + throw new NotImplementedException("Not implemented yet"); + } + + public OpenCGAResult create(String studyStr, WorkflowCreateParams workflow, QueryOptions options, String token) + throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); @@ -142,7 +150,7 @@ public OpenCGAResult create(String studyStr, ExternalTool external ObjectMap auditParams = new ObjectMap() .append("study", studyStr) - .append("workflow", externalTool) + .append("workflow", workflow) .append("options", options) .append("token", token); @@ -172,11 +180,11 @@ public OpenCGAResult create(String studyStr, ExternalTool external OpenCGAResult result = getWorkflowDBAdaptor(organizationId).get(query, options); insert.setResults(result.getResults()); } - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), externalTool.getUuid(), studyId, + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; } catch (CatalogException e) { - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), "", studyId, studyUuid, + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), "", studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -269,7 +277,7 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepos Query query = new Query() .append(STUDY_UID.key(), study.getUid()) .append(ID.key(), externalTool.getId()); - OpenCGAResult tmpResult = getWorkflowDBAdaptor(organizationId).get(query, INCLUDE_WORKFLOW_IDS); + OpenCGAResult tmpResult = getWorkflowDBAdaptor(organizationId).get(query, INCLUDE_IDS); if (tmpResult.getNumResults() > 0) { logger.warn("Workflow '" + workflowId + "' already exists. Updating with the latest workflow information."); try { @@ -297,11 +305,11 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepos OpenCGAResult tmpTmpResult = getWorkflowDBAdaptor(organizationId).get(query, options); result.setResults(tmpTmpResult.getResults()); } - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), externalTool.getUuid(), studyId, + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, workflowId, "", studyId, studyUuid, auditParams, + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, workflowId, "", studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -346,8 +354,8 @@ private void processMemoryRequirements(ExternalTool externalTool, WorkflowReposi String inputLine; long maxMemory = IOUtils.fromHumanReadableToByte(MAX_MEMORY); while ((inputLine = in.readLine()) != null) { - Matcher cpuMatcher = CPU_PATTERN.matcher(inputLine); - Matcher memoryMatcher = MEMORY_PATTERN.matcher(inputLine); + Matcher cpuMatcher = WORKFLOW_CPU_PATTERN.matcher(inputLine); + Matcher memoryMatcher = WORKFLOW_MEMORY_PATTERN.matcher(inputLine); if (cpuMatcher.find()) { String value = cpuMatcher.group(1); int intValue = Integer.parseInt(value); @@ -452,23 +460,23 @@ private void fillWithWorkflowManifest(ExternalTool externalTool, String rawline) case "name": externalTool.setId(value.replace("/", ".")); externalTool.setName(value.replace("/", " ")); - externalTool.getRepository().setId(value); + externalTool.getWorkflow().getRepository().setId(value); break; case "author": - externalTool.getRepository().setAuthor(value); + externalTool.getWorkflow().getRepository().setAuthor(value); break; case "description": externalTool.setDescription(value); - externalTool.getRepository().setDescription(value); + externalTool.getWorkflow().getRepository().setDescription(value); break; case "version": String version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); - externalTool.getRepository().setVersion(version); + externalTool.getWorkflow().getRepository().setVersion(version); break; case "nextflowVersion": // Nextflow version must start with a number version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); - externalTool.getManager().setVersion(version); + externalTool.getWorkflow().getManager().setVersion(version); break; default: break; @@ -521,7 +529,7 @@ public OpenCGAResult update(String studyStr, String workflowId, Ex studyUuid = study.getUuid(); ExternalTool externalTool = internalGet(organizationId, study.getUid(), Collections.singletonList(workflowId), null, - INCLUDE_WORKFLOW_IDS, userId, false).first(); + INCLUDE_IDS, userId, false).first(); id = externalTool.getId(); uuid = externalTool.getUuid(); @@ -558,11 +566,11 @@ public OpenCGAResult update(String studyStr, String workflowId, Ex OpenCGAResult result = getWorkflowDBAdaptor(organizationId).get(query, options); insert.setResults(result.getResults()); } - auditManager.auditUpdate(organizationId, userId, Enums.Resource.WORKFLOW, id, uuid, studyId, studyUuid, auditParams, + auditManager.auditUpdate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, id, uuid, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; } catch (CatalogException e) { - auditManager.auditUpdate(organizationId, userId, Enums.Resource.WORKFLOW, id, uuid, studyId, studyUuid, auditParams, + auditManager.auditUpdate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, id, uuid, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -631,12 +639,12 @@ public OpenCGAResult search(String studyStr, Query query, QueryOpt OpenCGAResult queryResult = getWorkflowDBAdaptor(organizationId).get(study.getUid(), query, options, userId); - auditManager.auditSearch(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditSearch(organizationId, userId, Enums.Resource.WORKFLOW, studyId, studyUuid, auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -664,12 +672,12 @@ public OpenCGAResult distinct(String studyStr, List fields, Query que query.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); OpenCGAResult result = getWorkflowDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -696,13 +704,13 @@ public OpenCGAResult count(String studyStr, Query query, String to query.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); OpenCGAResult queryResultAux = getWorkflowDBAdaptor(organizationId).count(query, userId); - auditManager.auditCount(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); } catch (CatalogException e) { - auditManager.auditCount(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -741,7 +749,7 @@ public OpenCGAResult delete(String studyStr, List ids, Obj long studyId = study.getUid(); checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.WORKFLOW, "", "", study.getId(), study.getUuid(), + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.EXTERNAL_TOOL, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -752,7 +760,7 @@ public OpenCGAResult delete(String studyStr, List ids, Obj String workflowId = id; String workflowUuid = ""; try { - OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_WORKFLOW_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Workflow '" + id + "' not found"); } @@ -772,7 +780,7 @@ public OpenCGAResult delete(String studyStr, List ids, Obj result.append(getWorkflowDBAdaptor(organizationId).delete(externalTool)); - auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -783,7 +791,7 @@ public OpenCGAResult delete(String studyStr, List ids, Obj result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.WORKFLOW, workflowId, workflowUuid, + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.EXTERNAL_TOOL, workflowId, workflowUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } @@ -832,13 +840,13 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool fixQueryObject(finalQuery); finalQuery.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = getWorkflowDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_WORKFLOW_IDS, userId); + iterator = getWorkflowDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_IDS, userId); // If the user is the owner or the admin, we won't check if he has permissions for every single entry long studyId = study.getUid(); checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, "", "", study.getId(), study.getUuid(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.EXTERNAL_TOOL, "", "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -858,7 +866,7 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool result.append(getWorkflowDBAdaptor(organizationId).delete(externalTool)); - auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, externalTool.getId(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -869,7 +877,7 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, externalTool.getId(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } @@ -910,17 +918,17 @@ private ExternalToolDBAdaptor.QueryParams getFieldFilter(List idList) th private void validateNewWorkflow(ExternalTool externalTool, String userId) throws CatalogParameterException { ParamUtils.checkIdentifier(externalTool.getId(), ID.key()); - if (externalTool.getManager() == null) { + if (externalTool.getWorkflow().getManager() == null) { externalTool.setManager(new WorkflowSystem()); } - if (externalTool.getManager().getId() == null) { - externalTool.getManager().setId(WorkflowSystem.SystemId.NEXTFLOW); + if (externalTool.getWorkflow().getManager().getId() == null) { + externalTool.getWorkflow().getManager().setId(WorkflowSystem.SystemId.NEXTFLOW); } externalTool.setScope(ParamUtils.defaultObject(externalTool.getScope(), ExternalTool.Scope.OTHER)); externalTool.setTags(externalTool.getTags() != null ? externalTool.getTags() : Collections.emptyList()); - externalTool.setScripts(externalTool.getScripts() != null ? externalTool.getScripts() : Collections.emptyList()); + externalTool.setScripts(externalTool.getWorkflow().getScripts() != null ? externalTool.getWorkflow().getScripts() : Collections.emptyList()); boolean main = false; - for (WorkflowScript script : externalTool.getScripts()) { + for (WorkflowScript script : externalTool.getWorkflow().getScripts()) { ParamUtils.checkIdentifier(script.getFileName(), SCRIPTS.key() + ".id"); ParamUtils.checkParameter(script.getContent(), SCRIPTS.key() + ".content"); if (script.isMain()) { @@ -930,14 +938,14 @@ private void validateNewWorkflow(ExternalTool externalTool, String userId) throw main = script.isMain(); } } - if (CollectionUtils.isNotEmpty(externalTool.getScripts()) && !main) { + if (CollectionUtils.isNotEmpty(externalTool.getWorkflow().getScripts()) && !main) { throw new CatalogParameterException("No main script found."); } - externalTool.setRepository(externalTool.getRepository() != null ? externalTool.getRepository() : new WorkflowRepository("")); - if (StringUtils.isEmpty(externalTool.getRepository().getId()) && CollectionUtils.isEmpty(externalTool.getScripts())) { + externalTool.setRepository(externalTool.getWorkflow().getRepository() != null ? externalTool.getWorkflow().getRepository() : new WorkflowRepository("")); + if (StringUtils.isEmpty(externalTool.getWorkflow().getRepository().getId()) && CollectionUtils.isEmpty(externalTool.getWorkflow().getScripts())) { throw new CatalogParameterException("No repository image or scripts found."); } - if (StringUtils.isNotEmpty(externalTool.getRepository().getId()) && CollectionUtils.isNotEmpty(externalTool.getScripts())) { + if (StringUtils.isNotEmpty(externalTool.getWorkflow().getRepository().getId()) && CollectionUtils.isNotEmpty(externalTool.getWorkflow().getScripts())) { throw new CatalogParameterException("Both repository image and scripts found. Please, either add scripts or a repository" + " image."); } @@ -981,7 +989,7 @@ public OpenCGAResult> getAcls(String study try { auditManager.initAuditBatch(operationId); InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), workflowList, - INCLUDE_WORKFLOW_IDS, userId, ignoreException); + INCLUDE_IDS, userId, ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -990,10 +998,10 @@ public OpenCGAResult> getAcls(String study List workflowUids = queryResult.getResults().stream().map(ExternalTool::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { - workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, members, Enums.Resource.WORKFLOW, + workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, members, Enums.Resource.EXTERNAL_TOOL, ExternalToolPermissions.class, userId); } else { - workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, Enums.Resource.WORKFLOW, + workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, Enums.Resource.EXTERNAL_TOOL, ExternalToolPermissions.class, userId); } @@ -1005,14 +1013,14 @@ public OpenCGAResult> getAcls(String study if (!missingMap.containsKey(workflowId)) { ExternalTool externalTool = queryResult.getResults().get(counter); resultList.add(workflowAcls.getResults().get(counter)); - auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.WORKFLOW, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, workflowId, missingMap.get(workflowId).getErrorMsg())); - auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.WORKFLOW, workflowId, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.EXTERNAL_TOOL, workflowId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", missingMap.get(workflowId).getErrorMsg())), new ObjectMap()); } @@ -1024,7 +1032,7 @@ public OpenCGAResult> getAcls(String study workflowAcls.setEvents(eventList); } catch (CatalogException e) { for (String workflowId : workflowList) { - auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.WORKFLOW, workflowId, "", + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.EXTERNAL_TOOL, workflowId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -1077,7 +1085,7 @@ public OpenCGAResult> updateAcl(String stu checkPermissions(params.getPermissions(), ExternalToolPermissions::valueOf); } - externalToolList = internalGet(organizationId, study.getUid(), params.getExternalToolIds(), INCLUDE_WORKFLOW_IDS, userId, false) + externalToolList = internalGet(organizationId, study.getUid(), params.getExternalToolIds(), INCLUDE_IDS, userId, false) .getResults(); authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); @@ -1092,7 +1100,7 @@ public OpenCGAResult> updateAcl(String stu } catch (CatalogException e) { if (CollectionUtils.isNotEmpty(params.getExternalToolIds())) { for (String workflowId : params.getExternalToolIds()) { - auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, workflowId, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.EXTERNAL_TOOL, workflowId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -1113,7 +1121,7 @@ public OpenCGAResult> updateAcl(String stu List workflowUids = batchExternalToolList.stream().map(ExternalTool::getUid).collect(Collectors.toList()); List workflowIds = batchExternalToolList.stream().map(ExternalTool::getId).collect(Collectors.toList()); List aclParamsList = new ArrayList<>(); - AuthorizationManager.CatalogAclParams.addToList(workflowUids, params.getPermissions(), Enums.Resource.WORKFLOW, aclParamsList); + AuthorizationManager.CatalogAclParams.addToList(workflowUids, params.getPermissions(), Enums.Resource.EXTERNAL_TOOL, aclParamsList); try { switch (action) { @@ -1138,7 +1146,7 @@ public OpenCGAResult> updateAcl(String stu OpenCGAResult> queryResults = authorizationManager.getAcls(organizationId, study.getUid(), - workflowUids, members, Enums.Resource.WORKFLOW, ExternalToolPermissions.class); + workflowUids, members, Enums.Resource.EXTERNAL_TOOL, ExternalToolPermissions.class); for (int i = 0; i < queryResults.getResults().size(); i++) { queryResults.getResults().get(i).setId(workflowIds.get(i)); @@ -1146,14 +1154,14 @@ public OpenCGAResult> updateAcl(String stu aclResultList.append(queryResults); for (ExternalTool externalTool : batchExternalToolList) { - auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } } catch (CatalogException e) { // Process current batch for (ExternalTool externalTool : batchExternalToolList) { - auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -1161,7 +1169,7 @@ public OpenCGAResult> updateAcl(String stu // Process remaining unprocessed batches while (numProcessed < externalToolList.size()) { ExternalTool externalTool = externalToolList.get(numProcessed); - auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java similarity index 72% rename from opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java rename to opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java index 6032bfb6f50..1afa7013f43 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java @@ -19,39 +19,39 @@ import static org.junit.Assert.*; @Category(MediumTests.class) -public class ExternalToolManagerTest extends AbstractManagerTest { +public class WorkflowManagerTest extends AbstractManagerTest { - private ExternalToolManager externalToolManager; + private WorkflowManager workflowManager; @Before public void setUp() throws Exception { super.setUp(); - externalToolManager = catalogManager.getWorkflowManager(); + workflowManager = catalogManager.getWorkflowManager(); } @Test public void importWorkflow() throws CatalogException { WorkflowRepositoryParams params = new WorkflowRepositoryParams("nf-core/rnaseq"); - OpenCGAResult result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + OpenCGAResult result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); // Update imported workflow - result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(2, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/proteinfold"); - result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/methylseq"); - result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/pacvar"); - result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); } @@ -62,7 +62,7 @@ public void createWorkflowTest() throws CatalogException { .setId("workflow") .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - OpenCGAResult result = externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + OpenCGAResult result = workflowManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertNotNull(result.first()); assertEquals(externalTool.getId(), result.first().getId()); @@ -71,12 +71,12 @@ public void createWorkflowTest() throws CatalogException { externalTool.setId("workflow2"); externalTool.setRepository(new WorkflowRepository("blabla")); CatalogException catalogException = assertThrows(CatalogException.class, - () -> externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); + () -> workflowManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("repository")); // Remove script from workflow externalTool.setScripts(Collections.emptyList()); - result = externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + result = workflowManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertNotNull(result.first()); assertEquals(externalTool.getId(), result.first().getId()); @@ -89,7 +89,7 @@ public void createWorkflowTest() throws CatalogException { new WorkflowScript("script2", "echo 'World'", true) )); catalogException = assertThrows(CatalogException.class, - () -> externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); + () -> workflowManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main")); // Add one single script without main @@ -97,7 +97,7 @@ public void createWorkflowTest() throws CatalogException { new WorkflowScript("script1", "echo 'Hello'", false) )); catalogException = assertThrows(CatalogException.class, - () -> externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); + () -> workflowManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main")); } @@ -107,26 +107,26 @@ public void workflowSearchTest() throws CatalogException { .setId("workflow") .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + workflowManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); externalTool = new ExternalTool() .setId("workflow2") .setScope(ExternalTool.Scope.OTHER) .setDraft(true) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + workflowManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); - OpenCGAResult search = externalToolManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); + OpenCGAResult search = workflowManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); assertEquals(2, search.getNumResults()); Query query = new Query(ExternalToolDBAdaptor.QueryParams.DRAFT.key(), true); - search = externalToolManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); + search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("workflow2", search.first().getId()); assertTrue(search.first().isDraft()); query = new Query(ExternalToolDBAdaptor.QueryParams.DRAFT.key(), false); - search = externalToolManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); + search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("workflow", search.first().getId()); assertFalse(search.first().isDraft()); @@ -138,7 +138,7 @@ public void updateWorkflowTest() throws CatalogException { .setId("workflow") .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + workflowManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); ExternalToolUpdateParams updateParams = new ExternalToolUpdateParams() .setName("newName") @@ -148,7 +148,7 @@ public void updateWorkflowTest() throws CatalogException { .setModificationDate("20240201000000") .setDescription("description"); - OpenCGAResult update = externalToolManager.update(studyFqn, externalTool.getId(), updateParams, INCLUDE_RESULT, ownerToken); + OpenCGAResult update = workflowManager.update(studyFqn, externalTool.getId(), updateParams, INCLUDE_RESULT, ownerToken); assertEquals(1, update.getNumUpdated()); ExternalTool updatedExternalTool = update.first(); assertEquals(updateParams.getName(), updatedExternalTool.getName()); @@ -164,13 +164,13 @@ public void deleteWorkflowTest() throws CatalogException { .setId("workflow") .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, QueryOptions.empty(), ownerToken); + workflowManager.create(studyFqn, externalTool, QueryOptions.empty(), ownerToken); - OpenCGAResult result = externalToolManager.delete(studyFqn, externalTool.getId(), QueryOptions.empty(), ownerToken); + OpenCGAResult result = workflowManager.delete(studyFqn, externalTool.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumDeleted()); CatalogException exception = assertThrows(CatalogException.class, - () -> externalToolManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), ownerToken)); + () -> workflowManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), ownerToken)); assertTrue(exception.getMessage().contains("not found")); } @@ -180,15 +180,15 @@ public void updateWorkflowAclTest() throws CatalogException { .setId("workflow") .setScope(ExternalTool.Scope.OTHER) .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, QueryOptions.empty(), ownerToken); + workflowManager.create(studyFqn, externalTool, QueryOptions.empty(), ownerToken); CatalogAuthorizationException catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, - () -> externalToolManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), noAccessToken1)); + () -> workflowManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), noAccessToken1)); assertTrue(catalogAuthorizationException.getMessage().contains("denied")); - externalToolManager.updateAcl(studyFqn, noAccessUserId1, new ExternalToolAclUpdateParams(Collections.singletonList(externalTool.getId()), Collections.singletonList("VIEW")), + workflowManager.updateAcl(studyFqn, noAccessUserId1, new ExternalToolAclUpdateParams(Collections.singletonList(externalTool.getId()), Collections.singletonList("VIEW")), ParamUtils.AclAction.ADD, ownerToken); - OpenCGAResult result = externalToolManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), noAccessToken1); + OpenCGAResult result = workflowManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), noAccessToken1); assertEquals(1, result.getNumResults()); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 82f64bb628a..dd8c748a877 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -335,20 +335,27 @@ public class FieldConstants { public static final String TOOL_INFO_EXTERNAL_EXECUTOR_DESCRIPTION = "Object containing the id and version of the external tool that" + " is being executed."; - // Workflow - public static final String WORKFLOW_ID_DESCRIPTION = "Workflow ID."; - public static final String WORKFLOW_NAME_DESCRIPTION = "Workflow name."; - public static final String WORKFLOW_SCOPE_DESCRIPTION = "Workflow scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, " - + "CLINICAL_INTERPRETATION_ANALYSIS or OTHER."; - public static final String WORKFLOW_MANAGER_DESCRIPTION = "Workflow system corresponding to the workflow."; - public static final String WORKFLOW_DRAFT_DESCRIPTION = "Flag indicating whether the workflow is still a draft or not."; - public static final String WORKFLOW_REPOSITORY_DESCRIPTION = "Workflow repository image to execute. If any, providing a list of" - + " scripts will not be mandatory."; - public static final String WORKFLOW_SCRIPTS_DESCRIPTION = "List of scripts used by the Workflow."; - public static final String WORKFLOW_TAGS_DESCRIPTION = "List of tags."; - public static final String WORKFLOW_VARIABLES_DESCRIPTION = "List of variables accepted by the Workflow."; + // Docker + public static final String DOCKER_ID_DESCRIPTION = "Docker ID."; + public static final String DOCKER_TAG_DESCRIPTION = "Docker tag."; + public static final String DOCKER_AUTHOR_DESCRIPTION = "Docker author."; + public static final String DOCKER_DESCRIPTION_DESCRIPTION = "Docker description."; + public static final String DOCKER_CLI_DESCRIPTION = "Docker CLI."; + + // External tool + public static final String EXTERNAL_TOOL_ID_DESCRIPTION = "External tool ID."; + public static final String EXTERNAL_TOOL_NAME_DESCRIPTION = "External tool name."; + public static final String EXTERNAL_TOOL_TYPE_DESCRIPTION = "External tool type. Valid values: TOOL, VARIANT_WALKER or WORKFLOW."; + public static final String EXTERNAL_TOOL_SCOPE_DESCRIPTION = "External tool scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS," + + " CLINICAL_INTERPRETATION_ANALYSIS or OTHER."; + public static final String EXTERNAL_TOOL_DRAFT_DESCRIPTION = "Flag indicating whether the external tool is a draft or not."; + public static final String EXTERNAL_TOOL_TAGS_DESCRIPTION = "List of tags."; + public static final String EXTERNAL_TOOL_VARIABLES_DESCRIPTION = "List of variables accepted by the External Tool."; public static final String MINIMUM_REQUIREMENTS_DESCRIPTION = "Minimum requirements to execute the process."; - public static final String WORKFLOW_INTERNAL_DESCRIPTION = "Workflow internal information."; + public static final String EXTERNAL_TOOL_INTERNAL_DESCRIPTION = "External tool internal information."; + public static final String EXTERNAL_TOOL_WORKFLOW_DESCRIPTION = "Workflow information if the external tool type is a workflow."; + public static final String EXTERNAL_TOOL_DOCKER_DESCRIPTION = "Docker information if the external tool type is of type TOOL or" + + " VARIANT_WALKER."; public static final String MIN_REQUIREMENTS_CPU_DESCRIPTION = "Minimum number of cpu cores required to execute the process."; public static final String MIN_REQUIREMENTS_MEMORY_DESCRIPTION = "Minimum memory required to execute the process."; @@ -357,6 +364,12 @@ public class FieldConstants { public static final String WORKFLOW_SYSTEM_ID_DESCRIPTION = "Workflow system id. Valid values: NEXTFLOW."; public static final String WORKFLOW_SYSTEM_VERSION_DESCRIPTION = "Workflow system version to use."; + // Workflow + public static final String WORKFLOW_MANAGER_DESCRIPTION = "Workflow system corresponding to the workflow."; + public static final String WORKFLOW_REPOSITORY_DESCRIPTION = "Workflow repository image to execute. If any, providing a list of" + + " scripts will not be mandatory."; + public static final String WORKFLOW_SCRIPTS_DESCRIPTION = "List of scripts used by the Workflow."; + //FileInternal public static final String FILE_INTERNAL_STATUS_DESCRIPTION = "File status can have the values READY, DELETED, " + "TRASHED, STAGE, MISSING, PENDING_DELETE, DELETING, REMOVED and MISSING_SAMPLES."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java index 794ac8555ed..d1ef08adf11 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java @@ -80,7 +80,7 @@ public enum Resource { EXPRESSION, RGA, FUNCTIONAL, - WORKFLOW, + EXTERNAL_TOOL, RESOURCE; public List getFullPermissionList() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Docker.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Docker.java new file mode 100644 index 00000000000..92798e164a3 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Docker.java @@ -0,0 +1,90 @@ +package org.opencb.opencga.core.models.externalTool; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +public class Docker { + + @DataField(id = "id", description = FieldConstants.DOCKER_ID_DESCRIPTION) + private String id; + + @DataField(id = "tag", description = FieldConstants.DOCKER_TAG_DESCRIPTION) + private String tag; + + @DataField(id = "author", description = FieldConstants.DOCKER_AUTHOR_DESCRIPTION) + private String author; + + @DataField(id = "description", description = FieldConstants.DOCKER_DESCRIPTION_DESCRIPTION) + private String description; + + @DataField(id = "cli", description = FieldConstants.DOCKER_CLI_DESCRIPTION) + private String cli; + + public Docker() { + } + + public Docker(String id, String tag, String author, String description, String cli) { + this.id = id; + this.tag = tag; + this.author = author; + this.description = description; + this.cli = cli; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ExternalToolDocker{"); + sb.append("id='").append(id).append('\''); + sb.append(", tag='").append(tag).append('\''); + sb.append(", author='").append(author).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", cli='").append(cli).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public Docker setId(String id) { + this.id = id; + return this; + } + + public String getTag() { + return tag; + } + + public Docker setTag(String tag) { + this.tag = tag; + return this; + } + + public String getAuthor() { + return author; + } + + public Docker setAuthor(String author) { + this.author = author; + return this; + } + + public String getDescription() { + return description; + } + + public Docker setDescription(String description) { + this.description = description; + return this; + } + + public String getCli() { + return cli; + } + + public Docker setCli(String cli) { + this.cli = cli; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java index 2ebc9ea82fc..4089adc3cb4 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java @@ -13,7 +13,7 @@ public class ExternalTool extends PrivateStudyUid { @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, - description = FieldConstants.WORKFLOW_ID_DESCRIPTION) + description = FieldConstants.EXTERNAL_TOOL_ID_DESCRIPTION) private String id; @DataField(id = "uuid", managed = true, indexed = true, unique = true, immutable = true, @@ -26,37 +26,37 @@ public class ExternalTool extends PrivateStudyUid { @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) private String description; - @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) + @DataField(id = "type", description = FieldConstants.EXTERNAL_TOOL_TYPE_DESCRIPTION) + private Type type; + + @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) private Scope scope; - @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) - private WorkflowSystem manager; + @DataField(id = "workflow", description = FieldConstants.EXTERNAL_TOOL_WORKFLOW_DESCRIPTION) + private Workflow workflow; + + @DataField(id = "docker", description = FieldConstants.EXTERNAL_TOOL_DOCKER_DESCRIPTION) + private Docker docker; - @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) + @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) private List tags; - @DataField(id = "variables", description = FieldConstants.WORKFLOW_VARIABLES_DESCRIPTION) + @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) private MinimumRequirements minimumRequirements; - @DataField(id = "draft", description = FieldConstants.WORKFLOW_DRAFT_DESCRIPTION) + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) private boolean draft; - @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) - private WorkflowRepository repository; - - @DataField(id = "scripts", description = FieldConstants.WORKFLOW_SCRIPTS_DESCRIPTION) - private List scripts; - @DataField(id = "version", managed = true, indexed = true, description = FieldConstants.GENERIC_VERSION_DESCRIPTION) private int version; @DataField(id = "release", managed = true, indexed = true, description = FieldConstants.GENERIC_RELEASE_DESCRIPTION) private int release; - @DataField(id = "internal", description = FieldConstants.WORKFLOW_INTERNAL_DESCRIPTION) + @DataField(id = "internal", description = FieldConstants.EXTERNAL_TOOL_INTERNAL_DESCRIPTION) private ExternalToolInternal internal; @DataField(id = "creationDate", indexed = true, description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) @@ -84,6 +84,7 @@ public enum Scope { public ExternalTool() { } + @Deprecated public ExternalTool(String id, String name, String description, Scope scope, WorkflowSystem manager, List tags, List variables, MinimumRequirements minimumRequirements, boolean draft, WorkflowRepository repository, List scripts, ExternalToolInternal internal, String creationDate, @@ -105,21 +106,45 @@ public ExternalTool(String id, String name, String description, Scope scope, Wor this.attributes = attributes; } + public ExternalTool(String id, String uuid, String name, String description, Type type, Scope scope, Workflow workflow, Docker docker, + List tags, List variables, MinimumRequirements minimumRequirements, boolean draft, + int version, int release, ExternalToolInternal internal, String creationDate, String modificationDate, + Map attributes) { + this.id = id; + this.uuid = uuid; + this.name = name; + this.description = description; + this.type = type; + this.scope = scope; + this.workflow = workflow; + this.docker = docker; + this.tags = tags; + this.variables = variables; + this.minimumRequirements = minimumRequirements; + this.draft = draft; + this.version = version; + this.release = release; + this.internal = internal; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.attributes = attributes; + } + @Override public String toString() { - final StringBuilder sb = new StringBuilder("Workflow{"); + final StringBuilder sb = new StringBuilder("ExternalTool{"); sb.append("id='").append(id).append('\''); sb.append(", uuid='").append(uuid).append('\''); sb.append(", name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); + sb.append(", type=").append(type); sb.append(", scope=").append(scope); - sb.append(", manager=").append(manager); + sb.append(", workflow=").append(workflow); + sb.append(", docker=").append(docker); sb.append(", tags=").append(tags); sb.append(", variables=").append(variables); sb.append(", minimumRequirements=").append(minimumRequirements); sb.append(", draft=").append(draft); - sb.append(", repository=").append(repository); - sb.append(", scripts=").append(scripts); sb.append(", version=").append(version); sb.append(", release=").append(release); sb.append(", internal=").append(internal); @@ -173,30 +198,12 @@ public ExternalTool setDescription(String description) { return this; } - public boolean isDraft() { - return draft; - } - - public ExternalTool setDraft(boolean draft) { - this.draft = draft; - return this; - } - - public int getVersion() { - return version; - } - - public ExternalTool setVersion(int version) { - this.version = version; - return this; - } - - public int getRelease() { - return release; + public Type getType() { + return type; } - public ExternalTool setRelease(int release) { - this.release = release; + public ExternalTool setType(Type type) { + this.type = type; return this; } @@ -209,30 +216,21 @@ public ExternalTool setScope(Scope scope) { return this; } - public WorkflowSystem getManager() { - return manager; - } - - public ExternalTool setManager(WorkflowSystem manager) { - this.manager = manager; - return this; - } - - public WorkflowRepository getRepository() { - return repository; + public Workflow getWorkflow() { + return workflow; } - public ExternalTool setRepository(WorkflowRepository repository) { - this.repository = repository; + public ExternalTool setWorkflow(Workflow workflow) { + this.workflow = workflow; return this; } - public List getScripts() { - return scripts; + public Docker getDocker() { + return docker; } - public ExternalTool setScripts(List scripts) { - this.scripts = scripts; + public ExternalTool setDocker(Docker docker) { + this.docker = docker; return this; } @@ -263,6 +261,33 @@ public ExternalTool setMinimumRequirements(MinimumRequirements minimumRequiremen return this; } + public boolean isDraft() { + return draft; + } + + public ExternalTool setDraft(boolean draft) { + this.draft = draft; + return this; + } + + public int getVersion() { + return version; + } + + public ExternalTool setVersion(int version) { + this.version = version; + return this; + } + + public int getRelease() { + return release; + } + + public ExternalTool setRelease(int release) { + this.release = release; + return this; + } + public ExternalToolInternal getInternal() { return internal; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java index 88ed58027d9..2b24a16e69a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java @@ -10,10 +10,10 @@ public class ExternalToolCreateParams { @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, - description = FieldConstants.WORKFLOW_ID_DESCRIPTION) + description = FieldConstants.EXTERNAL_TOOL_ID_DESCRIPTION) private String id; - @DataField(id = "name", description = FieldConstants.WORKFLOW_NAME_DESCRIPTION) + @DataField(id = "name", description = FieldConstants.EXTERNAL_TOOL_NAME_DESCRIPTION) private String name; @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) @@ -22,19 +22,19 @@ public class ExternalToolCreateParams { @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) private WorkflowSystem manager; - @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) + @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) private ExternalTool.Scope scope; - @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) + @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) private List tags; - @DataField(id = "draft", description = FieldConstants.WORKFLOW_DRAFT_DESCRIPTION) + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) private boolean draft; @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) private WorkflowRepository repository; - @DataField(id = "variables", description = FieldConstants.WORKFLOW_VARIABLES_DESCRIPTION) + @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java index 48e9bc6361a..fc72e95cb5a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java @@ -9,7 +9,7 @@ public class ExternalToolUpdateParams { - @DataField(id = "name", description = FieldConstants.WORKFLOW_NAME_DESCRIPTION) + @DataField(id = "name", description = FieldConstants.EXTERNAL_TOOL_NAME_DESCRIPTION) private String name; @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) @@ -18,13 +18,13 @@ public class ExternalToolUpdateParams { @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) private WorkflowSystem manager; - @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) + @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) private ExternalTool.Scope scope; - @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) + @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) private List tags; - @DataField(id = "draft", description = FieldConstants.WORKFLOW_DRAFT_DESCRIPTION) + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) private boolean draft; @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) @@ -33,7 +33,7 @@ public class ExternalToolUpdateParams { @DataField(id = "scripts", description = FieldConstants.WORKFLOW_SCRIPTS_DESCRIPTION) private List scripts; - @DataField(id = "variables", description = FieldConstants.WORKFLOW_VARIABLES_DESCRIPTION) + @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Workflow.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Workflow.java new file mode 100644 index 00000000000..416a8e8e819 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Workflow.java @@ -0,0 +1,65 @@ +package org.opencb.opencga.core.models.externalTool; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +import java.util.List; + +public class Workflow { + + @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) + private WorkflowSystem manager; + + @DataField(id = "scripts", description = FieldConstants.WORKFLOW_SCRIPTS_DESCRIPTION) + private List scripts; + + @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) + private WorkflowRepository repository; + + public Workflow() { + } + + public Workflow(WorkflowSystem manager, List scripts, WorkflowRepository repository) { + this.manager = manager; + this.scripts = scripts; + this.repository = repository; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Workflow{"); + sb.append("manager=").append(manager); + sb.append(", scripts=").append(scripts); + sb.append(", repository=").append(repository); + sb.append('}'); + return sb.toString(); + } + + public WorkflowSystem getManager() { + return manager; + } + + public Workflow setManager(WorkflowSystem manager) { + this.manager = manager; + return this; + } + + public List getScripts() { + return scripts; + } + + public Workflow setScripts(List scripts) { + this.scripts = scripts; + return this; + } + + public WorkflowRepository getRepository() { + return repository; + } + + public Workflow setRepository(WorkflowRepository repository) { + this.repository = repository; + return this; + } + +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java index ab692be189a..7ca958c5820 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java @@ -8,7 +8,7 @@ public class WorkflowSystem { @DataField(id = "id", description = FieldConstants.WORKFLOW_SYSTEM_ID_DESCRIPTION) private SystemId id; - @DataField(id = "version", description = FieldConstants.WORKFLOW_SYSTEM_ID_DESCRIPTION) + @DataField(id = "version", description = FieldConstants.WORKFLOW_SYSTEM_VERSION_DESCRIPTION) private String version; public enum SystemId { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowCreateParams.java new file mode 100644 index 00000000000..85677d46859 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowCreateParams.java @@ -0,0 +1,211 @@ +package org.opencb.opencga.core.models.externalTool.workflow; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.externalTool.*; +import org.opencb.opencga.core.models.job.MinimumRequirements; + +import java.util.List; +import java.util.Map; + +public class WorkflowCreateParams { + + @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, + description = FieldConstants.EXTERNAL_TOOL_ID_DESCRIPTION) + private String id; + + @DataField(id = "name", description = FieldConstants.GENERIC_UUID_DESCRIPTION) + private String name; + + @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) + private String description; + + @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) + private ExternalTool.Scope scope; + + @DataField(id = "workflow", description = FieldConstants.EXTERNAL_TOOL_WORKFLOW_DESCRIPTION) + private Workflow workflow; + + @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) + private List tags; + + @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) + private List variables; + + @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) + private MinimumRequirements minimumRequirements; + + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) + private boolean draft; + + @DataField(id = "internal", description = FieldConstants.EXTERNAL_TOOL_INTERNAL_DESCRIPTION) + private ExternalToolInternal internal; + + @DataField(id = "creationDate", indexed = true, description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; + + @DataField(id = "modificationDate", indexed = true, description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; + + @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) + private Map attributes; + + public WorkflowCreateParams() { + } + + public WorkflowCreateParams(String id, String name, String description, ExternalTool.Scope scope, Workflow workflow, List tags, + List variables, MinimumRequirements minimumRequirements, boolean draft, + ExternalToolInternal internal, String creationDate, String modificationDate, + Map attributes) { + this.id = id; + this.name = name; + this.description = description; + this.scope = scope; + this.workflow = workflow; + this.tags = tags; + this.variables = variables; + this.minimumRequirements = minimumRequirements; + this.draft = draft; + this.internal = internal; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.attributes = attributes; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("WorkflowCreateParams{"); + sb.append("id='").append(id).append('\''); + sb.append(", name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", scope=").append(scope); + sb.append(", workflow=").append(workflow); + sb.append(", tags=").append(tags); + sb.append(", variables=").append(variables); + sb.append(", minimumRequirements=").append(minimumRequirements); + sb.append(", draft=").append(draft); + sb.append(", internal=").append(internal); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", attributes=").append(attributes); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public WorkflowCreateParams setId(String id) { + this.id = id; + return this; + } + + public String getName() { + return name; + } + + public WorkflowCreateParams setName(String name) { + this.name = name; + return this; + } + + public String getDescription() { + return description; + } + + public WorkflowCreateParams setDescription(String description) { + this.description = description; + return this; + } + + public ExternalTool.Scope getScope() { + return scope; + } + + public WorkflowCreateParams setScope(ExternalTool.Scope scope) { + this.scope = scope; + return this; + } + + public Workflow getWorkflow() { + return workflow; + } + + public WorkflowCreateParams setWorkflow(Workflow workflow) { + this.workflow = workflow; + return this; + } + + public List getTags() { + return tags; + } + + public WorkflowCreateParams setTags(List tags) { + this.tags = tags; + return this; + } + + public List getVariables() { + return variables; + } + + public WorkflowCreateParams setVariables(List variables) { + this.variables = variables; + return this; + } + + public MinimumRequirements getMinimumRequirements() { + return minimumRequirements; + } + + public WorkflowCreateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { + this.minimumRequirements = minimumRequirements; + return this; + } + + public boolean isDraft() { + return draft; + } + + public WorkflowCreateParams setDraft(boolean draft) { + this.draft = draft; + return this; + } + + public ExternalToolInternal getInternal() { + return internal; + } + + public WorkflowCreateParams setInternal(ExternalToolInternal internal) { + this.internal = internal; + return this; + } + + public String getCreationDate() { + return creationDate; + } + + public WorkflowCreateParams setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public WorkflowCreateParams setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + public Map getAttributes() { + return attributes; + } + + public WorkflowCreateParams setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } +} diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java index 695942090ad..0613f71cab0 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java @@ -3,7 +3,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.managers.ExternalToolManager; +import org.opencb.opencga.catalog.managers.WorkflowManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -25,12 +25,12 @@ @Api(value = "Workflows", description = "Methods for working with 'workflows' endpoint") public class WorkflowWSServer extends OpenCGAWSServer { - private ExternalToolManager externalToolManager; + private WorkflowManager workflowManager; public WorkflowWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest, @Context HttpHeaders httpHeaders) throws IOException, VersionException { super(uriInfo, httpServletRequest, httpHeaders); - externalToolManager = catalogManager.getWorkflowManager(); + workflowManager = catalogManager.getWorkflowManager(); } @GET @@ -51,7 +51,7 @@ public Response workflowInfo( query.remove(ParamConstants.STUDY_PARAM); List workflowList = getIdList(workflowStr); - DataResult workflowDataResult = externalToolManager.get(studyStr, workflowList, query, queryOptions, true, token); + DataResult workflowDataResult = workflowManager.get(studyStr, workflowList, query, queryOptions, true, token); return createOkResponse(workflowDataResult); } catch (Exception e) { return createErrorResponse(e); @@ -76,7 +76,7 @@ public Response createWorkflowPOST( ExternalTool externalTool = params.toWorkflow(); - return createOkResponse(externalToolManager.create(studyStr, externalTool, queryOptions, token)); + return createOkResponse(workflowManager.create(studyStr, externalTool, queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -143,7 +143,7 @@ public Response search( ) { try { query.remove(ParamConstants.STUDY_PARAM); - return createOkResponse(externalToolManager.search(studyStr, query, queryOptions, token)); + return createOkResponse(workflowManager.search(studyStr, query, queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -173,7 +173,7 @@ public Response distinct( query.remove(ParamConstants.STUDY_PARAM); query.remove(ParamConstants.DISTINCT_FIELD_PARAM); List fields = split(field, ParamConstants.DISTINCT_FIELD_PARAM, true); - return createOkResponse(externalToolManager.distinct(studyStr, fields, query, token)); + return createOkResponse(workflowManager.distinct(studyStr, fields, query, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -199,7 +199,7 @@ public Response updateByPost( // Map actionMap = new HashMap<>(); // queryOptions.put(Constants.ACTIONS, actionMap); - return createOkResponse(externalToolManager.update(studyStr, workflowId, parameters, queryOptions, token), "Workflow update success"); + return createOkResponse(workflowManager.update(studyStr, workflowId, parameters, queryOptions, token), "Workflow update success"); } catch (Exception e) { return createErrorResponse(e); } @@ -212,7 +212,7 @@ public Response delete( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.WORKFLOWS_DESCRIPTION) @PathParam("workflows") String workflows) { try { - return createOkResponse(externalToolManager.delete(studyStr, getIdList(workflows), queryOptions, token)); + return createOkResponse(workflowManager.delete(studyStr, getIdList(workflows), queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -228,7 +228,7 @@ public Response getAcls(@ApiParam(value = ParamConstants.WORKFLOWS_DESCRIPTION, @ApiParam(value = ParamConstants.SILENT_DESCRIPTION, defaultValue = "false") @QueryParam(Constants.SILENT) boolean silent) { try { List idList = getIdList(workflowIdsStr); - return createOkResponse(externalToolManager.getAcls(studyStr, idList, member, silent, token)); + return createOkResponse(workflowManager.getAcls(studyStr, idList, member, silent, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -244,7 +244,7 @@ public Response updateAcl( @QueryParam(ParamConstants.ACL_ACTION_PARAM) ParamUtils.AclAction action, @ApiParam(value = "JSON containing the parameters to update the permissions.", required = true) ExternalToolAclUpdateParams params) { try { - return createOkResponse(externalToolManager.updateAcl(studyStr, memberIds, params, action, token)); + return createOkResponse(workflowManager.updateAcl(studyStr, memberIds, params, action, token)); } catch (Exception e) { return createErrorResponse(e); } From cdff2b5d64d4a9481b59b014b566599fcc326ee6 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 16 Apr 2025 15:59:11 +0200 Subject: [PATCH 04/22] catalog: external tool changes compile, #TASK-7610 --- .../analysis/workflow/NextFlowExecutor.java | 45 ++-- .../tools/ExternalToolExecutorTest.java | 27 +-- .../executors/WorkflowsCommandExecutor.java | 39 ++-- .../main/options/WorkflowsCommandOptions.java | 69 +++--- .../CatalogAuthorizationManager.java | 6 +- .../catalog/db/api/ExternalToolDBAdaptor.java | 2 + .../mongodb/AuthorizationMongoDBAdaptor.java | 12 +- .../db/mongodb/AuthorizationMongoDBUtils.java | 2 +- .../mongodb/ExternalToolMongoDBAdaptor.java | 4 +- .../catalog/managers/CatalogManager.java | 8 +- ...lToolManager.java => WorkflowManager.java} | 198 ++++++++++-------- ...agerTest.java => WorkflowManagerTest.java} | 111 +++++----- opencga-client/src/main/R/R/Admin-methods.R | 2 +- opencga-client/src/main/R/R/Study-methods.R | 2 +- .../client/rest/clients/WorkflowClient.java | 4 +- opencga-client/src/main/javascript/Admin.js | 2 +- opencga-client/src/main/javascript/Study.js | 2 +- .../pyopencga/rest_clients/admin_client.py | 2 +- .../pyopencga/rest_clients/study_client.py | 3 +- .../opencga/core/api/FieldConstants.java | 39 ++-- .../opencga/core/models/common/Enums.java | 2 +- .../core/models/externalTool/Docker.java | 90 ++++++++ .../models/externalTool/ExternalTool.java | 145 ++++++------- .../externalTool/ExternalToolScope.java | 8 + .../models/externalTool/ExternalToolType.java | 7 + .../ExternalToolUpdateParams.java | 18 +- .../core/models/externalTool/Workflow.java | 65 ++++++ .../externalTool/WorkflowRepository.java | 40 ++-- .../WorkflowRepositoryParams.java | 60 ++++-- .../models/externalTool/WorkflowScript.java | 18 +- .../models/externalTool/WorkflowSystem.java | 2 +- .../WorkflowCreateParams.java} | 132 +++++------- .../opencga/server/rest/WorkflowWSServer.java | 32 ++- 33 files changed, 704 insertions(+), 494 deletions(-) rename opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/{ExternalToolManager.java => WorkflowManager.java} (86%) rename opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/{ExternalToolManagerTest.java => WorkflowManagerTest.java} (50%) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Docker.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolScope.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolType.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Workflow.java rename opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/{ExternalToolCreateParams.java => workflow/WorkflowCreateParams.java} (52%) diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java index 5fd44aa1b96..5c70a8c9d85 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java @@ -18,11 +18,8 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.externalTool.ExternalTool; +import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.models.job.ToolInfoExecutor; -import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; -import org.opencb.opencga.core.models.externalTool.WorkflowScript; -import org.opencb.opencga.core.models.externalTool.ExternalToolVariable; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.ToolDependency; import org.opencb.opencga.core.tools.annotations.Tool; @@ -44,7 +41,7 @@ import java.util.*; import java.util.stream.Stream; -@Tool(id = NextFlowExecutor.ID, resource = Enums.Resource.WORKFLOW, description = NextFlowExecutor.DESCRIPTION) +@Tool(id = NextFlowExecutor.ID, resource = Enums.Resource.EXTERNAL_TOOL, description = NextFlowExecutor.DESCRIPTION) public class NextFlowExecutor extends OpenCgaDockerToolScopeStudy { public final static String ID = "nextflow"; @@ -106,26 +103,28 @@ protected void check() throws Exception { } } - if (StringUtils.isEmpty(externalTool.getManager().getVersion())) { - externalTool.getManager().setVersion(ParamConstants.DEFAULT_MIN_NEXTFLOW_VERSION); + if (StringUtils.isEmpty(externalTool.getWorkflow().getManager().getVersion())) { + externalTool.getWorkflow().getManager().setVersion(ParamConstants.DEFAULT_MIN_NEXTFLOW_VERSION); } // Update job tags and attributes - ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(externalTool.getManager().getId().name().toLowerCase(), - externalTool.getManager().getVersion()); + ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(externalTool.getWorkflow().getManager().getId().name().toLowerCase(), + externalTool.getWorkflow().getManager().getVersion()); Set tags = new HashSet<>(); tags.add(ID); - tags.add(externalTool.getManager().getId().name()); - tags.add(externalTool.getManager().getId() + ":" + externalTool.getManager().getVersion()); + tags.add(externalTool.getWorkflow().getManager().getId().name()); + tags.add(externalTool.getWorkflow().getManager().getId() + ":" + externalTool.getWorkflow().getManager().getVersion()); tags.add(externalTool.getId()); if (CollectionUtils.isNotEmpty(externalTool.getTags())) { tags.addAll(externalTool.getTags()); } List dependencyList = new ArrayList<>(4); dependencyList.add(new ToolDependency("opencb/opencga-workflow", GitRepositoryState.getInstance().getBuildVersion() + "-hdp3.1")); - dependencyList.add(new ToolDependency("nextflow", externalTool.getManager().getVersion())); + dependencyList.add(new ToolDependency("nextflow", externalTool.getWorkflow().getManager().getVersion())); dependencyList.add(new ToolDependency(externalTool.getId(), String.valueOf(externalTool.getVersion()))); - if (externalTool.getRepository() != null && StringUtils.isNotEmpty(externalTool.getRepository().getId())) { - dependencyList.add(new ToolDependency(externalTool.getRepository().getId(), externalTool.getRepository().getVersion())); + if (externalTool.getWorkflow().getRepository() != null) { + if (StringUtils.isNotEmpty(externalTool.getWorkflow().getRepository().getName())) { + dependencyList.add(new ToolDependency(externalTool.getWorkflow().getRepository().getName(), externalTool.getWorkflow().getRepository().getTag())); + } } addDependencies(dependencyList); updateJobInformation(new ArrayList<>(tags), toolInfoExecutor); @@ -190,9 +189,9 @@ protected void check() throws Exception { @Override protected void run() throws Exception { - for (WorkflowScript script : externalTool.getScripts()) { + for (WorkflowScript script : externalTool.getWorkflow().getScripts()) { // Write script files - Path path = temporalInputDir.resolve(script.getFileName()); + Path path = temporalInputDir.resolve(script.getName()); Files.write(path, script.getContent().getBytes()); dockerInputBindings.add(new AbstractMap.SimpleEntry<>(path.toString(), path.toString())); } @@ -216,23 +215,23 @@ protected void run() throws Exception { String dockerImage = "opencb/opencga-workflow:" + GitRepositoryState.getInstance().getBuildVersion() + "-hdp3.1"; StringBuilder stringBuilder = new StringBuilder() .append("bash -c \"NXF_VER=") - .append(externalTool.getManager().getVersion()) + .append(externalTool.getWorkflow().getManager().getVersion()) .append(" nextflow -c ") .append(nextflowConfigPath) .append(" run "); - if (externalTool.getRepository() != null && StringUtils.isNotEmpty(externalTool.getRepository().getId())) { + if (externalTool.getWorkflow().getRepository() != null && StringUtils.isNotEmpty(externalTool.getWorkflow().getRepository().getName())) { // stringBuilder.append(workflow.getRepository().getImage()).append(" -with-docker"); - stringBuilder.append(externalTool.getRepository().getId()); + stringBuilder.append(externalTool.getWorkflow().getRepository().getName()); // Add the repository version if we have it and the user is not specifying it - if (StringUtils.isNotEmpty(externalTool.getRepository().getVersion()) && !cliParams.contains("-r ")) { + if (StringUtils.isNotEmpty(externalTool.getWorkflow().getRepository().getTag()) && !cliParams.contains("-r ")) { stringBuilder .append(" -r ") - .append(externalTool.getRepository().getVersion()); + .append(externalTool.getWorkflow().getRepository().getTag()); } } else { - for (WorkflowScript script : externalTool.getScripts()) { + for (WorkflowScript script : externalTool.getWorkflow().getScripts()) { if (script.isMain()) { - stringBuilder.append(temporalInputDir.resolve(script.getFileName())); + stringBuilder.append(temporalInputDir.resolve(script.getName())); break; } } diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java index 832f0bc3fea..d4f61b561ba 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java @@ -13,6 +13,7 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.exceptions.ToolException; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileCreateParams; import org.opencb.opencga.core.models.externalTool.*; @@ -36,8 +37,8 @@ public void nextflowScriptTest() throws ToolException, CatalogException, IOExcep InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml"); StorageConfiguration storageConfiguration = StorageConfiguration.load(inputStream, "yml"); - ExternalToolCreateParams workflow = createDummyWorkflow(); - catalogManager.getWorkflowManager().create(studyFqn, workflow.toWorkflow(), QueryOptions.empty(), ownerToken); + WorkflowCreateParams workflow = createDummyWorkflow(); + catalogManager.getWorkflowManager().create(studyFqn, workflow, QueryOptions.empty(), ownerToken); Path outDir = Paths.get(catalogManagerResource.createTmpOutdir("_nextflow")); @@ -61,8 +62,8 @@ public void nextflowScriptWithFilesTest() throws ToolException, CatalogException catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setPath("myfile.txt").setContent("hello world").setType(File.Type.FILE), false, ownerToken); - ExternalToolCreateParams workflow = createDummyWorkflow("pipeline_cat_file.nf"); - catalogManager.getWorkflowManager().create(studyFqn, workflow.toWorkflow(), QueryOptions.empty(), ownerToken); + WorkflowCreateParams workflow = createDummyWorkflow("pipeline_cat_file.nf"); + catalogManager.getWorkflowManager().create(studyFqn, workflow, QueryOptions.empty(), ownerToken); Path outDir = Paths.get(catalogManagerResource.createTmpOutdir("_nextflow")); @@ -85,9 +86,9 @@ public void nextflowScriptWithFilesTest() throws ToolException, CatalogException public void nextflowDockerTest() throws ToolException, CatalogException, IOException { InputStream inputStream = StorageManager.class.getClassLoader().getResourceAsStream("storage-configuration.yml"); StorageConfiguration storageConfiguration = StorageConfiguration.load(inputStream, "yml"); - ExternalToolCreateParams workflow = new ExternalToolCreateParams() + WorkflowCreateParams workflow = new WorkflowCreateParams() .setId("workflow") - .setScope(ExternalTool.Scope.OTHER) + .setScope(ExternalToolScope.OTHER) .setVariables(Arrays.asList( new ExternalToolVariable() .setId("input") @@ -109,8 +110,8 @@ public void nextflowDockerTest() throws ToolException, CatalogException, IOExcep // .setRequired(true) // .setType(WorkflowVariable.WorkflowVariableType.FLAG) )) - .setRepository(new WorkflowRepository("nf-core/demo")); - catalogManager.getWorkflowManager().create(studyFqn, workflow.toWorkflow(), QueryOptions.empty(), ownerToken); + .setWorkflow(new Workflow().setRepository(new WorkflowRepository("nf-core/demo"))); + catalogManager.getWorkflowManager().create(studyFqn, workflow, QueryOptions.empty(), ownerToken); catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setPath("samplesheet.csv") @@ -138,16 +139,16 @@ public void nextflowDockerTest() throws ToolException, CatalogException, IOExcep System.out.println(stopWatch.getTime(TimeUnit.MILLISECONDS)); } - private ExternalToolCreateParams createDummyWorkflow() throws IOException { + private WorkflowCreateParams createDummyWorkflow() throws IOException { return createDummyWorkflow("pipeline.nf"); } - private ExternalToolCreateParams createDummyWorkflow(String pipelineId) throws IOException { + private WorkflowCreateParams createDummyWorkflow(String pipelineId) throws IOException { InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream("nextflow/" + pipelineId); String content = IOUtils.toString(inputStream, "UTF-8"); - return new ExternalToolCreateParams() + return new WorkflowCreateParams() .setId("workflow") - .setScope(ExternalTool.Scope.OTHER) - .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", content, true))); + .setScope(ExternalToolScope.OTHER) + .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", content, true)))); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java index 0246f81105a..a03abec53af 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java @@ -13,15 +13,19 @@ import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ClientException; +import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.externalTool.ExternalTool; import org.opencb.opencga.core.models.externalTool.ExternalToolAclEntryList; import org.opencb.opencga.core.models.externalTool.ExternalToolAclUpdateParams; -import org.opencb.opencga.core.models.externalTool.ExternalToolCreateParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolInternal; +import org.opencb.opencga.core.models.externalTool.ExternalToolScope; import org.opencb.opencga.core.models.externalTool.ExternalToolUpdateParams; import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; +import org.opencb.opencga.core.models.externalTool.Workflow; import org.opencb.opencga.core.models.externalTool.WorkflowRepository; import org.opencb.opencga.core.models.externalTool.WorkflowRepositoryParams; import org.opencb.opencga.core.models.externalTool.WorkflowSystem; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.job.MinimumRequirements; import org.opencb.opencga.core.response.QueryType; @@ -148,41 +152,38 @@ private RestResponse create() throws Exception { } - ExternalToolCreateParams externalToolCreateParams = null; + WorkflowCreateParams workflowCreateParams = null; if (commandOptions.jsonDataModel) { RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/workflows/create")); return res; } else if (commandOptions.jsonFile != null) { - externalToolCreateParams = JacksonUtils.getDefaultObjectMapper() - .readValue(new java.io.File(commandOptions.jsonFile), ExternalToolCreateParams.class); + workflowCreateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), WorkflowCreateParams.class); } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); - putNestedIfNotNull(beanParams, "manager.id", commandOptions.managerId, true); - putNestedIfNotEmpty(beanParams, "manager.version", commandOptions.managerVersion, true); putNestedIfNotNull(beanParams, "scope", commandOptions.scope, true); putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); - putNestedIfNotNull(beanParams, "draft", commandOptions.draft, true); - putNestedIfNotEmpty(beanParams, "repository.id", commandOptions.repositoryId, true); - putNestedIfNotEmpty(beanParams, "repository.version", commandOptions.repositoryVersion, true); - putNestedIfNotEmpty(beanParams, "repository.author", commandOptions.repositoryAuthor, true); - putNestedIfNotEmpty(beanParams, "repository.description", commandOptions.repositoryDescription, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); + putNestedIfNotNull(beanParams, "draft", commandOptions.draft, true); + putNestedIfNotEmpty(beanParams, "internal.registrationDate", commandOptions.internalRegistrationDate, true); + putNestedIfNotEmpty(beanParams, "internal.lastModified", commandOptions.internalLastModified, true); + putNestedIfNotEmpty(beanParams, "internal.registrationUserId", commandOptions.internalRegistrationUserId, true); putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); - externalToolCreateParams = JacksonUtils.getDefaultObjectMapper().copy() + workflowCreateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) - .readValue(beanParams.toJson(), ExternalToolCreateParams.class); + .readValue(beanParams.toJson(), WorkflowCreateParams.class); } - return openCGAClient.getWorkflowClient().create(externalToolCreateParams, queryParams); + return openCGAClient.getWorkflowClient().create(workflowCreateParams, queryParams); } private RestResponse distinct() throws Exception { @@ -236,8 +237,10 @@ private RestResponse importWorkflow() throws Exception { .readValue(new java.io.File(commandOptions.jsonFile), WorkflowRepositoryParams.class); } else { ObjectMap beanParams = new ObjectMap(); - putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); - putNestedIfNotEmpty(beanParams, "version", commandOptions.version, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "tag", commandOptions.tag, true); + putNestedIfNotEmpty(beanParams, "user", commandOptions.user, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); workflowRepositoryParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) @@ -353,8 +356,8 @@ private RestResponse update() throws Exception { putNestedIfNotNull(beanParams, "scope", commandOptions.scope, true); putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); putNestedIfNotNull(beanParams, "draft", commandOptions.draft, true); - putNestedIfNotEmpty(beanParams, "repository.id", commandOptions.repositoryId, true); - putNestedIfNotEmpty(beanParams, "repository.version", commandOptions.repositoryVersion, true); + putNestedIfNotEmpty(beanParams, "repository.name", commandOptions.repositoryName, true); + putNestedIfNotEmpty(beanParams, "repository.tag", commandOptions.repositoryTag, true); putNestedIfNotEmpty(beanParams, "repository.author", commandOptions.repositoryAuthor, true); putNestedIfNotEmpty(beanParams, "repository.description", commandOptions.repositoryDescription, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java index a46e29833a3..ee0c5f869e1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java @@ -115,42 +115,21 @@ public class CreateCommandOptions { @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; - @Parameter(names = {"--id"}, description = "Workflow ID.", required = true, arity = 1) + @Parameter(names = {"--id"}, description = "External tool ID.", required = true, arity = 1) public String id; - @Parameter(names = {"--name", "-n"}, description = "Workflow name.", required = false, arity = 1) + @Parameter(names = {"--name", "-n"}, description = "Unique 32-character identifier assigned automatically by OpenCGA.", required = false, arity = 1) public String name; @Parameter(names = {"--description"}, description = "Users may provide a description for the entry.", required = false, arity = 1) public String description; - @Parameter(names = {"--manager-id"}, description = "Workflow system id. Valid values: NEXTFLOW.", required = false, arity = 1) - public String managerId; - - @Parameter(names = {"--manager-version"}, description = "Workflow system id. Valid values: NEXTFLOW.", required = false, arity = 1) - public String managerVersion; - - @Parameter(names = {"--scope"}, description = "Workflow scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) + @Parameter(names = {"--scope"}, description = "External tool scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) public String scope; @Parameter(names = {"--tags"}, description = "List of tags.", required = false, arity = 1) public String tags; - @Parameter(names = {"--draft"}, description = "Flag indicating whether the workflow is still a draft or not.", required = false, help = true, arity = 0) - public boolean draft = false; - - @Parameter(names = {"--repository-id"}, description = "The body web service id parameter", required = false, arity = 1) - public String repositoryId; - - @Parameter(names = {"--repository-version"}, description = "The body web service version parameter", required = false, arity = 1) - public String repositoryVersion; - - @Parameter(names = {"--repository-author"}, description = "The body web service author parameter", required = false, arity = 1) - public String repositoryAuthor; - - @Parameter(names = {"--repository-description"}, description = "The body web service description parameter", required = false, arity = 1) - public String repositoryDescription; - @Parameter(names = {"--minimum-requirements-cpu"}, description = "Minimum number of cpu cores required to execute the process.", required = false, arity = 1) public String minimumRequirementsCpu; @@ -160,6 +139,18 @@ public class CreateCommandOptions { @Parameter(names = {"--minimum-requirements-disk"}, description = "Minimum disk required to execute the process.", required = false, arity = 1) public String minimumRequirementsDisk; + @Parameter(names = {"--draft"}, description = "Flag indicating whether the external tool is a draft or not.", required = false, help = true, arity = 0) + public boolean draft = false; + + @Parameter(names = {"--internal-registration-date"}, description = "Registration date of the internal object.", required = false, arity = 1) + public String internalRegistrationDate; + + @Parameter(names = {"--internal-last-modified"}, description = "Date of the last modification of the internal object.", required = false, arity = 1) + public String internalLastModified; + + @Parameter(names = {"--internal-registration-user-id"}, description = "The body web service registrationUserId parameter", required = false, arity = 1) + public String internalRegistrationUserId; + @Parameter(names = {"--creation-date", "--cd"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was first registered.", required = false, arity = 1) public String creationDate; @@ -242,11 +233,17 @@ public class ImportCommandOptions { @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) public String study; - @Parameter(names = {"--id"}, description = "The body web service id parameter", required = false, arity = 1) - public String id; + @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) + public String name; - @Parameter(names = {"--version"}, description = "The body web service version parameter", required = false, arity = 1) - public String version; + @Parameter(names = {"--tag"}, description = "The body web service tag parameter", required = false, arity = 1) + public String tag; + + @Parameter(names = {"--user", "-u"}, description = "The body web service user parameter", required = false, arity = 1) + public String user; + + @Parameter(names = {"--password"}, description = "The body web service password parameter", required = false, arity = 1) + public String password; } @@ -392,7 +389,7 @@ public class UpdateCommandOptions { @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) public boolean includeResult = false; - @Parameter(names = {"--name", "-n"}, description = "Workflow name.", required = false, arity = 1) + @Parameter(names = {"--name", "-n"}, description = "External tool name.", required = false, arity = 1) public String name; @Parameter(names = {"--description"}, description = "Users may provide a description for the entry.", required = false, arity = 1) @@ -401,23 +398,23 @@ public class UpdateCommandOptions { @Parameter(names = {"--manager-id"}, description = "Workflow system id. Valid values: NEXTFLOW.", required = false, arity = 1) public String managerId; - @Parameter(names = {"--manager-version"}, description = "Workflow system id. Valid values: NEXTFLOW.", required = false, arity = 1) + @Parameter(names = {"--manager-version"}, description = "Workflow system version to use.", required = false, arity = 1) public String managerVersion; - @Parameter(names = {"--scope"}, description = "Workflow scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) + @Parameter(names = {"--scope"}, description = "External tool scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) public String scope; @Parameter(names = {"--tags"}, description = "List of tags.", required = false, arity = 1) public String tags; - @Parameter(names = {"--draft"}, description = "Flag indicating whether the workflow is still a draft or not.", required = false, help = true, arity = 0) + @Parameter(names = {"--draft"}, description = "Flag indicating whether the external tool is a draft or not.", required = false, help = true, arity = 0) public boolean draft = false; - @Parameter(names = {"--repository-id"}, description = "The body web service id parameter", required = false, arity = 1) - public String repositoryId; + @Parameter(names = {"--repository-name"}, description = "The body web service name parameter", required = false, arity = 1) + public String repositoryName; - @Parameter(names = {"--repository-version"}, description = "The body web service version parameter", required = false, arity = 1) - public String repositoryVersion; + @Parameter(names = {"--repository-tag"}, description = "The body web service tag parameter", required = false, arity = 1) + public String repositoryTag; @Parameter(names = {"--repository-author"}, description = "The body web service author parameter", required = false, arity = 1) public String repositoryAuthor; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java index bb764152309..41f5624d38d 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/auth/authorization/CatalogAuthorizationManager.java @@ -754,7 +754,7 @@ private void setDependentPermissions(List aclParams) throws Ca .collect(Collectors.toSet()) ); break; - case WORKFLOW: + case EXTERNAL_TOOL: allPermissions.addAll(aclParam.getPermissions() .stream() .map(ExternalToolPermissions::valueOf) @@ -879,7 +879,7 @@ private void setImplicitPermissions(List aclParams) throws Cat .collect(Collectors.toSet()) ); break; - case WORKFLOW: + case EXTERNAL_TOOL: allPermissions.addAll(aclParam.getPermissions() .stream() .map(ExternalToolPermissions::valueOf) @@ -1002,7 +1002,7 @@ private List getImplicitPermissions(List permissions, Enums.Reso .collect(Collectors.toSet()) ); break; - case WORKFLOW: + case EXTERNAL_TOOL: allPermissions.addAll(permissions .stream() .map(ExternalToolPermissions::valueOf) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java index 00c6c28cc90..e707e0bd0d3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java @@ -23,9 +23,11 @@ enum QueryParams implements QueryParam { NAME("name", TEXT, ""), DESCRIPTION("description", TEXT, ""), DRAFT("draft", BOOLEAN, ""), + TYPE("type", TEXT, ""), SCOPE("scope", TEXT, ""), TAGS("tags", TEXT_ARRAY, ""), COMMAND_LINE("commandLine", TEXT, ""), + WORKFLOW("workflow", OBJECT, ""), MANAGER("manager", OBJECT, ""), MANAGER_ID("manager.id", TEXT, ""), REPOSITORY("repository", OBJECT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java index d160c9f6d6b..947647a6dca 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBAdaptor.java @@ -133,7 +133,7 @@ private void validateEntry(Enums.Resource entry) throws CatalogDBException { case FAMILY: case CLINICAL_ANALYSIS: case CLINICAL: - case WORKFLOW: + case EXTERNAL_TOOL: return; default: throw new CatalogDBException("Unexpected parameter received. " + entry + " has been received."); @@ -781,7 +781,7 @@ public OpenCGAResult resetMembersFromAllEntries(long studyId, List membe removePermissions(clientSession, studyId, members, Enums.Resource.DISEASE_PANEL); removePermissions(clientSession, studyId, members, Enums.Resource.FAMILY); removePermissions(clientSession, studyId, members, Enums.Resource.CLINICAL_ANALYSIS); - removePermissions(clientSession, studyId, members, Enums.Resource.WORKFLOW); + removePermissions(clientSession, studyId, members, Enums.Resource.EXTERNAL_TOOL); removeFromMembers(clientSession, Collections.singletonList(studyId), members, null, Enums.Resource.STUDY); return endWrite(tmpStartTime, -1, -1, null); @@ -1098,7 +1098,7 @@ private Bson parseQuery(Query query, Document rawQuery, Enums.Resource entry) th case CLINICAL_ANALYSIS: case CLINICAL: return dbAdaptorFactory.getClinicalAnalysisDBAdaptor().parseQuery(query, rawQuery); - case WORKFLOW: + case EXTERNAL_TOOL: return dbAdaptorFactory.getWorkflowDBAdaptor().parseQuery(query, rawQuery); default: throw new CatalogException("Unexpected parameter received. " + entry + " has been received."); @@ -1184,7 +1184,7 @@ private MongoDBCollection getMainCollection(Enums.Resource resource) throws Cata case CLINICAL_ANALYSIS: case CLINICAL: return dbAdaptorFactory.getClinicalAnalysisDBAdaptor().getCollection(); - case WORKFLOW: + case EXTERNAL_TOOL: return dbAdaptorFactory.getWorkflowDBAdaptor().getCollection(); default: throw new CatalogDBException("Unexpected resource '" + resource + "' parameter received."); @@ -1201,7 +1201,7 @@ private MongoDBCollection getArchiveCollection(Enums.Resource resource) throws C return dbAdaptorFactory.getCatalogPanelDBAdaptor().getPanelArchiveCollection(); case FAMILY: return dbAdaptorFactory.getCatalogFamilyDBAdaptor().getArchiveFamilyCollection(); - case WORKFLOW: + case EXTERNAL_TOOL: return dbAdaptorFactory.getWorkflowDBAdaptor().getArchiveCollection(); case CLINICAL: case CLINICAL_ANALYSIS: @@ -1214,6 +1214,6 @@ private MongoDBCollection getArchiveCollection(Enums.Resource resource) throws C private boolean hasArchiveCollection(Enums.Resource resource) { return resource == Enums.Resource.INDIVIDUAL || resource == Enums.Resource.SAMPLE || resource == Enums.Resource.DISEASE_PANEL || resource == Enums.Resource.FAMILY || resource == Enums.Resource.CLINICAL || resource == Enums.Resource.CLINICAL_ANALYSIS - || resource == Enums.Resource.WORKFLOW; + || resource == Enums.Resource.EXTERNAL_TOOL; } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java index 2218056e813..6913cbd465e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/AuthorizationMongoDBUtils.java @@ -191,7 +191,7 @@ private static int getPermissionType(Enums.Resource resource) throws CatalogPara return StudyPermissions.FAMILY; case CLINICAL_ANALYSIS: return StudyPermissions.CLINICAL_ANALYSIS; - case WORKFLOW: + case EXTERNAL_TOOL: return StudyPermissions.WORKFLOW; default: throw new CatalogParameterException("Unexpected resource '" + resource + "'."); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java index aab9cd5c7b4..7b7de5f79a3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java @@ -504,11 +504,11 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) boolean simplifyPermissions = simplifyPermissions(); if (query.containsKey(ParamConstants.ACL_PARAM)) { - andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.WORKFLOW, user, + andBsonList.addAll(AuthorizationMongoDBUtils.parseAclQuery(studyDocument, query, Enums.Resource.EXTERNAL_TOOL, user, simplifyPermissions)); } else { andBsonList.add(getQueryForAuthorisedEntries(studyDocument, user, ExternalToolPermissions.VIEW.name(), - Enums.Resource.WORKFLOW, simplifyPermissions)); + Enums.Resource.EXTERNAL_TOOL, simplifyPermissions)); } query.remove(ParamConstants.ACL_PARAM); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 876cd7c023d..f832c848e68 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -88,7 +88,7 @@ public class CatalogManager implements AutoCloseable { private ClinicalAnalysisManager clinicalAnalysisManager; private InterpretationManager interpretationManager; private PanelManager panelManager; - private ExternalToolManager externalToolManager; + private WorkflowManager workflowManager; private AuditManager auditManager; private AuthorizationManager authorizationManager; @@ -164,7 +164,7 @@ private void configureManagers(Configuration configuration) throws CatalogExcept clinicalAnalysisManager = new ClinicalAnalysisManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, configuration); interpretationManager = new InterpretationManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, configuration); - externalToolManager = new ExternalToolManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, ioManagerFactory, + workflowManager = new WorkflowManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, ioManagerFactory, catalogIOManager, configuration); } @@ -458,7 +458,7 @@ public MigrationManager getMigrationManager() { return migrationManager; } - public ExternalToolManager getWorkflowManager() { - return externalToolManager; + public WorkflowManager getWorkflowManager() { + return workflowManager; } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java similarity index 86% rename from opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java rename to opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java index 1651b9f40c9..fcfeb7c3b84 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.opencb.commons.datastore.core.*; import org.opencb.commons.datastore.core.result.Error; @@ -29,6 +30,7 @@ import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.job.JobType; import org.opencb.opencga.core.models.job.MinimumRequirements; @@ -54,36 +56,36 @@ import static org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor.QueryParams.*; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; -public class ExternalToolManager extends ResourceManager { +public class WorkflowManager extends ResourceManager { - public static final QueryOptions INCLUDE_WORKFLOW_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(ID.key(), UID.key(), + public static final QueryOptions INCLUDE_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(ID.key(), UID.key(), UUID.key(), VERSION.key(), DESCRIPTION.key(), STUDY_UID.key(), MANAGER.key())); private final CatalogIOManager catalogIOManager; private final IOManagerFactory ioManagerFactory; - private static final Pattern MEMORY_PATTERN = Pattern.compile("\\s*memory\\s*=\\s*\\{\\s*[^0-9]*([0-9]+\\.[A-Za-z]+)"); - private static final Pattern CPU_PATTERN = Pattern.compile("\\s*cpus\\s*=\\s*\\{\\s*[^0-9]*([0-9]+)"); + private static final Pattern WORKFLOW_MEMORY_PATTERN = Pattern.compile("\\s*memory\\s*=\\s*\\{\\s*[^0-9]*([0-9]+\\.[A-Za-z]+)"); + private static final Pattern WORKFLOW_CPU_PATTERN = Pattern.compile("\\s*cpus\\s*=\\s*\\{\\s*[^0-9]*([0-9]+)"); public static final int MAX_CPUS = 15; public static final String MAX_MEMORY = "64.GB"; // Format is important for Nextflow. It requires the dot symbol. private final Logger logger; - ExternalToolManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, - DBAdaptorFactory catalogDBAdaptorFactory, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager, - Configuration configuration) { + WorkflowManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, + DBAdaptorFactory catalogDBAdaptorFactory, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager, + Configuration configuration) { super(authorizationManager, auditManager, catalogManager, catalogDBAdaptorFactory, configuration); this.catalogIOManager = catalogIOManager; this.ioManagerFactory = ioManagerFactory; - this.logger = LoggerFactory.getLogger(ExternalToolManager.class); + this.logger = LoggerFactory.getLogger(WorkflowManager.class); } @Override Enums.Resource getEntity() { - return Enums.Resource.WORKFLOW; + return Enums.Resource.EXTERNAL_TOOL; } @Override @@ -99,6 +101,7 @@ InternalGetDataResult internalGet(String organizationId, long stud Query queryCopy = query == null ? new Query() : new Query(query); queryCopy.put(STUDY_UID.key(), studyUid); + queryCopy.put(TYPE.key(), ExternalToolType.WORKFLOW); boolean versioned = queryCopy.getBoolean(Constants.ALL_VERSIONS) || queryCopy.containsKey(VERSION.key()); @@ -135,6 +138,11 @@ InternalGetDataResult internalGet(String organizationId, long stud @Override public OpenCGAResult create(String studyStr, ExternalTool externalTool, QueryOptions options, String token) throws CatalogException { + throw new NotImplementedException("Not implemented yet"); + } + + public OpenCGAResult create(String studyStr, WorkflowCreateParams workflow, QueryOptions options, String token) + throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); @@ -142,7 +150,7 @@ public OpenCGAResult create(String studyStr, ExternalTool external ObjectMap auditParams = new ObjectMap() .append("study", studyStr) - .append("workflow", externalTool) + .append("workflow", workflow) .append("options", options) .append("token", token); @@ -159,6 +167,12 @@ public OpenCGAResult create(String studyStr, ExternalTool external authorizationManager.checkStudyPermission(organizationId, study.getUid(), tokenPayload, StudyPermissions.Permissions.WRITE_WORKFLOWS); + // Convert WorkflowCreateParams to ExternalTool + ExternalTool externalTool = new ExternalTool(workflow.getId(), workflow.getName(), workflow.getDescription(), + ExternalToolType.WORKFLOW, workflow.getScope(), workflow.getWorkflow(), null, workflow.getTags(), + workflow.getVariables(), workflow.getMinimumRequirements(), workflow.isDraft(), workflow.getInternal(), + workflow.getCreationDate(), workflow.getModificationDate(), workflow.getAttributes()); + // 2. Validate the workflow parameters validateNewWorkflow(externalTool, userId); @@ -172,11 +186,11 @@ public OpenCGAResult create(String studyStr, ExternalTool external OpenCGAResult result = getWorkflowDBAdaptor(organizationId).get(query, options); insert.setResults(result.getResults()); } - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), externalTool.getUuid(), studyId, - studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), + studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; } catch (CatalogException e) { - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), "", studyId, studyUuid, + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, workflow.getId(), "", studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -269,7 +283,7 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepos Query query = new Query() .append(STUDY_UID.key(), study.getUid()) .append(ID.key(), externalTool.getId()); - OpenCGAResult tmpResult = getWorkflowDBAdaptor(organizationId).get(query, INCLUDE_WORKFLOW_IDS); + OpenCGAResult tmpResult = getWorkflowDBAdaptor(organizationId).get(query, INCLUDE_IDS); if (tmpResult.getNumResults() > 0) { logger.warn("Workflow '" + workflowId + "' already exists. Updating with the latest workflow information."); try { @@ -297,11 +311,11 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepos OpenCGAResult tmpTmpResult = getWorkflowDBAdaptor(organizationId).get(query, options); result.setResults(tmpTmpResult.getResults()); } - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), externalTool.getUuid(), studyId, - studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), + studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditCreate(organizationId, userId, Enums.Resource.WORKFLOW, workflowId, "", studyId, studyUuid, auditParams, + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, workflowId, "", studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -309,13 +323,15 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepos private ExternalTool downloadWorkflow(WorkflowRepositoryParams repository) throws CatalogException { ParamUtils.checkObj(repository, "Workflow repository parameters"); - if (StringUtils.isEmpty(repository.getId())) { + if (StringUtils.isEmpty(repository.getName())) { throw new CatalogParameterException("Missing 'id' field in workflow import parameters"); } - String workflowId = repository.getId().replace("/", "."); - ExternalTool externalTool = new ExternalTool("", "", "", null, new WorkflowSystem(WorkflowSystem.SystemId.NEXTFLOW, ""), - new LinkedList<>(), new LinkedList<>(), new MinimumRequirements(), false, repository.toWorkflowRepository(), - new LinkedList<>(), new ExternalToolInternal(), TimeUtils.getTime(), TimeUtils.getTime(), new HashMap<>()); + String workflowId = repository.getName().replace("/", "."); + WorkflowSystem workflowSystem = new WorkflowSystem(WorkflowSystem.SystemId.NEXTFLOW, ""); + ExternalTool externalTool = new ExternalTool("", "", "", ExternalToolType.WORKFLOW, null, + new Workflow(workflowSystem, new LinkedList<>(), repository.toWorkflowRepository()), null, new LinkedList<>(), + new LinkedList<>(), new MinimumRequirements(), false, new ExternalToolInternal(), TimeUtils.getTime(), TimeUtils.getTime(), + new HashMap<>()); try { processNextflowConfig(externalTool, repository); @@ -330,10 +346,10 @@ private ExternalTool downloadWorkflow(WorkflowRepositoryParams repository) throw private void processMemoryRequirements(ExternalTool externalTool, WorkflowRepositoryParams repository) throws CatalogException { String urlStr; - if (StringUtils.isEmpty(repository.getVersion())) { - urlStr = "https://raw.githubusercontent.com/" + repository.getId() + "/refs/heads/master/conf/base.config"; + if (StringUtils.isEmpty(repository.getTag())) { + urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/heads/master/conf/base.config"; } else { - urlStr = "https://raw.githubusercontent.com/" + repository.getId() + "/refs/tags/" + repository.getVersion() + urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/tags/" + repository.getTag() + "/conf/base.config"; } @@ -346,8 +362,8 @@ private void processMemoryRequirements(ExternalTool externalTool, WorkflowReposi String inputLine; long maxMemory = IOUtils.fromHumanReadableToByte(MAX_MEMORY); while ((inputLine = in.readLine()) != null) { - Matcher cpuMatcher = CPU_PATTERN.matcher(inputLine); - Matcher memoryMatcher = MEMORY_PATTERN.matcher(inputLine); + Matcher cpuMatcher = WORKFLOW_CPU_PATTERN.matcher(inputLine); + Matcher memoryMatcher = WORKFLOW_MEMORY_PATTERN.matcher(inputLine); if (cpuMatcher.find()) { String value = cpuMatcher.group(1); int intValue = Integer.parseInt(value); @@ -385,10 +401,10 @@ private void processMemoryRequirements(ExternalTool externalTool, WorkflowReposi private void processNextflowConfig(ExternalTool externalTool, WorkflowRepositoryParams repository) throws CatalogException { String urlStr; - if (StringUtils.isEmpty(repository.getVersion())) { - urlStr = "https://raw.githubusercontent.com/" + repository.getId() + "/refs/heads/master/nextflow.config"; + if (StringUtils.isEmpty(repository.getTag())) { + urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/heads/master/nextflow.config"; } else { - urlStr = "https://raw.githubusercontent.com/" + repository.getId() + "/refs/tags/" + repository.getVersion() + urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/tags/" + repository.getTag() + "/nextflow.config"; } @@ -452,23 +468,23 @@ private void fillWithWorkflowManifest(ExternalTool externalTool, String rawline) case "name": externalTool.setId(value.replace("/", ".")); externalTool.setName(value.replace("/", " ")); - externalTool.getRepository().setId(value); + externalTool.getWorkflow().getRepository().setName(value); break; case "author": - externalTool.getRepository().setAuthor(value); + externalTool.getWorkflow().getRepository().setAuthor(value); break; case "description": externalTool.setDescription(value); - externalTool.getRepository().setDescription(value); + externalTool.getWorkflow().getRepository().setDescription(value); break; case "version": String version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); - externalTool.getRepository().setVersion(version); + externalTool.getWorkflow().getRepository().setTag(version); break; case "nextflowVersion": // Nextflow version must start with a number version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); - externalTool.getManager().setVersion(version); + externalTool.getWorkflow().getManager().setVersion(version); break; default: break; @@ -521,7 +537,7 @@ public OpenCGAResult update(String studyStr, String workflowId, Ex studyUuid = study.getUuid(); ExternalTool externalTool = internalGet(organizationId, study.getUid(), Collections.singletonList(workflowId), null, - INCLUDE_WORKFLOW_IDS, userId, false).first(); + INCLUDE_IDS, userId, false).first(); id = externalTool.getId(); uuid = externalTool.getUuid(); @@ -558,11 +574,11 @@ public OpenCGAResult update(String studyStr, String workflowId, Ex OpenCGAResult result = getWorkflowDBAdaptor(organizationId).get(query, options); insert.setResults(result.getResults()); } - auditManager.auditUpdate(organizationId, userId, Enums.Resource.WORKFLOW, id, uuid, studyId, studyUuid, auditParams, + auditManager.auditUpdate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, id, uuid, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return insert; } catch (CatalogException e) { - auditManager.auditUpdate(organizationId, userId, Enums.Resource.WORKFLOW, id, uuid, studyId, studyUuid, auditParams, + auditManager.auditUpdate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, id, uuid, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -631,12 +647,12 @@ public OpenCGAResult search(String studyStr, Query query, QueryOpt OpenCGAResult queryResult = getWorkflowDBAdaptor(organizationId).get(study.getUid(), query, options, userId); - auditManager.auditSearch(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return queryResult; } catch (CatalogException e) { - auditManager.auditSearch(organizationId, userId, Enums.Resource.WORKFLOW, studyId, studyUuid, auditParams, + auditManager.auditSearch(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -664,12 +680,12 @@ public OpenCGAResult distinct(String studyStr, List fields, Query que query.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); OpenCGAResult result = getWorkflowDBAdaptor(organizationId).distinct(study.getUid(), fields, query, userId); - auditManager.auditDistinct(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return result; } catch (CatalogException e) { - auditManager.auditDistinct(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditDistinct(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -696,13 +712,13 @@ public OpenCGAResult count(String studyStr, Query query, String to query.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); OpenCGAResult queryResultAux = getWorkflowDBAdaptor(organizationId).count(query, userId); - auditManager.auditCount(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); return new OpenCGAResult<>(queryResultAux.getTime(), queryResultAux.getEvents(), 0, Collections.emptyList(), queryResultAux.getNumMatches()); } catch (CatalogException e) { - auditManager.auditCount(organizationId, userId, Enums.Resource.WORKFLOW, study.getId(), study.getUuid(), auditParams, + auditManager.auditCount(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -741,8 +757,8 @@ public OpenCGAResult delete(String studyStr, List ids, Obj long studyId = study.getUid(); checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.WORKFLOW, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.EXTERNAL_TOOL, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -752,7 +768,7 @@ public OpenCGAResult delete(String studyStr, List ids, Obj String workflowId = id; String workflowUuid = ""; try { - OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_WORKFLOW_IDS, userId); + OpenCGAResult internalResult = internalGet(organizationId, study.getUid(), id, INCLUDE_IDS, userId); if (internalResult.getNumResults() == 0) { throw new CatalogException("Workflow '" + id + "' not found"); } @@ -772,7 +788,7 @@ public OpenCGAResult delete(String studyStr, List ids, Obj result.append(getWorkflowDBAdaptor(organizationId).delete(externalTool)); - auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.WORKFLOW, externalTool.getId(), + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -783,7 +799,7 @@ public OpenCGAResult delete(String studyStr, List ids, Obj result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.WORKFLOW, workflowId, workflowUuid, + auditManager.auditDelete(organizationId, operationId, userId, Enums.Resource.EXTERNAL_TOOL, workflowId, workflowUuid, study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } } @@ -832,14 +848,14 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool fixQueryObject(finalQuery); finalQuery.append(ExternalToolDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); - iterator = getWorkflowDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_WORKFLOW_IDS, userId); + iterator = getWorkflowDBAdaptor(organizationId).iterator(study.getUid(), finalQuery, INCLUDE_IDS, userId); // If the user is the owner or the admin, we won't check if he has permissions for every single entry long studyId = study.getUid(); checkPermissions = !authorizationManager.isAtLeastStudyAdministrator(organizationId, studyId, userId); } catch (CatalogException e) { - auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, "", "", study.getId(), study.getUuid(), - auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.EXTERNAL_TOOL, "", "", study.getId(), + study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); throw e; } @@ -858,7 +874,7 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool result.append(getWorkflowDBAdaptor(organizationId).delete(externalTool)); - auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, externalTool.getId(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); } catch (CatalogException e) { @@ -869,7 +885,7 @@ public OpenCGAResult delete(String studyStr, Query query, ObjectMap params, bool result.setNumErrors(result.getNumErrors() + 1); logger.error(errorMsg); - auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.WORKFLOW, externalTool.getId(), + auditManager.auditDelete(organizationId, operationUuid, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); } @@ -910,18 +926,23 @@ private ExternalToolDBAdaptor.QueryParams getFieldFilter(List idList) th private void validateNewWorkflow(ExternalTool externalTool, String userId) throws CatalogParameterException { ParamUtils.checkIdentifier(externalTool.getId(), ID.key()); - if (externalTool.getManager() == null) { - externalTool.setManager(new WorkflowSystem()); + ParamUtils.checkObj(externalTool.getWorkflow(), WORKFLOW.key()); + if (externalTool.getWorkflow().getManager() == null) { + externalTool.getWorkflow().setManager(new WorkflowSystem()); } - if (externalTool.getManager().getId() == null) { - externalTool.getManager().setId(WorkflowSystem.SystemId.NEXTFLOW); + if (externalTool.getWorkflow().getManager().getId() == null) { + externalTool.getWorkflow().getManager().setId(WorkflowSystem.SystemId.NEXTFLOW); } - externalTool.setScope(ParamUtils.defaultObject(externalTool.getScope(), ExternalTool.Scope.OTHER)); - externalTool.setTags(externalTool.getTags() != null ? externalTool.getTags() : Collections.emptyList()); - externalTool.setScripts(externalTool.getScripts() != null ? externalTool.getScripts() : Collections.emptyList()); + externalTool.setScope(ParamUtils.defaultObject(externalTool.getScope(), ExternalToolScope.OTHER)); + externalTool.setTags(externalTool.getTags() != null + ? externalTool.getTags() + : Collections.emptyList()); + externalTool.getWorkflow().setScripts(externalTool.getWorkflow().getScripts() != null + ? externalTool.getWorkflow().getScripts() + : Collections.emptyList()); boolean main = false; - for (WorkflowScript script : externalTool.getScripts()) { - ParamUtils.checkIdentifier(script.getFileName(), SCRIPTS.key() + ".id"); + for (WorkflowScript script : externalTool.getWorkflow().getScripts()) { + ParamUtils.checkIdentifier(script.getName(), SCRIPTS.key() + ".id"); ParamUtils.checkParameter(script.getContent(), SCRIPTS.key() + ".content"); if (script.isMain()) { if (main) { @@ -930,14 +951,18 @@ private void validateNewWorkflow(ExternalTool externalTool, String userId) throw main = script.isMain(); } } - if (CollectionUtils.isNotEmpty(externalTool.getScripts()) && !main) { + if (CollectionUtils.isNotEmpty(externalTool.getWorkflow().getScripts()) && !main) { throw new CatalogParameterException("No main script found."); } - externalTool.setRepository(externalTool.getRepository() != null ? externalTool.getRepository() : new WorkflowRepository("")); - if (StringUtils.isEmpty(externalTool.getRepository().getId()) && CollectionUtils.isEmpty(externalTool.getScripts())) { + externalTool.getWorkflow().setRepository(externalTool.getWorkflow().getRepository() != null + ? externalTool.getWorkflow().getRepository() + : new WorkflowRepository("")); + if (StringUtils.isEmpty(externalTool.getWorkflow().getRepository().getName()) + && CollectionUtils.isEmpty(externalTool.getWorkflow().getScripts())) { throw new CatalogParameterException("No repository image or scripts found."); } - if (StringUtils.isNotEmpty(externalTool.getRepository().getId()) && CollectionUtils.isNotEmpty(externalTool.getScripts())) { + if (StringUtils.isNotEmpty(externalTool.getWorkflow().getRepository().getName()) + && CollectionUtils.isNotEmpty(externalTool.getWorkflow().getScripts())) { throw new CatalogParameterException("Both repository image and scripts found. Please, either add scripts or a repository" + " image."); } @@ -951,6 +976,7 @@ private void validateNewWorkflow(ExternalTool externalTool, String userId) throw externalTool.setAttributes(ParamUtils.defaultObject(externalTool.getAttributes(), Collections.emptyMap())); externalTool.setInternal(new ExternalToolInternal(new InternalStatus(InternalStatus.READY), TimeUtils.getTime(), TimeUtils.getTime(), userId)); + externalTool.setDocker(null); } // ************************** ACLs ******************************** // @@ -981,7 +1007,7 @@ public OpenCGAResult> getAcls(String study try { auditManager.initAuditBatch(operationId); InternalGetDataResult queryResult = internalGet(organizationId, study.getUid(), workflowList, - INCLUDE_WORKFLOW_IDS, userId, ignoreException); + INCLUDE_IDS, userId, ignoreException); if (queryResult.getMissing() != null) { missingMap = queryResult.getMissing().stream() @@ -990,10 +1016,10 @@ public OpenCGAResult> getAcls(String study List workflowUids = queryResult.getResults().stream().map(ExternalTool::getUid).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(members)) { - workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, members, Enums.Resource.WORKFLOW, - ExternalToolPermissions.class, userId); + workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, members, + Enums.Resource.EXTERNAL_TOOL, ExternalToolPermissions.class, userId); } else { - workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, Enums.Resource.WORKFLOW, + workflowAcls = authorizationManager.getAcl(organizationId, study.getUid(), workflowUids, Enums.Resource.EXTERNAL_TOOL, ExternalToolPermissions.class, userId); } @@ -1005,15 +1031,16 @@ public OpenCGAResult> getAcls(String study if (!missingMap.containsKey(workflowId)) { ExternalTool externalTool = queryResult.getResults().get(counter); resultList.add(workflowAcls.getResults().get(counter)); - auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.WORKFLOW, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); counter++; } else { resultList.add(new AclEntryList<>()); eventList.add(new Event(Event.Type.ERROR, workflowId, missingMap.get(workflowId).getErrorMsg())); - auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.WORKFLOW, workflowId, - "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.EXTERNAL_TOOL, + workflowId, "", study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, "", missingMap.get(workflowId).getErrorMsg())), new ObjectMap()); } } @@ -1024,9 +1051,9 @@ public OpenCGAResult> getAcls(String study workflowAcls.setEvents(eventList); } catch (CatalogException e) { for (String workflowId : workflowList) { - auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.WORKFLOW, workflowId, "", - study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), - new ObjectMap()); + auditManager.audit(organizationId, operationId, userId, Enums.Action.FETCH_ACLS, Enums.Resource.EXTERNAL_TOOL, workflowId, + "", study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } if (!ignoreException) { throw e; @@ -1077,7 +1104,7 @@ public OpenCGAResult> updateAcl(String stu checkPermissions(params.getPermissions(), ExternalToolPermissions::valueOf); } - externalToolList = internalGet(organizationId, study.getUid(), params.getExternalToolIds(), INCLUDE_WORKFLOW_IDS, userId, false) + externalToolList = internalGet(organizationId, study.getUid(), params.getExternalToolIds(), INCLUDE_IDS, userId, false) .getResults(); authorizationManager.checkCanAssignOrSeePermissions(organizationId, study.getUid(), userId); @@ -1092,8 +1119,8 @@ public OpenCGAResult> updateAcl(String stu } catch (CatalogException e) { if (CollectionUtils.isNotEmpty(params.getExternalToolIds())) { for (String workflowId : params.getExternalToolIds()) { - auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, workflowId, - "", study.getId(), study.getUuid(), auditParams, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.EXTERNAL_TOOL, + workflowId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } } @@ -1113,7 +1140,8 @@ public OpenCGAResult> updateAcl(String stu List workflowUids = batchExternalToolList.stream().map(ExternalTool::getUid).collect(Collectors.toList()); List workflowIds = batchExternalToolList.stream().map(ExternalTool::getId).collect(Collectors.toList()); List aclParamsList = new ArrayList<>(); - AuthorizationManager.CatalogAclParams.addToList(workflowUids, params.getPermissions(), Enums.Resource.WORKFLOW, aclParamsList); + AuthorizationManager.CatalogAclParams.addToList(workflowUids, params.getPermissions(), Enums.Resource.EXTERNAL_TOOL, + aclParamsList); try { switch (action) { @@ -1138,7 +1166,7 @@ public OpenCGAResult> updateAcl(String stu OpenCGAResult> queryResults = authorizationManager.getAcls(organizationId, study.getUid(), - workflowUids, members, Enums.Resource.WORKFLOW, ExternalToolPermissions.class); + workflowUids, members, Enums.Resource.EXTERNAL_TOOL, ExternalToolPermissions.class); for (int i = 0; i < queryResults.getResults().size(); i++) { queryResults.getResults().get(i).setId(workflowIds.get(i)); @@ -1146,14 +1174,14 @@ public OpenCGAResult> updateAcl(String stu aclResultList.append(queryResults); for (ExternalTool externalTool : batchExternalToolList) { - auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS), new ObjectMap()); } } catch (CatalogException e) { // Process current batch for (ExternalTool externalTool : batchExternalToolList) { - auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } @@ -1161,7 +1189,7 @@ public OpenCGAResult> updateAcl(String stu // Process remaining unprocessed batches while (numProcessed < externalToolList.size()) { ExternalTool externalTool = externalToolList.get(numProcessed); - auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.WORKFLOW, + auditManager.audit(organizationId, operationId, userId, Enums.Action.UPDATE_ACLS, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()), new ObjectMap()); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java similarity index 50% rename from opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java rename to opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java index 6032bfb6f50..7f88072f910 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java @@ -10,6 +10,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.externalTool.*; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -19,114 +20,114 @@ import static org.junit.Assert.*; @Category(MediumTests.class) -public class ExternalToolManagerTest extends AbstractManagerTest { +public class WorkflowManagerTest extends AbstractManagerTest { - private ExternalToolManager externalToolManager; + private WorkflowManager workflowManager; @Before public void setUp() throws Exception { super.setUp(); - externalToolManager = catalogManager.getWorkflowManager(); + workflowManager = catalogManager.getWorkflowManager(); } @Test public void importWorkflow() throws CatalogException { WorkflowRepositoryParams params = new WorkflowRepositoryParams("nf-core/rnaseq"); - OpenCGAResult result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + OpenCGAResult result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); // Update imported workflow - result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(2, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/proteinfold"); - result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/methylseq"); - result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/pacvar"); - result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); } @Test public void createWorkflowTest() throws CatalogException { - ExternalTool externalTool = new ExternalTool() + WorkflowCreateParams workflowCreateParams = new WorkflowCreateParams() .setId("workflow") - .setScope(ExternalTool.Scope.OTHER) - .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - OpenCGAResult result = externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + .setScope(ExternalToolScope.OTHER) + .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); + OpenCGAResult result = workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertNotNull(result.first()); - assertEquals(externalTool.getId(), result.first().getId()); + assertEquals(workflowCreateParams.getId(), result.first().getId()); // Add repository to workflow - externalTool.setId("workflow2"); - externalTool.setRepository(new WorkflowRepository("blabla")); + workflowCreateParams.setId("workflow2"); + workflowCreateParams.getWorkflow().setRepository(new WorkflowRepository("blabla")); CatalogException catalogException = assertThrows(CatalogException.class, - () -> externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); + () -> workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("repository")); // Remove script from workflow - externalTool.setScripts(Collections.emptyList()); - result = externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + workflowCreateParams.getWorkflow().setScripts(Collections.emptyList()); + result = workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertNotNull(result.first()); - assertEquals(externalTool.getId(), result.first().getId()); + assertEquals(workflowCreateParams.getId(), result.first().getId()); // Remove script and add two scripts with 2 mains - externalTool.setId("workflow3"); - externalTool.setRepository(null); - externalTool.setScripts(Arrays.asList( + workflowCreateParams.setId("workflow3"); + workflowCreateParams.getWorkflow().setRepository(null); + workflowCreateParams.getWorkflow().setScripts(Arrays.asList( new WorkflowScript("script1", "echo 'Hello'", true), new WorkflowScript("script2", "echo 'World'", true) )); catalogException = assertThrows(CatalogException.class, - () -> externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); + () -> workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main")); // Add one single script without main - externalTool.setScripts(Collections.singletonList( + workflowCreateParams.getWorkflow().setScripts(Collections.singletonList( new WorkflowScript("script1", "echo 'Hello'", false) )); catalogException = assertThrows(CatalogException.class, - () -> externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken)); + () -> workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main")); } @Test public void workflowSearchTest() throws CatalogException { - ExternalTool externalTool = new ExternalTool() + WorkflowCreateParams workflowCreateParams = new WorkflowCreateParams() .setId("workflow") - .setScope(ExternalTool.Scope.OTHER) - .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + .setScope(ExternalToolScope.OTHER) + .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); + workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); - externalTool = new ExternalTool() + workflowCreateParams = new WorkflowCreateParams() .setId("workflow2") - .setScope(ExternalTool.Scope.OTHER) + .setScope(ExternalToolScope.OTHER) .setDraft(true) - .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); + workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); - OpenCGAResult search = externalToolManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); + OpenCGAResult search = workflowManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); assertEquals(2, search.getNumResults()); Query query = new Query(ExternalToolDBAdaptor.QueryParams.DRAFT.key(), true); - search = externalToolManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); + search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("workflow2", search.first().getId()); assertTrue(search.first().isDraft()); query = new Query(ExternalToolDBAdaptor.QueryParams.DRAFT.key(), false); - search = externalToolManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); + search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("workflow", search.first().getId()); assertFalse(search.first().isDraft()); @@ -134,21 +135,21 @@ public void workflowSearchTest() throws CatalogException { @Test public void updateWorkflowTest() throws CatalogException { - ExternalTool externalTool = new ExternalTool() + WorkflowCreateParams workflowCreateParams = new WorkflowCreateParams() .setId("workflow") - .setScope(ExternalTool.Scope.OTHER) - .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, INCLUDE_RESULT, ownerToken); + .setScope(ExternalToolScope.OTHER) + .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); + workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); ExternalToolUpdateParams updateParams = new ExternalToolUpdateParams() .setName("newName") - .setScope(ExternalTool.Scope.OTHER) + .setScope(ExternalToolScope.OTHER) .setDraft(true) .setCreationDate("20240101000000") .setModificationDate("20240201000000") .setDescription("description"); - OpenCGAResult update = externalToolManager.update(studyFqn, externalTool.getId(), updateParams, INCLUDE_RESULT, ownerToken); + OpenCGAResult update = workflowManager.update(studyFqn, workflowCreateParams.getId(), updateParams, INCLUDE_RESULT, ownerToken); assertEquals(1, update.getNumUpdated()); ExternalTool updatedExternalTool = update.first(); assertEquals(updateParams.getName(), updatedExternalTool.getName()); @@ -160,35 +161,35 @@ public void updateWorkflowTest() throws CatalogException { @Test public void deleteWorkflowTest() throws CatalogException { - ExternalTool externalTool = new ExternalTool() + WorkflowCreateParams workflowCreateParams = new WorkflowCreateParams() .setId("workflow") - .setScope(ExternalTool.Scope.OTHER) - .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, QueryOptions.empty(), ownerToken); + .setScope(ExternalToolScope.OTHER) + .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); + workflowManager.create(studyFqn, workflowCreateParams, QueryOptions.empty(), ownerToken); - OpenCGAResult result = externalToolManager.delete(studyFqn, externalTool.getId(), QueryOptions.empty(), ownerToken); + OpenCGAResult result = workflowManager.delete(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumDeleted()); CatalogException exception = assertThrows(CatalogException.class, - () -> externalToolManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), ownerToken)); + () -> workflowManager.get(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), ownerToken)); assertTrue(exception.getMessage().contains("not found")); } @Test public void updateWorkflowAclTest() throws CatalogException { - ExternalTool externalTool = new ExternalTool() + WorkflowCreateParams workflowCreateParams = new WorkflowCreateParams() .setId("workflow") - .setScope(ExternalTool.Scope.OTHER) - .setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true))); - externalToolManager.create(studyFqn, externalTool, QueryOptions.empty(), ownerToken); + .setScope(ExternalToolScope.OTHER) + .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); + workflowManager.create(studyFqn, workflowCreateParams, QueryOptions.empty(), ownerToken); CatalogAuthorizationException catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, - () -> externalToolManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), noAccessToken1)); + () -> workflowManager.get(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), noAccessToken1)); assertTrue(catalogAuthorizationException.getMessage().contains("denied")); - externalToolManager.updateAcl(studyFqn, noAccessUserId1, new ExternalToolAclUpdateParams(Collections.singletonList(externalTool.getId()), Collections.singletonList("VIEW")), + workflowManager.updateAcl(studyFqn, noAccessUserId1, new ExternalToolAclUpdateParams(Collections.singletonList(workflowCreateParams.getId()), Collections.singletonList("VIEW")), ParamUtils.AclAction.ADD, ownerToken); - OpenCGAResult result = externalToolManager.get(studyFqn, externalTool.getId(), QueryOptions.empty(), noAccessToken1); + OpenCGAResult result = workflowManager.get(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), noAccessToken1); assertEquals(1, result.getNumResults()); } diff --git a/opencga-client/src/main/R/R/Admin-methods.R b/opencga-client/src/main/R/R/Admin-methods.R index a723ea8f7a6..c17cb86cd2f 100644 --- a/opencga-client/src/main/R/R/Admin-methods.R +++ b/opencga-client/src/main/R/R/Admin-methods.R @@ -44,7 +44,7 @@ setMethod("adminClient", "OpencgaR", function(OpencgaR, user, endpointName, para #' @param count Count the number of elements matching the group. #' @param limit Maximum number of documents (groups) to be returned. #' @param fields Comma separated list of fields by which to group by. - #' @param entity Entity to be grouped by. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE'] + #' @param entity Entity to be grouped by. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL EXTERNAL_TOOL RESOURCE'] #' @param action Action performed. #' @param before Object before update. #' @param after Object after update. diff --git a/opencga-client/src/main/R/R/Study-methods.R b/opencga-client/src/main/R/R/Study-methods.R index 6bef965d132..521afc2a513 100644 --- a/opencga-client/src/main/R/R/Study-methods.R +++ b/opencga-client/src/main/R/R/Study-methods.R @@ -118,7 +118,7 @@ setMethod("studyClient", "OpencgaR", function(OpencgaR, group, id, members, stud #' @param operationId Audit operation UUID. #' @param userId User ID. #' @param action Action performed by the user. - #' @param resource Resource involved. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE'] + #' @param resource Resource involved. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL EXTERNAL_TOOL RESOURCE'] #' @param resourceId Resource ID. #' @param resourceUuid resource UUID. #' @param status Filter by status. Allowed values: ['SUCCESS ERROR'] diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java index d8ec5dc79a5..356ba7eef05 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java @@ -25,10 +25,10 @@ import org.opencb.opencga.core.models.externalTool.ExternalTool; import org.opencb.opencga.core.models.externalTool.ExternalToolAclEntryList; import org.opencb.opencga.core.models.externalTool.ExternalToolAclUpdateParams; -import org.opencb.opencga.core.models.externalTool.ExternalToolCreateParams; import org.opencb.opencga.core.models.externalTool.ExternalToolUpdateParams; import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; import org.opencb.opencga.core.models.externalTool.WorkflowRepositoryParams; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.response.RestResponse; @@ -82,7 +82,7 @@ public RestResponse updateAcl(String members, String a * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse create(ExternalToolCreateParams data, ObjectMap params) throws ClientException { + public RestResponse create(WorkflowCreateParams data, ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); params.put("body", data); return execute("workflows", null, null, null, "create", params, POST, ExternalTool.class); diff --git a/opencga-client/src/main/javascript/Admin.js b/opencga-client/src/main/javascript/Admin.js index 84eaeb04938..193e4df03e0 100644 --- a/opencga-client/src/main/javascript/Admin.js +++ b/opencga-client/src/main/javascript/Admin.js @@ -37,7 +37,7 @@ export default class Admin extends OpenCGAParentClass { /** Group by operation * @param {String} fields - Comma separated list of fields by which to group by. * @param {"AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS - * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE"} entity - Entity to be grouped by. + * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL EXTERNAL_TOOL RESOURCE"} entity - Entity to be grouped by. * @param {Object} [params] - The Object containing the following optional parameters: * @param {Boolean} [params.count] - Count the number of elements matching the group. * @param {Number} [params.limit = "50"] - Maximum number of documents (groups) to be returned. The default value is 50. diff --git a/opencga-client/src/main/javascript/Study.js b/opencga-client/src/main/javascript/Study.js index a4139fa1813..aa58a3ae457 100644 --- a/opencga-client/src/main/javascript/Study.js +++ b/opencga-client/src/main/javascript/Study.js @@ -119,7 +119,7 @@ export default class Study extends OpenCGAParentClass { * @param {String} [params.userId] - User ID. * @param {String} [params.action] - Action performed by the user. * @param {"AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS - * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE"} [params.resource] - Resource involved. + * INTERPRETATION VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL EXTERNAL_TOOL RESOURCE"} [params.resource] - Resource involved. * @param {String} [params.resourceId] - Resource ID. * @param {String} [params.resourceUuid] - resource UUID. * @param {"SUCCESS ERROR"} [params.status] - Filter by status. diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py index 1926c986c75..8261e9225e2 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/admin_client.py @@ -27,7 +27,7 @@ def group_by_audit(self, fields, entity, **options): :param str entity: Entity to be grouped by. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION - VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW + VARIANT ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL EXTERNAL_TOOL RESOURCE'] (REQUIRED) :param str fields: Comma separated list of fields by which to group by. (REQUIRED) diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py index 2ab53b7c02a..cae0d306b6c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/study_client.py @@ -139,7 +139,8 @@ def search_audit(self, study, **options): :param str resource: Resource involved. Allowed values: ['AUDIT NOTE ORGANIZATION USER PROJECT STUDY FILE SAMPLE JOB INDIVIDUAL COHORT DISEASE_PANEL FAMILY CLINICAL_ANALYSIS INTERPRETATION VARIANT - ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL WORKFLOW RESOURCE'] + ALIGNMENT CLINICAL EXPRESSION RGA FUNCTIONAL EXTERNAL_TOOL + RESOURCE'] :param str resource_id: Resource ID. :param str resource_uuid: resource UUID. :param str status: Filter by status. Allowed values: ['SUCCESS ERROR'] diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 82f64bb628a..177caae8f88 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -335,20 +335,27 @@ public class FieldConstants { public static final String TOOL_INFO_EXTERNAL_EXECUTOR_DESCRIPTION = "Object containing the id and version of the external tool that" + " is being executed."; - // Workflow - public static final String WORKFLOW_ID_DESCRIPTION = "Workflow ID."; - public static final String WORKFLOW_NAME_DESCRIPTION = "Workflow name."; - public static final String WORKFLOW_SCOPE_DESCRIPTION = "Workflow scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, " - + "CLINICAL_INTERPRETATION_ANALYSIS or OTHER."; - public static final String WORKFLOW_MANAGER_DESCRIPTION = "Workflow system corresponding to the workflow."; - public static final String WORKFLOW_DRAFT_DESCRIPTION = "Flag indicating whether the workflow is still a draft or not."; - public static final String WORKFLOW_REPOSITORY_DESCRIPTION = "Workflow repository image to execute. If any, providing a list of" - + " scripts will not be mandatory."; - public static final String WORKFLOW_SCRIPTS_DESCRIPTION = "List of scripts used by the Workflow."; - public static final String WORKFLOW_TAGS_DESCRIPTION = "List of tags."; - public static final String WORKFLOW_VARIABLES_DESCRIPTION = "List of variables accepted by the Workflow."; + // Docker + public static final String DOCKER_NAME_DESCRIPTION = "Docker name."; + public static final String DOCKER_TAG_DESCRIPTION = "Docker tag."; + public static final String DOCKER_COMMANDLINE_DESCRIPTION = "Docker CLI."; + public static final String DOCKER_USER_DESCRIPTION = "User that can access the private Docker repository."; + public static final String DOCKER_PASSWORD_DESCRIPTION = "Password corresponding to the user that can access the Docker repository."; + + // External tool + public static final String EXTERNAL_TOOL_ID_DESCRIPTION = "External tool ID."; + public static final String EXTERNAL_TOOL_NAME_DESCRIPTION = "External tool name."; + public static final String EXTERNAL_TOOL_TYPE_DESCRIPTION = "External tool type. Valid values: TOOL, VARIANT_WALKER or WORKFLOW."; + public static final String EXTERNAL_TOOL_SCOPE_DESCRIPTION = "External tool scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS," + + " CLINICAL_INTERPRETATION_ANALYSIS or OTHER."; + public static final String EXTERNAL_TOOL_DRAFT_DESCRIPTION = "Flag indicating whether the external tool is a draft or not."; + public static final String EXTERNAL_TOOL_TAGS_DESCRIPTION = "List of tags."; + public static final String EXTERNAL_TOOL_VARIABLES_DESCRIPTION = "List of variables accepted by the External Tool."; public static final String MINIMUM_REQUIREMENTS_DESCRIPTION = "Minimum requirements to execute the process."; - public static final String WORKFLOW_INTERNAL_DESCRIPTION = "Workflow internal information."; + public static final String EXTERNAL_TOOL_INTERNAL_DESCRIPTION = "External tool internal information."; + public static final String EXTERNAL_TOOL_WORKFLOW_DESCRIPTION = "Workflow information if the external tool type is a workflow."; + public static final String EXTERNAL_TOOL_DOCKER_DESCRIPTION = "Docker information if the external tool type is of type TOOL or" + + " VARIANT_WALKER."; public static final String MIN_REQUIREMENTS_CPU_DESCRIPTION = "Minimum number of cpu cores required to execute the process."; public static final String MIN_REQUIREMENTS_MEMORY_DESCRIPTION = "Minimum memory required to execute the process."; @@ -357,6 +364,12 @@ public class FieldConstants { public static final String WORKFLOW_SYSTEM_ID_DESCRIPTION = "Workflow system id. Valid values: NEXTFLOW."; public static final String WORKFLOW_SYSTEM_VERSION_DESCRIPTION = "Workflow system version to use."; + // Workflow + public static final String WORKFLOW_MANAGER_DESCRIPTION = "Workflow system corresponding to the workflow."; + public static final String WORKFLOW_REPOSITORY_DESCRIPTION = "Workflow repository image to execute. If any, providing a list of" + + " scripts will not be mandatory."; + public static final String WORKFLOW_SCRIPTS_DESCRIPTION = "List of scripts used by the Workflow."; + //FileInternal public static final String FILE_INTERNAL_STATUS_DESCRIPTION = "File status can have the values READY, DELETED, " + "TRASHED, STAGE, MISSING, PENDING_DELETE, DELETING, REMOVED and MISSING_SAMPLES."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java index 794ac8555ed..d1ef08adf11 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/Enums.java @@ -80,7 +80,7 @@ public enum Resource { EXPRESSION, RGA, FUNCTIONAL, - WORKFLOW, + EXTERNAL_TOOL, RESOURCE; public List getFullPermissionList() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Docker.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Docker.java new file mode 100644 index 00000000000..9d84e0a8cd1 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Docker.java @@ -0,0 +1,90 @@ +package org.opencb.opencga.core.models.externalTool; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +public class Docker { + + @DataField(id = "name", description = FieldConstants.DOCKER_NAME_DESCRIPTION) + private String name; + + @DataField(id = "tag", description = FieldConstants.DOCKER_TAG_DESCRIPTION) + private String tag; + + @DataField(id = "commandLine", description = FieldConstants.DOCKER_COMMANDLINE_DESCRIPTION) + private String commandLine; + + @DataField(id = "user", description = FieldConstants.DOCKER_USER_DESCRIPTION) + private String user; + + @DataField(id = "password", description = FieldConstants.DOCKER_PASSWORD_DESCRIPTION) + private String password; + + public Docker() { + } + + public Docker(String name, String tag, String commandLine, String user, String password) { + this.name = name; + this.tag = tag; + this.commandLine = commandLine; + this.user = user; + this.password = password; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Docker{"); + sb.append("name='").append(name).append('\''); + sb.append(", tag='").append(tag).append('\''); + sb.append(", commandLine='").append(commandLine).append('\''); + sb.append(", user='").append(user).append('\''); + sb.append(", password='").append(password).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getName() { + return name; + } + + public Docker setName(String name) { + this.name = name; + return this; + } + + public String getTag() { + return tag; + } + + public Docker setTag(String tag) { + this.tag = tag; + return this; + } + + public String getCommandLine() { + return commandLine; + } + + public Docker setCommandLine(String commandLine) { + this.commandLine = commandLine; + return this; + } + + public String getUser() { + return user; + } + + public Docker setUser(String user) { + this.user = user; + return this; + } + + public String getPassword() { + return password; + } + + public Docker setPassword(String password) { + this.password = password; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java index 2ebc9ea82fc..733276ce7a0 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalTool.java @@ -13,7 +13,7 @@ public class ExternalTool extends PrivateStudyUid { @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, - description = FieldConstants.WORKFLOW_ID_DESCRIPTION) + description = FieldConstants.EXTERNAL_TOOL_ID_DESCRIPTION) private String id; @DataField(id = "uuid", managed = true, indexed = true, unique = true, immutable = true, @@ -26,37 +26,37 @@ public class ExternalTool extends PrivateStudyUid { @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) private String description; - @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) - private Scope scope; + @DataField(id = "type", description = FieldConstants.EXTERNAL_TOOL_TYPE_DESCRIPTION) + private ExternalToolType type; - @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) - private WorkflowSystem manager; + @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) + private ExternalToolScope scope; - @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) + @DataField(id = "workflow", description = FieldConstants.EXTERNAL_TOOL_WORKFLOW_DESCRIPTION) + private Workflow workflow; + + @DataField(id = "docker", description = FieldConstants.EXTERNAL_TOOL_DOCKER_DESCRIPTION) + private Docker docker; + + @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) private List tags; - @DataField(id = "variables", description = FieldConstants.WORKFLOW_VARIABLES_DESCRIPTION) + @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) private MinimumRequirements minimumRequirements; - @DataField(id = "draft", description = FieldConstants.WORKFLOW_DRAFT_DESCRIPTION) + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) private boolean draft; - @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) - private WorkflowRepository repository; - - @DataField(id = "scripts", description = FieldConstants.WORKFLOW_SCRIPTS_DESCRIPTION) - private List scripts; - @DataField(id = "version", managed = true, indexed = true, description = FieldConstants.GENERIC_VERSION_DESCRIPTION) private int version; @DataField(id = "release", managed = true, indexed = true, description = FieldConstants.GENERIC_RELEASE_DESCRIPTION) private int release; - @DataField(id = "internal", description = FieldConstants.WORKFLOW_INTERNAL_DESCRIPTION) + @DataField(id = "internal", description = FieldConstants.EXTERNAL_TOOL_INTERNAL_DESCRIPTION) private ExternalToolInternal internal; @DataField(id = "creationDate", indexed = true, description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) @@ -68,37 +68,24 @@ public class ExternalTool extends PrivateStudyUid { @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) private Map attributes; - public enum Type { - TOOL, - VARIANT_WALKER, - WORKFLOW - } - - public enum Scope { - SECONDARY_ANALYSIS, - RESEARCH_ANALYSIS, - CLINICAL_INTERPRETATION_ANALYSIS, - OTHER - } - public ExternalTool() { } - public ExternalTool(String id, String name, String description, Scope scope, WorkflowSystem manager, List tags, - List variables, MinimumRequirements minimumRequirements, boolean draft, - WorkflowRepository repository, List scripts, ExternalToolInternal internal, String creationDate, + public ExternalTool(String id, String name, String description, ExternalToolType type, ExternalToolScope scope, + Workflow workflow, Docker docker, List tags, List variables, + MinimumRequirements minimumRequirements, boolean draft, ExternalToolInternal internal, String creationDate, String modificationDate, Map attributes) { this.id = id; this.name = name; this.description = description; - this.draft = draft; + this.type = type; this.scope = scope; - this.manager = manager; - this.repository = repository; - this.scripts = scripts; + this.workflow = workflow; + this.docker = docker; this.tags = tags; this.variables = variables; this.minimumRequirements = minimumRequirements; + this.draft = draft; this.internal = internal; this.creationDate = creationDate; this.modificationDate = modificationDate; @@ -107,19 +94,19 @@ public ExternalTool(String id, String name, String description, Scope scope, Wor @Override public String toString() { - final StringBuilder sb = new StringBuilder("Workflow{"); + final StringBuilder sb = new StringBuilder("ExternalTool{"); sb.append("id='").append(id).append('\''); sb.append(", uuid='").append(uuid).append('\''); sb.append(", name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); + sb.append(", type=").append(type); sb.append(", scope=").append(scope); - sb.append(", manager=").append(manager); + sb.append(", workflow=").append(workflow); + sb.append(", docker=").append(docker); sb.append(", tags=").append(tags); sb.append(", variables=").append(variables); sb.append(", minimumRequirements=").append(minimumRequirements); sb.append(", draft=").append(draft); - sb.append(", repository=").append(repository); - sb.append(", scripts=").append(scripts); sb.append(", version=").append(version); sb.append(", release=").append(release); sb.append(", internal=").append(internal); @@ -173,93 +160,93 @@ public ExternalTool setDescription(String description) { return this; } - public boolean isDraft() { - return draft; + public ExternalToolType getType() { + return type; } - public ExternalTool setDraft(boolean draft) { - this.draft = draft; + public ExternalTool setType(ExternalToolType type) { + this.type = type; return this; } - public int getVersion() { - return version; + public ExternalToolScope getScope() { + return scope; } - public ExternalTool setVersion(int version) { - this.version = version; + public ExternalTool setScope(ExternalToolScope scope) { + this.scope = scope; return this; } - public int getRelease() { - return release; + public Workflow getWorkflow() { + return workflow; } - public ExternalTool setRelease(int release) { - this.release = release; + public ExternalTool setWorkflow(Workflow workflow) { + this.workflow = workflow; return this; } - public Scope getScope() { - return scope; + public Docker getDocker() { + return docker; } - public ExternalTool setScope(Scope scope) { - this.scope = scope; + public ExternalTool setDocker(Docker docker) { + this.docker = docker; return this; } - public WorkflowSystem getManager() { - return manager; + public List getTags() { + return tags; } - public ExternalTool setManager(WorkflowSystem manager) { - this.manager = manager; + public ExternalTool setTags(List tags) { + this.tags = tags; return this; } - public WorkflowRepository getRepository() { - return repository; + public List getVariables() { + return variables; } - public ExternalTool setRepository(WorkflowRepository repository) { - this.repository = repository; + public ExternalTool setVariables(List variables) { + this.variables = variables; return this; } - public List getScripts() { - return scripts; + public MinimumRequirements getMinimumRequirements() { + return minimumRequirements; } - public ExternalTool setScripts(List scripts) { - this.scripts = scripts; + public ExternalTool setMinimumRequirements(MinimumRequirements minimumRequirements) { + this.minimumRequirements = minimumRequirements; return this; } - public List getTags() { - return tags; + public boolean isDraft() { + return draft; } - public ExternalTool setTags(List tags) { - this.tags = tags; + public ExternalTool setDraft(boolean draft) { + this.draft = draft; return this; } - public List getVariables() { - return variables; + public int getVersion() { + return version; } - public ExternalTool setVariables(List variables) { - this.variables = variables; + public ExternalTool setVersion(int version) { + this.version = version; return this; } - public MinimumRequirements getMinimumRequirements() { - return minimumRequirements; + public int getRelease() { + return release; } - public ExternalTool setMinimumRequirements(MinimumRequirements minimumRequirements) { - this.minimumRequirements = minimumRequirements; + public ExternalTool setRelease(int release) { + this.release = release; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolScope.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolScope.java new file mode 100644 index 00000000000..4d32957b01f --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolScope.java @@ -0,0 +1,8 @@ +package org.opencb.opencga.core.models.externalTool; + +public enum ExternalToolScope { + SECONDARY_ANALYSIS, + RESEARCH_ANALYSIS, + CLINICAL_INTERPRETATION_ANALYSIS, + OTHER +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolType.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolType.java new file mode 100644 index 00000000000..68bad6c4fcb --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolType.java @@ -0,0 +1,7 @@ +package org.opencb.opencga.core.models.externalTool; + +public enum ExternalToolType { + CUSTOM_TOOL, + VARIANT_WALKER, + WORKFLOW +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java index 48e9bc6361a..28a477b7f67 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java @@ -9,7 +9,7 @@ public class ExternalToolUpdateParams { - @DataField(id = "name", description = FieldConstants.WORKFLOW_NAME_DESCRIPTION) + @DataField(id = "name", description = FieldConstants.EXTERNAL_TOOL_NAME_DESCRIPTION) private String name; @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) @@ -18,13 +18,13 @@ public class ExternalToolUpdateParams { @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) private WorkflowSystem manager; - @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) - private ExternalTool.Scope scope; + @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) + private ExternalToolScope scope; - @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) + @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) private List tags; - @DataField(id = "draft", description = FieldConstants.WORKFLOW_DRAFT_DESCRIPTION) + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) private boolean draft; @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) @@ -33,7 +33,7 @@ public class ExternalToolUpdateParams { @DataField(id = "scripts", description = FieldConstants.WORKFLOW_SCRIPTS_DESCRIPTION) private List scripts; - @DataField(id = "variables", description = FieldConstants.WORKFLOW_VARIABLES_DESCRIPTION) + @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) @@ -51,7 +51,7 @@ public class ExternalToolUpdateParams { public ExternalToolUpdateParams() { } - public ExternalToolUpdateParams(String name, String description, WorkflowSystem manager, ExternalTool.Scope scope, List tags, + public ExternalToolUpdateParams(String name, String description, WorkflowSystem manager, ExternalToolScope scope, List tags, boolean draft, WorkflowRepository repository, List scripts, List variables, MinimumRequirements minimumRequirements, String creationDate, String modificationDate, Map attributes) { @@ -117,11 +117,11 @@ public ExternalToolUpdateParams setManager(WorkflowSystem manager) { return this; } - public ExternalTool.Scope getScope() { + public ExternalToolScope getScope() { return scope; } - public ExternalToolUpdateParams setScope(ExternalTool.Scope scope) { + public ExternalToolUpdateParams setScope(ExternalToolScope scope) { this.scope = scope; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Workflow.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Workflow.java new file mode 100644 index 00000000000..416a8e8e819 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/Workflow.java @@ -0,0 +1,65 @@ +package org.opencb.opencga.core.models.externalTool; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; + +import java.util.List; + +public class Workflow { + + @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) + private WorkflowSystem manager; + + @DataField(id = "scripts", description = FieldConstants.WORKFLOW_SCRIPTS_DESCRIPTION) + private List scripts; + + @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) + private WorkflowRepository repository; + + public Workflow() { + } + + public Workflow(WorkflowSystem manager, List scripts, WorkflowRepository repository) { + this.manager = manager; + this.scripts = scripts; + this.repository = repository; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("Workflow{"); + sb.append("manager=").append(manager); + sb.append(", scripts=").append(scripts); + sb.append(", repository=").append(repository); + sb.append('}'); + return sb.toString(); + } + + public WorkflowSystem getManager() { + return manager; + } + + public Workflow setManager(WorkflowSystem manager) { + this.manager = manager; + return this; + } + + public List getScripts() { + return scripts; + } + + public Workflow setScripts(List scripts) { + this.scripts = scripts; + return this; + } + + public WorkflowRepository getRepository() { + return repository; + } + + public Workflow setRepository(WorkflowRepository repository) { + this.repository = repository; + return this; + } + +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java index 06eb1cab136..e4136b7bea7 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java @@ -2,51 +2,57 @@ public class WorkflowRepository { - private String id; - private String version; + private String name; + private String tag; private String author; private String description; + private String user; + private String password; public WorkflowRepository() { } - public WorkflowRepository(String id) { - this.id = id; + public WorkflowRepository(String name) { + this.name = name; } - public WorkflowRepository(String id, String version, String author, String description) { - this.id = id; - this.version = version; + public WorkflowRepository(String name, String tag, String author, String description, String user, String password) { + this.name = name; + this.tag = tag; this.author = author; this.description = description; + this.user = user; + this.password = password; } @Override public String toString() { final StringBuilder sb = new StringBuilder("WorkflowRepository{"); - sb.append("id='").append(id).append('\''); - sb.append(", version='").append(version).append('\''); + sb.append("name='").append(name).append('\''); + sb.append(", tag='").append(tag).append('\''); sb.append(", author='").append(author).append('\''); sb.append(", description='").append(description).append('\''); + sb.append(", user='").append(user).append('\''); + sb.append(", password='").append(password).append('\''); sb.append('}'); return sb.toString(); } - public String getId() { - return id; + public String getName() { + return name; } - public WorkflowRepository setId(String id) { - this.id = id; + public WorkflowRepository setName(String name) { + this.name = name; return this; } - public String getVersion() { - return version; + public String getTag() { + return tag; } - public WorkflowRepository setVersion(String version) { - this.version = version; + public WorkflowRepository setTag(String tag) { + this.tag = tag; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepositoryParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepositoryParams.java index 0365490f761..bcff7ebda6b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepositoryParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepositoryParams.java @@ -2,49 +2,73 @@ public class WorkflowRepositoryParams { - private String id; - private String version; + private String name; + private String tag; + private String user; + private String password; public WorkflowRepositoryParams() { } - public WorkflowRepositoryParams(String id) { - this.id = id; + public WorkflowRepositoryParams(String name) { + this.name = name; } - public WorkflowRepositoryParams(String id, String version) { - this.id = id; - this.version = version; + public WorkflowRepositoryParams(String name, String tag, String user, String password) { + this.name = name; + this.tag = tag; + this.user = user; + this.password = password; } public WorkflowRepository toWorkflowRepository() { - return new WorkflowRepository(id, version, "", ""); + return new WorkflowRepository(name, tag, "", "", user, password); } @Override public String toString() { final StringBuilder sb = new StringBuilder("WorkflowRepositoryParams{"); - sb.append("id='").append(id).append('\''); - sb.append(", version='").append(version).append('\''); + sb.append("name='").append(name).append('\''); + sb.append(", tag='").append(tag).append('\''); + sb.append(", user='").append(user).append('\''); + sb.append(", password='").append(password).append('\''); sb.append('}'); return sb.toString(); } - public String getId() { - return id; + public String getName() { + return name; } - public WorkflowRepositoryParams setId(String id) { - this.id = id; + public WorkflowRepositoryParams setName(String name) { + this.name = name; return this; } - public String getVersion() { - return version; + public String getTag() { + return tag; } - public WorkflowRepositoryParams setVersion(String version) { - this.version = version; + public WorkflowRepositoryParams setTag(String tag) { + this.tag = tag; + return this; + } + + public String getUser() { + return user; + } + + public WorkflowRepositoryParams setUser(String user) { + this.user = user; + return this; + } + + public String getPassword() { + return password; + } + + public WorkflowRepositoryParams setPassword(String password) { + this.password = password; return this; } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowScript.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowScript.java index fbd37ebd0e0..13af6843c43 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowScript.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowScript.java @@ -4,8 +4,8 @@ public class WorkflowScript { - @DataField(id = "fileName", description = "File name") - private String fileName; + @DataField(id = "name", description = "File name") + private String name; @DataField(id = "content", description = "Script content") private String content; @@ -16,8 +16,8 @@ public class WorkflowScript { public WorkflowScript() { } - public WorkflowScript(String fileName, String content, boolean main) { - this.fileName = fileName; + public WorkflowScript(String name, String content, boolean main) { + this.name = name; this.content = content; this.main = main; } @@ -25,19 +25,19 @@ public WorkflowScript(String fileName, String content, boolean main) { @Override public String toString() { final StringBuilder sb = new StringBuilder("WorkflowScript{"); - sb.append("fileName='").append(fileName).append('\''); + sb.append("name='").append(name).append('\''); sb.append(", content='").append(content).append('\''); sb.append(", main=").append(main); sb.append('}'); return sb.toString(); } - public String getFileName() { - return fileName; + public String getName() { + return name; } - public WorkflowScript setFileName(String fileName) { - this.fileName = fileName; + public WorkflowScript setName(String name) { + this.name = name; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java index ab692be189a..7ca958c5820 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowSystem.java @@ -8,7 +8,7 @@ public class WorkflowSystem { @DataField(id = "id", description = FieldConstants.WORKFLOW_SYSTEM_ID_DESCRIPTION) private SystemId id; - @DataField(id = "version", description = FieldConstants.WORKFLOW_SYSTEM_ID_DESCRIPTION) + @DataField(id = "version", description = FieldConstants.WORKFLOW_SYSTEM_VERSION_DESCRIPTION) private String version; public enum SystemId { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowCreateParams.java similarity index 52% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowCreateParams.java index 88ed58027d9..1b324cf49e0 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolCreateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowCreateParams.java @@ -1,47 +1,45 @@ -package org.opencb.opencga.core.models.externalTool; +package org.opencb.opencga.core.models.externalTool.workflow; import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.models.job.MinimumRequirements; import java.util.List; import java.util.Map; -public class ExternalToolCreateParams { +public class WorkflowCreateParams { @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, - description = FieldConstants.WORKFLOW_ID_DESCRIPTION) + description = FieldConstants.EXTERNAL_TOOL_ID_DESCRIPTION) private String id; - @DataField(id = "name", description = FieldConstants.WORKFLOW_NAME_DESCRIPTION) + @DataField(id = "name", description = FieldConstants.GENERIC_UUID_DESCRIPTION) private String name; @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) private String description; - @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) - private WorkflowSystem manager; + @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) + private ExternalToolScope scope; - @DataField(id = "scope", description = FieldConstants.WORKFLOW_SCOPE_DESCRIPTION) - private ExternalTool.Scope scope; + @DataField(id = "workflow", description = FieldConstants.EXTERNAL_TOOL_WORKFLOW_DESCRIPTION) + private Workflow workflow; - @DataField(id = "tags", description = FieldConstants.WORKFLOW_TAGS_DESCRIPTION) + @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) private List tags; - @DataField(id = "draft", description = FieldConstants.WORKFLOW_DRAFT_DESCRIPTION) - private boolean draft; - - @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) - private WorkflowRepository repository; - - @DataField(id = "variables", description = FieldConstants.WORKFLOW_VARIABLES_DESCRIPTION) + @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) private MinimumRequirements minimumRequirements; - @DataField(id = "scripts", description = FieldConstants.WORKFLOW_SCRIPTS_DESCRIPTION) - private List scripts; + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) + private boolean draft; + + @DataField(id = "internal", description = FieldConstants.EXTERNAL_TOOL_INTERNAL_DESCRIPTION) + private ExternalToolInternal internal; @DataField(id = "creationDate", indexed = true, description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) private String creationDate; @@ -52,24 +50,23 @@ public class ExternalToolCreateParams { @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) private Map attributes; - public ExternalToolCreateParams() { + public WorkflowCreateParams() { } - public ExternalToolCreateParams(String id, String name, String description, WorkflowSystem manager, ExternalTool.Scope scope, List tags, - boolean draft, WorkflowRepository repository, List variables, - MinimumRequirements minimumRequirements, List scripts, String creationDate, - String modificationDate, Map attributes) { + public WorkflowCreateParams(String id, String name, String description, ExternalToolScope scope, Workflow workflow, List tags, + List variables, MinimumRequirements minimumRequirements, boolean draft, + ExternalToolInternal internal, String creationDate, String modificationDate, + Map attributes) { this.id = id; this.name = name; this.description = description; - this.manager = manager; this.scope = scope; + this.workflow = workflow; this.tags = tags; - this.draft = draft; - this.repository = repository; this.variables = variables; this.minimumRequirements = minimumRequirements; - this.scripts = scripts; + this.draft = draft; + this.internal = internal; this.creationDate = creationDate; this.modificationDate = modificationDate; this.attributes = attributes; @@ -81,14 +78,13 @@ public String toString() { sb.append("id='").append(id).append('\''); sb.append(", name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); - sb.append(", manager=").append(manager); sb.append(", scope=").append(scope); + sb.append(", workflow=").append(workflow); sb.append(", tags=").append(tags); - sb.append(", draft=").append(draft); - sb.append(", repository=").append(repository); sb.append(", variables=").append(variables); sb.append(", minimumRequirements=").append(minimumRequirements); - sb.append(", scripts=").append(scripts); + sb.append(", draft=").append(draft); + sb.append(", internal=").append(internal); sb.append(", creationDate='").append(creationDate).append('\''); sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", attributes=").append(attributes); @@ -96,16 +92,11 @@ public String toString() { return sb.toString(); } - public ExternalTool toWorkflow() { - return new ExternalTool(id, name, description, scope, manager, tags, variables, minimumRequirements, draft, repository, scripts, - new ExternalToolInternal(), creationDate, modificationDate, attributes); - } - public String getId() { return id; } - public ExternalToolCreateParams setId(String id) { + public WorkflowCreateParams setId(String id) { this.id = id; return this; } @@ -114,7 +105,7 @@ public String getName() { return name; } - public ExternalToolCreateParams setName(String name) { + public WorkflowCreateParams setName(String name) { this.name = name; return this; } @@ -123,26 +114,26 @@ public String getDescription() { return description; } - public ExternalToolCreateParams setDescription(String description) { + public WorkflowCreateParams setDescription(String description) { this.description = description; return this; } - public WorkflowSystem getManager() { - return manager; + public ExternalToolScope getScope() { + return scope; } - public ExternalToolCreateParams setManager(WorkflowSystem manager) { - this.manager = manager; + public WorkflowCreateParams setScope(ExternalToolScope scope) { + this.scope = scope; return this; } - public ExternalTool.Scope getScope() { - return scope; + public Workflow getWorkflow() { + return workflow; } - public ExternalToolCreateParams setScope(ExternalTool.Scope scope) { - this.scope = scope; + public WorkflowCreateParams setWorkflow(Workflow workflow) { + this.workflow = workflow; return this; } @@ -150,34 +141,16 @@ public List getTags() { return tags; } - public ExternalToolCreateParams setTags(List tags) { + public WorkflowCreateParams setTags(List tags) { this.tags = tags; return this; } - public boolean isDraft() { - return draft; - } - - public ExternalToolCreateParams setDraft(boolean draft) { - this.draft = draft; - return this; - } - - public WorkflowRepository getRepository() { - return repository; - } - - public ExternalToolCreateParams setRepository(WorkflowRepository repository) { - this.repository = repository; - return this; - } - public List getVariables() { return variables; } - public ExternalToolCreateParams setVariables(List variables) { + public WorkflowCreateParams setVariables(List variables) { this.variables = variables; return this; } @@ -186,17 +159,26 @@ public MinimumRequirements getMinimumRequirements() { return minimumRequirements; } - public ExternalToolCreateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { + public WorkflowCreateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { this.minimumRequirements = minimumRequirements; return this; } - public List getScripts() { - return scripts; + public boolean isDraft() { + return draft; + } + + public WorkflowCreateParams setDraft(boolean draft) { + this.draft = draft; + return this; + } + + public ExternalToolInternal getInternal() { + return internal; } - public ExternalToolCreateParams setScripts(List scripts) { - this.scripts = scripts; + public WorkflowCreateParams setInternal(ExternalToolInternal internal) { + this.internal = internal; return this; } @@ -204,7 +186,7 @@ public String getCreationDate() { return creationDate; } - public ExternalToolCreateParams setCreationDate(String creationDate) { + public WorkflowCreateParams setCreationDate(String creationDate) { this.creationDate = creationDate; return this; } @@ -213,7 +195,7 @@ public String getModificationDate() { return modificationDate; } - public ExternalToolCreateParams setModificationDate(String modificationDate) { + public WorkflowCreateParams setModificationDate(String modificationDate) { this.modificationDate = modificationDate; return this; } @@ -222,7 +204,7 @@ public Map getAttributes() { return attributes; } - public ExternalToolCreateParams setAttributes(Map attributes) { + public WorkflowCreateParams setAttributes(Map attributes) { this.attributes = attributes; return this; } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java index 695942090ad..809d67ee092 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java @@ -1,15 +1,15 @@ package org.opencb.opencga.server.rest; -import org.apache.commons.lang3.ObjectUtils; import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.managers.ExternalToolManager; +import org.opencb.opencga.catalog.managers.WorkflowManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.VersionException; -import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.externalTool.*; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; +import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.tools.annotations.*; import javax.servlet.http.HttpServletRequest; @@ -25,12 +25,12 @@ @Api(value = "Workflows", description = "Methods for working with 'workflows' endpoint") public class WorkflowWSServer extends OpenCGAWSServer { - private ExternalToolManager externalToolManager; + private WorkflowManager workflowManager; public WorkflowWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest, @Context HttpHeaders httpHeaders) throws IOException, VersionException { super(uriInfo, httpServletRequest, httpHeaders); - externalToolManager = catalogManager.getWorkflowManager(); + workflowManager = catalogManager.getWorkflowManager(); } @GET @@ -51,7 +51,7 @@ public Response workflowInfo( query.remove(ParamConstants.STUDY_PARAM); List workflowList = getIdList(workflowStr); - DataResult workflowDataResult = externalToolManager.get(studyStr, workflowList, query, queryOptions, true, token); + DataResult workflowDataResult = workflowManager.get(studyStr, workflowList, query, queryOptions, true, token); return createOkResponse(workflowDataResult); } catch (Exception e) { return createErrorResponse(e); @@ -70,13 +70,9 @@ public Response workflowInfo( public Response createWorkflowPOST( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, - @ApiParam(value = "JSON containing workflow information", required = true) ExternalToolCreateParams params) { + @ApiParam(value = "JSON containing workflow information", required = true) WorkflowCreateParams workflow) { try { - params = ObjectUtils.defaultIfNull(params, new ExternalToolCreateParams()); - - ExternalTool externalTool = params.toWorkflow(); - - return createOkResponse(externalToolManager.create(studyStr, externalTool, queryOptions, token)); + return createOkResponse(workflowManager.create(studyStr, workflow, queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -143,7 +139,7 @@ public Response search( ) { try { query.remove(ParamConstants.STUDY_PARAM); - return createOkResponse(externalToolManager.search(studyStr, query, queryOptions, token)); + return createOkResponse(workflowManager.search(studyStr, query, queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -173,7 +169,7 @@ public Response distinct( query.remove(ParamConstants.STUDY_PARAM); query.remove(ParamConstants.DISTINCT_FIELD_PARAM); List fields = split(field, ParamConstants.DISTINCT_FIELD_PARAM, true); - return createOkResponse(externalToolManager.distinct(studyStr, fields, query, token)); + return createOkResponse(workflowManager.distinct(studyStr, fields, query, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -199,7 +195,7 @@ public Response updateByPost( // Map actionMap = new HashMap<>(); // queryOptions.put(Constants.ACTIONS, actionMap); - return createOkResponse(externalToolManager.update(studyStr, workflowId, parameters, queryOptions, token), "Workflow update success"); + return createOkResponse(workflowManager.update(studyStr, workflowId, parameters, queryOptions, token), "Workflow update success"); } catch (Exception e) { return createErrorResponse(e); } @@ -212,7 +208,7 @@ public Response delete( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.WORKFLOWS_DESCRIPTION) @PathParam("workflows") String workflows) { try { - return createOkResponse(externalToolManager.delete(studyStr, getIdList(workflows), queryOptions, token)); + return createOkResponse(workflowManager.delete(studyStr, getIdList(workflows), queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -228,7 +224,7 @@ public Response getAcls(@ApiParam(value = ParamConstants.WORKFLOWS_DESCRIPTION, @ApiParam(value = ParamConstants.SILENT_DESCRIPTION, defaultValue = "false") @QueryParam(Constants.SILENT) boolean silent) { try { List idList = getIdList(workflowIdsStr); - return createOkResponse(externalToolManager.getAcls(studyStr, idList, member, silent, token)); + return createOkResponse(workflowManager.getAcls(studyStr, idList, member, silent, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -244,7 +240,7 @@ public Response updateAcl( @QueryParam(ParamConstants.ACL_ACTION_PARAM) ParamUtils.AclAction action, @ApiParam(value = "JSON containing the parameters to update the permissions.", required = true) ExternalToolAclUpdateParams params) { try { - return createOkResponse(externalToolManager.updateAcl(studyStr, memberIds, params, action, token)); + return createOkResponse(workflowManager.updateAcl(studyStr, memberIds, params, action, token)); } catch (Exception e) { return createErrorResponse(e); } From d418ccb2e9b4b18acc4eb60c28e8f01d90d459fb Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 7 May 2025 14:22:01 +0200 Subject: [PATCH 05/22] app: add external tool data model migration, #TASK-7610 --- .../ExternalToolTask7610Migration.java | 53 +++++++++++++++++++ .../catalog/db/api/ExternalToolDBAdaptor.java | 8 ++- .../mongodb/ExternalToolMongoDBAdaptor.java | 8 ++- .../OrganizationMongoDBAdaptorFactory.java | 30 ++++++----- .../catalog/managers/WorkflowManager.java | 15 +++--- 5 files changed, 84 insertions(+), 30 deletions(-) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ExternalToolTask7610Migration.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ExternalToolTask7610Migration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ExternalToolTask7610Migration.java new file mode 100644 index 00000000000..0acfe81353a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ExternalToolTask7610Migration.java @@ -0,0 +1,53 @@ +package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; + +import java.util.Arrays; + +@Migration(id = "externaltool__task_7610", + description = "Extend Workflow data model for new ExternalTool, #TASK-7610", version = "4.0.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20250506) +public class ExternalToolTask7610Migration extends MigrationTool { + + @Override + protected void run() throws Exception { + Bson query = Filters.exists("manager"); + Bson projection = new Document(); + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.DEPRECATED_WORKFLOW_COLLECTION, + OrganizationMongoDBAdaptorFactory.DEPRECATED_WORKFLOW_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DEPRECATED_DELETED_WORKFLOW_COLLECTION), query, projection, (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + + updateDocument.getSet().put("scope", document.get("type")); + updateDocument.getSet().put("type", "WORKFLOW"); + Document repository = document.get("repository", Document.class); + if (repository != null) { + repository.put("name", repository.get("id")); + repository.put("tag", repository.get("version")); + repository.put("user", ""); + repository.put("password", ""); + repository.remove("id"); + repository.remove("version"); + } + updateDocument.getSet().put("workflow", new Document() + .append("manager", document.get("manager")) + .append("repository", repository) + .append("scripts", document.get("scripts")) + ); + updateDocument.getUnset().add("manager"); + updateDocument.getUnset().add("repository"); + updateDocument.getUnset().add("scripts"); + updateDocument.getSet().put("docker", null); + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java index e707e0bd0d3..20f3dface9b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java @@ -26,12 +26,10 @@ enum QueryParams implements QueryParam { TYPE("type", TEXT, ""), SCOPE("scope", TEXT, ""), TAGS("tags", TEXT_ARRAY, ""), - COMMAND_LINE("commandLine", TEXT, ""), WORKFLOW("workflow", OBJECT, ""), - MANAGER("manager", OBJECT, ""), - MANAGER_ID("manager.id", TEXT, ""), - REPOSITORY("repository", OBJECT, ""), - SCRIPTS("scripts", OBJECT, ""), + WORKFLOW_REPOSITORY("workflow.repository", OBJECT, ""), + WORKFLOW_SCRIPTS("workflow.scripts", OBJECT, ""), + DOCKER("docker", OBJECT, ""), VARIABLES("variables", OBJECT, ""), MINIMUM_REQUIREMENTS("minimumRequirements", OBJECT, ""), INTERNAL_REGISTRATION_USER_ID("internal.registrationUserId", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java index 7b7de5f79a3..6b3a8e40f50 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java @@ -331,15 +331,14 @@ private UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, QueryO final String[] acceptedBooleanParams = {QueryParams.DRAFT.key()}; filterBooleanParams(parameters, document.getSet(), acceptedBooleanParams); - final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.DESCRIPTION.key(), QueryParams.COMMAND_LINE.key(), - QueryParams.SCOPE.key()}; + final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.DESCRIPTION.key(), QueryParams.SCOPE.key()}; filterStringParams(parameters, document.getSet(), acceptedParams); final String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key()}; filterMapParams(parameters, document.getSet(), acceptedMapParams); - final String[] acceptedListParams = {QueryParams.MANAGER.key(), QueryParams.SCRIPTS.key(), QueryParams.TAGS.key(), - QueryParams.REPOSITORY.key(), QueryParams.VARIABLES.key(), QueryParams.MINIMUM_REQUIREMENTS.key()}; + final String[] acceptedListParams = {QueryParams.DOCKER.key(), QueryParams.WORKFLOW_SCRIPTS.key(), QueryParams.TAGS.key(), + QueryParams.WORKFLOW_REPOSITORY.key(), QueryParams.VARIABLES.key(), QueryParams.MINIMUM_REQUIREMENTS.key()}; filterObjectParams(parameters, document.getSet(), acceptedListParams); // // Check if the scripts exist. @@ -573,7 +572,6 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) case RELEASE: case VERSION: case INTERNAL_REGISTRATION_USER_ID: - case MANAGER_ID: case SCOPE: case DRAFT: addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java index 3b0aaa63edf..eba9751c735 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java @@ -43,7 +43,9 @@ public class OrganizationMongoDBAdaptorFactory { public static final String PANEL_COLLECTION = "panel"; public static final String CLINICAL_ANALYSIS_COLLECTION = "clinical"; public static final String INTERPRETATION_COLLECTION = "interpretation"; - public static final String WORKFLOW_COLLECTION = "workflow"; + public static final String EXTERNAL_TOOL_COLLECTION = "workflow"; + @Deprecated + public static final String DEPRECATED_WORKFLOW_COLLECTION = "workflow"; public static final String NOTE_ARCHIVE_COLLECTION = "note_archive"; public static final String SAMPLE_ARCHIVE_COLLECTION = "sample_archive"; @@ -52,7 +54,9 @@ public class OrganizationMongoDBAdaptorFactory { public static final String PANEL_ARCHIVE_COLLECTION = "panel_archive"; public static final String CLINICAL_ANALYSIS_ARCHIVE_COLLECTION = "clinical_archive"; public static final String INTERPRETATION_ARCHIVE_COLLECTION = "interpretation_archive"; - public static final String WORKFLOW_ARCHIVE_COLLECTION = "workflow_archive"; + public static final String EXTERNAL_TOOL_ARCHIVE_COLLECTION = "workflow_archive"; + @Deprecated + public static final String DEPRECATED_WORKFLOW_ARCHIVE_COLLECTION = "workflow_archive"; public static final String DELETED_NOTE_COLLECTION = "note_deleted"; public static final String DELETED_USER_COLLECTION = "user_deleted"; @@ -67,7 +71,9 @@ public class OrganizationMongoDBAdaptorFactory { public static final String DELETED_PANEL_COLLECTION = "panel_deleted"; public static final String DELETED_CLINICAL_ANALYSIS_COLLECTION = "clinical_deleted"; public static final String DELETED_INTERPRETATION_COLLECTION = "interpretation_deleted"; - public static final String DELETED_WORKFLOW_COLLECTION = "workflow_deleted"; + public static final String DELETED_EXTERNAL_TOOL_COLLECTION = "workflow_deleted"; + @Deprecated + public static final String DEPRECATED_DELETED_WORKFLOW_COLLECTION = "workflow_deleted"; public static final String METADATA_COLLECTION = "metadata"; public static final String MIGRATION_COLLECTION = "migration"; @@ -88,7 +94,7 @@ public class OrganizationMongoDBAdaptorFactory { FAMILY_COLLECTION, CLINICAL_ANALYSIS_COLLECTION, INTERPRETATION_COLLECTION, - WORKFLOW_COLLECTION, + EXTERNAL_TOOL_COLLECTION, NOTE_ARCHIVE_COLLECTION, SAMPLE_ARCHIVE_COLLECTION, @@ -97,7 +103,7 @@ public class OrganizationMongoDBAdaptorFactory { PANEL_ARCHIVE_COLLECTION, CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, INTERPRETATION_ARCHIVE_COLLECTION, - WORKFLOW_ARCHIVE_COLLECTION, + EXTERNAL_TOOL_ARCHIVE_COLLECTION, DELETED_NOTE_COLLECTION, DELETED_USER_COLLECTION, @@ -112,7 +118,7 @@ public class OrganizationMongoDBAdaptorFactory { DELETED_FAMILY_COLLECTION, DELETED_CLINICAL_ANALYSIS_COLLECTION, DELETED_INTERPRETATION_COLLECTION, - DELETED_WORKFLOW_COLLECTION, + DELETED_EXTERNAL_TOOL_COLLECTION, MIGRATION_COLLECTION, // FIXME metadata collection is unused @@ -181,7 +187,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa MongoDBCollection familyCollection = mongoDataStore.getCollection(FAMILY_COLLECTION); MongoDBCollection clinicalCollection = mongoDataStore.getCollection(CLINICAL_ANALYSIS_COLLECTION); MongoDBCollection interpretationCollection = mongoDataStore.getCollection(INTERPRETATION_COLLECTION); - MongoDBCollection workflowCollection = mongoDataStore.getCollection(WORKFLOW_COLLECTION); + MongoDBCollection workflowCollection = mongoDataStore.getCollection(EXTERNAL_TOOL_COLLECTION); MongoDBCollection notesArchivedCollection = mongoDataStore.getCollection(NOTE_ARCHIVE_COLLECTION); MongoDBCollection sampleArchivedCollection = mongoDataStore.getCollection(SAMPLE_ARCHIVE_COLLECTION); @@ -190,7 +196,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa MongoDBCollection panelArchivedCollection = mongoDataStore.getCollection(PANEL_ARCHIVE_COLLECTION); MongoDBCollection clinicalArchivedCollection = mongoDataStore.getCollection(CLINICAL_ANALYSIS_ARCHIVE_COLLECTION); MongoDBCollection interpretationArchivedCollection = mongoDataStore.getCollection(INTERPRETATION_ARCHIVE_COLLECTION); - MongoDBCollection workflowArchivedCollection = mongoDataStore.getCollection(WORKFLOW_ARCHIVE_COLLECTION); + MongoDBCollection workflowArchivedCollection = mongoDataStore.getCollection(EXTERNAL_TOOL_ARCHIVE_COLLECTION); MongoDBCollection deletedNotesCollection = mongoDataStore.getCollection(DELETED_NOTE_COLLECTION); MongoDBCollection deletedUserCollection = mongoDataStore.getCollection(DELETED_USER_COLLECTION); @@ -205,7 +211,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa MongoDBCollection deletedFamilyCollection = mongoDataStore.getCollection(DELETED_FAMILY_COLLECTION); MongoDBCollection deletedClinicalCollection = mongoDataStore.getCollection(DELETED_CLINICAL_ANALYSIS_COLLECTION); MongoDBCollection deletedInterpretationCollection = mongoDataStore.getCollection(DELETED_INTERPRETATION_COLLECTION); - MongoDBCollection deletedWorkflowCollection = mongoDataStore.getCollection(DELETED_WORKFLOW_COLLECTION); + MongoDBCollection deletedWorkflowCollection = mongoDataStore.getCollection(DELETED_EXTERNAL_TOOL_COLLECTION); MongoDBCollection auditCollection = mongoDataStore.getCollection(AUDIT_COLLECTION); @@ -255,7 +261,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa mongoDBCollectionMap.put(FAMILY_COLLECTION, familyCollection); mongoDBCollectionMap.put(CLINICAL_ANALYSIS_COLLECTION, clinicalCollection); mongoDBCollectionMap.put(INTERPRETATION_COLLECTION, interpretationCollection); - mongoDBCollectionMap.put(WORKFLOW_COLLECTION, workflowCollection); + mongoDBCollectionMap.put(EXTERNAL_TOOL_COLLECTION, workflowCollection); mongoDBCollectionMap.put(NOTE_ARCHIVE_COLLECTION, notesArchivedCollection); mongoDBCollectionMap.put(SAMPLE_ARCHIVE_COLLECTION, sampleArchivedCollection); @@ -263,7 +269,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa mongoDBCollectionMap.put(FAMILY_ARCHIVE_COLLECTION, familyArchivedCollection); mongoDBCollectionMap.put(PANEL_ARCHIVE_COLLECTION, panelArchivedCollection); mongoDBCollectionMap.put(INTERPRETATION_ARCHIVE_COLLECTION, interpretationArchivedCollection); - mongoDBCollectionMap.put(WORKFLOW_ARCHIVE_COLLECTION, workflowArchivedCollection); + mongoDBCollectionMap.put(EXTERNAL_TOOL_ARCHIVE_COLLECTION, workflowArchivedCollection); mongoDBCollectionMap.put(DELETED_NOTE_COLLECTION, deletedNotesCollection); mongoDBCollectionMap.put(DELETED_USER_COLLECTION, deletedUserCollection); @@ -277,7 +283,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa mongoDBCollectionMap.put(DELETED_FAMILY_COLLECTION, deletedFamilyCollection); mongoDBCollectionMap.put(DELETED_CLINICAL_ANALYSIS_COLLECTION, deletedClinicalCollection); mongoDBCollectionMap.put(DELETED_INTERPRETATION_COLLECTION, deletedInterpretationCollection); - mongoDBCollectionMap.put(DELETED_WORKFLOW_COLLECTION, deletedWorkflowCollection); + mongoDBCollectionMap.put(DELETED_EXTERNAL_TOOL_COLLECTION, deletedWorkflowCollection); mongoDBCollectionMap.put(AUDIT_COLLECTION, auditCollection); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java index fcfeb7c3b84..7b2a296bf37 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java @@ -59,7 +59,7 @@ public class WorkflowManager extends ResourceManager { public static final QueryOptions INCLUDE_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(ID.key(), UID.key(), - UUID.key(), VERSION.key(), DESCRIPTION.key(), STUDY_UID.key(), MANAGER.key())); + UUID.key(), VERSION.key(), DESCRIPTION.key(), STUDY_UID.key(), WORKFLOW.key())); private final CatalogIOManager catalogIOManager; private final IOManagerFactory ioManagerFactory; @@ -89,19 +89,18 @@ Enums.Resource getEntity() { } @Override - InternalGetDataResult internalGet(String organizationId, long studyUid, List workflowIdList, + InternalGetDataResult internalGet(String organizationId, long studyUid, List externalToolIdList, @Nullable Query query, QueryOptions options, String user, boolean ignoreException) throws CatalogException { - if (ListUtils.isEmpty(workflowIdList)) { - throw new CatalogException("Missing workflow entries."); + if (ListUtils.isEmpty(externalToolIdList)) { + throw new CatalogException("Missing external tool entries."); } - List uniqueList = ListUtils.unique(workflowIdList); + List uniqueList = ListUtils.unique(externalToolIdList); QueryOptions queryOptions = new QueryOptions(ParamUtils.defaultObject(options, QueryOptions::new)); Query queryCopy = query == null ? new Query() : new Query(query); queryCopy.put(STUDY_UID.key(), studyUid); - queryCopy.put(TYPE.key(), ExternalToolType.WORKFLOW); boolean versioned = queryCopy.getBoolean(Constants.ALL_VERSIONS) || queryCopy.containsKey(VERSION.key()); @@ -942,8 +941,8 @@ private void validateNewWorkflow(ExternalTool externalTool, String userId) throw : Collections.emptyList()); boolean main = false; for (WorkflowScript script : externalTool.getWorkflow().getScripts()) { - ParamUtils.checkIdentifier(script.getName(), SCRIPTS.key() + ".id"); - ParamUtils.checkParameter(script.getContent(), SCRIPTS.key() + ".content"); + ParamUtils.checkIdentifier(script.getName(), WORKFLOW_SCRIPTS.key() + ".id"); + ParamUtils.checkParameter(script.getContent(), WORKFLOW_SCRIPTS.key() + ".content"); if (script.isMain()) { if (main) { throw new CatalogParameterException("More than one main script found."); From f7e6e3c058ec9a605cc16be71acd64f86f6fb547 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 15 May 2025 15:30:59 +0200 Subject: [PATCH 06/22] catalog: last changes to ExternalTool, #TASK-7610 --- .../customTool/CustomToolExecutor.java | 28 +- .../tools/OpenCgaDockerToolScopeStudy.java | 7 +- .../analysis/workflow/NextFlowExecutor.java | 20 +- .../tools/ExternalToolExecutorTest.java | 12 +- .../executors/WorkflowCommandExecutor.java | 4 +- .../executors/WorkflowsCommandExecutor.java | 30 +- .../main/options/WorkflowsCommandOptions.java | 6 + .../catalog/db/api/ExternalToolDBAdaptor.java | 3 + .../mongodb/ExternalToolMongoDBAdaptor.java | 45 +- .../catalog/managers/CatalogManager.java | 12 +- ...wManager.java => ExternalToolManager.java} | 488 +++++++++--------- .../opencga/catalog/utils/NextflowUtils.java | 220 ++++++++ .../opencga/catalog/utils/UuidUtils.java | 2 +- .../src/main/resources/catalog-indexes.txt | 5 +- ...Test.java => ExternalToolManagerTest.java} | 57 +- opencga-client/src/main/R/R/Job-methods.R | 2 +- .../src/main/R/R/Workflow-methods.R | 2 +- .../client/rest/clients/JobClient.java | 2 +- .../client/rest/clients/WorkflowClient.java | 11 +- opencga-client/src/main/javascript/Job.js | 2 +- .../src/main/javascript/Workflow.js | 2 +- .../pyopencga/rest_clients/job_client.py | 2 +- .../pyopencga/rest_clients/workflow_client.py | 2 +- .../opencga/core/api/ParamConstants.java | 31 +- ...Params.java => ExternalToolRunParams.java} | 18 +- .../ExternalToolUpdateParams.java | 92 +--- .../externalTool/WorkflowRepository.java | 18 + .../custom/CustomToolCreateParams.java | 214 ++++++++ .../custom/CustomToolUpdateParams.java | 106 ++++ .../DeprecatedWorkflowUpdateParams.java | 220 ++++++++ .../workflow/WorkflowUpdateParams.java | 106 ++++ .../monitor/daemons/ExecutionDaemon.java | 2 + .../server/rest/ExternalToolWSServer.java | 362 +++++++++++++ .../opencga/server/rest/JobWSServer.java | 4 +- .../opencga/server/rest/WorkflowWSServer.java | 34 +- 35 files changed, 1692 insertions(+), 479 deletions(-) rename opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/{WorkflowManager.java => ExternalToolManager.java} (77%) create mode 100644 opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/NextflowUtils.java rename opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/{WorkflowManagerTest.java => ExternalToolManagerTest.java} (70%) rename opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/{NextFlowRunParams.java => ExternalToolRunParams.java} (60%) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolCreateParams.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolUpdateParams.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/DeprecatedWorkflowUpdateParams.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowUpdateParams.java create mode 100644 opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolExecutor.java index 3fc79847202..8fa8275a49d 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolExecutor.java @@ -3,11 +3,11 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.opencb.opencga.analysis.tools.OpenCgaDockerToolScopeStudy; +import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.job.JobRunDockerParams; -import org.opencb.opencga.core.models.job.JobRunParams; +import org.opencb.opencga.core.models.externalTool.Docker; import org.opencb.opencga.core.models.job.ToolInfoExecutor; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; @@ -23,7 +23,7 @@ public class CustomToolExecutor extends OpenCgaDockerToolScopeStudy { public static final String DESCRIPTION = "Execute an analysis from a custom binary."; @ToolParams - protected JobRunParams runParams = new JobRunParams(); + protected Docker runParams = new Docker(); private String cliParams; private String dockerImage; @@ -41,19 +41,20 @@ protected void check() throws Exception { if (StringUtils.isEmpty(runParams.getCommandLine())) { throw new ToolException("Missing commandLine"); } - if (runParams.getDocker() == null || StringUtils.isEmpty(runParams.getDocker().getId())) { - runParams.setDocker(new JobRunDockerParams("opencb/opencga-ext-tools", "3.2.1", null)); + if (StringUtils.isEmpty(runParams.getName())) { + runParams.setName("opencb/opencga-ext-tools"); + runParams.setTag(GitRepositoryState.getInstance().getBuildVersion()); } - if (!runParams.getDocker().getId().contains("/")) { + if (!runParams.getName().contains("/")) { throw new ToolException("Missing repository organization. Format for the docker image should be 'organization/image'"); } - this.dockerImage = runParams.getDocker().getId(); - if (StringUtils.isNotEmpty(runParams.getDocker().getTag())) { - this.dockerImage += ":" + runParams.getDocker().getTag(); + this.dockerImage = runParams.getName(); + if (StringUtils.isNotEmpty(runParams.getTag())) { + this.dockerImage += ":" + runParams.getTag(); } // Update job tags and attributes - ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(runParams.getDocker().getId(), runParams.getDocker().getTag()); + ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(runParams.getName(), runParams.getTag()); List tags = new LinkedList<>(); tags.add(ID); tags.add(this.dockerImage); @@ -70,10 +71,11 @@ protected void run() throws Exception { Map dockerParams = new HashMap<>(); dockerParams.put("-e", "OPENCGA_TOKEN=" + getExpiringToken()); - String cmdline = runDocker(dockerImage, Collections.emptyList(), cliParams, dockerParams); + String cmdline = runDocker(dockerImage, Collections.emptyList(), cliParams, dockerParams, null, + runParams.getUser(), runParams.getPassword()); - logger.info("Docker command line: " + cmdline); - logger.info("Execution time: " + TimeUtils.durationToString(stopWatch)); + logger.info("Docker command line: {}", cmdline); + logger.info("Execution time: {}", TimeUtils.durationToString(stopWatch)); } } diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaDockerToolScopeStudy.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaDockerToolScopeStudy.java index e4dc9ea4ea1..4aaf11a0e17 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaDockerToolScopeStudy.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/tools/OpenCgaDockerToolScopeStudy.java @@ -155,6 +155,11 @@ protected String runDocker(String image, AbstractMap.SimpleEntry protected String runDocker(String image, List> userOutputBindings, String cmdParams, Map userDockerParams) throws IOException { + return runDocker(image, userOutputBindings, cmdParams, userDockerParams, null, null, null); + } + + protected String runDocker(String image, List> userOutputBindings, String cmdParams, + Map userDockerParams, String registry, String username, String password) throws IOException { List> outputBindings = CollectionUtils.isNotEmpty(userOutputBindings) ? userOutputBindings : Collections.singletonList(new AbstractMap.SimpleEntry<>(getOutDir().toAbsolutePath().toString(), getOutDir().toAbsolutePath().toString())); @@ -169,7 +174,7 @@ protected String runDocker(String image, List result; - if (nextflowParams.getVersion() != null) { - Query query = new Query(ExternalToolDBAdaptor.QueryParams.VERSION.key(), nextflowParams.getVersion()); - result = catalogManager.getWorkflowManager().get(study, Collections.singletonList(nextflowParams.getId()), query, + if (externalToolRunParams.getVersion() != null) { + Query query = new Query(ExternalToolDBAdaptor.QueryParams.VERSION.key(), externalToolRunParams.getVersion()); + result = catalogManager.getExternalToolManager().get(study, Collections.singletonList(externalToolRunParams.getId()), query, QueryOptions.empty(), false, token); } else { - result = catalogManager.getWorkflowManager().get(study, nextflowParams.getId(), QueryOptions.empty(), token); + result = catalogManager.getExternalToolManager().get(study, externalToolRunParams.getId(), QueryOptions.empty(), token); } if (result.getNumResults() == 0) { - throw new ToolException("Workflow '" + nextflowParams.getId() + "' not found"); + throw new ToolException("Workflow '" + externalToolRunParams.getId() + "' not found"); } externalTool = result.first(); if (externalTool == null) { - throw new ToolException("Workflow '" + nextflowParams.getId() + "' is null"); + throw new ToolException("Workflow '" + externalToolRunParams.getId() + "' is null"); } outDirPath = getOutDir().toAbsolutePath().toString(); @@ -130,8 +130,8 @@ protected void check() throws Exception { updateJobInformation(new ArrayList<>(tags), toolInfoExecutor); StringBuilder cliParamsBuilder = new StringBuilder(); - if (MapUtils.isNotEmpty(nextflowParams.getParams())) { - for (Map.Entry entry : nextflowParams.getParams().entrySet()) { + if (MapUtils.isNotEmpty(externalToolRunParams.getParams())) { + for (Map.Entry entry : externalToolRunParams.getParams().entrySet()) { String variableId = removePrefix(entry.getKey()); // Remove from the mandatoryParams set mandatoryParams.remove(variableId); diff --git a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java index d4f61b561ba..1c5d20f2d82 100644 --- a/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java +++ b/opencga-analysis/src/test/java/org/opencb/opencga/analysis/tools/ExternalToolExecutorTest.java @@ -38,13 +38,13 @@ public void nextflowScriptTest() throws ToolException, CatalogException, IOExcep StorageConfiguration storageConfiguration = StorageConfiguration.load(inputStream, "yml"); WorkflowCreateParams workflow = createDummyWorkflow(); - catalogManager.getWorkflowManager().create(studyFqn, workflow, QueryOptions.empty(), ownerToken); + catalogManager.getExternalToolManager().createWorkflow(studyFqn, workflow, QueryOptions.empty(), ownerToken); Path outDir = Paths.get(catalogManagerResource.createTmpOutdir("_nextflow")); StopWatch stopWatch = StopWatch.createStarted(); NextFlowExecutor nextFlowExecutorTest = new NextFlowExecutor(); - NextFlowRunParams runParams = new NextFlowRunParams(workflow.getId(), 1, Collections.emptyMap()); + ExternalToolRunParams runParams = new ExternalToolRunParams(workflow.getId(), 1, Collections.emptyMap()); ObjectMap params = runParams.toObjectMap(); params.put(ParamConstants.STUDY_PARAM, studyFqn); nextFlowExecutorTest.setUp(catalogManagerResource.getOpencgaHome().toString(), catalogManager, @@ -63,7 +63,7 @@ public void nextflowScriptWithFilesTest() throws ToolException, CatalogException catalogManager.getFileManager().create(studyFqn, new FileCreateParams().setPath("myfile.txt").setContent("hello world").setType(File.Type.FILE), false, ownerToken); WorkflowCreateParams workflow = createDummyWorkflow("pipeline_cat_file.nf"); - catalogManager.getWorkflowManager().create(studyFqn, workflow, QueryOptions.empty(), ownerToken); + catalogManager.getExternalToolManager().createWorkflow(studyFqn, workflow, QueryOptions.empty(), ownerToken); Path outDir = Paths.get(catalogManagerResource.createTmpOutdir("_nextflow")); @@ -71,7 +71,7 @@ public void nextflowScriptWithFilesTest() throws ToolException, CatalogException NextFlowExecutor nextFlowExecutorTest = new NextFlowExecutor(); Map workflowParams = new HashMap<>(); workflowParams.put("in", "ocga://myfile.txt"); - NextFlowRunParams runParams = new NextFlowRunParams(workflow.getId(), 1, workflowParams); + ExternalToolRunParams runParams = new ExternalToolRunParams(workflow.getId(), 1, workflowParams); ObjectMap params = runParams.toObjectMap(); params.put(ParamConstants.STUDY_PARAM, studyFqn); nextFlowExecutorTest.setUp(catalogManagerResource.getOpencgaHome().toString(), catalogManager, @@ -111,7 +111,7 @@ public void nextflowDockerTest() throws ToolException, CatalogException, IOExcep // .setType(WorkflowVariable.WorkflowVariableType.FLAG) )) .setWorkflow(new Workflow().setRepository(new WorkflowRepository("nf-core/demo"))); - catalogManager.getWorkflowManager().create(studyFqn, workflow, QueryOptions.empty(), ownerToken); + catalogManager.getExternalToolManager().createWorkflow(studyFqn, workflow, QueryOptions.empty(), ownerToken); catalogManager.getFileManager().create(studyFqn, new FileCreateParams() .setPath("samplesheet.csv") @@ -130,7 +130,7 @@ public void nextflowDockerTest() throws ToolException, CatalogException, IOExcep // cliParams.put("outdir", "$OUTPUT"); // cliParams.put("genome", "GRCh37"); // cliParams.put("-profile", "docker"); - NextFlowRunParams runParams = new NextFlowRunParams(workflow.getId(), 1, cliParams); + ExternalToolRunParams runParams = new ExternalToolRunParams(workflow.getId(), 1, cliParams); ObjectMap params = runParams.toObjectMap(); params.put(ParamConstants.STUDY_PARAM, studyFqn); nextFlowExecutorTest.setUp(catalogManagerResource.getOpencgaHome().toString(), catalogManager, diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/WorkflowCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/WorkflowCommandExecutor.java index a9b39a88729..02ad6171957 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/WorkflowCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/WorkflowCommandExecutor.java @@ -4,7 +4,7 @@ import org.opencb.opencga.analysis.workflow.NextFlowExecutor; import org.opencb.opencga.app.cli.internal.options.WorkflowCommandOptions; import org.opencb.opencga.core.exceptions.ToolException; -import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolRunParams; import java.nio.file.Path; import java.nio.file.Paths; @@ -46,7 +46,7 @@ private void nextflowRun() throws ToolException { Path outDir = Paths.get(options.outDir); - NextFlowRunParams nextFlowRunParams = new NextFlowRunParams(); + ExternalToolRunParams nextFlowRunParams = new ExternalToolRunParams(); nextFlowRunParams.setId(options.workflowId); toolRunner.execute(NextFlowExecutor.class, nextFlowRunParams, QueryOptions.empty(), outDir, jobId, dryRun, token); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java index a03abec53af..c6f67676d02 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java @@ -18,13 +18,13 @@ import org.opencb.opencga.core.models.externalTool.ExternalToolAclEntryList; import org.opencb.opencga.core.models.externalTool.ExternalToolAclUpdateParams; import org.opencb.opencga.core.models.externalTool.ExternalToolInternal; +import org.opencb.opencga.core.models.externalTool.ExternalToolRunParams; import org.opencb.opencga.core.models.externalTool.ExternalToolScope; -import org.opencb.opencga.core.models.externalTool.ExternalToolUpdateParams; -import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; import org.opencb.opencga.core.models.externalTool.Workflow; import org.opencb.opencga.core.models.externalTool.WorkflowRepository; import org.opencb.opencga.core.models.externalTool.WorkflowRepositoryParams; import org.opencb.opencga.core.models.externalTool.WorkflowSystem; +import org.opencb.opencga.core.models.externalTool.workflow.DeprecatedWorkflowUpdateParams; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.job.MinimumRequirements; @@ -268,26 +268,26 @@ private RestResponse run() throws Exception { } - NextFlowRunParams nextFlowRunParams = null; + ExternalToolRunParams externalToolRunParams = null; if (commandOptions.jsonDataModel) { RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/workflows/run")); return res; } else if (commandOptions.jsonFile != null) { - nextFlowRunParams = JacksonUtils.getDefaultObjectMapper() - .readValue(new java.io.File(commandOptions.jsonFile), NextFlowRunParams.class); + externalToolRunParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), ExternalToolRunParams.class); } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); putNestedIfNotNull(beanParams, "version", commandOptions.version, true); putNestedMapIfNotEmpty(beanParams, "params", commandOptions.params, true); - nextFlowRunParams = JacksonUtils.getDefaultObjectMapper().copy() + externalToolRunParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) - .readValue(beanParams.toJson(), NextFlowRunParams.class); + .readValue(beanParams.toJson(), ExternalToolRunParams.class); } - return openCGAClient.getWorkflowClient().run(nextFlowRunParams, queryParams); + return openCGAClient.getWorkflowClient().run(externalToolRunParams, queryParams); } private RestResponse search() throws Exception { @@ -338,15 +338,15 @@ private RestResponse update() throws Exception { } - ExternalToolUpdateParams externalToolUpdateParams = null; + DeprecatedWorkflowUpdateParams deprecatedWorkflowUpdateParams = null; if (commandOptions.jsonDataModel) { RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/workflows/{workflowId}/update")); return res; } else if (commandOptions.jsonFile != null) { - externalToolUpdateParams = JacksonUtils.getDefaultObjectMapper() - .readValue(new java.io.File(commandOptions.jsonFile), ExternalToolUpdateParams.class); + deprecatedWorkflowUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), DeprecatedWorkflowUpdateParams.class); } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); @@ -360,6 +360,8 @@ private RestResponse update() throws Exception { putNestedIfNotEmpty(beanParams, "repository.tag", commandOptions.repositoryTag, true); putNestedIfNotEmpty(beanParams, "repository.author", commandOptions.repositoryAuthor, true); putNestedIfNotEmpty(beanParams, "repository.description", commandOptions.repositoryDescription, true); + putNestedIfNotEmpty(beanParams, "repository.user", commandOptions.repositoryUser, true); + putNestedIfNotEmpty(beanParams, "repository.password", commandOptions.repositoryPassword, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); @@ -367,11 +369,11 @@ private RestResponse update() throws Exception { putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); - externalToolUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + deprecatedWorkflowUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) - .readValue(beanParams.toJson(), ExternalToolUpdateParams.class); + .readValue(beanParams.toJson(), DeprecatedWorkflowUpdateParams.class); } - return openCGAClient.getWorkflowClient().update(commandOptions.workflowId, externalToolUpdateParams, queryParams); + return openCGAClient.getWorkflowClient().update(commandOptions.workflowId, deprecatedWorkflowUpdateParams, queryParams); } private RestResponse acl() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java index ee0c5f869e1..5e7be6091d1 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java @@ -422,6 +422,12 @@ public class UpdateCommandOptions { @Parameter(names = {"--repository-description"}, description = "The body web service description parameter", required = false, arity = 1) public String repositoryDescription; + @Parameter(names = {"--repository-user"}, description = "The body web service user parameter", required = false, arity = 1) + public String repositoryUser; + + @Parameter(names = {"--repository-password"}, description = "The body web service password parameter", required = false, arity = 1) + public String repositoryPassword; + @Parameter(names = {"--minimum-requirements-cpu"}, description = "Minimum number of cpu cores required to execute the process.", required = false, arity = 1) public String minimumRequirementsCpu; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java index 20f3dface9b..bd978ba9721 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ExternalToolDBAdaptor.java @@ -27,9 +27,12 @@ enum QueryParams implements QueryParam { SCOPE("scope", TEXT, ""), TAGS("tags", TEXT_ARRAY, ""), WORKFLOW("workflow", OBJECT, ""), + WORKFLOW_MANAGER("workflow.manager", OBJECT, ""), WORKFLOW_REPOSITORY("workflow.repository", OBJECT, ""), + WORKFLOW_REPOSITORY_NAME("workflow.repository.name", TEXT, ""), WORKFLOW_SCRIPTS("workflow.scripts", OBJECT, ""), DOCKER("docker", OBJECT, ""), + DOCKER_NAME("docker.name", TEXT, ""), VARIABLES("variables", OBJECT, ""), MINIMUM_REQUIREMENTS("minimumRequirements", OBJECT, ""), INTERNAL_REGISTRATION_USER_ID("internal.registrationUserId", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java index 6b3a8e40f50..010c05ef226 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ExternalToolMongoDBAdaptor.java @@ -337,34 +337,11 @@ private UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, QueryO final String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key()}; filterMapParams(parameters, document.getSet(), acceptedMapParams); - final String[] acceptedListParams = {QueryParams.DOCKER.key(), QueryParams.WORKFLOW_SCRIPTS.key(), QueryParams.TAGS.key(), - QueryParams.WORKFLOW_REPOSITORY.key(), QueryParams.VARIABLES.key(), QueryParams.MINIMUM_REQUIREMENTS.key()}; + final String[] acceptedListParams = {QueryParams.DOCKER.key(), QueryParams.WORKFLOW_MANAGER.key(), + QueryParams.WORKFLOW_SCRIPTS.key(), QueryParams.WORKFLOW_REPOSITORY.key(), QueryParams.TAGS.key(), + QueryParams.VARIABLES.key(), QueryParams.MINIMUM_REQUIREMENTS.key()}; filterObjectParams(parameters, document.getSet(), acceptedListParams); -// // Check if the scripts exist. -// if (parameters.containsKey(QueryParams.SCRIPTS.key())) { -// List scriptList = parameters.getAsList(QueryParams.SCRIPTS.key(), WorkflowScript.class); -// -// if (!scriptList.isEmpty()) { -// Map actionMap = queryOptions.getMap(Constants.ACTIONS, new HashMap<>()); -// ParamUtils.BasicUpdateAction operation = -// ParamUtils.BasicUpdateAction.from(actionMap, QueryParams.SCRIPTS.key(), ParamUtils.BasicUpdateAction.ADD); -// switch (operation) { -// case SET: -// document.getSet().put(QueryParams.SCRIPTS.key(), scriptList); -// break; -// case REMOVE: -// document.getPullAll().put(QueryParams.SCRIPTS.key(), scriptList); -// break; -// case ADD: -// document.getAddToSet().put(QueryParams.SCRIPTS.key(), scriptList); -// break; -// default: -// throw new IllegalArgumentException("Unknown update action " + operation); -// } -// } -// } - if (!document.toFinalUpdateDocument().isEmpty()) { String time = TimeUtils.getTime(); if (StringUtils.isEmpty(parameters.getString(QueryParams.MODIFICATION_DATE.key()))) { @@ -552,26 +529,16 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) case STUDY_UID: addAutoOrQuery(PRIVATE_STUDY_UID, queryParam.key(), queryCopy, queryParam.type(), andBsonList); break; -// case STATUS: -// case STATUS_ID: -// addAutoOrQuery(WorkflowDBAdaptor.QueryParams.STATUS_ID.key(), queryParam.key(), query, -// WorkflowDBAdaptor.QueryParams.STATUS_ID.type(), andBsonList); -// break; -// case INTERNAL_STATUS: -// case INTERNAL_STATUS_ID: -// // Convert the status to a positive status -// query.put(queryParam.key(), InternalStatus.getPositiveStatus(InternalStatus.STATUS_LIST, -// query.getString(queryParam.key()))); -// addAutoOrQuery(WorkflowDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), queryParam.key(), query, -// WorkflowDBAdaptor.QueryParams.INTERNAL_STATUS_ID.type(), andBsonList); -// break; case ID: case UUID: case NAME: + case WORKFLOW_REPOSITORY_NAME: + case DOCKER_NAME: case TAGS: case RELEASE: case VERSION: case INTERNAL_REGISTRATION_USER_ID: + case TYPE: case SCOPE: case DRAFT: addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index f832c848e68..712f0a2f740 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -37,8 +37,8 @@ import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.migration.MigrationManager; import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.core.common.JwtUtils; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.common.JwtUtils; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.config.Configuration; @@ -88,7 +88,7 @@ public class CatalogManager implements AutoCloseable { private ClinicalAnalysisManager clinicalAnalysisManager; private InterpretationManager interpretationManager; private PanelManager panelManager; - private WorkflowManager workflowManager; + private ExternalToolManager externalToolManager; private AuditManager auditManager; private AuthorizationManager authorizationManager; @@ -164,8 +164,8 @@ private void configureManagers(Configuration configuration) throws CatalogExcept clinicalAnalysisManager = new ClinicalAnalysisManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, configuration); interpretationManager = new InterpretationManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, configuration); - workflowManager = new WorkflowManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, ioManagerFactory, - catalogIOManager, configuration); + externalToolManager = new ExternalToolManager(authorizationManager, auditManager, this, catalogDBAdaptorFactory, + configuration); } private void initializeAdmin(Configuration configuration) throws CatalogDBException { @@ -458,7 +458,7 @@ public MigrationManager getMigrationManager() { return migrationManager; } - public WorkflowManager getWorkflowManager() { - return workflowManager; + public ExternalToolManager getExternalToolManager() { + return externalToolManager; } } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java similarity index 77% rename from opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java rename to opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java index 7b2a296bf37..af2afecd8c8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/WorkflowManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java @@ -14,15 +14,9 @@ import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.catalog.exceptions.CatalogParameterException; -import org.opencb.opencga.catalog.io.CatalogIOManager; -import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.models.InternalGetDataResult; -import org.opencb.opencga.catalog.utils.CatalogFqn; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.catalog.utils.ParamUtils; -import org.opencb.opencga.catalog.utils.UuidUtils; +import org.opencb.opencga.catalog.utils.*; import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.common.IOUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.models.AclEntryList; @@ -30,59 +24,42 @@ import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.externalTool.*; +import org.opencb.opencga.core.models.externalTool.custom.CustomToolCreateParams; +import org.opencb.opencga.core.models.externalTool.custom.CustomToolUpdateParams; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowUpdateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.job.JobType; -import org.opencb.opencga.core.models.job.MinimumRequirements; import org.opencb.opencga.core.models.job.ToolInfo; import org.opencb.opencga.core.models.study.Study; import org.opencb.opencga.core.models.study.StudyPermissions; -import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.response.OpenCGAResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.URL; import java.util.*; import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager.checkPermissions; import static org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor.QueryParams.*; import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; -public class WorkflowManager extends ResourceManager { +public class ExternalToolManager extends ResourceManager { public static final QueryOptions INCLUDE_IDS = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(ID.key(), UID.key(), - UUID.key(), VERSION.key(), DESCRIPTION.key(), STUDY_UID.key(), WORKFLOW.key())); - - private final CatalogIOManager catalogIOManager; - private final IOManagerFactory ioManagerFactory; - - private static final Pattern WORKFLOW_MEMORY_PATTERN = Pattern.compile("\\s*memory\\s*=\\s*\\{\\s*[^0-9]*([0-9]+\\.[A-Za-z]+)"); - private static final Pattern WORKFLOW_CPU_PATTERN = Pattern.compile("\\s*cpus\\s*=\\s*\\{\\s*[^0-9]*([0-9]+)"); - - public static final int MAX_CPUS = 15; - public static final String MAX_MEMORY = "64.GB"; // Format is important for Nextflow. It requires the dot symbol. + UUID.key(), VERSION.key(), DESCRIPTION.key(), STUDY_UID.key(), WORKFLOW.key(), DOCKER.key())); private final Logger logger; - WorkflowManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, - DBAdaptorFactory catalogDBAdaptorFactory, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager, - Configuration configuration) { + ExternalToolManager(AuthorizationManager authorizationManager, AuditManager auditManager, CatalogManager catalogManager, + DBAdaptorFactory catalogDBAdaptorFactory, Configuration configuration) { super(authorizationManager, auditManager, catalogManager, catalogDBAdaptorFactory, configuration); - - this.catalogIOManager = catalogIOManager; - this.ioManagerFactory = ioManagerFactory; - this.logger = LoggerFactory.getLogger(WorkflowManager.class); + this.logger = LoggerFactory.getLogger(ExternalToolManager.class); } - @Override Enums.Resource getEntity() { return Enums.Resource.EXTERNAL_TOOL; @@ -105,7 +82,7 @@ InternalGetDataResult internalGet(String organizationId, long stud boolean versioned = queryCopy.getBoolean(Constants.ALL_VERSIONS) || queryCopy.containsKey(VERSION.key()); if (versioned && uniqueList.size() > 1) { - throw new CatalogException("Only one workflow allowed when requesting multiple versions"); + throw new CatalogException("Only one external tool allowed when requesting multiple versions"); } ExternalToolDBAdaptor.QueryParams idQueryParam = getFieldFilter(uniqueList); @@ -114,23 +91,25 @@ InternalGetDataResult internalGet(String organizationId, long stud // Ensure the field by which we are querying for will be kept in the results queryOptions = keepFieldInQueryOptions(queryOptions, idQueryParam.key()); - OpenCGAResult workflowResult = getWorkflowDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); + OpenCGAResult externalToolResult = getWorkflowDBAdaptor(organizationId).get(studyUid, queryCopy, queryOptions, user); - Function workflowStringFunction = ExternalTool::getId; + Function externalToolStringFunction = ExternalTool::getId; if (idQueryParam.equals(UUID)) { - workflowStringFunction = ExternalTool::getUuid; + externalToolStringFunction = ExternalTool::getUuid; } - if (ignoreException || workflowResult.getNumResults() >= uniqueList.size()) { - return keepOriginalOrder(uniqueList, workflowStringFunction, workflowResult, ignoreException, versioned); + if (ignoreException || externalToolResult.getNumResults() >= uniqueList.size()) { + return keepOriginalOrder(uniqueList, externalToolStringFunction, externalToolResult, ignoreException, versioned); } // Query without adding the user check OpenCGAResult resultsNoCheck = getWorkflowDBAdaptor(organizationId).get(queryCopy, queryOptions); - if (resultsNoCheck.getNumResults() == workflowResult.getNumResults()) { - throw CatalogException.notFound("workflows", getMissingFields(uniqueList, workflowResult.getResults(), workflowStringFunction)); + if (resultsNoCheck.getNumResults() == externalToolResult.getNumResults()) { + throw CatalogException.notFound("external tools", getMissingFields(uniqueList, externalToolResult.getResults(), + externalToolStringFunction)); } else { - throw new CatalogAuthorizationException("Permission denied. " + user + " is not allowed to see some or none of the workflows."); + throw new CatalogAuthorizationException("Permission denied. " + user + " is not allowed to see some or none of the external" + + " tools."); } } @@ -140,7 +119,7 @@ public OpenCGAResult create(String studyStr, ExternalTool external throw new NotImplementedException("Not implemented yet"); } - public OpenCGAResult create(String studyStr, WorkflowCreateParams workflow, QueryOptions options, String token) + public OpenCGAResult createWorkflow(String studyStr, WorkflowCreateParams workflow, QueryOptions options, String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); @@ -195,9 +174,86 @@ public OpenCGAResult create(String studyStr, WorkflowCreateParams } } - public OpenCGAResult submit(String studyStr, NextFlowRunParams runParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, - Boolean dryRun, String token) throws CatalogException { + public OpenCGAResult createCustomTool(String studyStr, CustomToolCreateParams toolCreateParams, QueryOptions options, + String token) throws CatalogException { + options = ParamUtils.defaultObject(options, QueryOptions::new); + + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + + ObjectMap auditParams = new ObjectMap() + .append("study", studyStr) + .append("customTool", toolCreateParams) + .append("options", options) + .append("token", token); + + String organizationId = studyFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + String studyId = studyFqn.getStudyId(); + String studyUuid = studyFqn.getStudyUuid(); + try { + Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); + studyId = study.getId(); + studyUuid = study.getUuid(); + + // 1. Check permissions + authorizationManager.checkStudyPermission(organizationId, study.getUid(), tokenPayload, + StudyPermissions.Permissions.WRITE_WORKFLOWS); + + // Convert WorkflowCreateParams to ExternalTool + ExternalTool externalTool = new ExternalTool(toolCreateParams.getId(), toolCreateParams.getName(), + toolCreateParams.getDescription(), ExternalToolType.WORKFLOW, toolCreateParams.getScope(), null, + toolCreateParams.getDocker(), toolCreateParams.getTags(), toolCreateParams.getVariables(), + toolCreateParams.getMinimumRequirements(), toolCreateParams.isDraft(), toolCreateParams.getInternal(), + toolCreateParams.getCreationDate(), toolCreateParams.getModificationDate(), toolCreateParams.getAttributes()); + + // 2. Validate the workflow parameters + validateNewCustomTool(externalTool, userId); + + // 3. We insert the workflow + OpenCGAResult insert = getWorkflowDBAdaptor(organizationId).insert(study.getUid(), externalTool, options); + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { + // Fetch created workflow + Query query = new Query() + .append(STUDY_UID.key(), study.getUid()) + .append(UID.key(), externalTool.getUid()); + OpenCGAResult result = getWorkflowDBAdaptor(organizationId).get(query, options); + insert.setResults(result.getResults()); + } + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, externalTool.getId(), externalTool.getUuid(), + studyId, studyUuid, auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + return insert; + } catch (CatalogException e) { + auditManager.auditCreate(organizationId, userId, Enums.Resource.EXTERNAL_TOOL, toolCreateParams.getId(), "", studyId, studyUuid, + auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); + throw e; + } + } + + public OpenCGAResult submitWorkflow(String studyStr, ExternalToolRunParams runParams, String jobId, String jobDescription, + String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, + Boolean dryRun, String token) throws CatalogException { + return submit(studyStr, runParams, ExternalToolType.WORKFLOW, jobId, jobDescription, jobDependsOnStr, jobTagsStr, + jobScheduledStartTime, jobPriority, dryRun, token); + } + + public OpenCGAResult submitCustomTool(String studyStr, ExternalToolRunParams runParams, String jobId, String jobDescription, + String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, + Boolean dryRun, String token) throws CatalogException { + return submit(studyStr, runParams, ExternalToolType.CUSTOM_TOOL, jobId, jobDescription, jobDependsOnStr, jobTagsStr, + jobScheduledStartTime, jobPriority, dryRun, token); + } + + public OpenCGAResult submitVariantWalker(String studyStr, ExternalToolRunParams runParams, String jobId, String jobDescription, + String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, + Boolean dryRun, String token) throws CatalogException { + return submit(studyStr, runParams, ExternalToolType.VARIANT_WALKER, jobId, jobDescription, jobDependsOnStr, jobTagsStr, + jobScheduledStartTime, jobPriority, dryRun, token); + } + + private OpenCGAResult submit(String studyStr, ExternalToolRunParams runParams, ExternalToolType type, String jobId, + String jobDescription, String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, + String jobPriority, Boolean dryRun, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); @@ -242,7 +298,21 @@ public OpenCGAResult submit(String studyStr, NextFlowRunParams runParams, S jobTags = Collections.emptyList(); } - return catalogManager.getJobManager().submit(study.getFqn(), JobType.WORKFLOW, toolInfo, priority, paramsMap, jobId, jobDescription, + JobType jobType; + switch (type) { + case CUSTOM_TOOL: + jobType = JobType.CUSTOM; + break; + case VARIANT_WALKER: + jobType = JobType.WALKER; + break; + case WORKFLOW: + jobType = JobType.WORKFLOW; + break; + default: + throw new CatalogException("Unknown external tool type: " + type); + } + return catalogManager.getJobManager().submit(study.getFqn(), jobType, toolInfo, priority, paramsMap, jobId, jobDescription, jobDependsOn, jobTags, null, jobScheduledStartTime, dryRun, Collections.emptyMap(), token); } @@ -273,7 +343,7 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepos StudyPermissions.Permissions.WRITE_WORKFLOWS); // 2. Download workflow - ExternalTool externalTool = downloadWorkflow(repository); + ExternalTool externalTool = NextflowUtils.importRepository(repository); validateNewWorkflow(externalTool, userId); workflowId = externalTool.getId(); @@ -320,205 +390,82 @@ public OpenCGAResult importWorkflow(String studyStr, WorkflowRepos } } - private ExternalTool downloadWorkflow(WorkflowRepositoryParams repository) throws CatalogException { - ParamUtils.checkObj(repository, "Workflow repository parameters"); - if (StringUtils.isEmpty(repository.getName())) { - throw new CatalogParameterException("Missing 'id' field in workflow import parameters"); - } - String workflowId = repository.getName().replace("/", "."); - WorkflowSystem workflowSystem = new WorkflowSystem(WorkflowSystem.SystemId.NEXTFLOW, ""); - ExternalTool externalTool = new ExternalTool("", "", "", ExternalToolType.WORKFLOW, null, - new Workflow(workflowSystem, new LinkedList<>(), repository.toWorkflowRepository()), null, new LinkedList<>(), - new LinkedList<>(), new MinimumRequirements(), false, new ExternalToolInternal(), TimeUtils.getTime(), TimeUtils.getTime(), - new HashMap<>()); - - try { - processNextflowConfig(externalTool, repository); - processMemoryRequirements(externalTool, repository); - } catch (CatalogException e) { - throw new CatalogException("Could not process repository information from workflow '" + workflowId + "'.", e); - } - - return externalTool; + @FunctionalInterface + private interface CatalogExceptionThrowingConsumer { + void execute(T t) throws CatalogException; } - private void processMemoryRequirements(ExternalTool externalTool, WorkflowRepositoryParams repository) throws CatalogException { - String urlStr; - - if (StringUtils.isEmpty(repository.getTag())) { - urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/heads/master/conf/base.config"; - } else { - urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/tags/" + repository.getTag() - + "/conf/base.config"; - } - - try { - URL url = new URL(urlStr); - BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream())); - - int cpus = 0; - String memory = null; - String inputLine; - long maxMemory = IOUtils.fromHumanReadableToByte(MAX_MEMORY); - while ((inputLine = in.readLine()) != null) { - Matcher cpuMatcher = WORKFLOW_CPU_PATTERN.matcher(inputLine); - Matcher memoryMatcher = WORKFLOW_MEMORY_PATTERN.matcher(inputLine); - if (cpuMatcher.find()) { - String value = cpuMatcher.group(1); - int intValue = Integer.parseInt(value); - if (intValue > cpus) { - cpus = Math.min(intValue, MAX_CPUS); - } - } else if (memoryMatcher.find()) { - String value = memoryMatcher.group(1); - if (memory == null) { - memory = value; - } else { - long memoryBytes = IOUtils.fromHumanReadableToByte(value); - long currentMemoryBytes = IOUtils.fromHumanReadableToByte(memory); - if (memoryBytes > currentMemoryBytes) { - if (memoryBytes > maxMemory) { - memory = MAX_MEMORY; - } else { - memory = value; - } - } - } + public OpenCGAResult updateWorkflow(String studyStr, String externalToolId, WorkflowUpdateParams updateParams, + QueryOptions options, String token) throws CatalogException { + return privateUpdate(studyStr, externalToolId, updateParams, externalTool -> { + Workflow workflow = updateParams.getWorkflow(); + if (workflow == null) { + return; + } + if (workflow.getManager() != null) { + if (workflow.getManager().getId() == null) { + throw new CatalogException("Workflow manager id cannot be left empty."); + } + if (StringUtils.isEmpty(workflow.getManager().getVersion())) { + throw new CatalogException("Workflow manager version cannot be left empty."); } } - if (cpus > 0 && memory != null) { - externalTool.getMinimumRequirements().setCpu(String.valueOf(cpus)); - externalTool.getMinimumRequirements().setMemory(memory); - } else { - logger.warn("Could not find the minimum requirements for the workflow " + externalTool.getId()); + if (CollectionUtils.isNotEmpty(workflow.getScripts()) && workflow.getRepository() != null) { + throw new CatalogParameterException("Both workflow repository and scripts objects are provided. Please, choose one."); } - in.close(); - } catch (Exception e) { - throw new CatalogException("Could not process nextflow.config file.", e); - } - } - - private void processNextflowConfig(ExternalTool externalTool, WorkflowRepositoryParams repository) throws CatalogException { - String urlStr; - if (StringUtils.isEmpty(repository.getTag())) { - urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/heads/master/nextflow.config"; - } else { - urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/tags/" + repository.getTag() - + "/nextflow.config"; - } - - try { - URL url = new URL(urlStr); - BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream())); - - // We add the bracket close strings because they are expected to be properly indented. That way, we will be able to know when - // that section has been properly closed. Otherwise, we may get confused by some other subsections that could be closed before - // the actual section closure. - String manifestBracketClose = null; - String profilesBracketClose = null; - String gitpodBracketClose = null; - String inputLine; - while ((inputLine = in.readLine()) != null) { - if (manifestBracketClose != null) { - if (manifestBracketClose.equals(inputLine)) { - manifestBracketClose = null; - } else { - // Process manifest line - fillWithWorkflowManifest(externalTool, inputLine); - } - } else if (profilesBracketClose != null) { - if (gitpodBracketClose != null) { - if (gitpodBracketClose.equals(inputLine)) { - gitpodBracketClose = null; - } else { - // Process gitpod line - fillWithGitpodManifest(externalTool, inputLine); + if (CollectionUtils.isEmpty(externalTool.getWorkflow().getScripts()) + && CollectionUtils.isNotEmpty(workflow.getScripts())) { + throw new CatalogParameterException("Cannot add scripts to a workflow that was created with a repository."); + } + if (externalTool.getWorkflow().getRepository() == null && workflow.getRepository() != null) { + throw new CatalogParameterException("Cannot add a repository to a workflow that was created with scripts."); + } + if (CollectionUtils.isNotEmpty(workflow.getScripts())) { + boolean main = false; + for (WorkflowScript script : workflow.getScripts()) { + ParamUtils.checkIdentifier(script.getName(), WORKFLOW_SCRIPTS.key() + ".id"); + ParamUtils.checkParameter(script.getContent(), WORKFLOW_SCRIPTS.key() + ".content"); + if (script.isMain()) { + if (main) { + throw new CatalogParameterException("More than one workflow main script found."); } - } else if (inputLine.trim().startsWith("gitpod {")) { - int position = inputLine.indexOf("gitpod {"); - gitpodBracketClose = StringUtils.repeat(" ", position) + "}"; - } else if (profilesBracketClose.equals(inputLine)) { - profilesBracketClose = null; + main = script.isMain(); } - } else if (inputLine.trim().startsWith("manifest {")) { - int position = inputLine.indexOf("profiles {"); - manifestBracketClose = StringUtils.repeat(" ", position) + "}"; - } else if (inputLine.trim().startsWith("profiles {")) { - int position = inputLine.indexOf("profiles {"); - profilesBracketClose = StringUtils.repeat(" ", position) + "}"; - + } + if (CollectionUtils.isNotEmpty(workflow.getScripts()) && !main) { + throw new CatalogParameterException("No workflow main script found."); } } - in.close(); - } catch (Exception e) { - throw new CatalogException("Could not process nextflow.config file.", e); - } + if (workflow.getRepository() != null) { + validateWorkflowRepository(workflow.getRepository()); + } + }, options, token); } - private void fillWithWorkflowManifest(ExternalTool externalTool, String rawline) { - String[] split = rawline.split("= "); - if (split.length != 2) { - return; - } - String key = split[0].trim(); -// String key = split[0].replaceAll(" ", ""); - String value = split[1].replace("\"", "").replace("'", "").trim(); - switch (key) { - case "name": - externalTool.setId(value.replace("/", ".")); - externalTool.setName(value.replace("/", " ")); - externalTool.getWorkflow().getRepository().setName(value); - break; - case "author": - externalTool.getWorkflow().getRepository().setAuthor(value); - break; - case "description": - externalTool.setDescription(value); - externalTool.getWorkflow().getRepository().setDescription(value); - break; - case "version": - String version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); - externalTool.getWorkflow().getRepository().setTag(version); - break; - case "nextflowVersion": - // Nextflow version must start with a number - version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); - externalTool.getWorkflow().getManager().setVersion(version); - break; - default: - break; - } + public OpenCGAResult updateCustomTool(String studyStr, String externalToolId, CustomToolUpdateParams updateParams, + QueryOptions options, String token) throws CatalogException { + return privateUpdate(studyStr, externalToolId, updateParams, externalTool -> { + if (updateParams.getDocker() != null) { + validateDocker(updateParams.getDocker()); + } + }, options, token); } - private void fillWithGitpodManifest(ExternalTool externalTool, String rawline) { - String[] split = rawline.split("="); - if (split.length != 2) { - return; - } - String key = split[0].replaceAll(" ", ""); - String value = split[1].replace("\"", "").replace("'", "").trim(); - switch (key) { - case "executor.cpus": - externalTool.getMinimumRequirements().setCpu(value); - break; - case "executor.memory": - externalTool.getMinimumRequirements().setMemory(value); - break; - default: - break; - } + public OpenCGAResult update(String studyStr, String externalToolId, ExternalToolUpdateParams updateParams, + QueryOptions options, String token) throws CatalogException { + return privateUpdate(studyStr, externalToolId, updateParams, (et) -> { }, options, token); } - - public OpenCGAResult update(String studyStr, String workflowId, ExternalToolUpdateParams updateParams, - QueryOptions options, String token) throws CatalogException { + private OpenCGAResult privateUpdate(String studyStr, String externalToolId, ExternalToolUpdateParams updateParams, + CatalogExceptionThrowingConsumer validateFunction, QueryOptions options, + String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); - JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); ObjectMap auditParams = new ObjectMap() - .append("workflowId", workflowId) + .append("study", studyStr) + .append("externalToolId", externalToolId) .append("updateParams", updateParams) .append("options", options) .append("token", token); @@ -528,15 +475,15 @@ public OpenCGAResult update(String studyStr, String workflowId, Ex String studyId = studyFqn.getStudyId(); String studyUuid = studyFqn.getStudyUuid(); - String id = UuidUtils.isOpenCgaUuid(workflowId) ? "" : workflowId; - String uuid = UuidUtils.isOpenCgaUuid(workflowId) ? workflowId : ""; + String id = UuidUtils.isOpenCgaUuid(externalToolId) ? "" : externalToolId; + String uuid = UuidUtils.isOpenCgaUuid(externalToolId) ? externalToolId : ""; try { Study study = catalogManager.getStudyManager().resolveId(studyFqn, StudyManager.INCLUDE_VARIABLE_SET, tokenPayload); studyId = study.getId(); studyUuid = study.getUuid(); - ExternalTool externalTool = internalGet(organizationId, study.getUid(), Collections.singletonList(workflowId), null, - INCLUDE_IDS, userId, false).first(); + ExternalTool externalTool = internalGet(organizationId, study.getUid(), Collections.singletonList(externalToolId), null, + QueryOptions.empty(), userId, false).first(); id = externalTool.getId(); uuid = externalTool.getUuid(); @@ -547,15 +494,7 @@ public OpenCGAResult update(String studyStr, String workflowId, Ex if (updateParams == null) { throw new CatalogException("Missing parameters to update the workflow."); } - - if (updateParams.getManager() != null) { - if (updateParams.getManager().getId() == null) { - throw new CatalogException("Manager id cannot be left empty."); - } - if (StringUtils.isEmpty(updateParams.getManager().getVersion())) { - throw new CatalogException("Manager version cannot be left empty."); - } - } + validateFunction.execute(externalTool); ObjectMap updateMap; try { @@ -583,6 +522,39 @@ public OpenCGAResult update(String studyStr, String workflowId, Ex } } + private void validateWorkflowRepository(WorkflowRepository repository) throws CatalogParameterException { + if (repository == null) { + return; + } + ParamUtils.checkParameter(repository.getName(), "workflow repository name"); + repository.setTag(ParamUtils.defaultString(repository.getTag(), "")); + repository.setDescription(ParamUtils.defaultString(repository.getDescription(), "")); + repository.setAuthor(ParamUtils.defaultString(repository.getAuthor(), "")); + repository.setUser(ParamUtils.defaultString(repository.getUser(), "")); + repository.setPassword(ParamUtils.defaultString(repository.getPassword(), "")); + + if ((StringUtils.isNotEmpty(repository.getUser()) && StringUtils.isEmpty(repository.getPassword())) + || (StringUtils.isEmpty(repository.getUser()) && StringUtils.isNotEmpty(repository.getPassword()))) { + throw new CatalogParameterException("User and password must be provided together."); + } + } + + private void validateDocker(Docker docker) throws CatalogParameterException { + if (docker == null) { + return; + } + ParamUtils.checkParameter(docker.getName(), DOCKER.key() + ".name"); + ParamUtils.checkParameter(docker.getTag(), DOCKER.key() + ".tag"); + docker.setCommandLine(ParamUtils.defaultString(docker.getCommandLine(), "")); + docker.setUser(ParamUtils.defaultString(docker.getUser(), "")); + docker.setPassword(ParamUtils.defaultString(docker.getPassword(), "")); + if ((StringUtils.isEmpty(docker.getPassword()) && StringUtils.isNotEmpty(docker.getUser())) + || (StringUtils.isNotEmpty(docker.getPassword()) + && StringUtils.isEmpty(docker.getUser()))) { + throw new CatalogParameterException("Docker user and password must be set together."); + } + } + @Override public DBIterator iterator(String studyStr, Query query, QueryOptions options, String token) throws CatalogException { query = ParamUtils.defaultObject(query, Query::new); @@ -924,7 +896,6 @@ private ExternalToolDBAdaptor.QueryParams getFieldFilter(List idList) th } private void validateNewWorkflow(ExternalTool externalTool, String userId) throws CatalogParameterException { - ParamUtils.checkIdentifier(externalTool.getId(), ID.key()); ParamUtils.checkObj(externalTool.getWorkflow(), WORKFLOW.key()); if (externalTool.getWorkflow().getManager() == null) { externalTool.getWorkflow().setManager(new WorkflowSystem()); @@ -932,10 +903,6 @@ private void validateNewWorkflow(ExternalTool externalTool, String userId) throw if (externalTool.getWorkflow().getManager().getId() == null) { externalTool.getWorkflow().getManager().setId(WorkflowSystem.SystemId.NEXTFLOW); } - externalTool.setScope(ParamUtils.defaultObject(externalTool.getScope(), ExternalToolScope.OTHER)); - externalTool.setTags(externalTool.getTags() != null - ? externalTool.getTags() - : Collections.emptyList()); externalTool.getWorkflow().setScripts(externalTool.getWorkflow().getScripts() != null ? externalTool.getWorkflow().getScripts() : Collections.emptyList()); @@ -956,6 +923,9 @@ private void validateNewWorkflow(ExternalTool externalTool, String userId) throw externalTool.getWorkflow().setRepository(externalTool.getWorkflow().getRepository() != null ? externalTool.getWorkflow().getRepository() : new WorkflowRepository("")); + if (externalTool.getWorkflow().getRepository() != null) { + validateWorkflowRepository(externalTool.getWorkflow().getRepository()); + } if (StringUtils.isEmpty(externalTool.getWorkflow().getRepository().getName()) && CollectionUtils.isEmpty(externalTool.getWorkflow().getScripts())) { throw new CatalogParameterException("No repository image or scripts found."); @@ -966,16 +936,40 @@ private void validateNewWorkflow(ExternalTool externalTool, String userId) throw + " image."); } + externalTool.setDocker(null); + validateNewExternalTool(externalTool, userId); + } + + private void validateNewCustomTool(ExternalTool externalTool, String userId) throws CatalogParameterException { + ParamUtils.checkObj(externalTool.getDocker(), DOCKER.key()); + validateDocker(externalTool.getDocker()); + externalTool.setWorkflow(null); + validateNewExternalTool(externalTool, userId); + } + + private void validateNewExternalTool(ExternalTool externalTool, String userId) throws CatalogParameterException { + ParamUtils.checkIdentifier(externalTool.getId(), ID.key()); + externalTool.setScope(ParamUtils.defaultObject(externalTool.getScope(), ExternalToolScope.OTHER)); + externalTool.setTags(externalTool.getTags() != null + ? externalTool.getTags() + : Collections.emptyList()); + externalTool.getWorkflow().setScripts(externalTool.getWorkflow().getScripts() != null + ? externalTool.getWorkflow().getScripts() + : Collections.emptyList()); + externalTool.setName(ParamUtils.defaultString(externalTool.getName(), externalTool.getId())); externalTool.setDescription(ParamUtils.defaultString(externalTool.getDescription(), "")); - externalTool.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.WORKFLOW)); + externalTool.setUuid(UuidUtils.generateOpenCgaUuid(UuidUtils.Entity.EXTERNAL_TOOL)); externalTool.setVersion(1); externalTool.setCreationDate(ParamUtils.checkDateOrGetCurrentDate(externalTool.getCreationDate(), CREATION_DATE.key())); externalTool.setModificationDate(ParamUtils.checkDateOrGetCurrentDate(externalTool.getModificationDate(), MODIFICATION_DATE.key())); externalTool.setAttributes(ParamUtils.defaultObject(externalTool.getAttributes(), Collections.emptyMap())); externalTool.setInternal(new ExternalToolInternal(new InternalStatus(InternalStatus.READY), TimeUtils.getTime(), TimeUtils.getTime(), userId)); - externalTool.setDocker(null); + + if (externalTool.getWorkflow() == null && externalTool.getDocker() == null) { + throw new CatalogParameterException("Missing expected workflow or docker object"); + } } // ************************** ACLs ******************************** // diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/NextflowUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/NextflowUtils.java new file mode 100644 index 00000000000..ab2cc23bc26 --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/NextflowUtils.java @@ -0,0 +1,220 @@ +package org.opencb.opencga.catalog.utils; + +import org.apache.commons.lang3.StringUtils; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.core.common.IOUtils; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.models.externalTool.*; +import org.opencb.opencga.core.models.job.MinimumRequirements; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class NextflowUtils { + + private static final Pattern WORKFLOW_MEMORY_PATTERN = Pattern.compile("\\s*memory\\s*=\\s*\\{\\s*[^0-9]*([0-9]+\\.[A-Za-z]+)"); + private static final Pattern WORKFLOW_CPU_PATTERN = Pattern.compile("\\s*cpus\\s*=\\s*\\{\\s*[^0-9]*([0-9]+)"); + + public static final int MAX_CPUS = 15; + public static final String MAX_MEMORY = "64.GB"; // Format is important for Nextflow. It requires the dot symbol. + + private static final Logger LOGGER = LoggerFactory.getLogger(NextflowUtils.class); + + public static ExternalTool importRepository(WorkflowRepositoryParams repository) throws CatalogException { + ParamUtils.checkObj(repository, "Nextflow repository parameters"); + if (StringUtils.isEmpty(repository.getName())) { + throw new CatalogParameterException("Missing 'name' field in workflow import parameters"); + } + String workflowId = repository.getName().replace("/", "."); + WorkflowSystem workflowSystem = new WorkflowSystem(WorkflowSystem.SystemId.NEXTFLOW, ""); + ExternalTool externalTool = new ExternalTool("", "", "", ExternalToolType.WORKFLOW, null, + new Workflow(workflowSystem, new LinkedList<>(), repository.toWorkflowRepository()), null, new LinkedList<>(), + new LinkedList<>(), new MinimumRequirements(), false, new ExternalToolInternal(), TimeUtils.getTime(), TimeUtils.getTime(), + new HashMap<>()); + + try { + processNextflowConfig(externalTool, repository); + processMemoryRequirements(externalTool, repository); + } catch (CatalogException e) { + throw new CatalogException("Could not process repository information from workflow '" + workflowId + "'.", e); + } + + return externalTool; + } + + private static void processMemoryRequirements(ExternalTool externalTool, WorkflowRepositoryParams repository) throws CatalogException { + String urlStr; + + if (StringUtils.isEmpty(repository.getTag())) { + urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/heads/master/conf/base.config"; + } else { + urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/tags/" + repository.getTag() + + "/conf/base.config"; + } + + try { + URL url = new URL(urlStr); + BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream())); + + int cpus = 0; + String memory = null; + String inputLine; + long maxMemory = IOUtils.fromHumanReadableToByte(MAX_MEMORY); + while ((inputLine = in.readLine()) != null) { + Matcher cpuMatcher = WORKFLOW_CPU_PATTERN.matcher(inputLine); + Matcher memoryMatcher = WORKFLOW_MEMORY_PATTERN.matcher(inputLine); + if (cpuMatcher.find()) { + String value = cpuMatcher.group(1); + int intValue = Integer.parseInt(value); + if (intValue > cpus) { + cpus = Math.min(intValue, MAX_CPUS); + } + } else if (memoryMatcher.find()) { + String value = memoryMatcher.group(1); + if (memory == null) { + memory = value; + } else { + long memoryBytes = IOUtils.fromHumanReadableToByte(value); + long currentMemoryBytes = IOUtils.fromHumanReadableToByte(memory); + if (memoryBytes > currentMemoryBytes) { + if (memoryBytes > maxMemory) { + memory = MAX_MEMORY; + } else { + memory = value; + } + } + } + } + } + if (cpus > 0 && memory != null) { + externalTool.getMinimumRequirements().setCpu(String.valueOf(cpus)); + externalTool.getMinimumRequirements().setMemory(memory); + } else { + LOGGER.warn("Could not find the minimum requirements for the workflow " + externalTool.getId()); + } + in.close(); + } catch (Exception e) { + throw new CatalogException("Could not process nextflow.config file.", e); + } + } + + private static void processNextflowConfig(ExternalTool externalTool, WorkflowRepositoryParams repository) throws CatalogException { + String urlStr; + if (StringUtils.isEmpty(repository.getTag())) { + urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/heads/master/nextflow.config"; + } else { + urlStr = "https://raw.githubusercontent.com/" + repository.getName() + "/refs/tags/" + repository.getTag() + + "/nextflow.config"; + } + + try { + URL url = new URL(urlStr); + BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream())); + + // We add the bracket close strings because they are expected to be properly indented. That way, we will be able to know when + // that section has been properly closed. Otherwise, we may get confused by some other subsections that could be closed before + // the actual section closure. + String manifestBracketClose = null; + String profilesBracketClose = null; + String gitpodBracketClose = null; + String inputLine; + while ((inputLine = in.readLine()) != null) { + if (manifestBracketClose != null) { + if (manifestBracketClose.equals(inputLine)) { + manifestBracketClose = null; + } else { + // Process manifest line + fillWithWorkflowManifest(externalTool, inputLine); + } + } else if (profilesBracketClose != null) { + if (gitpodBracketClose != null) { + if (gitpodBracketClose.equals(inputLine)) { + gitpodBracketClose = null; + } else { + // Process gitpod line + fillWithGitpodManifest(externalTool, inputLine); + } + } else if (inputLine.trim().startsWith("gitpod {")) { + int position = inputLine.indexOf("gitpod {"); + gitpodBracketClose = StringUtils.repeat(" ", position) + "}"; + } else if (profilesBracketClose.equals(inputLine)) { + profilesBracketClose = null; + } + } else if (inputLine.trim().startsWith("manifest {")) { + int position = inputLine.indexOf("profiles {"); + manifestBracketClose = StringUtils.repeat(" ", position) + "}"; + } else if (inputLine.trim().startsWith("profiles {")) { + int position = inputLine.indexOf("profiles {"); + profilesBracketClose = StringUtils.repeat(" ", position) + "}"; + + } + } + in.close(); + } catch (Exception e) { + throw new CatalogException("Could not process nextflow.config file.", e); + } + } + + private static void fillWithWorkflowManifest(ExternalTool externalTool, String rawline) { + String[] split = rawline.split("= "); + if (split.length != 2) { + return; + } + String key = split[0].trim(); +// String key = split[0].replaceAll(" ", ""); + String value = split[1].replace("\"", "").replace("'", "").trim(); + switch (key) { + case "name": + externalTool.setId(value.replace("/", ".")); + externalTool.setName(value.replace("/", " ")); + externalTool.getWorkflow().getRepository().setName(value); + break; + case "author": + externalTool.getWorkflow().getRepository().setAuthor(value); + break; + case "description": + externalTool.setDescription(value); + externalTool.getWorkflow().getRepository().setDescription(value); + break; + case "version": + String version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); + externalTool.getWorkflow().getRepository().setTag(version); + break; + case "nextflowVersion": + // Nextflow version must start with a number + version = value.replaceAll("^[^0-9]+|[^0-9.]+$", ""); + externalTool.getWorkflow().getManager().setVersion(version); + break; + default: + break; + } + } + + private static void fillWithGitpodManifest(ExternalTool externalTool, String rawline) { + String[] split = rawline.split("="); + if (split.length != 2) { + return; + } + String key = split[0].replaceAll(" ", ""); + String value = split[1].replace("\"", "").replace("'", "").trim(); + switch (key) { + case "executor.cpus": + externalTool.getMinimumRequirements().setCpu(value); + break; + case "executor.memory": + externalTool.getMinimumRequirements().setMemory(value); + break; + default: + break; + } + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/UuidUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/UuidUtils.java index ad0b1e4b0d1..e99db491766 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/UuidUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/UuidUtils.java @@ -51,7 +51,7 @@ public enum Entity { INTERPRETATION(11), ORGANIZATION(12), NOTES(13), - WORKFLOW(14); + EXTERNAL_TOOL(14); private final int mask; diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index eab521fc907..d0bfb323ddb 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -277,10 +277,11 @@ {"collections": ["workflow", "workflow_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["workflow", "workflow_archive"], "fields": {"name": 1, "studyUid": 1}, "options": {"unique": true}} {"collections": ["workflow", "workflow_archive"], "fields": {"type": 1, "studyUid": 1}, "options": {}} -{"collections": ["workflow", "workflow_archive"], "fields": {"manager.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["workflow", "workflow_archive"], "fields": {"scope": 1, "studyUid": 1}, "options": {}} +{"collections": ["workflow", "workflow_archive"], "fields": {"workflow.repository.name": 1, "studyUid": 1}, "options": {}} +{"collections": ["workflow", "workflow_archive"], "fields": {"docker.name": 1, "studyUid": 1}, "options": {}} {"collections": ["workflow", "workflow_archive"], "fields": {"tags": 1, "studyUid": 1}, "options": {}} {"collections": ["workflow", "workflow_archive"], "fields": {"draft": 1, "studyUid": 1}, "options": {}} -{"collections": ["workflow", "workflow_archive"], "fields": {"scripts.fileName": 1, "studyUid": 1}, "options": {}} {"collections": ["workflow", "workflow_archive"], "fields": {"internal.registrationUserId": 1, "studyUid": 1}, "options": {}} {"collections": ["workflow", "workflow_archive"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["workflow", "workflow_archive"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java similarity index 70% rename from opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java rename to opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java index 7f88072f910..01fa67567ec 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/WorkflowManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ExternalToolManagerTest.java @@ -11,6 +11,7 @@ import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowUpdateParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.testclassification.duration.MediumTests; @@ -20,39 +21,39 @@ import static org.junit.Assert.*; @Category(MediumTests.class) -public class WorkflowManagerTest extends AbstractManagerTest { +public class ExternalToolManagerTest extends AbstractManagerTest { - private WorkflowManager workflowManager; + private ExternalToolManager externalToolManager; @Before public void setUp() throws Exception { super.setUp(); - workflowManager = catalogManager.getWorkflowManager(); + externalToolManager = catalogManager.getExternalToolManager(); } @Test public void importWorkflow() throws CatalogException { WorkflowRepositoryParams params = new WorkflowRepositoryParams("nf-core/rnaseq"); - OpenCGAResult result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + OpenCGAResult result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); // Update imported workflow - result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(2, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/proteinfold"); - result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/methylseq"); - result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); params = new WorkflowRepositoryParams("nf-core/pacvar"); - result = workflowManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); + result = externalToolManager.importWorkflow(studyFqn, params, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumInserted()); assertEquals(1, result.first().getVersion()); } @@ -63,7 +64,7 @@ public void createWorkflowTest() throws CatalogException { .setId("workflow") .setScope(ExternalToolScope.OTHER) .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); - OpenCGAResult result = workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); + OpenCGAResult result = externalToolManager.createWorkflow(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertNotNull(result.first()); assertEquals(workflowCreateParams.getId(), result.first().getId()); @@ -72,12 +73,12 @@ public void createWorkflowTest() throws CatalogException { workflowCreateParams.setId("workflow2"); workflowCreateParams.getWorkflow().setRepository(new WorkflowRepository("blabla")); CatalogException catalogException = assertThrows(CatalogException.class, - () -> workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken)); + () -> externalToolManager.createWorkflow(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("repository")); // Remove script from workflow workflowCreateParams.getWorkflow().setScripts(Collections.emptyList()); - result = workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); + result = externalToolManager.createWorkflow(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); assertEquals(1, result.getNumResults()); assertNotNull(result.first()); assertEquals(workflowCreateParams.getId(), result.first().getId()); @@ -90,7 +91,7 @@ public void createWorkflowTest() throws CatalogException { new WorkflowScript("script2", "echo 'World'", true) )); catalogException = assertThrows(CatalogException.class, - () -> workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken)); + () -> externalToolManager.createWorkflow(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main")); // Add one single script without main @@ -98,7 +99,7 @@ public void createWorkflowTest() throws CatalogException { new WorkflowScript("script1", "echo 'Hello'", false) )); catalogException = assertThrows(CatalogException.class, - () -> workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken)); + () -> externalToolManager.createWorkflow(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken)); assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main")); } @@ -108,26 +109,26 @@ public void workflowSearchTest() throws CatalogException { .setId("workflow") .setScope(ExternalToolScope.OTHER) .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); - workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); + externalToolManager.createWorkflow(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); workflowCreateParams = new WorkflowCreateParams() .setId("workflow2") .setScope(ExternalToolScope.OTHER) .setDraft(true) .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); - workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); + externalToolManager.createWorkflow(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); - OpenCGAResult search = workflowManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); + OpenCGAResult search = externalToolManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken); assertEquals(2, search.getNumResults()); Query query = new Query(ExternalToolDBAdaptor.QueryParams.DRAFT.key(), true); - search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); + search = externalToolManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("workflow2", search.first().getId()); assertTrue(search.first().isDraft()); query = new Query(ExternalToolDBAdaptor.QueryParams.DRAFT.key(), false); - search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); + search = externalToolManager.search(studyFqn, query, QueryOptions.empty(), ownerToken); assertEquals(1, search.getNumResults()); assertEquals("workflow", search.first().getId()); assertFalse(search.first().isDraft()); @@ -139,9 +140,9 @@ public void updateWorkflowTest() throws CatalogException { .setId("workflow") .setScope(ExternalToolScope.OTHER) .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); - workflowManager.create(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); + externalToolManager.createWorkflow(studyFqn, workflowCreateParams, INCLUDE_RESULT, ownerToken); - ExternalToolUpdateParams updateParams = new ExternalToolUpdateParams() + WorkflowUpdateParams updateParams = new WorkflowUpdateParams() .setName("newName") .setScope(ExternalToolScope.OTHER) .setDraft(true) @@ -149,7 +150,7 @@ public void updateWorkflowTest() throws CatalogException { .setModificationDate("20240201000000") .setDescription("description"); - OpenCGAResult update = workflowManager.update(studyFqn, workflowCreateParams.getId(), updateParams, INCLUDE_RESULT, ownerToken); + OpenCGAResult update = externalToolManager.updateWorkflow(studyFqn, workflowCreateParams.getId(), updateParams, INCLUDE_RESULT, ownerToken); assertEquals(1, update.getNumUpdated()); ExternalTool updatedExternalTool = update.first(); assertEquals(updateParams.getName(), updatedExternalTool.getName()); @@ -165,13 +166,13 @@ public void deleteWorkflowTest() throws CatalogException { .setId("workflow") .setScope(ExternalToolScope.OTHER) .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); - workflowManager.create(studyFqn, workflowCreateParams, QueryOptions.empty(), ownerToken); + externalToolManager.createWorkflow(studyFqn, workflowCreateParams, QueryOptions.empty(), ownerToken); - OpenCGAResult result = workflowManager.delete(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), ownerToken); + OpenCGAResult result = externalToolManager.delete(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), ownerToken); assertEquals(1, result.getNumDeleted()); CatalogException exception = assertThrows(CatalogException.class, - () -> workflowManager.get(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), ownerToken)); + () -> externalToolManager.get(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), ownerToken)); assertTrue(exception.getMessage().contains("not found")); } @@ -181,15 +182,15 @@ public void updateWorkflowAclTest() throws CatalogException { .setId("workflow") .setScope(ExternalToolScope.OTHER) .setWorkflow(new Workflow().setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)))); - workflowManager.create(studyFqn, workflowCreateParams, QueryOptions.empty(), ownerToken); + externalToolManager.createWorkflow(studyFqn, workflowCreateParams, QueryOptions.empty(), ownerToken); CatalogAuthorizationException catalogAuthorizationException = assertThrows(CatalogAuthorizationException.class, - () -> workflowManager.get(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), noAccessToken1)); + () -> externalToolManager.get(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), noAccessToken1)); assertTrue(catalogAuthorizationException.getMessage().contains("denied")); - workflowManager.updateAcl(studyFqn, noAccessUserId1, new ExternalToolAclUpdateParams(Collections.singletonList(workflowCreateParams.getId()), Collections.singletonList("VIEW")), + externalToolManager.updateAcl(studyFqn, noAccessUserId1, new ExternalToolAclUpdateParams(Collections.singletonList(workflowCreateParams.getId()), Collections.singletonList("VIEW")), ParamUtils.AclAction.ADD, ownerToken); - OpenCGAResult result = workflowManager.get(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), noAccessToken1); + OpenCGAResult result = externalToolManager.get(studyFqn, workflowCreateParams.getId(), QueryOptions.empty(), noAccessToken1); assertEquals(1, result.getNumResults()); } diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index 7f8ba86e665..29ae224c800 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -187,7 +187,7 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpoi #' @param jobScheduledStartTime Time when the job is scheduled to start. #' @param jobPriority Priority of the job. #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. - #' @param data NextFlow run parameters. + #' @param data External tool run parameters. runTool=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=NULL, subcategory="tool", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/R/Workflow-methods.R b/opencga-client/src/main/R/R/Workflow-methods.R index 830be126768..c8a2b7a1e9a 100644 --- a/opencga-client/src/main/R/R/Workflow-methods.R +++ b/opencga-client/src/main/R/R/Workflow-methods.R @@ -97,7 +97,7 @@ setMethod("workflowClient", "OpencgaR", function(OpencgaR, members, workflowId, #' @param jobScheduledStartTime Time when the job is scheduled to start. #' @param jobPriority Priority of the job. #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. - #' @param data NextFlow run parameters. + #' @param data External tool run parameters. run=fetchOpenCGA(object=OpencgaR, category="workflows", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index f3eb67bf579..9a94ba413c5 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -251,7 +251,7 @@ public RestResponse buildTool(JobToolBuildParams data, ObjectMap params) th /** * Execute an analysis from a custom binary. - * @param data NextFlow run parameters. + * @param data External tool run parameters. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java index 356ba7eef05..93be5408ec7 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java @@ -25,9 +25,9 @@ import org.opencb.opencga.core.models.externalTool.ExternalTool; import org.opencb.opencga.core.models.externalTool.ExternalToolAclEntryList; import org.opencb.opencga.core.models.externalTool.ExternalToolAclUpdateParams; -import org.opencb.opencga.core.models.externalTool.ExternalToolUpdateParams; -import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolRunParams; import org.opencb.opencga.core.models.externalTool.WorkflowRepositoryParams; +import org.opencb.opencga.core.models.externalTool.workflow.DeprecatedWorkflowUpdateParams; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.response.RestResponse; @@ -136,7 +136,7 @@ public RestResponse importWorkflow(WorkflowRepositoryParams data, /** * Execute a workflow analysis. - * @param data NextFlow run parameters. + * @param data External tool run parameters. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. @@ -150,7 +150,7 @@ public RestResponse importWorkflow(WorkflowRepositoryParams data, * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse run(NextFlowRunParams data, ObjectMap params) throws ClientException { + public RestResponse run(ExternalToolRunParams data, ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); params.put("body", data); return execute("workflows", null, null, null, "run", params, POST, Job.class); @@ -203,7 +203,8 @@ public RestResponse search(ObjectMap params) throws ClientExceptio * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse update(String workflowId, ExternalToolUpdateParams data, ObjectMap params) throws ClientException { + public RestResponse update(String workflowId, DeprecatedWorkflowUpdateParams data, ObjectMap params) + throws ClientException { params = params != null ? params : new ObjectMap(); params.put("body", data); return execute("workflows", workflowId, null, null, "update", params, POST, ExternalTool.class); diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index 8469a01bde6..b8f7ab15d62 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -209,7 +209,7 @@ export default class Job extends OpenCGAParentClass { } /** Execute an analysis from a custom binary. - * @param {Object} data - NextFlow run parameters. + * @param {Object} data - External tool run parameters. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not diff --git a/opencga-client/src/main/javascript/Workflow.js b/opencga-client/src/main/javascript/Workflow.js index 9379a568ab3..70645ad3655 100644 --- a/opencga-client/src/main/javascript/Workflow.js +++ b/opencga-client/src/main/javascript/Workflow.js @@ -99,7 +99,7 @@ export default class Workflow extends OpenCGAParentClass { } /** Execute a workflow analysis - * @param {Object} data - NextFlow run parameters. + * @param {Object} data - External tool run parameters. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 6e43d071508..5680548587c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -265,7 +265,7 @@ def run_tool(self, data=None, **options): Execute an analysis from a custom binary. PATH: /{apiVersion}/jobs/tool/run - :param dict data: NextFlow run parameters. (REQUIRED) + :param dict data: External tool run parameters. (REQUIRED) :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/workflow_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/workflow_client.py index 3ce08f2e135..9f86abba04b 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/workflow_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/workflow_client.py @@ -118,7 +118,7 @@ def run(self, data=None, **options): Execute a workflow analysis. PATH: /{apiVersion}/workflows/run - :param dict data: NextFlow run parameters. (REQUIRED) + :param dict data: External tool run parameters. (REQUIRED) :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 7f98cf64e69..a26f3b6174a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -600,6 +600,33 @@ public class ParamConstants { + " not actually run."; // --------------------------------------------- + // EXTERNAL TOOLS + public static final String EXTERNAL_TOOLS_DESCRIPTION = "Comma separated of external tool ids."; + public static final String EXTERNAL_TOOL_VERSION_PARAM = "version"; + public static final String EXTERNAL_TOOL_VERSION_DESCRIPTION = "Comma separated list of external tool versions. " + + "'all' to get all the external tool versions. Not supported if multiple external tool ids are provided"; + public static final String EXTERNAL_TOOL_ID_PARAM = "id"; + public static final String EXTERNAL_TOOL_UUID_PARAM = "uuid"; + public static final String EXTERNAL_TOOL_NAME_PARAM = "name"; + public static final String EXTERNAL_TOOL_TYPE_PARAM = "type"; + public static final String EXTERNAL_TOOL_SCOPE_PARAM = "scope"; + public static final String EXTERNAL_TOOL_TAGS_PARAM = "tags"; + public static final String EXTERNAL_TOOL_DRAFT_PARAM = "draft"; + public static final String EXTERNAL_TOOL_INTERNAL_REGISTRATION_USER_ID_PARAM = "internal.registrationUserId"; + public static final String EXTERNAL_TOOL_WORKFLOW_REPOSITORY_NAME_PARAM = "workflowRepositoryName"; + public static final String EXTERNAL_TOOL_DOCKER_NAME_PARAM = "dockerName"; + public static final String EXTERNAL_TOOL_ID_DESCRIPTION = "Comma separated list of external tool IDs" + UP_TO_100 + REGEX_SUPPORT; + public static final String EXTERNAL_TOOL_NAME_DESCRIPTION = "Comma separated list of external tool names" + UP_TO_100 + REGEX_SUPPORT; + public static final String EXTERNAL_TOOL_UUID_DESCRIPTION = "Comma separated list of external tool UUIDs" + UP_TO_100; + public static final String EXTERNAL_TOOL_TYPE_DESCRIPTION = "External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]"; + public static final String EXTERNAL_TOOL_SCOPE_DESCRIPTION = "External tool scope. Allowed types: [CLINICAL_INTERPRETATION," + + " SECONDARY_ANALYSIS, RESEARCH or OTHER]"; + public static final String EXTERNAL_TOOL_TAGS_DESCRIPTION = "Comma separated list of tags"; + public static final String EXTERNAL_TOOL_WORKFLOW_REPOSITORY_NAME_DESCRIPTION = "Workflow repository name"; + public static final String EXTERNAL_TOOL_DOCKER_NAME_DESCRIPTION = "Docker name"; + public static final String EXTERNAL_TOOL_DRAFT_DESCRIPTION = "Boolean field indicating whether the workflow is a draft or not."; + public static final String EXTERNAL_TOOL_INTERNAL_REGISTRATION_USER_ID_DESCRIPTION = "UserId that created the workflow."; + // WORKFLOWS public static final String WORKFLOW_VERSION_PARAM = "version"; public static final String WORKFLOW_VERSION_DESCRIPTION = "Comma separated list of workflow versions. 'all' to get all the workflow " @@ -621,9 +648,7 @@ public class ParamConstants { public static final String WORKFLOWS_DRAFT_DESCRIPTION = "Boolean field indicating whether the workflow is a draft or not."; public static final String WORKFLOWS_INTERNAL_REGISTRATION_USER_ID_DESCRIPTION = "UserId that created the workflow."; public static final String WORKFLOWS_MANAGER_ID_DESCRIPTION = "Id of the workflow system (Allowed values: NEXTFLOW)."; - public static final String WORKFLOW_SCRIPTS_ACTION_DESCRIPTION = "Action to be performed if the array of scripts is being updated " - + "[SET, ADD, REMOVE]"; - public static final String WORKFLOW_SCRIPTS_ACTION_PARAM = "scriptsAction"; + // --------------------------------------------- public static final String JOB_INPUT_FILES_PARAM = "input"; public static final String JOB_INPUT_FILES_DESCRIPTION = "Comma separated list of file IDs used as input."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/NextFlowRunParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolRunParams.java similarity index 60% rename from opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/NextFlowRunParams.java rename to opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolRunParams.java index c05043b9b59..4913e9510c4 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/NextFlowRunParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolRunParams.java @@ -4,18 +4,18 @@ import java.util.Map; -public class NextFlowRunParams extends ToolParams { +public class ExternalToolRunParams extends ToolParams { - public static final String DESCRIPTION = "NextFlow run parameters"; + public static final String DESCRIPTION = "External tool run parameters"; private String id; private Integer version; private Map params; - public NextFlowRunParams() { + public ExternalToolRunParams() { } - public NextFlowRunParams(String id, Integer version, Map params) { + public ExternalToolRunParams(String id, Integer version, Map params) { this.id = id; this.version = version; this.params = params; @@ -23,10 +23,10 @@ public NextFlowRunParams(String id, Integer version, Map params) @Override public String toString() { - final StringBuilder sb = new StringBuilder("NextFlowRunParams{"); + final StringBuilder sb = new StringBuilder("ExternalToolRunParams{"); sb.append("id='").append(id).append('\''); sb.append(", version=").append(version); - sb.append(", params='").append(params).append('\''); + sb.append(", params=").append(params); sb.append('}'); return sb.toString(); } @@ -35,7 +35,7 @@ public String getId() { return id; } - public NextFlowRunParams setId(String id) { + public ExternalToolRunParams setId(String id) { this.id = id; return this; } @@ -44,7 +44,7 @@ public Integer getVersion() { return version; } - public NextFlowRunParams setVersion(Integer version) { + public ExternalToolRunParams setVersion(Integer version) { this.version = version; return this; } @@ -53,7 +53,7 @@ public Map getParams() { return params; } - public NextFlowRunParams setParams(Map params) { + public ExternalToolRunParams setParams(Map params) { this.params = params; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java index 28a477b7f67..b42ee140efa 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolUpdateParams.java @@ -7,38 +7,29 @@ import java.util.List; import java.util.Map; -public class ExternalToolUpdateParams { +public abstract class ExternalToolUpdateParams { - @DataField(id = "name", description = FieldConstants.EXTERNAL_TOOL_NAME_DESCRIPTION) + @DataField(id = "name", description = FieldConstants.GENERIC_UUID_DESCRIPTION) private String name; @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) private String description; - @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) - private WorkflowSystem manager; - @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) private ExternalToolScope scope; @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) private List tags; - @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) - private boolean draft; - - @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) - private WorkflowRepository repository; - - @DataField(id = "scripts", description = FieldConstants.WORKFLOW_SCRIPTS_DESCRIPTION) - private List scripts; - @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) private List variables; @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) private MinimumRequirements minimumRequirements; + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) + private boolean draft; + @DataField(id = "creationDate", indexed = true, description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) private String creationDate; @@ -51,20 +42,16 @@ public class ExternalToolUpdateParams { public ExternalToolUpdateParams() { } - public ExternalToolUpdateParams(String name, String description, WorkflowSystem manager, ExternalToolScope scope, List tags, - boolean draft, WorkflowRepository repository, List scripts, - List variables, MinimumRequirements minimumRequirements, String creationDate, - String modificationDate, Map attributes) { + public ExternalToolUpdateParams(String name, String description, ExternalToolScope scope, List tags, + List variables, MinimumRequirements minimumRequirements, boolean draft, + String creationDate, String modificationDate, Map attributes) { this.name = name; this.description = description; - this.manager = manager; this.scope = scope; this.tags = tags; - this.draft = draft; - this.repository = repository; - this.scripts = scripts; this.variables = variables; this.minimumRequirements = minimumRequirements; + this.draft = draft; this.creationDate = creationDate; this.modificationDate = modificationDate; this.attributes = attributes; @@ -72,21 +59,17 @@ public ExternalToolUpdateParams(String name, String description, WorkflowSystem @Override public String toString() { - final StringBuilder sb = new StringBuilder("WorkflowUpdateParams{"); + final StringBuilder sb = new StringBuilder(); sb.append("name='").append(name).append('\''); sb.append(", description='").append(description).append('\''); - sb.append(", manager=").append(manager); sb.append(", scope=").append(scope); sb.append(", tags=").append(tags); - sb.append(", draft=").append(draft); - sb.append(", repository=").append(repository); - sb.append(", scripts=").append(scripts); sb.append(", variables=").append(variables); sb.append(", minimumRequirements=").append(minimumRequirements); + sb.append(", draft=").append(draft); sb.append(", creationDate='").append(creationDate).append('\''); sb.append(", modificationDate='").append(modificationDate).append('\''); - sb.append(", attributes=").append(attributes); - sb.append('}'); + sb.append(", attributes=").append(attributes).append('\''); return sb.toString(); } @@ -108,15 +91,6 @@ public ExternalToolUpdateParams setDescription(String description) { return this; } - public WorkflowSystem getManager() { - return manager; - } - - public ExternalToolUpdateParams setManager(WorkflowSystem manager) { - this.manager = manager; - return this; - } - public ExternalToolScope getScope() { return scope; } @@ -135,39 +109,30 @@ public ExternalToolUpdateParams setTags(List tags) { return this; } - public boolean isDraft() { - return draft; - } - - public ExternalToolUpdateParams setDraft(boolean draft) { - this.draft = draft; - return this; - } - - public WorkflowRepository getRepository() { - return repository; + public List getVariables() { + return variables; } - public ExternalToolUpdateParams setRepository(WorkflowRepository repository) { - this.repository = repository; + public ExternalToolUpdateParams setVariables(List variables) { + this.variables = variables; return this; } - public List getScripts() { - return scripts; + public MinimumRequirements getMinimumRequirements() { + return minimumRequirements; } - public ExternalToolUpdateParams setScripts(List scripts) { - this.scripts = scripts; + public ExternalToolUpdateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { + this.minimumRequirements = minimumRequirements; return this; } - public List getVariables() { - return variables; + public boolean isDraft() { + return draft; } - public ExternalToolUpdateParams setVariables(List variables) { - this.variables = variables; + public ExternalToolUpdateParams setDraft(boolean draft) { + this.draft = draft; return this; } @@ -189,15 +154,6 @@ public ExternalToolUpdateParams setModificationDate(String modificationDate) { return this; } - public MinimumRequirements getMinimumRequirements() { - return minimumRequirements; - } - - public ExternalToolUpdateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { - this.minimumRequirements = minimumRequirements; - return this; - } - public Map getAttributes() { return attributes; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java index e4136b7bea7..df4a3bd9ad2 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/WorkflowRepository.java @@ -73,4 +73,22 @@ public WorkflowRepository setDescription(String description) { this.description = description; return this; } + + public String getUser() { + return user; + } + + public WorkflowRepository setUser(String user) { + this.user = user; + return this; + } + + public String getPassword() { + return password; + } + + public WorkflowRepository setPassword(String password) { + this.password = password; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolCreateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolCreateParams.java new file mode 100644 index 00000000000..064846de835 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolCreateParams.java @@ -0,0 +1,214 @@ +package org.opencb.opencga.core.models.externalTool.custom; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.externalTool.Docker; +import org.opencb.opencga.core.models.externalTool.ExternalToolInternal; +import org.opencb.opencga.core.models.externalTool.ExternalToolScope; +import org.opencb.opencga.core.models.externalTool.ExternalToolVariable; +import org.opencb.opencga.core.models.job.MinimumRequirements; + +import java.util.List; +import java.util.Map; + +public class CustomToolCreateParams { + + @DataField(id = "id", required = true, indexed = true, unique = true, immutable = true, + description = FieldConstants.EXTERNAL_TOOL_ID_DESCRIPTION) + private String id; + + @DataField(id = "name", description = FieldConstants.GENERIC_UUID_DESCRIPTION) + private String name; + + @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) + private String description; + + @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) + private ExternalToolScope scope; + + @DataField(id = "docker", description = FieldConstants.EXTERNAL_TOOL_DOCKER_DESCRIPTION) + private Docker docker; + + @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) + private List tags; + + @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) + private List variables; + + @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) + private MinimumRequirements minimumRequirements; + + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) + private boolean draft; + + @DataField(id = "internal", description = FieldConstants.EXTERNAL_TOOL_INTERNAL_DESCRIPTION) + private ExternalToolInternal internal; + + @DataField(id = "creationDate", indexed = true, description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; + + @DataField(id = "modificationDate", indexed = true, description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; + + @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) + private Map attributes; + + public CustomToolCreateParams() { + } + + public CustomToolCreateParams(String id, String name, String description, ExternalToolScope scope, Docker docker, List tags, + List variables, MinimumRequirements minimumRequirements, boolean draft, + ExternalToolInternal internal, String creationDate, String modificationDate, + Map attributes) { + this.id = id; + this.name = name; + this.description = description; + this.scope = scope; + this.docker = docker; + this.tags = tags; + this.variables = variables; + this.minimumRequirements = minimumRequirements; + this.draft = draft; + this.internal = internal; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.attributes = attributes; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CustomToolCreateParams{"); + sb.append("id='").append(id).append('\''); + sb.append(", name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", scope=").append(scope); + sb.append(", docker=").append(docker); + sb.append(", tags=").append(tags); + sb.append(", variables=").append(variables); + sb.append(", minimumRequirements=").append(minimumRequirements); + sb.append(", draft=").append(draft); + sb.append(", internal=").append(internal); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", attributes=").append(attributes); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public CustomToolCreateParams setId(String id) { + this.id = id; + return this; + } + + public String getName() { + return name; + } + + public CustomToolCreateParams setName(String name) { + this.name = name; + return this; + } + + public String getDescription() { + return description; + } + + public CustomToolCreateParams setDescription(String description) { + this.description = description; + return this; + } + + public ExternalToolScope getScope() { + return scope; + } + + public CustomToolCreateParams setScope(ExternalToolScope scope) { + this.scope = scope; + return this; + } + + public Docker getDocker() { + return docker; + } + + public CustomToolCreateParams setDocker(Docker docker) { + this.docker = docker; + return this; + } + + public List getTags() { + return tags; + } + + public CustomToolCreateParams setTags(List tags) { + this.tags = tags; + return this; + } + + public List getVariables() { + return variables; + } + + public CustomToolCreateParams setVariables(List variables) { + this.variables = variables; + return this; + } + + public MinimumRequirements getMinimumRequirements() { + return minimumRequirements; + } + + public CustomToolCreateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { + this.minimumRequirements = minimumRequirements; + return this; + } + + public boolean isDraft() { + return draft; + } + + public CustomToolCreateParams setDraft(boolean draft) { + this.draft = draft; + return this; + } + + public ExternalToolInternal getInternal() { + return internal; + } + + public CustomToolCreateParams setInternal(ExternalToolInternal internal) { + this.internal = internal; + return this; + } + + public String getCreationDate() { + return creationDate; + } + + public CustomToolCreateParams setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public CustomToolCreateParams setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + public Map getAttributes() { + return attributes; + } + + public CustomToolCreateParams setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolUpdateParams.java new file mode 100644 index 00000000000..ee986d08109 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolUpdateParams.java @@ -0,0 +1,106 @@ +package org.opencb.opencga.core.models.externalTool.custom; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.externalTool.Docker; +import org.opencb.opencga.core.models.externalTool.ExternalToolScope; +import org.opencb.opencga.core.models.externalTool.ExternalToolUpdateParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolVariable; +import org.opencb.opencga.core.models.job.MinimumRequirements; + +import java.util.List; +import java.util.Map; + +public class CustomToolUpdateParams extends ExternalToolUpdateParams { + + @DataField(id = "docker", description = FieldConstants.EXTERNAL_TOOL_DOCKER_DESCRIPTION) + private Docker docker; + + public CustomToolUpdateParams() { + } + + public CustomToolUpdateParams(String name, String description, ExternalToolScope scope, List tags, + List variables, MinimumRequirements minimumRequirements, boolean draft, + String creationDate, String modificationDate, Map attributes, Docker docker) { + super(name, description, scope, tags, variables, minimumRequirements, draft, creationDate, modificationDate, attributes); + this.docker = docker; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CustomToolUpdateParams{"); + sb.append(super.toString()); + sb.append(", docker=").append(docker); + sb.append('}'); + return sb.toString(); + } + + public Docker getDocker() { + return docker; + } + + public CustomToolUpdateParams setDocker(Docker docker) { + this.docker = docker; + return this; + } + + @Override + public CustomToolUpdateParams setName(String name) { + super.setName(name); + return this; + } + + @Override + public CustomToolUpdateParams setDescription(String description) { + super.setDescription(description); + return this; + } + + @Override + public CustomToolUpdateParams setScope(ExternalToolScope scope) { + super.setScope(scope); + return this; + } + + @Override + public CustomToolUpdateParams setTags(List tags) { + super.setTags(tags); + return this; + } + + @Override + public CustomToolUpdateParams setVariables(List variables) { + super.setVariables(variables); + return this; + } + + @Override + public CustomToolUpdateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { + super.setMinimumRequirements(minimumRequirements); + return this; + } + + @Override + public CustomToolUpdateParams setDraft(boolean draft) { + super.setDraft(draft); + return this; + } + + @Override + public CustomToolUpdateParams setCreationDate(String creationDate) { + super.setCreationDate(creationDate); + return this; + } + + @Override + public CustomToolUpdateParams setModificationDate(String modificationDate) { + super.setModificationDate(modificationDate); + return this; + } + + @Override + public CustomToolUpdateParams setAttributes(Map attributes) { + super.setAttributes(attributes); + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/DeprecatedWorkflowUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/DeprecatedWorkflowUpdateParams.java new file mode 100644 index 00000000000..2219b33861f --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/DeprecatedWorkflowUpdateParams.java @@ -0,0 +1,220 @@ +package org.opencb.opencga.core.models.externalTool.workflow; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.externalTool.*; +import org.opencb.opencga.core.models.job.MinimumRequirements; + +import java.util.List; +import java.util.Map; + +@Deprecated +public class DeprecatedWorkflowUpdateParams { + + @DataField(id = "name", description = FieldConstants.EXTERNAL_TOOL_NAME_DESCRIPTION) + private String name; + + @DataField(id = "description", description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) + private String description; + + @DataField(id = "manager", description = FieldConstants.WORKFLOW_MANAGER_DESCRIPTION) + private WorkflowSystem manager; + + @DataField(id = "scope", description = FieldConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) + private ExternalToolScope scope; + + @DataField(id = "tags", description = FieldConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) + private List tags; + + @DataField(id = "draft", description = FieldConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) + private boolean draft; + + @DataField(id = "repository", description = FieldConstants.WORKFLOW_REPOSITORY_DESCRIPTION) + private WorkflowRepository repository; + + @DataField(id = "scripts", description = FieldConstants.WORKFLOW_SCRIPTS_DESCRIPTION) + private List scripts; + + @DataField(id = "variables", description = FieldConstants.EXTERNAL_TOOL_VARIABLES_DESCRIPTION) + private List variables; + + @DataField(id = "minimumRequirements", description = FieldConstants.MINIMUM_REQUIREMENTS_DESCRIPTION) + private MinimumRequirements minimumRequirements; + + @DataField(id = "creationDate", indexed = true, description = FieldConstants.GENERIC_CREATION_DATE_DESCRIPTION) + private String creationDate; + + @DataField(id = "modificationDate", indexed = true, description = FieldConstants.GENERIC_MODIFICATION_DATE_DESCRIPTION) + private String modificationDate; + + @DataField(id = "attributes", description = FieldConstants.GENERIC_ATTRIBUTES_DESCRIPTION) + private Map attributes; + + public DeprecatedWorkflowUpdateParams() { + } + + public DeprecatedWorkflowUpdateParams(String name, String description, WorkflowSystem manager, ExternalToolScope scope, List tags, + boolean draft, WorkflowRepository repository, List scripts, + List variables, MinimumRequirements minimumRequirements, String creationDate, + String modificationDate, Map attributes) { + this.name = name; + this.description = description; + this.manager = manager; + this.scope = scope; + this.tags = tags; + this.draft = draft; + this.repository = repository; + this.scripts = scripts; + this.variables = variables; + this.minimumRequirements = minimumRequirements; + this.creationDate = creationDate; + this.modificationDate = modificationDate; + this.attributes = attributes; + } + + public WorkflowUpdateParams toWorkflowUpdateParams() { + Workflow workflow = null; + if (manager != null || scripts != null || repository != null) { + workflow = new Workflow(manager, scripts, repository); + } + return new WorkflowUpdateParams(name, description, scope, tags, variables, minimumRequirements, draft, creationDate, + modificationDate, attributes, workflow); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("WorkflowUpdateParams{"); + sb.append("name='").append(name).append('\''); + sb.append(", description='").append(description).append('\''); + sb.append(", manager=").append(manager); + sb.append(", scope=").append(scope); + sb.append(", tags=").append(tags); + sb.append(", draft=").append(draft); + sb.append(", repository=").append(repository); + sb.append(", scripts=").append(scripts); + sb.append(", variables=").append(variables); + sb.append(", minimumRequirements=").append(minimumRequirements); + sb.append(", creationDate='").append(creationDate).append('\''); + sb.append(", modificationDate='").append(modificationDate).append('\''); + sb.append(", attributes=").append(attributes); + sb.append('}'); + return sb.toString(); + } + + public String getName() { + return name; + } + + public DeprecatedWorkflowUpdateParams setName(String name) { + this.name = name; + return this; + } + + public String getDescription() { + return description; + } + + public DeprecatedWorkflowUpdateParams setDescription(String description) { + this.description = description; + return this; + } + + public WorkflowSystem getManager() { + return manager; + } + + public DeprecatedWorkflowUpdateParams setManager(WorkflowSystem manager) { + this.manager = manager; + return this; + } + + public ExternalToolScope getScope() { + return scope; + } + + public DeprecatedWorkflowUpdateParams setScope(ExternalToolScope scope) { + this.scope = scope; + return this; + } + + public List getTags() { + return tags; + } + + public DeprecatedWorkflowUpdateParams setTags(List tags) { + this.tags = tags; + return this; + } + + public boolean isDraft() { + return draft; + } + + public DeprecatedWorkflowUpdateParams setDraft(boolean draft) { + this.draft = draft; + return this; + } + + public WorkflowRepository getRepository() { + return repository; + } + + public DeprecatedWorkflowUpdateParams setRepository(WorkflowRepository repository) { + this.repository = repository; + return this; + } + + public List getScripts() { + return scripts; + } + + public DeprecatedWorkflowUpdateParams setScripts(List scripts) { + this.scripts = scripts; + return this; + } + + public List getVariables() { + return variables; + } + + public DeprecatedWorkflowUpdateParams setVariables(List variables) { + this.variables = variables; + return this; + } + + public String getCreationDate() { + return creationDate; + } + + public DeprecatedWorkflowUpdateParams setCreationDate(String creationDate) { + this.creationDate = creationDate; + return this; + } + + public String getModificationDate() { + return modificationDate; + } + + public DeprecatedWorkflowUpdateParams setModificationDate(String modificationDate) { + this.modificationDate = modificationDate; + return this; + } + + public MinimumRequirements getMinimumRequirements() { + return minimumRequirements; + } + + public DeprecatedWorkflowUpdateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { + this.minimumRequirements = minimumRequirements; + return this; + } + + public Map getAttributes() { + return attributes; + } + + public DeprecatedWorkflowUpdateParams setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowUpdateParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowUpdateParams.java new file mode 100644 index 00000000000..7250d4b0010 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/workflow/WorkflowUpdateParams.java @@ -0,0 +1,106 @@ +package org.opencb.opencga.core.models.externalTool.workflow; + +import org.opencb.commons.annotations.DataField; +import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.models.externalTool.ExternalToolScope; +import org.opencb.opencga.core.models.externalTool.ExternalToolUpdateParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolVariable; +import org.opencb.opencga.core.models.externalTool.Workflow; +import org.opencb.opencga.core.models.job.MinimumRequirements; + +import java.util.List; +import java.util.Map; + +public class WorkflowUpdateParams extends ExternalToolUpdateParams { + + @DataField(id = "workflow", description = FieldConstants.EXTERNAL_TOOL_WORKFLOW_DESCRIPTION) + private Workflow workflow; + + public WorkflowUpdateParams() { + } + + public WorkflowUpdateParams(String name, String description, ExternalToolScope scope, List tags, + List variables, MinimumRequirements minimumRequirements, boolean draft, + String creationDate, String modificationDate, Map attributes, Workflow workflow) { + super(name, description, scope, tags, variables, minimumRequirements, draft, creationDate, modificationDate, attributes); + this.workflow = workflow; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("WorkflowUpdateParams{"); + sb.append(super.toString()); + sb.append(", workflow=").append(workflow); + sb.append('}'); + return sb.toString(); + } + + public Workflow getWorkflow() { + return workflow; + } + + public WorkflowUpdateParams setWorkflow(Workflow workflow) { + this.workflow = workflow; + return this; + } + + @Override + public WorkflowUpdateParams setName(String name) { + super.setName(name); + return this; + } + + @Override + public WorkflowUpdateParams setDescription(String description) { + super.setDescription(description); + return this; + } + + @Override + public WorkflowUpdateParams setScope(ExternalToolScope scope) { + super.setScope(scope); + return this; + } + + @Override + public WorkflowUpdateParams setTags(List tags) { + super.setTags(tags); + return this; + } + + @Override + public WorkflowUpdateParams setVariables(List variables) { + super.setVariables(variables); + return this; + } + + @Override + public WorkflowUpdateParams setMinimumRequirements(MinimumRequirements minimumRequirements) { + super.setMinimumRequirements(minimumRequirements); + return this; + } + + @Override + public WorkflowUpdateParams setDraft(boolean draft) { + super.setDraft(draft); + return this; + } + + @Override + public WorkflowUpdateParams setCreationDate(String creationDate) { + super.setCreationDate(creationDate); + return this; + } + + @Override + public WorkflowUpdateParams setModificationDate(String modificationDate) { + super.setModificationDate(modificationDate); + return this; + } + + @Override + public WorkflowUpdateParams setAttributes(Map attributes) { + super.setAttributes(attributes); + return this; + } +} diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java index 287b818bd67..4a29322a0fa 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java @@ -414,6 +414,8 @@ protected int checkRunningJob(Job job) { return 0; } + + } protected void checkQueuedJobs(List organizationIds) { diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java new file mode 100644 index 00000000000..645a0bceb35 --- /dev/null +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java @@ -0,0 +1,362 @@ +package org.opencb.opencga.server.rest; + +import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.analysis.customTool.CustomToolBuilder; +import org.opencb.opencga.analysis.customTool.CustomToolExecutor; +import org.opencb.opencga.catalog.managers.ExternalToolManager; +import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.api.ParamConstants; +import org.opencb.opencga.core.exceptions.VersionException; +import org.opencb.opencga.core.models.externalTool.*; +import org.opencb.opencga.core.models.externalTool.custom.CustomToolCreateParams; +import org.opencb.opencga.core.models.externalTool.custom.CustomToolUpdateParams; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowUpdateParams; +import org.opencb.opencga.core.models.job.*; +import org.opencb.opencga.core.tools.annotations.*; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.*; +import java.io.IOException; +import java.util.List; + +import static org.opencb.opencga.core.api.ParamConstants.JOB_DEPENDS_ON; + +@Path("/{apiVersion}/tools") +@Produces(MediaType.APPLICATION_JSON) +@Api(value = "External Tools", description = "Methods for working with 'tools' endpoint") +public class ExternalToolWSServer extends OpenCGAWSServer { + + private ExternalToolManager externalToolManager; + + public ExternalToolWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest, @Context HttpHeaders httpHeaders) + throws IOException, VersionException { + super(uriInfo, httpServletRequest, httpHeaders); + externalToolManager = catalogManager.getExternalToolManager(); + } + + @GET + @Path("/{tools}/info") + @ApiOperation(value = "Get external tool information", response = ExternalTool.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, format = "", example = "name,attributes", + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, example = "id,status", dataType = + "string", paramType = "query") + }) + public Response info( + @ApiParam(value = ParamConstants.EXTERNAL_TOOLS_DESCRIPTION, required = true) @PathParam("tools") String toolsStr, + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_VERSION_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_VERSION_PARAM) String version, + @ApiParam(value = ParamConstants.DELETED_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.DELETED_PARAM) boolean deleted) { + return run(() -> { + query.remove(ParamConstants.STUDY_PARAM); + List toolList = getIdList(toolsStr); + return externalToolManager.get(studyStr, toolList, query, queryOptions, true, token); + }); + } + + @GET + @Path("/search") + @ApiOperation(value = "External tool search method", response = ExternalTool.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, example = "name,attributes", + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, example = "id,status", dataType = + "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.LIMIT, value = ParamConstants.LIMIT_DESCRIPTION, dataType = "integer", paramType = + "query"), + @ApiImplicitParam(name = QueryOptions.SKIP, value = ParamConstants.SKIP_DESCRIPTION, dataType = "integer", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.COUNT, value = ParamConstants.COUNT_DESCRIPTION, defaultValue = "false", dataType = + "boolean", paramType = "query") + }) + public Response search( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_ID_PARAM) String id, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_NAME_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_NAME_PARAM) String name, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_UUID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_UUID_PARAM) String uuid, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_TAGS_PARAM) String tags, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_DRAFT_PARAM) Boolean draft, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_INTERNAL_REGISTRATION_USER_ID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_INTERNAL_REGISTRATION_USER_ID_PARAM) String userId, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_TYPE_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_TYPE_PARAM) String type, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_SCOPE_PARAM) String scope, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_WORKFLOW_REPOSITORY_NAME_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_WORKFLOW_REPOSITORY_NAME_PARAM) String repositoryName, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_DOCKER_NAME_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_DOCKER_NAME_PARAM) String dockerName, + @ApiParam(value = ParamConstants.CREATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.CREATION_DATE_PARAM) String creationDate, + @ApiParam(value = ParamConstants.MODIFICATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.MODIFICATION_DATE_PARAM) String modificationDate, + @ApiParam(value = ParamConstants.ACL_DESCRIPTION) @QueryParam(ParamConstants.ACL_PARAM) String acl, + @ApiParam(value = ParamConstants.RELEASE_DESCRIPTION) @QueryParam(ParamConstants.RELEASE_PARAM) String release, + @ApiParam(value = ParamConstants.SNAPSHOT_DESCRIPTION) @QueryParam(ParamConstants.SNAPSHOT_PARAM) int snapshot, + @ApiParam(value = ParamConstants.DELETED_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.DELETED_PARAM) boolean deleted + ) { + return run(() -> { + query.remove(ParamConstants.STUDY_PARAM); + return externalToolManager.search(studyStr, query, queryOptions, token); + }); + } + + @GET + @Path("/aggregationStats") + @ApiOperation(value = "Fetch external tool stats", response = FacetField.class) + public Response getAggregationStats( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_ID_PARAM) String id, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_NAME_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_NAME_PARAM) String name, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_UUID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_UUID_PARAM) String uuid, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_TAGS_PARAM) String tags, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_DRAFT_PARAM) Boolean draft, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_INTERNAL_REGISTRATION_USER_ID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_INTERNAL_REGISTRATION_USER_ID_PARAM) String userId, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_TYPE_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_TYPE_PARAM) String type, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_SCOPE_PARAM) String scope, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_WORKFLOW_REPOSITORY_NAME_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_WORKFLOW_REPOSITORY_NAME_PARAM) String repositoryName, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_DOCKER_NAME_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_DOCKER_NAME_PARAM) String dockerName, + @ApiParam(value = ParamConstants.CREATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.CREATION_DATE_PARAM) String creationDate, + @ApiParam(value = ParamConstants.MODIFICATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.MODIFICATION_DATE_PARAM) String modificationDate, + @ApiParam(value = ParamConstants.ACL_DESCRIPTION) @QueryParam(ParamConstants.ACL_PARAM) String acl, + @ApiParam(value = ParamConstants.RELEASE_DESCRIPTION) @QueryParam(ParamConstants.RELEASE_PARAM) String release, + @ApiParam(value = ParamConstants.SNAPSHOT_DESCRIPTION) @QueryParam(ParamConstants.SNAPSHOT_PARAM) int snapshot, + @ApiParam(value = ParamConstants.DELETED_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.DELETED_PARAM) boolean deleted, + + // Facet field + @ApiParam(value = ParamConstants.FACET_DESCRIPTION) @QueryParam(ParamConstants.FACET_PARAM) String facet) { + return run(() -> { + query.remove(ParamConstants.STUDY_PARAM); + query.remove(ParamConstants.FACET_PARAM); + return catalogManager.getExternalToolManager().facet(studyStr, query, facet, token); + }); + } + + @GET + @Path("/distinct") + @ApiOperation(value = "External tool distinct method", response = Object.class) + public Response distinct( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_ID_PARAM) String id, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_NAME_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_NAME_PARAM) String name, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_UUID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_UUID_PARAM) String uuid, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_TAGS_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_TAGS_PARAM) String tags, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_DRAFT_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_DRAFT_PARAM) Boolean draft, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_INTERNAL_REGISTRATION_USER_ID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_INTERNAL_REGISTRATION_USER_ID_PARAM) String userId, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_TYPE_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_TYPE_PARAM) String type, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_SCOPE_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_SCOPE_PARAM) String scope, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_WORKFLOW_REPOSITORY_NAME_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_WORKFLOW_REPOSITORY_NAME_PARAM) String repositoryName, + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_DOCKER_NAME_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_DOCKER_NAME_PARAM) String dockerName, + @ApiParam(value = ParamConstants.CREATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.CREATION_DATE_PARAM) String creationDate, + @ApiParam(value = ParamConstants.MODIFICATION_DATE_DESCRIPTION) @QueryParam(ParamConstants.MODIFICATION_DATE_PARAM) String modificationDate, + @ApiParam(value = ParamConstants.ACL_DESCRIPTION) @QueryParam(ParamConstants.ACL_PARAM) String acl, + @ApiParam(value = ParamConstants.RELEASE_DESCRIPTION) @QueryParam(ParamConstants.RELEASE_PARAM) String release, + @ApiParam(value = ParamConstants.SNAPSHOT_DESCRIPTION) @QueryParam(ParamConstants.SNAPSHOT_PARAM) int snapshot, + @ApiParam(value = ParamConstants.DELETED_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.DELETED_PARAM) boolean deleted, + @ApiParam(value = ParamConstants.DISTINCT_FIELD_DESCRIPTION, required = true) @QueryParam(ParamConstants.DISTINCT_FIELD_PARAM) String field) { + return run(() -> { + query.remove(ParamConstants.STUDY_PARAM); + query.remove(ParamConstants.DISTINCT_FIELD_PARAM); + List fields = split(field, ParamConstants.DISTINCT_FIELD_PARAM, true); + return externalToolManager.distinct(studyStr, fields, query, token); + }); + } + + @GET + @Path("/{tools}/acl") + @ApiOperation(value = "Returns the acl of the external tools. If member is provided, it will only return the acl for the member.", + response = ExternalToolAclEntryList.class) + public Response getAcls(@ApiParam(value = ParamConstants.EXTERNAL_TOOLS_DESCRIPTION, required = true) @PathParam("tools") String toolsStr, + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = "User or group id") @QueryParam("member") String member, + @ApiParam(value = ParamConstants.SILENT_DESCRIPTION, defaultValue = "false") @QueryParam(Constants.SILENT) boolean silent) { + return run(() -> { + List idList = getIdList(toolsStr); + return externalToolManager.getAcls(studyStr, idList, member, silent, token); + }); + } + + @POST + @Path("/acl/{members}/update") + @ApiOperation(value = "Update the set of external tool permissions granted for the member", response = ExternalToolAclEntryList.class) + public Response updateAcl( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = "Comma separated list of user or group ids", required = true) @PathParam("members") String memberIds, + @ApiParam(value = ParamConstants.ACL_ACTION_DESCRIPTION, required = true, defaultValue = "ADD") + @QueryParam(ParamConstants.ACL_ACTION_PARAM) ParamUtils.AclAction action, + @ApiParam(value = "JSON containing the parameters to update the permissions.", required = true) ExternalToolAclUpdateParams params) { + return run(() -> externalToolManager.updateAcl(studyStr, memberIds, params, action, token)); + } + + @DELETE + @Path("/{tools}/delete") + @ApiOperation(value = "Delete external tools", response = ExternalTool.class) + public Response delete( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.EXTERNAL_TOOLS_DESCRIPTION) @PathParam("tools") String toolsStr) { + return run(() -> externalToolManager.delete(studyStr, getIdList(toolsStr), queryOptions, token)); + } + + // ********************************** CUSTOM TOOL WS ENDPOINTS ********************************** + + @POST + @Path("/custom/build") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = CustomToolExecutor.DESCRIPTION, response = Job.class) + public Response dockerBuildByPost( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, + @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, + @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, + @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, + @ApiParam(value = "body", required = true) JobToolBuildParams params) { + return submitJob(study, JobType.NATIVE, CustomToolBuilder.ID, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); + } + + @POST + @Path("/custom/create") + @ApiOperation(value = "Register a new external tool of type CUSTOM_TOOL", response = ExternalTool.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, + dataType = "string", paramType = "query") + }) + public Response createCustomTool( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing workflow information", required = true) CustomToolCreateParams toolCreateParams) { + return run(() -> externalToolManager.createCustomTool(studyStr, toolCreateParams, queryOptions, token)); + } + + @POST + @Path("/custom/{toolId}/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update some custom external tool attributes", response = ExternalTool.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, + dataType = "string", paramType = "query") + }) + public Response updateCustomTool( + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION, required = true) @PathParam("toolId") String toolId, + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "body") CustomToolUpdateParams parameters) { + return run(() -> externalToolManager.updateCustomTool(studyStr, toolId, parameters, queryOptions, token)); + } + + @POST + @Path("/custom/run") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = CustomToolExecutor.DESCRIPTION, response = Job.class) + public Response runCustomTool( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, + @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, + @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, + @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, + @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) JobRunParams params) { + ToolInfo toolInfo = new ToolInfo() + .setId(params.getDocker().getId()); + return submitJob(study, JobType.CUSTOM, toolInfo, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, + jobPriority, dryRun); + } + + @POST + @Path("/custom/{toolId}/run") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = CustomToolExecutor.DESCRIPTION, response = Job.class) + public Response runCustomToolByToolId( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, + @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, + @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, + @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, + @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) JobRunParams params) { + return null; + } + + // ********************************** WORKFLOW WS ENDPOINTS ********************************** + + @POST + @Path("/workflow/create") + @ApiOperation(value = "Register a new external tool of type WORKFLOW", response = ExternalTool.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, + dataType = "string", paramType = "query") + }) + public Response createWorkflow( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "JSON containing workflow information", required = true) WorkflowCreateParams workflow) { + return run(() -> externalToolManager.createWorkflow(studyStr, workflow, queryOptions, token)); + } + + @POST + @Path("/workflow/{toolId}/update") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Update some external tool attributes", response = ExternalTool.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, + dataType = "string", paramType = "query"), + @ApiImplicitParam(name = QueryOptions.EXCLUDE, value = ParamConstants.EXCLUDE_DESCRIPTION, + dataType = "string", paramType = "query") + }) + public Response updateWorkflow( + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION, required = true) @PathParam("toolId") String toolId, + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, +// @ApiParam(value = ParamConstants.WORKFLOW_SCRIPTS_ACTION_DESCRIPTION, allowableValues = "ADD,SET,REMOVE", defaultValue = "ADD") @QueryParam(ParamConstants.WORKFLOW_SCRIPTS_ACTION_PARAM) ParamUtils.BasicUpdateAction workflowScriptsAction, + @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, + @ApiParam(value = "body") WorkflowUpdateParams parameters) { + try { + return createOkResponse(externalToolManager.updateWorkflow(studyStr, toolId, parameters, queryOptions, token), "Workflow update success"); + } catch (Exception e) { + return createErrorResponse(e); + } + } + + @POST + @Path("/workflow/import") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Import an external tool of type WORKFLOW", response = ExternalTool.class) + public Response importWorkflow( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = "Repository parameters", required = true) WorkflowRepositoryParams params) { + return run(() -> catalogManager.getExternalToolManager().importWorkflow(study, params, queryOptions, token)); + } + + @POST + @Path("/workflow/{workflowId}/run") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Execute an external tool of type WORKFLOW", response = Job.class) + public Response executeWorkflow( + @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, + @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, + @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, + @ApiParam(value = ParamConstants.JOB_DEPENDS_ON_DESCRIPTION) @QueryParam(JOB_DEPENDS_ON) String dependsOn, + @ApiParam(value = ParamConstants.JOB_TAGS_DESCRIPTION) @QueryParam(ParamConstants.JOB_TAGS) String jobTags, + @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, + @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, + @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, + @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) ExternalToolRunParams params) { + return run(() -> catalogManager.getExternalToolManager().submitWorkflow(study, params, jobName, jobDescription, dependsOn, jobTags, + scheduledStartTime, jobPriority, dryRun, token)); + } + + // ********************************** VARIANT WALKER WS ENDPOINTS ********************************** + +///tools/walker/{build ?? | create | update} +///tools/walker/run x2? + + +} diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java index efa484e89c2..7c12e33ec39 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java @@ -30,9 +30,9 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.AclParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolRunParams; import org.opencb.opencga.core.models.file.FileContent; import org.opencb.opencga.core.models.job.*; -import org.opencb.opencga.core.models.externalTool.NextFlowRunParams; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.annotations.*; @@ -135,7 +135,7 @@ public Response runByPost( @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, - @ApiParam(value = NextFlowRunParams.DESCRIPTION, required = true) JobRunParams params) { + @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) JobRunParams params) { ToolInfo toolInfo = new ToolInfo() .setId(params.getDocker().getId()); return submitJob(study, JobType.CUSTOM, toolInfo, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java index 809d67ee092..a058d82d3c5 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java @@ -2,12 +2,13 @@ import org.opencb.commons.datastore.core.DataResult; import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.managers.WorkflowManager; +import org.opencb.opencga.catalog.managers.ExternalToolManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.externalTool.*; +import org.opencb.opencga.core.models.externalTool.workflow.DeprecatedWorkflowUpdateParams; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.tools.annotations.*; @@ -20,17 +21,18 @@ import static org.opencb.opencga.core.api.ParamConstants.JOB_DEPENDS_ON; +@Deprecated @Path("/{apiVersion}/workflows") @Produces(MediaType.APPLICATION_JSON) -@Api(value = "Workflows", description = "Methods for working with 'workflows' endpoint") +@Api(value = "Workflows", description = "[DEPRECATED] Moved to /tools category") public class WorkflowWSServer extends OpenCGAWSServer { - private WorkflowManager workflowManager; + private ExternalToolManager externalToolManager; public WorkflowWSServer(@Context UriInfo uriInfo, @Context HttpServletRequest httpServletRequest, @Context HttpHeaders httpHeaders) throws IOException, VersionException { super(uriInfo, httpServletRequest, httpHeaders); - workflowManager = catalogManager.getWorkflowManager(); + externalToolManager = catalogManager.getExternalToolManager(); } @GET @@ -51,7 +53,7 @@ public Response workflowInfo( query.remove(ParamConstants.STUDY_PARAM); List workflowList = getIdList(workflowStr); - DataResult workflowDataResult = workflowManager.get(studyStr, workflowList, query, queryOptions, true, token); + DataResult workflowDataResult = externalToolManager.get(studyStr, workflowList, query, queryOptions, true, token); return createOkResponse(workflowDataResult); } catch (Exception e) { return createErrorResponse(e); @@ -72,7 +74,7 @@ public Response createWorkflowPOST( @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, @ApiParam(value = "JSON containing workflow information", required = true) WorkflowCreateParams workflow) { try { - return createOkResponse(workflowManager.create(studyStr, workflow, queryOptions, token)); + return createOkResponse(externalToolManager.createWorkflow(studyStr, workflow, queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -91,8 +93,8 @@ public Response updateByPost( @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, - @ApiParam(value = NextFlowRunParams.DESCRIPTION, required = true) NextFlowRunParams params) { - return run(() -> catalogManager.getWorkflowManager().submit(study, params, jobName, jobDescription, dependsOn, jobTags, + @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) ExternalToolRunParams params) { + return run(() -> catalogManager.getExternalToolManager().submitWorkflow(study, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, token)); } @@ -103,7 +105,7 @@ public Response updateByPost( public Response importWorkflow( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "Repository parameters", required = true) WorkflowRepositoryParams params) { - return run(() -> catalogManager.getWorkflowManager().importWorkflow(study, params, queryOptions, token)); + return run(() -> catalogManager.getExternalToolManager().importWorkflow(study, params, queryOptions, token)); } @GET @@ -139,7 +141,7 @@ public Response search( ) { try { query.remove(ParamConstants.STUDY_PARAM); - return createOkResponse(workflowManager.search(studyStr, query, queryOptions, token)); + return createOkResponse(externalToolManager.search(studyStr, query, queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -169,7 +171,7 @@ public Response distinct( query.remove(ParamConstants.STUDY_PARAM); query.remove(ParamConstants.DISTINCT_FIELD_PARAM); List fields = split(field, ParamConstants.DISTINCT_FIELD_PARAM, true); - return createOkResponse(workflowManager.distinct(studyStr, fields, query, token)); + return createOkResponse(externalToolManager.distinct(studyStr, fields, query, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -190,12 +192,12 @@ public Response updateByPost( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, // @ApiParam(value = ParamConstants.WORKFLOW_SCRIPTS_ACTION_DESCRIPTION, allowableValues = "ADD,SET,REMOVE", defaultValue = "ADD") @QueryParam(ParamConstants.WORKFLOW_SCRIPTS_ACTION_PARAM) ParamUtils.BasicUpdateAction workflowScriptsAction, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, - @ApiParam(value = "body") ExternalToolUpdateParams parameters) { + @ApiParam(value = "body") DeprecatedWorkflowUpdateParams parameters) { try { // Map actionMap = new HashMap<>(); // queryOptions.put(Constants.ACTIONS, actionMap); - return createOkResponse(workflowManager.update(studyStr, workflowId, parameters, queryOptions, token), "Workflow update success"); + return createOkResponse(externalToolManager.update(studyStr, workflowId, parameters.toWorkflowUpdateParams(), queryOptions, token), "Workflow update success"); } catch (Exception e) { return createErrorResponse(e); } @@ -208,7 +210,7 @@ public Response delete( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.WORKFLOWS_DESCRIPTION) @PathParam("workflows") String workflows) { try { - return createOkResponse(workflowManager.delete(studyStr, getIdList(workflows), queryOptions, token)); + return createOkResponse(externalToolManager.delete(studyStr, getIdList(workflows), queryOptions, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -224,7 +226,7 @@ public Response getAcls(@ApiParam(value = ParamConstants.WORKFLOWS_DESCRIPTION, @ApiParam(value = ParamConstants.SILENT_DESCRIPTION, defaultValue = "false") @QueryParam(Constants.SILENT) boolean silent) { try { List idList = getIdList(workflowIdsStr); - return createOkResponse(workflowManager.getAcls(studyStr, idList, member, silent, token)); + return createOkResponse(externalToolManager.getAcls(studyStr, idList, member, silent, token)); } catch (Exception e) { return createErrorResponse(e); } @@ -240,7 +242,7 @@ public Response updateAcl( @QueryParam(ParamConstants.ACL_ACTION_PARAM) ParamUtils.AclAction action, @ApiParam(value = "JSON containing the parameters to update the permissions.", required = true) ExternalToolAclUpdateParams params) { try { - return createOkResponse(workflowManager.updateAcl(studyStr, memberIds, params, action, token)); + return createOkResponse(externalToolManager.updateAcl(studyStr, memberIds, params, action, token)); } catch (Exception e) { return createErrorResponse(e); } From 4d157160cbf936aeee4b6f3e77b6360851e43ab0 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 15 May 2025 15:38:20 +0200 Subject: [PATCH 07/22] app: deprecate old migrations, #TASK-7610 --- .../AuthOriginSimplificationMigration.java | 53 +-------- .../migrations/v3/v3_1_0/NoteMigration.java | 40 +------ .../v3/v3_1_0/UserBanMigration.java | 36 +----- .../v3/v3_2_0/AddNewJobFieldsMigration.java | 21 +--- .../TASK_5964/AddInterpretationName.java | 24 +--- .../TASK_5964/AddNewClinicalStatusValues.java | 99 +---------------- ...AddVersionToClinicalAnalysisMigration.java | 51 +-------- .../TASK_5964/RemoveStatusNameMigration.java | 103 +----------------- .../TASK_5964/RenamePanelLockMigration.java | 15 +-- .../UpdateClinicalStudyConfiguration.java | 55 +--------- .../v3/v3_2_0/VariantSetupMigration.java | 42 +------ .../v3_2_1/AddPasswordHistoryMigration.java | 33 +----- .../MoveUserAccountToInternalMigration.java | 51 +-------- .../catalog/AddNewNoteTypeMigration.java | 32 +----- .../catalog/AssociateAlignmentFiles.java | 21 +--- .../catalog/ClinicalCvdbIndexMigration.java | 35 +----- .../catalog/FederationChangesMigration.java | 49 +-------- .../v4/v4_0_0/catalog/IndexesTask6445.java | 4 +- .../JobModelChangesMigrationTask6445.java | 31 +----- .../v4/v4_0_0/catalog/ResourcesMigration.java | 69 +----------- ...mpleIndexConfigurationIsAlwaysDefined.java | 69 +----------- .../ExternalToolTask7610Migration.java | 4 +- 22 files changed, 43 insertions(+), 894 deletions(-) rename opencga-app/src/main/java/org/opencb/opencga/app/migrations/{v4/v4_0_0 => v5}/catalog/ExternalToolTask7610Migration.java (96%) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java index e1b331fa987..1137fbea09d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/AuthOriginSimplificationMigration.java @@ -1,64 +1,15 @@ package org.opencb.opencga.app.migrations.v3.v3_1_0; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.Updates; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.api.OrganizationDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.PasswordUtils; - -import java.util.List; @Migration(id = "hide_secret_key", description = "Hide secret key #TASK-5923", version = "3.1.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240410) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240410, + deprecatedSince = "4.0.0") public class AuthOriginSimplificationMigration extends MigrationTool { @Override protected void run() throws Exception { - Bson query = Filters.exists("configuration.token", false); - Bson projection = Projections.include("_id", "configuration"); - migrateCollection(OrganizationMongoDBAdaptorFactory.ORGANIZATION_COLLECTION, query, projection, (orgDocument, bulk) -> { - Document configuration = orgDocument.get(OrganizationDBAdaptor.QueryParams.CONFIGURATION.key(), Document.class); - if (configuration != null) { - String secretKey = null; - List authenticationOrigins = configuration.getList("authenticationOrigins", Document.class); - if (CollectionUtils.isNotEmpty(authenticationOrigins)) { - for (Document authenticationOrigin : authenticationOrigins) { - if (authenticationOrigin.getString("type").equals("OPENCGA")) { - secretKey = authenticationOrigin.getString("secretKey"); - if (authenticationOrigin.getString("id").equals("internal")) { - authenticationOrigin.put("id", "OPENCGA"); - } - } - authenticationOrigin.remove("secretKey"); - authenticationOrigin.remove("algorithm"); - authenticationOrigin.remove("expiration"); - } - } - secretKey = StringUtils.isNotEmpty(secretKey) ? secretKey : PasswordUtils.getStrongRandomPassword(32); - Document token = new Document() - .append("secretKey", secretKey) - .append("algorithm", "HS256") - .append("expiration", 3600L); - configuration.put("token", token); - bulk.add(new UpdateOneModel<>( - Filters.eq("_id", orgDocument.get("_id")), - Updates.set("configuration", configuration))); - } - }); - - // Change authOrigin id from all "internal" users - getMongoCollection(OrganizationMongoDBAdaptorFactory.USER_COLLECTION) - .updateMany( - new Document("account.authentication.id", "internal"), - Updates.set("account.authentication.id", "OPENCGA")); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java index c3e49a2d85e..61026c0caad 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/NoteMigration.java @@ -1,51 +1,15 @@ package org.opencb.opencga.app.migrations.v3.v3_1_0; -import com.mongodb.MongoNamespace; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.RenameCollectionOptions; -import com.mongodb.client.model.Updates; -import com.mongodb.client.result.UpdateResult; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.commons.datastore.mongodb.MongoDataStore; -import org.opencb.opencga.catalog.db.api.NoteDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.notes.Note; @Migration(id = "migrate_notes", description = "Migrate notes #TASK-5836", version = "3.1.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240315) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240315, + deprecatedSince = "4.0.0") public class NoteMigration extends MigrationTool { @Override protected void run() throws Exception { - // First migrate to add the new values - MongoCollection collection = getMongoCollection(organizationId, "notes"); - Bson query = Filters.exists(NoteDBAdaptor.QueryParams.STUDY_UID.key(), false); - Bson update = Updates.combine( - Updates.set(NoteDBAdaptor.QueryParams.STUDY_UID.key(), -1L), - Updates.set(NoteDBAdaptor.QueryParams.SCOPE.key(), Note.Scope.ORGANIZATION.name()), - Updates.set(NoteDBAdaptor.QueryParams.STUDY.key(), ""), - Updates.set(NoteDBAdaptor.QueryParams.VISIBILITY.key(), Note.Visibility.PRIVATE.name()) - ); - UpdateResult updateResult = collection.updateMany(query, update); - if (updateResult.getModifiedCount() == 0) { - logger.info("Note data model could not be updated. Notes found in organization '{}': {}", organizationId, updateResult.getMatchedCount()); - } - - // Rename Note collection - RenameCollectionOptions options = new RenameCollectionOptions().dropTarget(true); - // Rename collection - String databaseName = dbAdaptorFactory.getMongoDataStore(organizationId).getDatabaseName(); - logger.info("Renaming notes collection for organization '{}' -> Database: '{}'", organizationId, databaseName); - MongoDataStore mongoDataStore = dbAdaptorFactory.getMongoDataStore(organizationId); - mongoDataStore.getDb().getCollection("notes").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.NOTE_COLLECTION), options); - mongoDataStore.getDb().getCollection("notes_archive").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.NOTE_ARCHIVE_COLLECTION), options); - mongoDataStore.getDb().getCollection("notes_deleted").renameCollection(new MongoNamespace(databaseName, OrganizationMongoDBAdaptorFactory.DELETED_NOTE_COLLECTION), options); - - catalogManager.installIndexes(organizationId, token); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java index aa0834da33a..ee9d277c455 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_1_0/UserBanMigration.java @@ -1,47 +1,15 @@ package org.opencb.opencga.app.migrations.v3.v3_1_0; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.catalog.utils.Constants; -import org.opencb.opencga.core.common.TimeUtils; @Migration(id = "addFailedLoginAttemptsMigration", description = "Add failedAttempts to User #TASK-6013", version = "3.2.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240419, patch = 2) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240419, patch = 2, + deprecatedSince = "4.0.0") public class UserBanMigration extends MigrationTool { @Override protected void run() throws Exception { - String lastModified = TimeUtils.getTime(); - migrateCollection(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, - Filters.exists("internal.registrationDate", false), - Projections.include("_id", "internal", "account"), - (document, bulk) -> { - Document internal = document.get("internal", Document.class); - Document account = document.get("account", Document.class); - internal.put("failedAttempts", 0); - internal.put("registrationDate", account.get("creationDate")); - internal.put("lastModified", lastModified); - account.put("expirationDate", Constants.DEFAULT_USER_EXPIRATION_DATE); - - bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), - Updates.combine( - Updates.set("internal", internal), - Updates.set("account", account)) - ) - ); - }); - - // Patch 2. Organization 'configuration.defaultUserExpirationDate' field was not set for existing organizations - MongoCollection orgCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.ORGANIZATION_COLLECTION); - orgCollection.updateMany(Filters.exists("configuration.defaultUserExpirationDate", false), - Updates.set("configuration.defaultUserExpirationDate", Constants.DEFAULT_USER_EXPIRATION_DATE)); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java index 8e196973bba..cd67e8a5b28 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/AddNewJobFieldsMigration.java @@ -1,34 +1,17 @@ package org.opencb.opencga.app.migrations.v3.v3_2_0; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.Arrays; - @Migration(id = "add_jobParentId_scheduledStartTime", description = "Add 'jobParentId' and 'scheduledStartTime' to existing jobs #TASK-6171 #TASK-6089", version = "3.2.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240506) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240506, + deprecatedSince = "4.0.0") public class AddNewJobFieldsMigration extends MigrationTool { @Override protected void run() throws Exception { - for (String jobCollection : Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_JOB_COLLECTION)) { - MongoCollection mongoCollection = getMongoCollection(jobCollection); - Bson query = Filters.exists("parentId", false); - Bson update = Updates.combine( - Updates.set("parentId", ""), - Updates.set("scheduledStartTime", "") - ); - mongoCollection.updateMany(query, update); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java index 372130c0a5e..77e69315a95 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddInterpretationName.java @@ -1,35 +1,15 @@ package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.Arrays; - @Migration(id = "add_interpretation_name", description = "Add Interpretation name #TASK-5964", version = "3.2.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610, + deprecatedSince = "4.0.0") public class AddInterpretationName extends MigrationTool { @Override protected void run() throws Exception { - // Add new Interpretation name field - Bson nameDoesNotExistQuery = Filters.exists("name", false); - Bson projection = Projections.include("id"); - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { - migrateCollection(collection, nameDoesNotExistQuery, projection, - (document, bulk) -> { - Document updateDocument = new Document() - .append("name", document.get("id")); - bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), new Document("$set", updateDocument))); - }); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java index 775cf2c64d7..d5099c9bd9b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddNewClinicalStatusValues.java @@ -1,112 +1,17 @@ package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.lang3.StringUtils; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.common.GitRepositoryState; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.clinical.ClinicalStatus; -import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; -import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; - -import java.util.Arrays; -import java.util.List; @Migration(id = "add_new_clinical_status", description = "Add new ClinicalStatus to ClinicalAnalysis and Interpretation, #TASK-5964", version = "3.2.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240611) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240611, + deprecatedSince = "4.0.0") public class AddNewClinicalStatusValues extends MigrationTool { @Override protected void run() throws Exception { - ClinicalAnalysisStudyConfiguration defaultConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); - - Bson query = Filters.exists("status.author", false); - Bson projection = Projections.include("status", "id"); - - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { - migrateCollection(collection, query, projection, - (document, bulk) -> { - MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); - String clinicalId = document.getString("id"); - Document status = document.get("status", Document.class); - - Document newStatus = fillStatusValues("Clinical Analysis", clinicalId, status, defaultConfiguration.getStatus()); - updateDocument.getSet().put("status", newStatus); - - Document effectiveUpdate = updateDocument.toFinalUpdateDocument(); - - logger.debug("Updating clinical analysis '{}': {}", document.get("id"), effectiveUpdate.toBsonDocument()); - bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), effectiveUpdate)); - }); - } - - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { - migrateCollection(collection, query, projection, - (document, bulk) -> { - MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); - String interpretationId = document.getString("id"); - Document status = document.get("status", Document.class); - - Document newStatus = fillStatusValues("Interpretation", interpretationId, status, - defaultConfiguration.getInterpretation().getStatus()); - updateDocument.getSet().put("status", newStatus); - - Document effectiveUpdate = updateDocument.toFinalUpdateDocument(); - - logger.debug("Updating interpretation '{}': {}", document.get("id"), effectiveUpdate.toBsonDocument()); - bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), effectiveUpdate)); - }); - } - } - - private Document fillStatusValues(String entity, String id, Document status, List statusValueList) { - ClinicalStatus clinicalStatus = new ClinicalStatus(); - String clinicalStatusId = status != null ? status.getString("id") : null; - if (status == null || StringUtils.isEmpty(clinicalStatusId)) { - logger.warn("Status is empty or does not contain 'id' field. Setting default status value for {} '{}'", entity, id); - for (ClinicalStatusValue clinicalStatusValue : statusValueList) { - if (clinicalStatusValue.getType().equals(ClinicalStatusValue.ClinicalStatusType.NOT_STARTED)) { - clinicalStatus.setId(clinicalStatusValue.getId()); - clinicalStatus.setDescription(clinicalStatusValue.getDescription()); - clinicalStatus.setType(clinicalStatusValue.getType()); - } - } - } else { - for (ClinicalStatusValue clinicalStatusValue : statusValueList) { - if (clinicalStatusValue.getId().equals(clinicalStatusId)) { - clinicalStatus.setId(clinicalStatusValue.getId()); - clinicalStatus.setDescription(clinicalStatusValue.getDescription()); - clinicalStatus.setType(clinicalStatusValue.getType()); - break; - } - } - if (clinicalStatus.getType() == null) { - logger.warn("Status '{}' not found in the list of available status values. Keeping original status for {} '{}'", - clinicalStatusId, entity, id); - clinicalStatus.setId(clinicalStatusId); - clinicalStatus.setDescription(status.getString("description")); - } - } - clinicalStatus.setDate(TimeUtils.getTime()); - clinicalStatus.setVersion(GitRepositoryState.getInstance().getBuildVersion()); - clinicalStatus.setCommit(GitRepositoryState.getInstance().getCommitId()); - clinicalStatus.setAuthor("opencga"); - - Document document = convertToDocument(clinicalStatus); - return document; } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java index 302637cf968..cbb5f851d06 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/AddVersionToClinicalAnalysisMigration.java @@ -1,62 +1,15 @@ package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.apache.commons.collections4.CollectionUtils; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.Collections; -import java.util.List; - @Migration(id = "add_version_to_clinicalAnalysis", description = "Add version to Clinical Analysis #TASK-5964", version = "3.2.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240527) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240527, + deprecatedSince = "4.0.0") public class AddVersionToClinicalAnalysisMigration extends MigrationTool { @Override protected void run() throws Exception { - // Add missing mandatory "versioning" fields to Clinical Analysis - logger.info("Adding missing 'versioning' fields to Clinical Analysis..."); - Bson versionDoesNotExistQuery = Filters.exists("version", false); - Bson projection = Projections.include("release", "interpretation", "secondaryInterpretations"); - migrateCollection(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, versionDoesNotExistQuery, projection, - (document, bulk) -> { - int version = 1; - Document interpretation = document.get("interpretation", Document.class); - if (interpretation != null) { - int interpretationVersion = interpretation.get("version", Number.class).intValue(); - version = Math.max(version, interpretationVersion + 1); - } - List secondaryInterpretations = document.getList("secondaryInterpretations", Document.class); - if (CollectionUtils.isNotEmpty(secondaryInterpretations)) { - for (Document secondaryInterpretation : secondaryInterpretations) { - int secondaryInterpretationVersion = secondaryInterpretation.get("version", Number.class).intValue(); - version = Math.max(version, secondaryInterpretationVersion + 1); - } - } - int release = document.get("release", Number.class).intValue(); - Document updateDocument = new Document() - .append("version", version) - .append("_releaseFromVersion", Collections.singletonList(release)) - .append("_lastOfVersion", true) - .append("_lastOfRelease", true); - bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), new Document("$set", updateDocument))); - }); - - // Recalculate indexes - logger.info("Installing new indexes..."); - catalogManager.installIndexes(organizationId, token); - - // Copy all Clinical Analyses to the archive collection - logger.info("Copying all the data from the main collection to the archive one..."); - copyData(new Document(), OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION); - - } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java index 80692ae910c..48083f2476a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RemoveStatusNameMigration.java @@ -1,114 +1,15 @@ package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - @Migration(id = "remove_status_name", description = "Remove status name #TASK-5964", version = "3.2.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240612) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240612, + deprecatedSince = "4.0.0") public class RemoveStatusNameMigration extends MigrationTool { @Override protected void run() throws Exception { - // Remove from user - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION)) { - removeStatusField(collection, Collections.singletonList("internal.status.name")); - } - - // Remove from project - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.PROJECT_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_PROJECT_COLLECTION)) { - removeStatusField(collection, Collections.singletonList("internal.status.name")); - } - - // Remove from study - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { - List statusList = Arrays.asList("status.name", "internal.status.name"); - removeStatusField(collection, statusList); - } - - // Remove from sample - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, - OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_SAMPLE_COLLECTION)) { - List statusList = Arrays.asList("status.name", "internal.status.name", "internal.variant.index.status.name", - "internal.variant.sampleGenotypeIndex.status.name", "internal.variant.sampleGenotypeIndex.familyStatus.name", - "internal.variant.secondarySampleIndex.status.name", "internal.variant.secondarySampleIndex.familyStatus.name", - "internal.variant.annotationIndex.status.name", "internal.variant.secondaryAnnotationIndex.status.name"); - removeStatusField(collection, statusList); - } - - // Remove from file - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_FILE_COLLECTION)) { - List statusList = Arrays.asList("status.name", "internal.status.name", "internal.variant.index.status.name", - "internal.variant.annotationIndex.status.name", "internal.variant.secondaryAnnotationIndex.status.name", - "internal.alignment.index.status.name"); - removeStatusField(collection, statusList); - } - - // Remove from individual - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, - OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_INDIVIDUAL_COLLECTION)) { - List statusList = Arrays.asList("status.name", "internal.status.name"); - removeStatusField(collection, statusList); - } - - // Remove from family - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, - OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_FAMILY_COLLECTION)) { - List statusList = Arrays.asList("status.name", "internal.status.name"); - removeStatusField(collection, statusList); - } - - // Remove from cohort - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.COHORT_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_COHORT_COLLECTION)) { - List statusList = Arrays.asList("status.name", "internal.status.name"); - removeStatusField(collection, statusList); - } - - // Remove from panel - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.PANEL_COLLECTION, - OrganizationMongoDBAdaptorFactory.PANEL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_PANEL_COLLECTION)) { - List statusList = Arrays.asList("status.name", "internal.status.name"); - removeStatusField(collection, statusList); - } - - // Remove from job - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_JOB_COLLECTION)) { - removeStatusField(collection, Collections.singletonList("internal.status.name")); - } - - // Remove from clinical - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { - removeStatusField(collection, Collections.singletonList("internal.status.name")); - } - - // Remove from interpretation - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.INTERPRETATION_COLLECTION, - OrganizationMongoDBAdaptorFactory.INTERPRETATION_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_INTERPRETATION_COLLECTION)) { - removeStatusField(collection, Collections.singletonList("internal.status.name")); - } - } - - private void removeStatusField(String collection, List statusList) throws CatalogDBException { - logger.info("Removing status.name fields from collection {}...", collection); - List exists = statusList.stream().map(Filters::exists).collect(Collectors.toList()); - List unsetList = statusList.stream().map(Updates::unset).collect(Collectors.toList()); - getMongoCollection(collection).updateMany(Filters.or(exists), Updates.combine(unsetList)); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java index 32c10c3fab7..3f67ca3ce1f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/RenamePanelLockMigration.java @@ -1,26 +1,15 @@ package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.Arrays; - @Migration(id = "rename_panel_lock_from_clinical__task_5964", description = "Rename 'panelLock' to 'panelLocked' #TASK-5964", - version = "3.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240710) + version = "3.2.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240710, + deprecatedSince = "4.0.0") public class RenamePanelLockMigration extends MigrationTool { @Override protected void run() throws Exception { - Bson query = new Document(); - - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { - getMongoCollection(collection).updateMany(query, new Document("$rename", new Document("panelLock", "panelLocked"))); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java index 35960978d4c..74a57692a6c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/TASK_5964/UpdateClinicalStudyConfiguration.java @@ -1,68 +1,17 @@ package org.opencb.opencga.app.migrations.v3.v3_2_0.TASK_5964; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.catalog.migration.MigrationRuntimeException; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.study.configuration.ClinicalAnalysisStudyConfiguration; - -import java.util.Arrays; -import java.util.List; @Migration(id = "update_clinical_study_configuration", description = "Setting new default Clinical Study Configuration status values, #TASK-5964", version = "3.2.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240610, + deprecatedSince = "4.0.0") public class UpdateClinicalStudyConfiguration extends MigrationTool { @Override protected void run() throws Exception { - ClinicalAnalysisStudyConfiguration clinicalAnalysisStudyConfiguration = ClinicalAnalysisStudyConfiguration.defaultConfiguration(); - Document clinicalConfigurationDocument = convertToDocument(clinicalAnalysisStudyConfiguration); - - Bson query = new Document(); - Bson projection = Projections.include("internal.configuration.clinical", "fqn"); - - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { - migrateCollection(collection, query, projection, - (document, bulk) -> { - Document internal = document.get("internal", Document.class); - if (internal == null) { - throw new MigrationRuntimeException("'internal' field not found in study '" + document.get("fqn") + "'"); - } - Document configuration = internal.get("configuration", Document.class); - if (configuration == null) { - throw new MigrationRuntimeException("'internal.configuration' field not found in study '" + document.get("fqn") + "'"); - } - Document clinicalDocument = configuration.get("clinical", Document.class); - - MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); - if (clinicalDocument == null) { - logger.warn("Found empty 'internal.configuration.clinical' field in study '{}'. Creating a new one...", document.get("fqn")); - updateDocument.getSet().put("internal.configuration.clinical", clinicalConfigurationDocument); - } else { - Object statusObject = clinicalDocument.get("status"); - if (statusObject instanceof List) { - // The study seems to be already migrated. Skipping... - logger.warn("Study '{}' seems to be already migrated. Skipping...", document.get("fqn")); - return; - } - // Study needs to be migrated - logger.info("Migrating study '{}'", document.get("fqn")); - updateDocument.getSet().put("internal.configuration.clinical.status", clinicalConfigurationDocument.get("status")); - updateDocument.getSet().put("internal.configuration.clinical.flags", clinicalConfigurationDocument.get("flags")); - updateDocument.getSet().put("internal.configuration.clinical.interpretation.status", clinicalConfigurationDocument.get("interpretation", Document.class).get("status")); - } - logger.debug("Updating study '{}': {}", document.get("fqn"), updateDocument.toFinalUpdateDocument()); - bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); - }); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java index f7d6c03be9d..73c3f846a4b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_0/VariantSetupMigration.java @@ -1,53 +1,13 @@ package org.opencb.opencga.app.migrations.v3.v3_2_0; -import org.opencb.commons.datastore.core.ObjectMap; -import org.opencb.opencga.analysis.variant.manager.VariantStorageManager; import org.opencb.opencga.app.migrations.StorageMigrationTool; import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.models.study.VariantSetupResult; -import org.opencb.opencga.storage.core.metadata.VariantStorageMetadataManager; - -import java.util.LinkedHashSet; -import java.util.List; @Migration(id = "variant_setup", description = "Add a dummy variant setup for studies with data", version = "3.2.0", - domain = Migration.MigrationDomain.STORAGE, date = 20240516) + domain = Migration.MigrationDomain.STORAGE, date = 20240516, deprecatedSince = "4.0.0") public class VariantSetupMigration extends StorageMigrationTool { @Override protected void run() throws Exception { - VariantStorageManager variantStorageManager = getVariantStorageManager(); - List storageStudies = getVariantStorageStudies(); - if (storageStudies.isEmpty()) { - logger.info("No studies with variant storage found on organization '{}'", organizationId); - return; - } - for (String study : storageStudies) { - logger.info("--- Checking study '{}'", study); - if (variantStorageManager.hasVariantSetup(study, token)) { - logger.info("Study '{}' already has a variant setup", study); - continue; - } - - String projectFqn = catalogManager.getStudyManager().getProjectFqn(study); - VariantStorageMetadataManager metadataManager = getVariantStorageEngineByProject(projectFqn).getMetadataManager(); - int studyId = metadataManager.getStudyId(study); - LinkedHashSet indexedFiles = metadataManager.getIndexedFiles(studyId); - if (indexedFiles.isEmpty()) { - logger.info("Study '{}' does not have any indexed files. Skipping variant setup", study); - continue; - } - logger.info("Study '{}' doesn't have a variant setup, but it has {} indexed files. Creating a dummy variant setup", - study, indexedFiles.size()); - logger.info("Creating a dummy variant setup for study '{}'", study); - VariantSetupResult dummy = new VariantSetupResult(); - dummy.setDate(TimeUtils.getTime()); - dummy.setUserId(catalogManager.getUserManager().getUserId(token)); - dummy.setParams(new ObjectMap("executed_from_migration", getId())); - dummy.setStatus(VariantSetupResult.Status.READY); - dummy.setOptions(new ObjectMap()); - catalogManager.getStudyManager().setVariantEngineSetupOptions(study, dummy, token); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java index f016e564c77..921fdbf95e0 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/AddPasswordHistoryMigration.java @@ -1,45 +1,16 @@ package org.opencb.opencga.app.migrations.v3.v3_2_1; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.Arrays; -import java.util.Collections; - -import static org.opencb.opencga.catalog.db.mongodb.UserMongoDBAdaptor.*; - @Migration(id = "add_archivePasswords_array", description = "Add password history #6494", version = "3.2.1", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723, + deprecatedSince = "4.0.0") public class AddPasswordHistoryMigration extends MigrationTool { @Override protected void run() throws Exception { - Bson query = Filters.exists(PRIVATE_PASSWORD_ARCHIVE, false); - Bson projection = Projections.include(PRIVATE_PASSWORD); - migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), - query, projection, (document, bulk) -> { - String currentPassword = document.getString("_password"); - - Document passwordDoc = new Document() - .append(HASH, currentPassword) - .append(SALT, ""); - Document privatePassword = new Document(); - privatePassword.put(CURRENT, passwordDoc); - privatePassword.put(ARCHIVE, Collections.singletonList(passwordDoc)); - - MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); - updateDocument.getSet().put(PRIVATE_PASSWORD, privatePassword); - - bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); - }); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java index d2a3b680e1a..6b7a41cc240 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v3/v3_2_1/MoveUserAccountToInternalMigration.java @@ -1,62 +1,15 @@ package org.opencb.opencga.app.migrations.v3.v3_2_1; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.UpdateOneModel; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.catalog.utils.Constants; - -import java.util.Arrays; @Migration(id = "move_user_account_to_internal", description = "Move account to internal.account #6494", version = "3.2.1", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20240723, + deprecatedSince = "4.0.0") public class MoveUserAccountToInternalMigration extends MigrationTool { @Override protected void run() throws Exception { - Bson query = Filters.exists("account", true); - Bson projection = Projections.include("internal", "account"); - migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION), - query, projection, (document, bulk) -> { - MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); - - Document account = document.get("account", Document.class); - Document internal = document.get("internal", Document.class); - internal.put("account", account); - - updateDocument.getSet().put("modificationDate", internal.get("lastModified")); - updateDocument.getSet().put("creationDate", account.get("creationDate")); - account.remove("creationDate"); - - Document password = new Document() - .append("expirationDate", null) - .append("lastModified", internal.get("lastModified")); - account.put("password", password); - account.put("failedAttempts", internal.get("failedAttempts")); - internal.remove("failedAttempts"); - - updateDocument.getSet().put("internal", internal); - updateDocument.getUnset().add("account"); - - bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); - }); - - // Due to patch 2 from TASK-6013, default user expirationDate for some users was not set. - MongoCollection userCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.USER_COLLECTION); - userCollection.updateMany( - Filters.or( - Filters.eq("internal.account.expirationDate", null), - Filters.eq("internal.account.expirationDate", "") - ), - Updates.set("internal.account.expirationDate", Constants.DEFAULT_USER_EXPIRATION_DATE)); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java index b18c1350268..07122848aca 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AddNewNoteTypeMigration.java @@ -1,43 +1,15 @@ package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.api.ParamConstants; -import org.opencb.opencga.core.models.notes.NoteType; - -import java.util.Arrays; @Migration(id = "addNewNoteType__task_7046", description = "Add new Note type #7046", version = "4.0.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241030) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241030, + deprecatedSince = "5.0.0") public class AddNewNoteTypeMigration extends MigrationTool { @Override protected void run() throws Exception { - NoteType type = NoteType.UNKNOWN; - if (ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { - type = NoteType.ORGANIZATION; - } - MongoCollection collection = getMongoCollection(OrganizationMongoDBAdaptorFactory.NOTE_COLLECTION); - Bson query = Filters.exists("type", false); - Bson update = Updates.set("type", type); - logger.info("Setting all notes from organization '{}' to type '{}'", organizationId, type); - collection.updateMany(query, update); - - // Drop old index - Document oldIndex = new Document() - .append("id", 1) - .append("studyUid", 1) - .append("version", 1); - dropIndex(Arrays.asList(OrganizationMongoDBAdaptorFactory.NOTE_COLLECTION, OrganizationMongoDBAdaptorFactory.NOTE_ARCHIVE_COLLECTION), - oldIndex); - // Generate new indexes - catalogManager.installIndexes(organizationId, token); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AssociateAlignmentFiles.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AssociateAlignmentFiles.java index 875e1eff894..e5261767f6a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AssociateAlignmentFiles.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/AssociateAlignmentFiles.java @@ -1,33 +1,16 @@ package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; -import com.mongodb.client.model.Projections; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.ArrayList; -import java.util.List; - @Migration(id = "associate_alignment_files__task_5662", description = "Associate BAM files with BAI and BIGWIG files and CRAM files with CRAI files #5662", version = "4.0.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241028) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241028, + deprecatedSince = "5.0.0") public class AssociateAlignmentFiles extends MigrationTool { @Override protected void run() throws Exception { - Bson studyProjection = Projections.include(StudyDBAdaptor.QueryParams.FQN.key()); - List studyFqns = new ArrayList<>(); - queryMongo(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, new Document(), studyProjection, document -> { - studyFqns.add(document.getString(StudyDBAdaptor.QueryParams.FQN.key())); - }); - - for (String studyFqn : studyFqns) { - logger.info("Checking alignment files from study '{}'...", studyFqn); - catalogManager.getFileManager().associateAlignmentFiles(studyFqn, token); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ClinicalCvdbIndexMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ClinicalCvdbIndexMigration.java index 9cb73ade19e..f9f6a1c488c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ClinicalCvdbIndexMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ClinicalCvdbIndexMigration.java @@ -1,47 +1,16 @@ package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; -import org.opencb.opencga.core.models.clinical.CvdbIndexStatus; - -import java.util.Arrays; @Migration(id = "add_cvdb_index_to_clinical_analysis", description = "Add CVDB index status to Clinical Analysis #TASK-5610", version = "4.0.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241118) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241118, + deprecatedSince = "5.0.0") public class ClinicalCvdbIndexMigration extends MigrationTool { @Override protected void run() throws Exception { - Bson closedCaseQuery = Filters.and( - Filters.exists(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), false), - Filters.eq("status.type", ClinicalStatusValue.ClinicalStatusType.CLOSED) - ); - Bson openCaseQuery = Filters.and( - Filters.exists(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), false), - Filters.ne("status.type", ClinicalStatusValue.ClinicalStatusType.CLOSED) - ); - CvdbIndexStatus pendingCvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING); - Document pendingCvdbIndexDoc = convertToDocument(pendingCvdbIndexStatus); - Bson updateClosedCase = Updates.set(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), pendingCvdbIndexDoc); - - CvdbIndexStatus noneCvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.NONE); - Document noneCvdbIndexDoc = convertToDocument(noneCvdbIndexStatus); - Bson updateOpenCase = Updates.set(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), noneCvdbIndexDoc); - - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { - getMongoCollection(collection).updateMany(closedCaseQuery, updateClosedCase); - getMongoCollection(collection).updateMany(openCaseQuery, updateOpenCase); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/FederationChangesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/FederationChangesMigration.java index 47ce704da78..e204d1da7eb 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/FederationChangesMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/FederationChangesMigration.java @@ -1,20 +1,12 @@ package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.Arrays; -import java.util.Collections; - @Migration(id = "federationChanges__task_7192", description = "Federation changes, #TASK-7192", version = "4.0.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20250120) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20250120, + deprecatedSince = "5.0.0") public class FederationChangesMigration extends MigrationTool { /* @@ -28,42 +20,5 @@ public class FederationChangesMigration extends MigrationTool { @Override protected void run() throws Exception { - // Organization update - MongoCollection orgCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.ORGANIZATION_COLLECTION); - Bson query = Filters.exists("federation", false); - Bson update = Updates.set("federation", new Document() - .append("clients", Collections.emptyList()) - .append("servers", Collections.emptyList()) - ); - orgCollection.updateMany(query, update); - - // Project and Study - Bson projectStudyQuery = Filters.exists("federation", false); - Bson projectStudyUpdate = Updates.combine( - Updates.set("federation", new Document() - .append("id", "") - .append("description", "") - .append("version", "") - ), - Updates.set("internal.federated", false) - ); - for (String collectionStr : Arrays.asList(OrganizationMongoDBAdaptorFactory.PROJECT_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_PROJECT_COLLECTION, OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION)) { - getMongoCollection(collectionStr).updateMany(projectStudyQuery, projectStudyUpdate); - } - - // User - Bson userQuery = Filters.exists("internal.account.authentication.federation", false); - Bson userUpdate = Updates.set("internal.account.authentication.federation", false); - for (String collectionStr : Arrays.asList(OrganizationMongoDBAdaptorFactory.USER_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_USER_COLLECTION)) { - getMongoCollection(collectionStr).updateMany(userQuery, userUpdate); - } - - // Drop project id index (no longer unique) - Document oldIndex = new Document() - .append("id", 1); - dropIndex(OrganizationMongoDBAdaptorFactory.PROJECT_COLLECTION, oldIndex); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/IndexesTask6445.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/IndexesTask6445.java index 06862c5b8c8..b4f876787c6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/IndexesTask6445.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/IndexesTask6445.java @@ -5,12 +5,12 @@ @Migration(id = "new_indexes_6445", description = "New indexes from #TASK-6445", version = "4.0.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241113) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241113, + deprecatedSince = "5.0.0") public class IndexesTask6445 extends MigrationTool { @Override protected void run() throws Exception { - catalogManager.installIndexes(organizationId, token); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/JobModelChangesMigrationTask6445.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/JobModelChangesMigrationTask6445.java index aa1ade35e87..d74000ac694 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/JobModelChangesMigrationTask6445.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/JobModelChangesMigrationTask6445.java @@ -1,44 +1,17 @@ package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import java.util.Arrays; -import java.util.Collections; - @Migration(id = "job_model_changes_6445", description = "Job data model changes #TASK-6445", version = "4.0.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241113) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241113, + deprecatedSince = "5.0.0") public class JobModelChangesMigrationTask6445 extends MigrationTool { @Override protected void run() throws Exception { - Bson typeQuery = Filters.exists("type", false); - Bson typeUpdate = Updates.set("type", "NATIVE"); - - Bson requirementsQuery = Filters.and( - Filters.exists("tool.id"), - Filters.exists("tool.minimumRequirements", false) - ); - Bson requirementsUpdate = Updates.set("tool.minimumRequirements", new Document()); - - Bson dependenciesQuery = Filters.and( - Filters.exists("execution.executor"), - Filters.exists("execution.dependencies", false) - ); - Bson dependenciesUpdate = Updates.set("execution.dependencies", Collections.emptyList()); - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_JOB_COLLECTION)) { - getMongoCollection(collection).updateMany(typeQuery, typeUpdate); - getMongoCollection(collection).updateMany(requirementsQuery, requirementsUpdate); - getMongoCollection(collection).updateMany(dependenciesQuery, dependenciesUpdate); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ResourcesMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ResourcesMigration.java index 4057ed778bb..56a37f28764 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ResourcesMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ResourcesMigration.java @@ -1,80 +1,15 @@ package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.commons.datastore.core.QueryOptions; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogDBRuntimeException; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogRuntimeException; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileInternal; - -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Paths; -import java.util.Collections; @Migration(id = "add_missing_resources", description = "Add missing RESOURCES folder and dependencies #TASK-6442", version = "4.0.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241016) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241016, + deprecatedSince = "5.0.0") public class ResourcesMigration extends MigrationTool { @Override protected void run() throws Exception { - MongoCollection fileCollection = getMongoCollection(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION); - - queryMongo(OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, new Document(), - Projections.include("fqn", "uid", "uri"), studyDoc -> { - String studyFqn = studyDoc.getString("fqn"); - long studyUid = studyDoc.get("uid", Number.class).longValue(); - String studyUriStr = studyDoc.getString("uri"); - - // Check if the resources folder already exists - Bson fileQuery = Filters.and( - Filters.eq("studyUid", studyUid), - Filters.eq("path", "RESOURCES/") - ); - if (fileCollection.countDocuments(fileQuery) > 0) { - // Nothing to do. Resources folder already exists. - return; - } - - // Obtain JOBS folder to get the release number that should be associated to the RESOURCES folder - fileQuery = Filters.and( - Filters.eq("studyUid", studyUid), - Filters.eq("path", "JOBS/") - ); - Bson projection = Projections.include("release"); - try { - URI studyUri = new URI(studyUriStr); - - queryMongo(OrganizationMongoDBAdaptorFactory.FILE_COLLECTION, fileQuery, projection, fileDoc -> { - int release = fileDoc.get("release", Number.class).intValue(); - - // Create the RESOURCES folder - File file = new File("RESOURCES", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, - "RESOURCES/", Paths.get(studyUri).resolve("RESOURCES").toUri(), "Default resources folder", - FileInternal.init(), true, 0, release); - try { - dbAdaptorFactory.getCatalogFileDBAdaptor(organizationId).insert(studyUid, file, Collections.emptyList(), - Collections.emptyList(), Collections.emptyList(), QueryOptions.empty()); - logger.info("Missing RESOURCES folder created for study '{}'", studyFqn); - } catch (CatalogException e) { - throw new CatalogRuntimeException("Could not create missing RESOURCES folder for study '" + studyFqn + "'.", - e); - } - }); - } catch (CatalogDBException | URISyntaxException e) { - throw new CatalogDBRuntimeException(e); - } - - }); } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/storage/EnsureSampleIndexConfigurationIsAlwaysDefined.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/storage/EnsureSampleIndexConfigurationIsAlwaysDefined.java index b3badf02c5f..5847ca15a24 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/storage/EnsureSampleIndexConfigurationIsAlwaysDefined.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/storage/EnsureSampleIndexConfigurationIsAlwaysDefined.java @@ -1,83 +1,18 @@ package org.opencb.opencga.app.migrations.v4.v4_0_0.storage; -import org.apache.commons.collections4.CollectionUtils; import org.opencb.opencga.app.migrations.StorageMigrationTool; import org.opencb.opencga.catalog.migration.Migration; -import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; -import org.opencb.opencga.core.models.study.Study; -import org.opencb.opencga.storage.core.metadata.models.StudyMetadata; -import org.opencb.opencga.storage.core.variant.VariantStorageEngine; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.function.Supplier; @Migration(id = "ensure_sample_index_configuration_is_defined", description = "Ensure that the SampleIndexConfiguration object is correctly defined. #TASK-6765", version = "4.0.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.STORAGE, patch = 2, - date = 20240910) + date = 20240910, + deprecatedSince = "5.0.0") public class EnsureSampleIndexConfigurationIsAlwaysDefined extends StorageMigrationTool { @Override protected void run() throws Exception { - - for (String variantStorageProject : getVariantStorageProjects()) { - VariantStorageEngine engine = getVariantStorageEngineByProject(variantStorageProject); - logger.info("Checking project '" + variantStorageProject + "'"); - if (engine.getMetadataManager().exists()) { - logger.info("Project '" + variantStorageProject + "' exists in variant storage. Checking studies."); - for (Integer studyId : engine.getMetadataManager().getStudyIds()) { - StudyMetadata studyMetadata = engine.getMetadataManager().getStudyMetadata(studyId); - if (CollectionUtils.isEmpty(studyMetadata.getSampleIndexConfigurations())) { - engine.getMetadataManager().updateStudyMetadata(studyId, sm -> { - if (CollectionUtils.isEmpty(sm.getSampleIndexConfigurations())) { - List configurations = new ArrayList<>(1); - logger.info("Creating default SampleIndexConfiguration for study '" + studyMetadata.getName() + "'" - + " (" + studyId + ")"); - configurations.add(new StudyMetadata.SampleIndexConfigurationVersioned( - preFileDataConfiguration(), - StudyMetadata.DEFAULT_SAMPLE_INDEX_VERSION, - Date.from(Instant.now()), StudyMetadata.SampleIndexConfigurationVersioned.Status.ACTIVE)); - sm.setSampleIndexConfigurations(configurations); - } - }); - } - Study study = catalogManager.getStudyManager().get(studyMetadata.getName(), null, token).first(); - SampleIndexConfiguration sampleIndexConfFromStorage = studyMetadata.getSampleIndexConfigurationLatest().getConfiguration(); - SampleIndexConfiguration sampleIndexConfFromCatalog = orDefault(() -> study.getInternal() - .getConfiguration().getVariantEngine().getSampleIndex(), null); - - if (sampleIndexConfFromStorage.equals(sampleIndexConfFromCatalog)) { - logger.info("SampleIndexConfiguration for study '" + studyMetadata.getName() + "' (" + studyId + ") is up to date."); - } else { - logger.info("Updating SampleIndexConfiguration for study '" + studyMetadata.getName() + "' (" + studyId + ")"); - catalogManager.getStudyManager().setVariantEngineConfigurationSampleIndex(studyMetadata.getName(), - sampleIndexConfFromStorage, token); - } - } - } else { - logger.info("Project '" + variantStorageProject + "' does not exist in variant storage. Skipping."); - } - } - } - - public static T orDefault(Supplier supplier, T defaultValue) { - try { - return supplier.get(); - } catch (Exception e) { - return defaultValue; - } - } - - public static SampleIndexConfiguration preFileDataConfiguration() { - // If missing, it was assuming cellbase v5 - SampleIndexConfiguration sampleIndexConfiguration = SampleIndexConfiguration.defaultConfiguration(false); - sampleIndexConfiguration.getFileDataConfiguration().setIncludeOriginalCall(false); - sampleIndexConfiguration.getFileDataConfiguration().setIncludeSecondaryAlternates(false); - return sampleIndexConfiguration; } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ExternalToolTask7610Migration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/catalog/ExternalToolTask7610Migration.java similarity index 96% rename from opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ExternalToolTask7610Migration.java rename to opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/catalog/ExternalToolTask7610Migration.java index 0acfe81353a..60891362043 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ExternalToolTask7610Migration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/catalog/ExternalToolTask7610Migration.java @@ -1,4 +1,4 @@ -package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; +package org.opencb.opencga.app.migrations.v5.catalog; import com.mongodb.client.model.Filters; import com.mongodb.client.model.UpdateOneModel; @@ -12,7 +12,7 @@ import java.util.Arrays; @Migration(id = "externaltool__task_7610", - description = "Extend Workflow data model for new ExternalTool, #TASK-7610", version = "4.0.0", + description = "Extend Workflow data model for new ExternalTool, #TASK-7610", version = "5.0.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20250506) public class ExternalToolTask7610Migration extends MigrationTool { From b264089d6ddaad5bb75ecab51566acf09971edc4 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 21 May 2025 11:06:14 +0200 Subject: [PATCH 08/22] analysis: implement new options from CustomToolExecutor, #TASK-7610 --- .../customTool/CustomToolExecutor.java | 112 +++++++++++++++--- .../analysis/workflow/NextFlowExecutor.java | 28 ++--- .../executors/WorkflowsCommandExecutor.java | 14 +-- .../ExternalToolTask7610Migration.java | 2 +- .../catalog/managers/ExternalToolManager.java | 35 +++--- .../opencga/catalog/utils/InputFileUtils.java | 80 ++++++++++++- opencga-client/src/main/R/R/Job-methods.R | 2 +- .../src/main/R/R/Workflow-methods.R | 2 +- .../client/rest/clients/JobClient.java | 2 +- .../client/rest/clients/WorkflowClient.java | 6 +- opencga-client/src/main/javascript/Job.js | 2 +- .../src/main/javascript/Workflow.js | 2 +- .../pyopencga/rest_clients/job_client.py | 2 +- .../pyopencga/rest_clients/workflow_client.py | 2 +- .../DeprecatedWorkflowRunParams.java | 59 +++++++++ .../externalTool/ExternalToolRunParams.java | 34 ++++++ .../custom/CustomToolRunParams.java | 44 +++++++ .../server/rest/ExternalToolWSServer.java | 32 +++-- .../opencga/server/rest/JobWSServer.java | 4 +- .../opencga/server/rest/WorkflowWSServer.java | 6 +- 20 files changed, 388 insertions(+), 82 deletions(-) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/DeprecatedWorkflowRunParams.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolRunParams.java diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolExecutor.java index 8fa8275a49d..e22f92f1e20 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/customTool/CustomToolExecutor.java @@ -1,14 +1,21 @@ package org.opencb.opencga.analysis.customTool; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.tools.OpenCgaDockerToolScopeStudy; +import org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor; +import org.opencb.opencga.catalog.exceptions.CatalogException; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.externalTool.Docker; +import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.models.job.ToolInfoExecutor; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.tools.ToolDependency; import org.opencb.opencga.core.tools.annotations.Tool; import org.opencb.opencga.core.tools.annotations.ToolParams; import org.slf4j.Logger; @@ -23,11 +30,15 @@ public class CustomToolExecutor extends OpenCgaDockerToolScopeStudy { public static final String DESCRIPTION = "Execute an analysis from a custom binary."; @ToolParams - protected Docker runParams = new Docker(); + protected ExternalToolRunParams runParams = new ExternalToolRunParams(); private String cliParams; private String dockerImage; + // Docker credentials + private String username; + private String password; + private final static Logger logger = LoggerFactory.getLogger(CustomToolExecutor.class); @Override @@ -36,43 +47,110 @@ protected void check() throws Exception { // Check any condition if (runParams == null) { - throw new ToolException("Missing runParams"); + throw new ToolException("Missing ExternalToolRunParams object."); + } else if (StringUtils.isNotEmpty(runParams.getId()) && runParams.getDocker() == null) { + Docker docker = generateDockerObject(runParams); + checkDockerObject(docker); + } else if (runParams.getDocker() != null && StringUtils.isEmpty(runParams.getId())) { + checkDockerObject(runParams.getDocker()); + } else { + throw new ToolException("Unexpected ExternalToolRunParams object. " + + "Either 'id' and 'params/commandLine' or 'docker' should be set."); + } + } + + private Docker generateDockerObject(ExternalToolRunParams runParams) throws CatalogException, ToolException { + OpenCGAResult result; + if (runParams.getVersion() != null) { + Query query = new Query(ExternalToolDBAdaptor.QueryParams.VERSION.key(), runParams.getVersion()); + result = catalogManager.getExternalToolManager().get(study, Collections.singletonList(runParams.getId()), query, + QueryOptions.empty(), false, token); + } else { + result = catalogManager.getExternalToolManager().get(study, runParams.getId(), QueryOptions.empty(), token); + } + if (result.getNumResults() == 0) { + throw new ToolException("Custom tool '" + runParams.getId() + "' not found"); + } + ExternalTool externalTool = result.first(); + + if (externalTool == null) { + throw new ToolException("Custom tool '" + runParams.getId() + "' is null"); + } + if (externalTool.getType() != ExternalToolType.CUSTOM_TOOL) { + throw new ToolException("External tool '" + runParams.getId() + "' is not of type " + ExternalToolType.CUSTOM_TOOL); + } + if (externalTool.getDocker() == null) { + throw new ToolException("External tool '" + runParams.getId() + "' does not have a docker object"); } - if (StringUtils.isEmpty(runParams.getCommandLine())) { + + Docker docker = externalTool.getDocker(); + String commandLine = StringUtils.isNotEmpty(runParams.getCommandLine()) + ? runParams.getCommandLine() + : docker.getCommandLine(); + + // Process docker command line to replace variables + Map params = new HashMap<>(); + if (runParams.getParams() != null) { + params.putAll(runParams.getParams()); + } + if (CollectionUtils.isNotEmpty(externalTool.getVariables())) { + for (ExternalToolVariable variable : externalTool.getVariables()) { + String variableId = removePrefix(variable.getId()); + if (!params.containsKey(variableId) && StringUtils.isNotEmpty(variable.getDefaultValue())) { + params.put(variableId, variable.getDefaultValue()); + } + } + } + String processedCli = inputFileUtils.processCommandLine(study, commandLine, params, temporalInputDir, dockerInputBindings, + getOutDir().toString(), token); + docker.setCommandLine(processedCli); + + return docker; + } + + private void checkDockerObject(Docker docker) throws ToolException, CatalogException { + if (StringUtils.isEmpty(docker.getCommandLine())) { throw new ToolException("Missing commandLine"); } - if (StringUtils.isEmpty(runParams.getName())) { - runParams.setName("opencb/opencga-ext-tools"); - runParams.setTag(GitRepositoryState.getInstance().getBuildVersion()); + if (StringUtils.isEmpty(docker.getName())) { + docker.setName("opencb/opencga-ext-tools"); + docker.setTag(GitRepositoryState.getInstance().getBuildVersion()); } - if (!runParams.getName().contains("/")) { + if (!docker.getName().contains("/")) { throw new ToolException("Missing repository organization. Format for the docker image should be 'organization/image'"); } - this.dockerImage = runParams.getName(); - if (StringUtils.isNotEmpty(runParams.getTag())) { - this.dockerImage += ":" + runParams.getTag(); + this.dockerImage = docker.getName(); + if (StringUtils.isNotEmpty(docker.getTag())) { + this.dockerImage += ":" + docker.getTag(); } // Update job tags and attributes - ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(runParams.getName(), runParams.getTag()); + ToolInfoExecutor toolInfoExecutor = new ToolInfoExecutor(docker.getName(), docker.getTag()); List tags = new LinkedList<>(); tags.add(ID); tags.add(this.dockerImage); + + List dependencyList = new ArrayList<>(1); + dependencyList.add(new ToolDependency(docker.getName(), docker.getTag())); + addDependencies(dependencyList); updateJobInformation(tags, toolInfoExecutor); - StringBuilder cliParamsBuilder = new StringBuilder(); - processInputParams(runParams.getCommandLine(), cliParamsBuilder); - this.cliParams = cliParamsBuilder.toString(); + this.cliParams = docker.getCommandLine(); + this.username = docker.getUser(); + this.password = docker.getPassword(); } @Override protected void run() throws Exception { StopWatch stopWatch = StopWatch.createStarted(); + List> outputBindings = new ArrayList<>(1); + String outDirPath = getOutDir().toAbsolutePath().toString(); + outputBindings.add(new AbstractMap.SimpleEntry<>(outDirPath, outDirPath)); + Map dockerParams = new HashMap<>(); dockerParams.put("-e", "OPENCGA_TOKEN=" + getExpiringToken()); - String cmdline = runDocker(dockerImage, Collections.emptyList(), cliParams, dockerParams, null, - runParams.getUser(), runParams.getPassword()); + String cmdline = runDocker(dockerImage, outputBindings, cliParams, dockerParams, null, username, password); logger.info("Docker command line: {}", cmdline); logger.info("Execution time: {}", TimeUtils.durationToString(stopWatch)); diff --git a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java index aaefd47eeb7..db1944af6d5 100644 --- a/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java +++ b/opencga-analysis/src/main/java/org/opencb/opencga/analysis/workflow/NextFlowExecutor.java @@ -11,14 +11,16 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.opencga.analysis.tools.OpenCgaDockerToolScopeStudy; import org.opencb.opencga.catalog.db.api.ExternalToolDBAdaptor; -import org.opencb.opencga.catalog.utils.InputFileUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.GitRepositoryState; import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.exceptions.ToolException; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.externalTool.*; +import org.opencb.opencga.core.models.externalTool.ExternalTool; +import org.opencb.opencga.core.models.externalTool.ExternalToolRunParams; +import org.opencb.opencga.core.models.externalTool.ExternalToolVariable; +import org.opencb.opencga.core.models.externalTool.WorkflowScript; import org.opencb.opencga.core.models.job.ToolInfoExecutor; import org.opencb.opencga.core.response.OpenCGAResult; import org.opencb.opencga.core.tools.ToolDependency; @@ -48,7 +50,7 @@ public class NextFlowExecutor extends OpenCgaDockerToolScopeStudy { public static final String DESCRIPTION = "Execute a Nextflow analysis."; @ToolParams - protected ExternalToolRunParams externalToolRunParams = new ExternalToolRunParams(); + protected ExternalToolRunParams runParams = new ExternalToolRunParams(); private ExternalTool externalTool; private String cliParams; @@ -65,27 +67,25 @@ public class NextFlowExecutor extends OpenCgaDockerToolScopeStudy { protected void check() throws Exception { super.check(); - if (externalToolRunParams.getId() == null) { + if (runParams.getId() == null) { throw new IllegalArgumentException("Missing Nextflow ID"); } - InputFileUtils inputFileUtils = new InputFileUtils(catalogManager); - OpenCGAResult result; - if (externalToolRunParams.getVersion() != null) { - Query query = new Query(ExternalToolDBAdaptor.QueryParams.VERSION.key(), externalToolRunParams.getVersion()); - result = catalogManager.getExternalToolManager().get(study, Collections.singletonList(externalToolRunParams.getId()), query, + if (runParams.getVersion() != null) { + Query query = new Query(ExternalToolDBAdaptor.QueryParams.VERSION.key(), runParams.getVersion()); + result = catalogManager.getExternalToolManager().get(study, Collections.singletonList(runParams.getId()), query, QueryOptions.empty(), false, token); } else { - result = catalogManager.getExternalToolManager().get(study, externalToolRunParams.getId(), QueryOptions.empty(), token); + result = catalogManager.getExternalToolManager().get(study, runParams.getId(), QueryOptions.empty(), token); } if (result.getNumResults() == 0) { - throw new ToolException("Workflow '" + externalToolRunParams.getId() + "' not found"); + throw new ToolException("Workflow '" + runParams.getId() + "' not found"); } externalTool = result.first(); if (externalTool == null) { - throw new ToolException("Workflow '" + externalToolRunParams.getId() + "' is null"); + throw new ToolException("Workflow '" + runParams.getId() + "' is null"); } outDirPath = getOutDir().toAbsolutePath().toString(); @@ -130,8 +130,8 @@ protected void check() throws Exception { updateJobInformation(new ArrayList<>(tags), toolInfoExecutor); StringBuilder cliParamsBuilder = new StringBuilder(); - if (MapUtils.isNotEmpty(externalToolRunParams.getParams())) { - for (Map.Entry entry : externalToolRunParams.getParams().entrySet()) { + if (MapUtils.isNotEmpty(runParams.getParams())) { + for (Map.Entry entry : runParams.getParams().entrySet()) { String variableId = removePrefix(entry.getKey()); // Remove from the mandatoryParams set mandatoryParams.remove(variableId); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java index c6f67676d02..09c693490d4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java @@ -14,11 +14,11 @@ import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.exceptions.ClientException; import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.externalTool.DeprecatedWorkflowRunParams; import org.opencb.opencga.core.models.externalTool.ExternalTool; import org.opencb.opencga.core.models.externalTool.ExternalToolAclEntryList; import org.opencb.opencga.core.models.externalTool.ExternalToolAclUpdateParams; import org.opencb.opencga.core.models.externalTool.ExternalToolInternal; -import org.opencb.opencga.core.models.externalTool.ExternalToolRunParams; import org.opencb.opencga.core.models.externalTool.ExternalToolScope; import org.opencb.opencga.core.models.externalTool.Workflow; import org.opencb.opencga.core.models.externalTool.WorkflowRepository; @@ -268,26 +268,26 @@ private RestResponse run() throws Exception { } - ExternalToolRunParams externalToolRunParams = null; + DeprecatedWorkflowRunParams deprecatedWorkflowRunParams = null; if (commandOptions.jsonDataModel) { RestResponse res = new RestResponse<>(); res.setType(QueryType.VOID); PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/workflows/run")); return res; } else if (commandOptions.jsonFile != null) { - externalToolRunParams = JacksonUtils.getDefaultObjectMapper() - .readValue(new java.io.File(commandOptions.jsonFile), ExternalToolRunParams.class); + deprecatedWorkflowRunParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), DeprecatedWorkflowRunParams.class); } else { ObjectMap beanParams = new ObjectMap(); putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); putNestedIfNotNull(beanParams, "version", commandOptions.version, true); putNestedMapIfNotEmpty(beanParams, "params", commandOptions.params, true); - externalToolRunParams = JacksonUtils.getDefaultObjectMapper().copy() + deprecatedWorkflowRunParams = JacksonUtils.getDefaultObjectMapper().copy() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) - .readValue(beanParams.toJson(), ExternalToolRunParams.class); + .readValue(beanParams.toJson(), DeprecatedWorkflowRunParams.class); } - return openCGAClient.getWorkflowClient().run(externalToolRunParams, queryParams); + return openCGAClient.getWorkflowClient().run(deprecatedWorkflowRunParams, queryParams); } private RestResponse search() throws Exception { diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/catalog/ExternalToolTask7610Migration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/catalog/ExternalToolTask7610Migration.java index 60891362043..e3c0869d968 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/catalog/ExternalToolTask7610Migration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/catalog/ExternalToolTask7610Migration.java @@ -11,7 +11,7 @@ import java.util.Arrays; -@Migration(id = "externaltool__task_7610", +@Migration(id = "external_tool__task_7610", description = "Extend Workflow data model for new ExternalTool, #TASK-7610", version = "5.0.0", language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20250506) public class ExternalToolTask7610Migration extends MigrationTool { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java index af2afecd8c8..5e35f8d3b1c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java @@ -26,6 +26,7 @@ import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.models.externalTool.custom.CustomToolCreateParams; +import org.opencb.opencga.core.models.externalTool.custom.CustomToolRunParams; import org.opencb.opencga.core.models.externalTool.custom.CustomToolUpdateParams; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowUpdateParams; @@ -230,30 +231,35 @@ public OpenCGAResult createCustomTool(String studyStr, CustomToolC } } - public OpenCGAResult submitWorkflow(String studyStr, ExternalToolRunParams runParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, - Boolean dryRun, String token) throws CatalogException { + public OpenCGAResult submitWorkflow(String studyStr, String externalToolId, Integer version, Map params, + String jobId, String jobDescription, String jobDependsOnStr, String jobTagsStr, + String jobScheduledStartTime, String jobPriority, Boolean dryRun, String token) + throws CatalogException { + ExternalToolRunParams runParams = new ExternalToolRunParams(externalToolId, version, params); return submit(studyStr, runParams, ExternalToolType.WORKFLOW, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun, token); } - public OpenCGAResult submitCustomTool(String studyStr, ExternalToolRunParams runParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, - Boolean dryRun, String token) throws CatalogException { + public OpenCGAResult submitCustomTool(String studyStr, String externalToolId, Integer version, CustomToolRunParams params, + String jobId, String jobDescription, String jobDependsOnStr, String jobTagsStr, + String jobScheduledStartTime, String jobPriority, Boolean dryRun, String token) + throws CatalogException { + ExternalToolRunParams runParams = new ExternalToolRunParams(externalToolId, version, params.getParams(), params.getCommandLine()); return submit(studyStr, runParams, ExternalToolType.CUSTOM_TOOL, jobId, jobDescription, jobDependsOnStr, jobTagsStr, jobScheduledStartTime, jobPriority, dryRun, token); } - public OpenCGAResult submitVariantWalker(String studyStr, ExternalToolRunParams runParams, String jobId, String jobDescription, - String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, String jobPriority, - Boolean dryRun, String token) throws CatalogException { - return submit(studyStr, runParams, ExternalToolType.VARIANT_WALKER, jobId, jobDescription, jobDependsOnStr, jobTagsStr, - jobScheduledStartTime, jobPriority, dryRun, token); - } +// public OpenCGAResult submitVariantWalker(String studyStr, String externalToolId, Integer version, Map params, +// String jobId, String jobDescription, String jobDependsOnStr, String jobTagsStr, +// String jobScheduledStartTime, String jobPriority, Boolean dryRun, String token) +// throws CatalogException { +// return submit(studyStr, externalToolId, version, params, ExternalToolType.VARIANT_WALKER, jobId, jobDescription, jobDependsOnStr, +// jobTagsStr, jobScheduledStartTime, jobPriority, dryRun, token); +// } private OpenCGAResult submit(String studyStr, ExternalToolRunParams runParams, ExternalToolType type, String jobId, - String jobDescription, String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, - String jobPriority, Boolean dryRun, String token) throws CatalogException { + String jobDescription, String jobDependsOnStr, String jobTagsStr, String jobScheduledStartTime, + String jobPriority, Boolean dryRun, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); @@ -276,6 +282,7 @@ private OpenCGAResult submit(String studyStr, ExternalToolRunParams runPara : configuration.getAnalysis().getWorkflow().getMinRequirements()) .setDescription(externalTool.getDescription()); + // Build ExternalToolRunParams Map paramsMap = runParams.toParams(); paramsMap.putIfAbsent(ParamConstants.STUDY_PARAM, study.getFqn()); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/InputFileUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/InputFileUtils.java index 11ca9cc0bd0..bdae14bd7f0 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/InputFileUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/InputFileUtils.java @@ -11,9 +11,7 @@ import java.io.*; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -23,6 +21,8 @@ public class InputFileUtils { private static final Pattern OPENCGA_PATH_PATTERN = Pattern.compile("^(?i)(ocga://|opencga://|file://)(.+)$"); private static final Pattern OPENCGA_PATH_IN_LINE_PATTERN = Pattern.compile("(?i)(ocga://|opencga://|file://)([^\\s,;]+)"); + + private static final Pattern VARIABLE_PATTERN = Pattern.compile("\\$\\{([^{}]+)}"); private static final String OUTPUT = "$OUTPUT"; private static final List OUTPUT_LIST = Arrays.asList("$OUTPUT", "$JOB_OUTPUT"); private static final String FLAG = "$FLAG"; @@ -131,4 +131,78 @@ public String appendSubpath(String path, String subpath) { } } + /** + * Given a command line that may contain OpenCGA file paths and variables using between ${xx}, obtain the final command line. + * + * Example: + * commandLineTemplate: "run ${INPUT} -o ${OUTPUT}" + * params: {INPUT=ocga://user@study/1.txt, OUTPUT=/tmp/output.txt} + * Result: "run /path/to/1.txt -o /tmp/output.txt" + * + * @param study Study where the files are located. + * @param commandLineTemplate Command line template. + * @param params Map with the parameters to be replaced in the command line template. + * @param temporalInputDir Empty directory where files that require any changes will be copied. + * @param inputBindings Map where it will store the input bindings that need to be mounted in the Docker. + * @param outDir Output directory. + * @param token User token. + * @return Final command line with the parameters replaced. + * @throws CatalogException If any error occurs while processing the command line. + */ + public String processCommandLine(String study, String commandLineTemplate, Map params, Path temporalInputDir, + List> inputBindings, String outDir, String token) + throws CatalogException { + // Replace variables + Matcher matcher = VARIABLE_PATTERN.matcher(commandLineTemplate); + StringBuffer sb = new StringBuffer(); + while (matcher.find()) { + String variableName = matcher.group(1); + if (params.containsKey(variableName)) { + throw new CatalogException("Variable '" + variableName + "' not found in the params object"); + } + String replacement = params.get(variableName); + matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement)); + } + matcher.appendTail(sb); + String variablesReplaced = sb.toString(); + + // Replace input variables + matcher = OPENCGA_PATH_IN_LINE_PATTERN.matcher(variablesReplaced); + sb = new StringBuffer(); + while (matcher.find()) { + String group = matcher.group(0); + File file = findOpenCGAFileFromPattern(study, group, token); + + if (fileMayContainReferencesToOtherFiles(file)) { + Path outputFile = temporalInputDir.resolve(file.getName()); + List files = findAndReplaceFilePathToUrisFromFile(study, file, outputFile, token); + + // Write outputFile as inputBinding + inputBindings.add(new AbstractMap.SimpleEntry<>(outputFile.toString(), outputFile.toString())); + logger.info("Params: OpenCGA input file: '{}'", outputFile); + matcher.appendReplacement(sb, Matcher.quoteReplacement(outputFile.toString())); + + // Add files to inputBindings to ensure they are also mounted (if any) + for (File tmpFile : files) { + inputBindings.add(new AbstractMap.SimpleEntry<>(tmpFile.getUri().getPath(), tmpFile.getUri().getPath())); + logger.info("Inner files from '{}': OpenCGA input file: '{}'", outputFile, tmpFile.getUri().getPath()); + } + } else { + String path = file.getUri().getPath(); + inputBindings.add(new AbstractMap.SimpleEntry<>(path, path)); + logger.info("Params: OpenCGA input file: '{}'", path); + matcher.appendReplacement(sb, Matcher.quoteReplacement(path)); + } + } + matcher.appendTail(sb); + String finalCli = sb.toString(); + + // Replace output variables + for (String outputVariable : OUTPUT_LIST) { + finalCli = finalCli.replaceAll(outputVariable, outDir); + } + + return finalCli; + } + } diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index 29ae224c800..129351da8b9 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -187,7 +187,7 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpoi #' @param jobScheduledStartTime Time when the job is scheduled to start. #' @param jobPriority Priority of the job. #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. - #' @param data External tool run parameters. + #' @param data Workflow tool run parameters. runTool=fetchOpenCGA(object=OpencgaR, category="jobs", categoryId=NULL, subcategory="tool", subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/R/R/Workflow-methods.R b/opencga-client/src/main/R/R/Workflow-methods.R index c8a2b7a1e9a..991b35ed3c6 100644 --- a/opencga-client/src/main/R/R/Workflow-methods.R +++ b/opencga-client/src/main/R/R/Workflow-methods.R @@ -97,7 +97,7 @@ setMethod("workflowClient", "OpencgaR", function(OpencgaR, members, workflowId, #' @param jobScheduledStartTime Time when the job is scheduled to start. #' @param jobPriority Priority of the job. #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. - #' @param data External tool run parameters. + #' @param data Workflow tool run parameters. run=fetchOpenCGA(object=OpencgaR, category="workflows", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index 9a94ba413c5..9de7218019a 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -251,7 +251,7 @@ public RestResponse buildTool(JobToolBuildParams data, ObjectMap params) th /** * Execute an analysis from a custom binary. - * @param data External tool run parameters. + * @param data Workflow tool run parameters. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java index 93be5408ec7..99889fe1c07 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/WorkflowClient.java @@ -22,10 +22,10 @@ import org.opencb.opencga.core.client.ParentClient; import org.opencb.opencga.core.config.client.ClientConfiguration; import org.opencb.opencga.core.exceptions.ClientException; +import org.opencb.opencga.core.models.externalTool.DeprecatedWorkflowRunParams; import org.opencb.opencga.core.models.externalTool.ExternalTool; import org.opencb.opencga.core.models.externalTool.ExternalToolAclEntryList; import org.opencb.opencga.core.models.externalTool.ExternalToolAclUpdateParams; -import org.opencb.opencga.core.models.externalTool.ExternalToolRunParams; import org.opencb.opencga.core.models.externalTool.WorkflowRepositoryParams; import org.opencb.opencga.core.models.externalTool.workflow.DeprecatedWorkflowUpdateParams; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; @@ -136,7 +136,7 @@ public RestResponse importWorkflow(WorkflowRepositoryParams data, /** * Execute a workflow analysis. - * @param data External tool run parameters. + * @param data Workflow tool run parameters. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. @@ -150,7 +150,7 @@ public RestResponse importWorkflow(WorkflowRepositoryParams data, * @return a RestResponse object. * @throws ClientException ClientException if there is any server error. */ - public RestResponse run(ExternalToolRunParams data, ObjectMap params) throws ClientException { + public RestResponse run(DeprecatedWorkflowRunParams data, ObjectMap params) throws ClientException { params = params != null ? params : new ObjectMap(); params.put("body", data); return execute("workflows", null, null, null, "run", params, POST, Job.class); diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index b8f7ab15d62..43a10255a18 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -209,7 +209,7 @@ export default class Job extends OpenCGAParentClass { } /** Execute an analysis from a custom binary. - * @param {Object} data - External tool run parameters. + * @param {Object} data - Workflow tool run parameters. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not diff --git a/opencga-client/src/main/javascript/Workflow.js b/opencga-client/src/main/javascript/Workflow.js index 70645ad3655..41495c54179 100644 --- a/opencga-client/src/main/javascript/Workflow.js +++ b/opencga-client/src/main/javascript/Workflow.js @@ -99,7 +99,7 @@ export default class Workflow extends OpenCGAParentClass { } /** Execute a workflow analysis - * @param {Object} data - External tool run parameters. + * @param {Object} data - Workflow tool run parameters. * @param {Object} [params] - The Object containing the following optional parameters: * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 5680548587c..8f0714751fa 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -265,7 +265,7 @@ def run_tool(self, data=None, **options): Execute an analysis from a custom binary. PATH: /{apiVersion}/jobs/tool/run - :param dict data: External tool run parameters. (REQUIRED) + :param dict data: Workflow tool run parameters. (REQUIRED) :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/workflow_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/workflow_client.py index 9f86abba04b..bfcf079632c 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/workflow_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/workflow_client.py @@ -118,7 +118,7 @@ def run(self, data=None, **options): Execute a workflow analysis. PATH: /{apiVersion}/workflows/run - :param dict data: External tool run parameters. (REQUIRED) + :param dict data: Workflow tool run parameters. (REQUIRED) :param str study: Study [[organization@]project:]study where study and project can be either the ID or UUID. :param str job_id: Job ID. It must be a unique string within the diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/DeprecatedWorkflowRunParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/DeprecatedWorkflowRunParams.java new file mode 100644 index 00000000000..603c27ae77a --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/DeprecatedWorkflowRunParams.java @@ -0,0 +1,59 @@ +package org.opencb.opencga.core.models.externalTool; + +import java.util.Map; + +@Deprecated +public class DeprecatedWorkflowRunParams { + + public static final String DESCRIPTION = "Workflow tool run parameters"; + + private String id; + private Integer version; + private Map params; + + public DeprecatedWorkflowRunParams() { + } + + public DeprecatedWorkflowRunParams(String id, Integer version, Map params) { + this.id = id; + this.version = version; + this.params = params; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("DeprecatedWorkflowRunParams{"); + sb.append("id='").append(id).append('\''); + sb.append(", version=").append(version); + sb.append(", params=").append(params); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public DeprecatedWorkflowRunParams setId(String id) { + this.id = id; + return this; + } + + public Integer getVersion() { + return version; + } + + public DeprecatedWorkflowRunParams setVersion(Integer version) { + this.version = version; + return this; + } + + public Map getParams() { + return params; + } + + public DeprecatedWorkflowRunParams setParams(Map params) { + this.params = params; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolRunParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolRunParams.java index 4913e9510c4..e5e5a5b3b66 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolRunParams.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/ExternalToolRunParams.java @@ -11,6 +11,9 @@ public class ExternalToolRunParams extends ToolParams { private String id; private Integer version; private Map params; + private String commandLine; + + private Docker docker; public ExternalToolRunParams() { } @@ -21,12 +24,25 @@ public ExternalToolRunParams(String id, Integer version, Map par this.params = params; } + public ExternalToolRunParams(String id, Integer version, Map params, String commandLine) { + this.id = id; + this.version = version; + this.params = params; + this.commandLine = commandLine; + } + + public ExternalToolRunParams(Docker docker) { + this.docker = docker; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("ExternalToolRunParams{"); sb.append("id='").append(id).append('\''); sb.append(", version=").append(version); sb.append(", params=").append(params); + sb.append(", commandLine='").append(commandLine).append('\''); + sb.append(", docker=").append(docker); sb.append('}'); return sb.toString(); } @@ -57,4 +73,22 @@ public ExternalToolRunParams setParams(Map params) { this.params = params; return this; } + + public String getCommandLine() { + return commandLine; + } + + public ExternalToolRunParams setCommandLine(String commandLine) { + this.commandLine = commandLine; + return this; + } + + public Docker getDocker() { + return docker; + } + + public ExternalToolRunParams setDocker(Docker docker) { + this.docker = docker; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolRunParams.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolRunParams.java new file mode 100644 index 00000000000..fe0cad28729 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/externalTool/custom/CustomToolRunParams.java @@ -0,0 +1,44 @@ +package org.opencb.opencga.core.models.externalTool.custom; + +import java.util.Map; + +public class CustomToolRunParams { + + private String commandLine; + private Map params; + + public CustomToolRunParams() { + } + + public CustomToolRunParams(String commandLine, Map params) { + this.commandLine = commandLine; + this.params = params; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CustomToolRunParams{"); + sb.append("commandLine='").append(commandLine).append('\''); + sb.append(", params=").append(params); + sb.append('}'); + return sb.toString(); + } + + public String getCommandLine() { + return commandLine; + } + + public CustomToolRunParams setCommandLine(String commandLine) { + this.commandLine = commandLine; + return this; + } + + public Map getParams() { + return params; + } + + public CustomToolRunParams setParams(Map params) { + this.params = params; + return this; + } +} diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java index 645a0bceb35..7fc804cea6f 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java @@ -11,10 +11,14 @@ import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.externalTool.*; import org.opencb.opencga.core.models.externalTool.custom.CustomToolCreateParams; +import org.opencb.opencga.core.models.externalTool.custom.CustomToolRunParams; import org.opencb.opencga.core.models.externalTool.custom.CustomToolUpdateParams; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; import org.opencb.opencga.core.models.externalTool.workflow.WorkflowUpdateParams; -import org.opencb.opencga.core.models.job.*; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.job.JobToolBuildParams; +import org.opencb.opencga.core.models.job.JobType; +import org.opencb.opencga.core.models.job.ToolInfo; import org.opencb.opencga.core.tools.annotations.*; import javax.servlet.http.HttpServletRequest; @@ -22,6 +26,7 @@ import javax.ws.rs.core.*; import java.io.IOException; import java.util.List; +import java.util.Map; import static org.opencb.opencga.core.api.ParamConstants.JOB_DEPENDS_ON; @@ -260,10 +265,11 @@ public Response runCustomTool( @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, - @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) JobRunParams params) { + @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) Docker dockerParams) { ToolInfo toolInfo = new ToolInfo() - .setId(params.getDocker().getId()); - return submitJob(study, JobType.CUSTOM, toolInfo, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, + .setId(dockerParams.getName()); + ExternalToolRunParams runParams = new ExternalToolRunParams(dockerParams); + return submitJob(study, JobType.CUSTOM, toolInfo, runParams, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun); } @@ -272,6 +278,8 @@ public Response runCustomTool( @Consumes(MediaType.APPLICATION_JSON) @ApiOperation(value = CustomToolExecutor.DESCRIPTION, response = Job.class) public Response runCustomToolByToolId( + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION, required = true) @PathParam("toolId") String toolId, + @ApiParam(value = "Tool version. If not provided, the latest version will be used.") @QueryParam(ParamConstants.EXTERNAL_TOOL_VERSION_PARAM) Integer version, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @@ -280,8 +288,9 @@ public Response runCustomToolByToolId( @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, - @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) JobRunParams params) { - return null; + @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) CustomToolRunParams params) { + return run(() -> catalogManager.getExternalToolManager().submitCustomTool(study, toolId, version, params, jobName, jobDescription, + dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, token)); } // ********************************** WORKFLOW WS ENDPOINTS ********************************** @@ -315,7 +324,6 @@ public Response createWorkflow( public Response updateWorkflow( @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION, required = true) @PathParam("toolId") String toolId, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, -// @ApiParam(value = ParamConstants.WORKFLOW_SCRIPTS_ACTION_DESCRIPTION, allowableValues = "ADD,SET,REMOVE", defaultValue = "ADD") @QueryParam(ParamConstants.WORKFLOW_SCRIPTS_ACTION_PARAM) ParamUtils.BasicUpdateAction workflowScriptsAction, @ApiParam(value = ParamConstants.INCLUDE_RESULT_DESCRIPTION, defaultValue = "false") @QueryParam(ParamConstants.INCLUDE_RESULT_PARAM) boolean includeResult, @ApiParam(value = "body") WorkflowUpdateParams parameters) { try { @@ -336,10 +344,12 @@ public Response importWorkflow( } @POST - @Path("/workflow/{workflowId}/run") + @Path("/workflow/{toolId}/run") @Consumes(MediaType.APPLICATION_JSON) @ApiOperation(value = "Execute an external tool of type WORKFLOW", response = Job.class) public Response executeWorkflow( + @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION, required = true) @PathParam("toolId") String toolId, + @ApiParam(value = "Tool version. If not provided, the latest version will be used.") @QueryParam(ParamConstants.EXTERNAL_TOOL_VERSION_PARAM) Integer version, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = ParamConstants.JOB_ID_CREATION_DESCRIPTION) @QueryParam(ParamConstants.JOB_ID) String jobName, @ApiParam(value = ParamConstants.JOB_DESCRIPTION_DESCRIPTION) @QueryParam(ParamConstants.JOB_DESCRIPTION) String jobDescription, @@ -348,9 +358,9 @@ public Response executeWorkflow( @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, - @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) ExternalToolRunParams params) { - return run(() -> catalogManager.getExternalToolManager().submitWorkflow(study, params, jobName, jobDescription, dependsOn, jobTags, - scheduledStartTime, jobPriority, dryRun, token)); + @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) Map params) { + return run(() -> catalogManager.getExternalToolManager().submitWorkflow(study, toolId, version, params, jobName, jobDescription, + dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, token)); } // ********************************** VARIANT WALKER WS ENDPOINTS ********************************** diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java index 7c12e33ec39..48281145c32 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java @@ -30,7 +30,7 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.exceptions.VersionException; import org.opencb.opencga.core.models.AclParams; -import org.opencb.opencga.core.models.externalTool.ExternalToolRunParams; +import org.opencb.opencga.core.models.externalTool.DeprecatedWorkflowRunParams; import org.opencb.opencga.core.models.file.FileContent; import org.opencb.opencga.core.models.job.*; import org.opencb.opencga.core.response.OpenCGAResult; @@ -135,7 +135,7 @@ public Response runByPost( @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, - @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) JobRunParams params) { + @ApiParam(value = DeprecatedWorkflowRunParams.DESCRIPTION, required = true) JobRunParams params) { ToolInfo toolInfo = new ToolInfo() .setId(params.getDocker().getId()); return submitJob(study, JobType.CUSTOM, toolInfo, params, jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java index a058d82d3c5..04596c06436 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/WorkflowWSServer.java @@ -93,9 +93,9 @@ public Response updateByPost( @ApiParam(value = ParamConstants.JOB_SCHEDULED_START_TIME_DESCRIPTION) @QueryParam(ParamConstants.JOB_SCHEDULED_START_TIME) String scheduledStartTime, @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.SUBMIT_JOB_PRIORITY_PARAM) String jobPriority, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN) Boolean dryRun, - @ApiParam(value = ExternalToolRunParams.DESCRIPTION, required = true) ExternalToolRunParams params) { - return run(() -> catalogManager.getExternalToolManager().submitWorkflow(study, params, jobName, jobDescription, dependsOn, jobTags, - scheduledStartTime, jobPriority, dryRun, token)); + @ApiParam(value = DeprecatedWorkflowRunParams.DESCRIPTION, required = true) DeprecatedWorkflowRunParams params) { + return run(() -> catalogManager.getExternalToolManager().submitWorkflow(study, params.getId(), params.getVersion(), params.getParams(), + jobName, jobDescription, dependsOn, jobTags, scheduledStartTime, jobPriority, dryRun, token)); } @POST From 2f47ee8dd169c1f8b27c313f38a8108f6736f14d Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 21 May 2025 12:24:53 +0200 Subject: [PATCH 09/22] server: fix generator to support special naming for WS method, #TASK-7610 --- .../app/misc/clients/r_client_generator.py | 1 + .../app/misc/clients/rest_client_generator.py | 4 +- .../app/cli/main/OpenCgaCompleter.java | 8 +- .../app/cli/main/OpencgaCliOptionsParser.java | 26 + .../ExternalToolsCommandExecutor.java | 704 ++++++++++++++ .../options/ExternalToolsCommandOptions.java | 898 ++++++++++++++++++ opencga-client/src/main/R/R/AllGenerics.R | 5 + .../src/main/R/R/ExternalTool-methods.R | 268 ++++++ .../opencga/client/rest/OpenCGAClient.java | 4 + .../rest/clients/ExternalToolClient.java | 410 ++++++++ .../src/main/javascript/ExternalTool.js | 335 +++++++ .../rest_clients/external_tool_client.py | 441 +++++++++ .../generator/commons/ApiCommonsImpl.java | 1 + .../server/generator/config/ApiConfig.java | 14 +- .../generator/config/RestMethodParse.java | 42 + .../writers/ParentClientRestApiWriter.java | 31 +- .../src/main/resources/cli-config.yaml | 6 + .../ParentClientRestApiWriterTest.java | 28 +- .../src/test/resources/cli-config.yaml | 221 +++++ 19 files changed, 3436 insertions(+), 11 deletions(-) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExternalToolsCommandExecutor.java create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ExternalToolsCommandOptions.java create mode 100644 opencga-client/src/main/R/R/ExternalTool-methods.R create mode 100644 opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ExternalToolClient.java create mode 100644 opencga-client/src/main/javascript/ExternalTool.js create mode 100644 opencga-client/src/main/python/pyopencga/rest_clients/external_tool_client.py create mode 100644 opencga-server/src/main/java/org/opencb/opencga/server/generator/config/RestMethodParse.java create mode 100644 opencga-server/src/test/resources/cli-config.yaml diff --git a/opencga-app/app/misc/clients/r_client_generator.py b/opencga-app/app/misc/clients/r_client_generator.py index 78e9c36c47d..1955147435b 100644 --- a/opencga-app/app/misc/clients/r_client_generator.py +++ b/opencga-app/app/misc/clients/r_client_generator.py @@ -19,6 +19,7 @@ def __init__(self, server_url, output_dir): 'Files': 'File', 'Jobs': 'Job', 'Workflows': 'Workflow', + 'External Tools': 'ExternalTool', 'Samples': 'Sample', 'Individuals': 'Individual', 'Families': 'Family', diff --git a/opencga-app/app/misc/clients/rest_client_generator.py b/opencga-app/app/misc/clients/rest_client_generator.py index 71e169a3839..73e0a8af9c5 100644 --- a/opencga-app/app/misc/clients/rest_client_generator.py +++ b/opencga-app/app/misc/clients/rest_client_generator.py @@ -33,7 +33,8 @@ def __init__(self, rest_api_file, output_dir): 'analysis/clinical/{clinicalAnalysis}/interpretation/{interpretations}/delete': { 'method_name': 'delete_interpretation'}, 'panels/import': {'method_name': 'import_panels'}, - 'workflows/import': {'method_name': 'import_workflow'} + 'workflows/import': {'method_name': 'import_workflow'}, + 'tools/custom/run': {'method_name': 'run_custom_docker'} } self.categories = { 'Federations': 'Federation', @@ -50,6 +51,7 @@ def __init__(self, rest_api_file, output_dir): 'Disease Panels': 'DiseasePanel', 'Notes': 'Note', 'Workflows': 'Workflow', + 'External Tools': 'ExternalTool', 'Analysis - Alignment': 'Alignment', 'Analysis - Variant': 'Variant', 'Analysis - Clinical': 'ClinicalAnalysis', diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java index 358986f7fae..2682a17976b 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java @@ -24,7 +24,7 @@ public abstract class OpenCgaCompleter implements Completer { - protected List commands = asList("login","logout","help","use","variant","projects","panels","clinical","jobs","admin","workflows","individuals","families","users","samples","alignments","meta","organizations","studies","files","operations","cohorts") + protected List commands = asList("login","logout","help","use","variant","projects","panels","clinical","jobs","admin","workflows","individuals","families","tools","users","samples","alignments","meta","organizations","studies","files","operations","cohorts") .stream() .map(Candidate::new) .collect(toList()); @@ -74,6 +74,11 @@ public abstract class OpenCgaCompleter implements Completer { .map(Candidate::new) .collect(toList()); + private List toolsList = asList( "acl-update","aggregationstats","custom-build","custom-create","custom-docker-run","custom-run","custom-update","distinct","search","workflow-create","workflow-import","workflow-run","workflow-update","acl","delete","info") + .stream() + .map(Candidate::new) + .collect(toList()); + private List usersList = asList( "anonymous","create","login","password","search","info","configs","configs-update","filters","password-reset","update") .stream() .map(Candidate::new) @@ -136,6 +141,7 @@ public void complete(LineReader lineReader, ParsedLine parsedLine, List updateAcl() throws Exception { + logger.debug("Executing updateAcl in External Tools command line"); + + ExternalToolsCommandOptions.UpdateAclCommandOptions commandOptions = externalToolsCommandOptions.updateAclCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + ExternalToolAclUpdateParams externalToolAclUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/acl/{members}/update")); + return res; + } else if (commandOptions.jsonFile != null) { + externalToolAclUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), ExternalToolAclUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotNull(beanParams, "externalToolIds", commandOptions.externalToolIds, true); + putNestedIfNotNull(beanParams, "permissions", commandOptions.permissions, true); + + externalToolAclUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), ExternalToolAclUpdateParams.class); + } + return openCGAClient.getExternalToolClient().updateAcl(commandOptions.members, commandOptions.action, externalToolAclUpdateParams, queryParams); + } + + private RestResponse aggregationStats() throws Exception { + logger.debug("Executing aggregationStats in External Tools command line"); + + ExternalToolsCommandOptions.AggregationStatsCommandOptions commandOptions = externalToolsCommandOptions.aggregationStatsCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("id", commandOptions.id); + queryParams.putIfNotEmpty("name", commandOptions.name); + queryParams.putIfNotEmpty("uuid", commandOptions.uuid); + queryParams.putIfNotEmpty("tags", commandOptions.tags); + queryParams.putIfNotNull("draft", commandOptions.draft); + queryParams.putIfNotEmpty("internalRegistrationUserId", commandOptions.internalRegistrationUserId); + queryParams.putIfNotEmpty("type", commandOptions.type); + queryParams.putIfNotEmpty("scope", commandOptions.scope); + queryParams.putIfNotEmpty("workflowRepositoryName", commandOptions.workflowRepositoryName); + queryParams.putIfNotEmpty("dockerName", commandOptions.dockerName); + queryParams.putIfNotEmpty("creationDate", commandOptions.creationDate); + queryParams.putIfNotEmpty("modificationDate", commandOptions.modificationDate); + queryParams.putIfNotEmpty("acl", commandOptions.acl); + queryParams.putIfNotEmpty("release", commandOptions.release); + queryParams.putIfNotNull("snapshot", commandOptions.snapshot); + queryParams.putIfNotNull("deleted", commandOptions.deleted); + queryParams.putIfNotEmpty("field", commandOptions.field); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + return openCGAClient.getExternalToolClient().aggregationStats(queryParams); + } + + private RestResponse buildCustom() throws Exception { + logger.debug("Executing buildCustom in External Tools command line"); + + ExternalToolsCommandOptions.BuildCustomCommandOptions commandOptions = externalToolsCommandOptions.buildCustomCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("jobId", commandOptions.jobId); + queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); + queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); + queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + JobToolBuildParams jobToolBuildParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/custom/build")); + return res; + } else if (commandOptions.jsonFile != null) { + jobToolBuildParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), JobToolBuildParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "gitRepository", commandOptions.gitRepository, true); + putNestedIfNotEmpty(beanParams, "aptGet", commandOptions.aptGet, true); + putNestedIfNotNull(beanParams, "installR", commandOptions.installR, true); + putNestedIfNotEmpty(beanParams, "docker.organisation", commandOptions.dockerOrganisation, true); + putNestedIfNotEmpty(beanParams, "docker.name", commandOptions.dockerName, true); + putNestedIfNotEmpty(beanParams, "docker.tag", commandOptions.dockerTag, true); + putNestedIfNotEmpty(beanParams, "docker.user", commandOptions.dockerUser, true); + putNestedIfNotEmpty(beanParams, "docker.password", commandOptions.dockerPassword, true); + + jobToolBuildParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), JobToolBuildParams.class); + } + return openCGAClient.getExternalToolClient().buildCustom(jobToolBuildParams, queryParams); + } + + private RestResponse createCustom() throws Exception { + logger.debug("Executing createCustom in External Tools command line"); + + ExternalToolsCommandOptions.CreateCustomCommandOptions commandOptions = externalToolsCommandOptions.createCustomCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + CustomToolCreateParams customToolCreateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/custom/create")); + return res; + } else if (commandOptions.jsonFile != null) { + customToolCreateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), CustomToolCreateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "scope", commandOptions.scope, true); + putNestedIfNotEmpty(beanParams, "docker.name", commandOptions.dockerName, true); + putNestedIfNotEmpty(beanParams, "docker.tag", commandOptions.dockerTag, true); + putNestedIfNotEmpty(beanParams, "docker.commandLine", commandOptions.dockerCommandLine, true); + putNestedIfNotEmpty(beanParams, "docker.user", commandOptions.dockerUser, true); + putNestedIfNotEmpty(beanParams, "docker.password", commandOptions.dockerPassword, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); + putNestedIfNotNull(beanParams, "draft", commandOptions.draft, true); + putNestedIfNotEmpty(beanParams, "internal.registrationDate", commandOptions.internalRegistrationDate, true); + putNestedIfNotEmpty(beanParams, "internal.lastModified", commandOptions.internalLastModified, true); + putNestedIfNotEmpty(beanParams, "internal.registrationUserId", commandOptions.internalRegistrationUserId, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + + customToolCreateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), CustomToolCreateParams.class); + } + return openCGAClient.getExternalToolClient().createCustom(customToolCreateParams, queryParams); + } + + private RestResponse runCustomDocker() throws Exception { + logger.debug("Executing runCustomDocker in External Tools command line"); + + ExternalToolsCommandOptions.RunCustomDockerCommandOptions commandOptions = externalToolsCommandOptions.runCustomDockerCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("jobId", commandOptions.jobId); + queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); + queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); + queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + Docker docker = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/custom/run")); + return res; + } else if (commandOptions.jsonFile != null) { + docker = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), Docker.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "tag", commandOptions.tag, true); + putNestedIfNotEmpty(beanParams, "commandLine", commandOptions.commandLine, true); + putNestedIfNotEmpty(beanParams, "user", commandOptions.user, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + + docker = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), Docker.class); + } + return openCGAClient.getExternalToolClient().runCustomDocker(docker, queryParams); + } + + private RestResponse runCustom() throws Exception { + logger.debug("Executing runCustom in External Tools command line"); + + ExternalToolsCommandOptions.RunCustomCommandOptions commandOptions = externalToolsCommandOptions.runCustomCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotNull("version", commandOptions.version); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("jobId", commandOptions.jobId); + queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); + queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); + queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + CustomToolRunParams customToolRunParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/custom/{toolId}/run")); + return res; + } else if (commandOptions.jsonFile != null) { + customToolRunParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), CustomToolRunParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "commandLine", commandOptions.commandLine, true); + putNestedMapIfNotEmpty(beanParams, "params", commandOptions.params, true); + + customToolRunParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), CustomToolRunParams.class); + } + return openCGAClient.getExternalToolClient().runCustom(commandOptions.toolId, customToolRunParams, queryParams); + } + + private RestResponse updateCustom() throws Exception { + logger.debug("Executing updateCustom in External Tools command line"); + + ExternalToolsCommandOptions.UpdateCustomCommandOptions commandOptions = externalToolsCommandOptions.updateCustomCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + CustomToolUpdateParams customToolUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/custom/{toolId}/update")); + return res; + } else if (commandOptions.jsonFile != null) { + customToolUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), CustomToolUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "scope", commandOptions.scope, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); + putNestedIfNotNull(beanParams, "draft", commandOptions.draft, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + putNestedIfNotEmpty(beanParams, "docker.name", commandOptions.dockerName, true); + putNestedIfNotEmpty(beanParams, "docker.tag", commandOptions.dockerTag, true); + putNestedIfNotEmpty(beanParams, "docker.commandLine", commandOptions.dockerCommandLine, true); + putNestedIfNotEmpty(beanParams, "docker.user", commandOptions.dockerUser, true); + putNestedIfNotEmpty(beanParams, "docker.password", commandOptions.dockerPassword, true); + + customToolUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), CustomToolUpdateParams.class); + } + return openCGAClient.getExternalToolClient().updateCustom(commandOptions.toolId, customToolUpdateParams, queryParams); + } + + private RestResponse distinct() throws Exception { + logger.debug("Executing distinct in External Tools command line"); + + ExternalToolsCommandOptions.DistinctCommandOptions commandOptions = externalToolsCommandOptions.distinctCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("id", commandOptions.id); + queryParams.putIfNotEmpty("name", commandOptions.name); + queryParams.putIfNotEmpty("uuid", commandOptions.uuid); + queryParams.putIfNotEmpty("tags", commandOptions.tags); + queryParams.putIfNotNull("draft", commandOptions.draft); + queryParams.putIfNotEmpty("internalRegistrationUserId", commandOptions.internalRegistrationUserId); + queryParams.putIfNotEmpty("type", commandOptions.type); + queryParams.putIfNotEmpty("scope", commandOptions.scope); + queryParams.putIfNotEmpty("workflowRepositoryName", commandOptions.workflowRepositoryName); + queryParams.putIfNotEmpty("dockerName", commandOptions.dockerName); + queryParams.putIfNotEmpty("creationDate", commandOptions.creationDate); + queryParams.putIfNotEmpty("modificationDate", commandOptions.modificationDate); + queryParams.putIfNotEmpty("acl", commandOptions.acl); + queryParams.putIfNotEmpty("release", commandOptions.release); + queryParams.putIfNotNull("snapshot", commandOptions.snapshot); + queryParams.putIfNotNull("deleted", commandOptions.deleted); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + return openCGAClient.getExternalToolClient().distinct(commandOptions.field, queryParams); + } + + private RestResponse search() throws Exception { + logger.debug("Executing search in External Tools command line"); + + ExternalToolsCommandOptions.SearchCommandOptions commandOptions = externalToolsCommandOptions.searchCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotNull("limit", commandOptions.limit); + queryParams.putIfNotNull("skip", commandOptions.skip); + queryParams.putIfNotNull("count", commandOptions.count); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("id", commandOptions.id); + queryParams.putIfNotEmpty("name", commandOptions.name); + queryParams.putIfNotEmpty("uuid", commandOptions.uuid); + queryParams.putIfNotEmpty("tags", commandOptions.tags); + queryParams.putIfNotNull("draft", commandOptions.draft); + queryParams.putIfNotEmpty("internalRegistrationUserId", commandOptions.internalRegistrationUserId); + queryParams.putIfNotEmpty("type", commandOptions.type); + queryParams.putIfNotEmpty("scope", commandOptions.scope); + queryParams.putIfNotEmpty("workflowRepositoryName", commandOptions.workflowRepositoryName); + queryParams.putIfNotEmpty("dockerName", commandOptions.dockerName); + queryParams.putIfNotEmpty("creationDate", commandOptions.creationDate); + queryParams.putIfNotEmpty("modificationDate", commandOptions.modificationDate); + queryParams.putIfNotEmpty("acl", commandOptions.acl); + queryParams.putIfNotEmpty("release", commandOptions.release); + queryParams.putIfNotNull("snapshot", commandOptions.snapshot); + queryParams.putIfNotNull("deleted", commandOptions.deleted); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + return openCGAClient.getExternalToolClient().search(queryParams); + } + + private RestResponse createWorkflow() throws Exception { + logger.debug("Executing createWorkflow in External Tools command line"); + + ExternalToolsCommandOptions.CreateWorkflowCommandOptions commandOptions = externalToolsCommandOptions.createWorkflowCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + WorkflowCreateParams workflowCreateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/workflow/create")); + return res; + } else if (commandOptions.jsonFile != null) { + workflowCreateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), WorkflowCreateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "id", commandOptions.id, true); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "scope", commandOptions.scope, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); + putNestedIfNotNull(beanParams, "draft", commandOptions.draft, true); + putNestedIfNotEmpty(beanParams, "internal.registrationDate", commandOptions.internalRegistrationDate, true); + putNestedIfNotEmpty(beanParams, "internal.lastModified", commandOptions.internalLastModified, true); + putNestedIfNotEmpty(beanParams, "internal.registrationUserId", commandOptions.internalRegistrationUserId, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + + workflowCreateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), WorkflowCreateParams.class); + } + return openCGAClient.getExternalToolClient().createWorkflow(workflowCreateParams, queryParams); + } + + private RestResponse importWorkflow() throws Exception { + logger.debug("Executing importWorkflow in External Tools command line"); + + ExternalToolsCommandOptions.ImportWorkflowCommandOptions commandOptions = externalToolsCommandOptions.importWorkflowCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + WorkflowRepositoryParams workflowRepositoryParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/workflow/import")); + return res; + } else if (commandOptions.jsonFile != null) { + workflowRepositoryParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), WorkflowRepositoryParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "tag", commandOptions.tag, true); + putNestedIfNotEmpty(beanParams, "user", commandOptions.user, true); + putNestedIfNotEmpty(beanParams, "password", commandOptions.password, true); + + workflowRepositoryParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), WorkflowRepositoryParams.class); + } + return openCGAClient.getExternalToolClient().importWorkflow(workflowRepositoryParams, queryParams); + } + + private RestResponse runWorkflow() throws Exception { + logger.debug("Executing runWorkflow in External Tools command line"); + + ExternalToolsCommandOptions.RunWorkflowCommandOptions commandOptions = externalToolsCommandOptions.runWorkflowCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotNull("version", commandOptions.version); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("jobId", commandOptions.jobId); + queryParams.putIfNotEmpty("jobDescription", commandOptions.jobDescription); + queryParams.putIfNotEmpty("jobDependsOn", commandOptions.jobDependsOn); + queryParams.putIfNotEmpty("jobTags", commandOptions.jobTags); + queryParams.putIfNotEmpty("jobScheduledStartTime", commandOptions.jobScheduledStartTime); + queryParams.putIfNotEmpty("jobPriority", commandOptions.jobPriority); + queryParams.putIfNotNull("jobDryRun", commandOptions.jobDryRun); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + ObjectMap objectMap = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/workflow/{toolId}/run")); + return res; + } else if (commandOptions.jsonFile != null) { + objectMap = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), ObjectMap.class); + } + return openCGAClient.getExternalToolClient().runWorkflow(commandOptions.toolId, objectMap, queryParams); + } + + private RestResponse updateWorkflow() throws Exception { + logger.debug("Executing updateWorkflow in External Tools command line"); + + ExternalToolsCommandOptions.UpdateWorkflowCommandOptions commandOptions = externalToolsCommandOptions.updateWorkflowCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotNull("includeResult", commandOptions.includeResult); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + + WorkflowUpdateParams workflowUpdateParams = null; + if (commandOptions.jsonDataModel) { + RestResponse res = new RestResponse<>(); + res.setType(QueryType.VOID); + PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/tools/workflow/{toolId}/update")); + return res; + } else if (commandOptions.jsonFile != null) { + workflowUpdateParams = JacksonUtils.getDefaultObjectMapper() + .readValue(new java.io.File(commandOptions.jsonFile), WorkflowUpdateParams.class); + } else { + ObjectMap beanParams = new ObjectMap(); + putNestedIfNotEmpty(beanParams, "name", commandOptions.name, true); + putNestedIfNotEmpty(beanParams, "description", commandOptions.description, true); + putNestedIfNotNull(beanParams, "scope", commandOptions.scope, true); + putNestedIfNotNull(beanParams, "tags", commandOptions.tags, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); + putNestedIfNotNull(beanParams, "draft", commandOptions.draft, true); + putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); + putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); + putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); + + workflowUpdateParams = JacksonUtils.getDefaultObjectMapper().copy() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + .readValue(beanParams.toJson(), WorkflowUpdateParams.class); + } + return openCGAClient.getExternalToolClient().updateWorkflow(commandOptions.toolId, workflowUpdateParams, queryParams); + } + + private RestResponse acl() throws Exception { + logger.debug("Executing acl in External Tools command line"); + + ExternalToolsCommandOptions.AclCommandOptions commandOptions = externalToolsCommandOptions.aclCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("member", commandOptions.member); + queryParams.putIfNotNull("silent", commandOptions.silent); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + return openCGAClient.getExternalToolClient().acl(commandOptions.tools, queryParams); + } + + private RestResponse delete() throws Exception { + logger.debug("Executing delete in External Tools command line"); + + ExternalToolsCommandOptions.DeleteCommandOptions commandOptions = externalToolsCommandOptions.deleteCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("study", commandOptions.study); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + return openCGAClient.getExternalToolClient().delete(commandOptions.tools, queryParams); + } + + private RestResponse info() throws Exception { + logger.debug("Executing info in External Tools command line"); + + ExternalToolsCommandOptions.InfoCommandOptions commandOptions = externalToolsCommandOptions.infoCommandOptions; + + ObjectMap queryParams = new ObjectMap(); + queryParams.putIfNotEmpty("include", commandOptions.include); + queryParams.putIfNotEmpty("exclude", commandOptions.exclude); + queryParams.putIfNotEmpty("study", commandOptions.study); + queryParams.putIfNotEmpty("version", commandOptions.version); + queryParams.putIfNotNull("deleted", commandOptions.deleted); + if (queryParams.get("study") == null && OpencgaMain.isShellMode()) { + queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); + } + + return openCGAClient.getExternalToolClient().info(commandOptions.tools, queryParams); + } +} \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ExternalToolsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ExternalToolsCommandOptions.java new file mode 100644 index 00000000000..2ab26242efd --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ExternalToolsCommandOptions.java @@ -0,0 +1,898 @@ +package org.opencb.opencga.app.cli.main.options; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.beust.jcommander.DynamicParameter; +import com.beust.jcommander.ParametersDelegate; + +import java.util.HashMap; +import java.util.Map; +import java.util.List; + +import static org.opencb.opencga.app.cli.GeneralCliOptions.*; + + +/* +* WARNING: AUTOGENERATED CODE +* +* This code was generated by a tool. +* +* Manual changes to this file may cause unexpected behavior in your application. +* Manual changes to this file will be overwritten if the code is regenerated. +* +*/ + +/** + * This class contains methods for the External Tools command line. + * PATH: /{apiVersion}/tools + */ +@Parameters(commandNames = {"tools"}, commandDescription = "External Tools commands") +public class ExternalToolsCommandOptions { + + public JCommander jCommander; + public CommonCommandOptions commonCommandOptions; + + public UpdateAclCommandOptions updateAclCommandOptions; + public AggregationStatsCommandOptions aggregationStatsCommandOptions; + public BuildCustomCommandOptions buildCustomCommandOptions; + public CreateCustomCommandOptions createCustomCommandOptions; + public RunCustomDockerCommandOptions runCustomDockerCommandOptions; + public RunCustomCommandOptions runCustomCommandOptions; + public UpdateCustomCommandOptions updateCustomCommandOptions; + public DistinctCommandOptions distinctCommandOptions; + public SearchCommandOptions searchCommandOptions; + public CreateWorkflowCommandOptions createWorkflowCommandOptions; + public ImportWorkflowCommandOptions importWorkflowCommandOptions; + public RunWorkflowCommandOptions runWorkflowCommandOptions; + public UpdateWorkflowCommandOptions updateWorkflowCommandOptions; + public AclCommandOptions aclCommandOptions; + public DeleteCommandOptions deleteCommandOptions; + public InfoCommandOptions infoCommandOptions; + + + public ExternalToolsCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) { + + this.jCommander = jCommander; + this.commonCommandOptions = commonCommandOptions; + this.updateAclCommandOptions = new UpdateAclCommandOptions(); + this.aggregationStatsCommandOptions = new AggregationStatsCommandOptions(); + this.buildCustomCommandOptions = new BuildCustomCommandOptions(); + this.createCustomCommandOptions = new CreateCustomCommandOptions(); + this.runCustomDockerCommandOptions = new RunCustomDockerCommandOptions(); + this.runCustomCommandOptions = new RunCustomCommandOptions(); + this.updateCustomCommandOptions = new UpdateCustomCommandOptions(); + this.distinctCommandOptions = new DistinctCommandOptions(); + this.searchCommandOptions = new SearchCommandOptions(); + this.createWorkflowCommandOptions = new CreateWorkflowCommandOptions(); + this.importWorkflowCommandOptions = new ImportWorkflowCommandOptions(); + this.runWorkflowCommandOptions = new RunWorkflowCommandOptions(); + this.updateWorkflowCommandOptions = new UpdateWorkflowCommandOptions(); + this.aclCommandOptions = new AclCommandOptions(); + this.deleteCommandOptions = new DeleteCommandOptions(); + this.infoCommandOptions = new InfoCommandOptions(); + + } + + @Parameters(commandNames = {"acl-update"}, commandDescription ="Update the set of external tool permissions granted for the member") + public class UpdateAclCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--members"}, description = "Comma separated list of user or group ids", required = true, arity = 1) + public String members; + + @Parameter(names = {"--action"}, description = "Action to be performed [ADD, SET, REMOVE or RESET].", required = true, arity = 1) + public String action = "ADD"; + + @Parameter(names = {"--external-tool-ids"}, description = "The body web service externalToolIds parameter", required = false, arity = 1) + public String externalToolIds; + + @Parameter(names = {"--permissions"}, description = "The body web service permissions parameter", required = false, arity = 1) + public String permissions; + + } + + @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch external tool stats") + public class AggregationStatsCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--id"}, description = "Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) + public String id; + + @Parameter(names = {"--name", "-n"}, description = "Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--uuid"}, description = "Comma separated list of external tool UUIDs up to a maximum of 100", required = false, arity = 1) + public String uuid; + + @Parameter(names = {"--tags"}, description = "Comma separated list of tags", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--draft"}, description = "Boolean field indicating whether the workflow is a draft or not.", required = false, arity = 1) + public Boolean draft; + + @Parameter(names = {"--internal.registration-user-id"}, description = "UserId that created the workflow.", required = false, arity = 1) + public String internalRegistrationUserId; + + @Parameter(names = {"--type"}, description = "External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]", required = false, arity = 1) + public String type; + + @Parameter(names = {"--scope"}, description = "External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]", required = false, arity = 1) + public String scope; + + @Parameter(names = {"--workflow-repository-name"}, description = "Workflow repository name", required = false, arity = 1) + public String workflowRepositoryName; + + @Parameter(names = {"--docker-name"}, description = "Docker name", required = false, arity = 1) + public String dockerName; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String modificationDate; + + @Parameter(names = {"--acl"}, description = "Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS permissions. Only study owners or administrators can query by this field. ", required = false, arity = 1) + public String acl; + + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) + public String release; + + @Parameter(names = {"--snapshot"}, description = "Snapshot value (Latest version of the entry in the specified release)", required = false, arity = 1) + public Integer snapshot; + + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) + public boolean deleted = false; + + @Parameter(names = {"--field"}, description = "Field to apply aggregation statistics to (or a list of fields separated by semicolons), e.g.: studies;type;numSamples[0..10]:1;format:sum(size)", required = false, arity = 1) + public String field; + + } + + @Parameters(commandNames = {"custom-build"}, commandDescription ="Execute an analysis from a custom binary.") + public class BuildCustomCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) + public String jobId; + + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) + public String jobTags; + + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--git-repository"}, description = "The body web service gitRepository parameter", required = false, arity = 1) + public String gitRepository; + + @Parameter(names = {"--apt-get"}, description = "The body web service aptGet parameter", required = false, arity = 1) + public String aptGet; + + @Parameter(names = {"--install-r"}, description = "The body web service installR parameter", required = false, arity = 1) + public Boolean installR; + + @Parameter(names = {"--docker-organisation"}, description = "The body web service organisation parameter", required = false, arity = 1) + public String dockerOrganisation; + + @Parameter(names = {"--docker-name"}, description = "The body web service name parameter", required = false, arity = 1) + public String dockerName; + + @Parameter(names = {"--docker-tag"}, description = "The body web service tag parameter", required = false, arity = 1) + public String dockerTag; + + @Parameter(names = {"--docker-user"}, description = "The body web service user parameter", required = false, arity = 1) + public String dockerUser; + + @Parameter(names = {"--docker-password"}, description = "The body web service password parameter", required = false, arity = 1) + public String dockerPassword; + + } + + @Parameters(commandNames = {"custom-create"}, commandDescription ="Register a new external tool of type CUSTOM_TOOL") + public class CreateCustomCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--id"}, description = "External tool ID.", required = false, arity = 1) + public String id; + + @Parameter(names = {"--name", "-n"}, description = "Unique 32-character identifier assigned automatically by OpenCGA.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--description"}, description = "Users may provide a description for the entry.", required = false, arity = 1) + public String description; + + @Parameter(names = {"--scope"}, description = "External tool scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) + public String scope; + + @Parameter(names = {"--docker-name"}, description = "Docker name.", required = false, arity = 1) + public String dockerName; + + @Parameter(names = {"--docker-tag"}, description = "Docker tag.", required = false, arity = 1) + public String dockerTag; + + @Parameter(names = {"--docker-command-line"}, description = "Docker CLI.", required = false, arity = 1) + public String dockerCommandLine; + + @Parameter(names = {"--docker-user"}, description = "User that can access the private Docker repository.", required = false, arity = 1) + public String dockerUser; + + @Parameter(names = {"--docker-password"}, description = "Password corresponding to the user that can access the Docker repository.", required = false, arity = 1) + public String dockerPassword; + + @Parameter(names = {"--tags"}, description = "List of tags.", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--minimum-requirements-cpu"}, description = "Minimum number of cpu cores required to execute the process.", required = false, arity = 1) + public String minimumRequirementsCpu; + + @Parameter(names = {"--minimum-requirements-memory"}, description = "Minimum memory required to execute the process.", required = false, arity = 1) + public String minimumRequirementsMemory; + + @Parameter(names = {"--minimum-requirements-disk"}, description = "Minimum disk required to execute the process.", required = false, arity = 1) + public String minimumRequirementsDisk; + + @Parameter(names = {"--draft"}, description = "Flag indicating whether the external tool is a draft or not.", required = false, help = true, arity = 0) + public boolean draft = false; + + @Parameter(names = {"--internal-registration-date"}, description = "Registration date of the internal object.", required = false, arity = 1) + public String internalRegistrationDate; + + @Parameter(names = {"--internal-last-modified"}, description = "Date of the last modification of the internal object.", required = false, arity = 1) + public String internalLastModified; + + @Parameter(names = {"--internal-registration-user-id"}, description = "The body web service registrationUserId parameter", required = false, arity = 1) + public String internalRegistrationUserId; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was first registered.", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was last modified.", required = false, arity = 1) + public String modificationDate; + + @DynamicParameter(names = {"--attributes"}, description = "You can use this field to store any other information, keep in mind this is not indexed so you cannot search by attributes.. Use: --attributes key=value", required = false) + public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; + + } + + @Parameters(commandNames = {"custom-docker-run"}, commandDescription ="Execute an analysis from a custom binary.") + public class RunCustomDockerCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) + public String jobId; + + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) + public String jobTags; + + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--name", "-n"}, description = "Docker name.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--tag"}, description = "Docker tag.", required = false, arity = 1) + public String tag; + + @Parameter(names = {"--command-line"}, description = "Docker CLI.", required = false, arity = 1) + public String commandLine; + + @Parameter(names = {"--user", "-u"}, description = "User that can access the private Docker repository.", required = false, arity = 1) + public String user; + + @Parameter(names = {"--password"}, description = "Password corresponding to the user that can access the Docker repository.", required = false, arity = 1) + public String password; + + } + + @Parameters(commandNames = {"custom-run"}, commandDescription ="Execute an analysis from a custom binary.") + public class RunCustomCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--tool-id"}, description = "Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = true, arity = 1) + public String toolId; + + @Parameter(names = {"--version"}, description = "Tool version. If not provided, the latest version will be used.", required = false, arity = 1) + public Integer version; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) + public String jobId; + + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) + public String jobTags; + + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + @Parameter(names = {"--command-line"}, description = "The body web service commandLine parameter", required = false, arity = 1) + public String commandLine; + + @DynamicParameter(names = {"--params"}, description = "The body web service params parameter. Use: --params key=value", required = false) + public java.util.Map params = new HashMap<>(); //Dynamic parameters must be initialized; + + } + + @Parameters(commandNames = {"custom-update"}, commandDescription ="Update some custom external tool attributes") + public class UpdateCustomCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--tool-id"}, description = "Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = true, arity = 1) + public String toolId; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--name", "-n"}, description = "Unique 32-character identifier assigned automatically by OpenCGA.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--description"}, description = "Users may provide a description for the entry.", required = false, arity = 1) + public String description; + + @Parameter(names = {"--scope"}, description = "External tool scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) + public String scope; + + @Parameter(names = {"--tags"}, description = "List of tags.", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--minimum-requirements-cpu"}, description = "Minimum number of cpu cores required to execute the process.", required = false, arity = 1) + public String minimumRequirementsCpu; + + @Parameter(names = {"--minimum-requirements-memory"}, description = "Minimum memory required to execute the process.", required = false, arity = 1) + public String minimumRequirementsMemory; + + @Parameter(names = {"--minimum-requirements-disk"}, description = "Minimum disk required to execute the process.", required = false, arity = 1) + public String minimumRequirementsDisk; + + @Parameter(names = {"--draft"}, description = "Flag indicating whether the external tool is a draft or not.", required = false, help = true, arity = 0) + public boolean draft = false; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was first registered.", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was last modified.", required = false, arity = 1) + public String modificationDate; + + @DynamicParameter(names = {"--attributes"}, description = "You can use this field to store any other information, keep in mind this is not indexed so you cannot search by attributes.. Use: --attributes key=value", required = false) + public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; + + @Parameter(names = {"--docker-name"}, description = "Docker name.", required = false, arity = 1) + public String dockerName; + + @Parameter(names = {"--docker-tag"}, description = "Docker tag.", required = false, arity = 1) + public String dockerTag; + + @Parameter(names = {"--docker-command-line"}, description = "Docker CLI.", required = false, arity = 1) + public String dockerCommandLine; + + @Parameter(names = {"--docker-user"}, description = "User that can access the private Docker repository.", required = false, arity = 1) + public String dockerUser; + + @Parameter(names = {"--docker-password"}, description = "Password corresponding to the user that can access the Docker repository.", required = false, arity = 1) + public String dockerPassword; + + } + + @Parameters(commandNames = {"distinct"}, commandDescription ="External tool distinct method") + public class DistinctCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--id"}, description = "Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) + public String id; + + @Parameter(names = {"--name", "-n"}, description = "Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--uuid"}, description = "Comma separated list of external tool UUIDs up to a maximum of 100", required = false, arity = 1) + public String uuid; + + @Parameter(names = {"--tags"}, description = "Comma separated list of tags", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--draft"}, description = "Boolean field indicating whether the workflow is a draft or not.", required = false, arity = 1) + public Boolean draft; + + @Parameter(names = {"--internal.registration-user-id"}, description = "UserId that created the workflow.", required = false, arity = 1) + public String internalRegistrationUserId; + + @Parameter(names = {"--type"}, description = "External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]", required = false, arity = 1) + public String type; + + @Parameter(names = {"--scope"}, description = "External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]", required = false, arity = 1) + public String scope; + + @Parameter(names = {"--workflow-repository-name"}, description = "Workflow repository name", required = false, arity = 1) + public String workflowRepositoryName; + + @Parameter(names = {"--docker-name"}, description = "Docker name", required = false, arity = 1) + public String dockerName; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String modificationDate; + + @Parameter(names = {"--acl"}, description = "Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS permissions. Only study owners or administrators can query by this field. ", required = false, arity = 1) + public String acl; + + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) + public String release; + + @Parameter(names = {"--snapshot"}, description = "Snapshot value (Latest version of the entry in the specified release)", required = false, arity = 1) + public Integer snapshot; + + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) + public boolean deleted = false; + + @Parameter(names = {"--field"}, description = "Comma separated list of fields for which to obtain the distinct values", required = true, arity = 1) + public String field; + + } + + @Parameters(commandNames = {"search"}, commandDescription ="External tool search method") + public class SearchCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--limit"}, description = "Number of results to be returned", required = false, arity = 1) + public Integer limit; + + @Parameter(names = {"--skip"}, description = "Number of results to skip", required = false, arity = 1) + public Integer skip; + + @Parameter(names = {"--count"}, description = "Get the total number of results matching the query. Deactivated by default.", required = false, help = true, arity = 0) + public boolean count = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--id"}, description = "Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) + public String id; + + @Parameter(names = {"--name", "-n"}, description = "Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--uuid"}, description = "Comma separated list of external tool UUIDs up to a maximum of 100", required = false, arity = 1) + public String uuid; + + @Parameter(names = {"--tags"}, description = "Comma separated list of tags", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--draft"}, description = "Boolean field indicating whether the workflow is a draft or not.", required = false, arity = 1) + public Boolean draft; + + @Parameter(names = {"--internal.registration-user-id"}, description = "UserId that created the workflow.", required = false, arity = 1) + public String internalRegistrationUserId; + + @Parameter(names = {"--type"}, description = "External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]", required = false, arity = 1) + public String type; + + @Parameter(names = {"--scope"}, description = "External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]", required = false, arity = 1) + public String scope; + + @Parameter(names = {"--workflow-repository-name"}, description = "Workflow repository name", required = false, arity = 1) + public String workflowRepositoryName; + + @Parameter(names = {"--docker-name"}, description = "Docker name", required = false, arity = 1) + public String dockerName; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805", required = false, arity = 1) + public String modificationDate; + + @Parameter(names = {"--acl"}, description = "Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS permissions. Only study owners or administrators can query by this field. ", required = false, arity = 1) + public String acl; + + @Parameter(names = {"--release"}, description = "Release when it was created", required = false, arity = 1) + public String release; + + @Parameter(names = {"--snapshot"}, description = "Snapshot value (Latest version of the entry in the specified release)", required = false, arity = 1) + public Integer snapshot; + + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) + public boolean deleted = false; + + } + + @Parameters(commandNames = {"workflow-create"}, commandDescription ="Register a new external tool of type WORKFLOW") + public class CreateWorkflowCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--id"}, description = "External tool ID.", required = false, arity = 1) + public String id; + + @Parameter(names = {"--name", "-n"}, description = "Unique 32-character identifier assigned automatically by OpenCGA.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--description"}, description = "Users may provide a description for the entry.", required = false, arity = 1) + public String description; + + @Parameter(names = {"--scope"}, description = "External tool scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) + public String scope; + + @Parameter(names = {"--tags"}, description = "List of tags.", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--minimum-requirements-cpu"}, description = "Minimum number of cpu cores required to execute the process.", required = false, arity = 1) + public String minimumRequirementsCpu; + + @Parameter(names = {"--minimum-requirements-memory"}, description = "Minimum memory required to execute the process.", required = false, arity = 1) + public String minimumRequirementsMemory; + + @Parameter(names = {"--minimum-requirements-disk"}, description = "Minimum disk required to execute the process.", required = false, arity = 1) + public String minimumRequirementsDisk; + + @Parameter(names = {"--draft"}, description = "Flag indicating whether the external tool is a draft or not.", required = false, help = true, arity = 0) + public boolean draft = false; + + @Parameter(names = {"--internal-registration-date"}, description = "Registration date of the internal object.", required = false, arity = 1) + public String internalRegistrationDate; + + @Parameter(names = {"--internal-last-modified"}, description = "Date of the last modification of the internal object.", required = false, arity = 1) + public String internalLastModified; + + @Parameter(names = {"--internal-registration-user-id"}, description = "The body web service registrationUserId parameter", required = false, arity = 1) + public String internalRegistrationUserId; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was first registered.", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was last modified.", required = false, arity = 1) + public String modificationDate; + + @DynamicParameter(names = {"--attributes"}, description = "You can use this field to store any other information, keep in mind this is not indexed so you cannot search by attributes.. Use: --attributes key=value", required = false) + public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; + + } + + @Parameters(commandNames = {"workflow-import"}, commandDescription ="Import an external tool of type WORKFLOW") + public class ImportWorkflowCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--name", "-n"}, description = "The body web service name parameter", required = false, arity = 1) + public String name; + + @Parameter(names = {"--tag"}, description = "The body web service tag parameter", required = false, arity = 1) + public String tag; + + @Parameter(names = {"--user", "-u"}, description = "The body web service user parameter", required = false, arity = 1) + public String user; + + @Parameter(names = {"--password"}, description = "The body web service password parameter", required = false, arity = 1) + public String password; + + } + + @Parameters(commandNames = {"workflow-run"}, commandDescription ="Execute an external tool of type WORKFLOW") + public class RunWorkflowCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--tool-id"}, description = "Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = true, arity = 1) + public String toolId; + + @Parameter(names = {"--version"}, description = "Tool version. If not provided, the latest version will be used.", required = false, arity = 1) + public Integer version; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--job-id"}, description = "Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided.", required = false, arity = 1) + public String jobId; + + @Parameter(names = {"--job-description"}, description = "Job description", required = false, arity = 1) + public String jobDescription; + + @Parameter(names = {"--job-depends-on"}, description = "Comma separated list of existing job IDs the job will depend on.", required = false, arity = 1) + public String jobDependsOn; + + @Parameter(names = {"--job-tags"}, description = "Job tags", required = false, arity = 1) + public String jobTags; + + @Parameter(names = {"--job-scheduled-start-time"}, description = "Time when the job is scheduled to start.", required = false, arity = 1) + public String jobScheduledStartTime; + + @Parameter(names = {"--job-priority"}, description = "Priority of the job", required = false, arity = 1) + public String jobPriority; + + @Parameter(names = {"--job-dry-run"}, description = "Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run.", required = false, arity = 1) + public Boolean jobDryRun; + + } + + @Parameters(commandNames = {"workflow-update"}, commandDescription ="Update some external tool attributes") + public class UpdateWorkflowCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1) + public String jsonFile; + + @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0) + public Boolean jsonDataModel = false; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--tool-id"}, description = "Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search.", required = true, arity = 1) + public String toolId; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--include-result"}, description = "Flag indicating to include the created or updated document result in the response", required = false, help = true, arity = 0) + public boolean includeResult = false; + + @Parameter(names = {"--name", "-n"}, description = "Unique 32-character identifier assigned automatically by OpenCGA.", required = false, arity = 1) + public String name; + + @Parameter(names = {"--description"}, description = "Users may provide a description for the entry.", required = false, arity = 1) + public String description; + + @Parameter(names = {"--scope"}, description = "External tool scope. Valid values: SECONDARY_ANALYSIS, RESEARCH_ANALYSIS, CLINICAL_INTERPRETATION_ANALYSIS or OTHER.", required = false, arity = 1) + public String scope; + + @Parameter(names = {"--tags"}, description = "List of tags.", required = false, arity = 1) + public String tags; + + @Parameter(names = {"--minimum-requirements-cpu"}, description = "Minimum number of cpu cores required to execute the process.", required = false, arity = 1) + public String minimumRequirementsCpu; + + @Parameter(names = {"--minimum-requirements-memory"}, description = "Minimum memory required to execute the process.", required = false, arity = 1) + public String minimumRequirementsMemory; + + @Parameter(names = {"--minimum-requirements-disk"}, description = "Minimum disk required to execute the process.", required = false, arity = 1) + public String minimumRequirementsDisk; + + @Parameter(names = {"--draft"}, description = "Flag indicating whether the external tool is a draft or not.", required = false, help = true, arity = 0) + public boolean draft = false; + + @Parameter(names = {"--creation-date", "--cd"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was first registered.", required = false, arity = 1) + public String creationDate; + + @Parameter(names = {"--modification-date", "--md"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was last modified.", required = false, arity = 1) + public String modificationDate; + + @DynamicParameter(names = {"--attributes"}, description = "You can use this field to store any other information, keep in mind this is not indexed so you cannot search by attributes.. Use: --attributes key=value", required = false) + public java.util.Map attributes = new HashMap<>(); //Dynamic parameters must be initialized; + + } + + @Parameters(commandNames = {"acl"}, commandDescription ="Returns the acl of the external tools. If member is provided, it will only return the acl for the member.") + public class AclCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--tools"}, description = "Comma separated of external tool ids.", required = true, arity = 1) + public String tools; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--member"}, description = "User or group id", required = false, arity = 1) + public String member; + + @Parameter(names = {"--silent"}, description = "Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason", required = false, help = true, arity = 0) + public boolean silent = false; + + } + + @Parameters(commandNames = {"delete"}, commandDescription ="Delete external tools") + public class DeleteCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--tools"}, description = "Comma separated of external tool ids.", required = true, arity = 1) + public String tools; + + } + + @Parameters(commandNames = {"info"}, commandDescription ="Get external tool information") + public class InfoCommandOptions { + + @ParametersDelegate + public CommonCommandOptions commonOptions = commonCommandOptions; + + @Parameter(names = {"--include", "-I"}, description = "Fields included in the response, whole JSON path must be provided", required = false, arity = 1) + public String include; + + @Parameter(names = {"--exclude", "-E"}, description = "Fields excluded in the response, whole JSON path must be provided", required = false, arity = 1) + public String exclude; + + @Parameter(names = {"--tools"}, description = "Comma separated of external tool ids.", required = true, arity = 1) + public String tools; + + @Parameter(names = {"--study", "-s"}, description = "Study [[organization@]project:]study where study and project can be either the ID or UUID", required = false, arity = 1) + public String study; + + @Parameter(names = {"--version"}, description = "Comma separated list of external tool versions. 'all' to get all the external tool versions. Not supported if multiple external tool ids are provided", required = false, arity = 1) + public String version; + + @Parameter(names = {"--deleted"}, description = "Boolean to retrieve deleted entries", required = false, help = true, arity = 0) + public boolean deleted = false; + + } + +} \ No newline at end of file diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index d6e8959a00c..f8ee99409c4 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -33,6 +33,11 @@ setGeneric("jobClient", function(OpencgaR, job, jobs, members, endpointName, par setGeneric("workflowClient", function(OpencgaR, members, workflowId, workflows, endpointName, params=NULL, ...) standardGeneric("workflowClient")) +# ############################################################################## +## ExternalToolClient +setGeneric("externaltoolClient", function(OpencgaR, members, toolId, tools, endpointName, params=NULL, ...) + standardGeneric("externaltoolClient")) + # ############################################################################## ## SampleClient setGeneric("sampleClient", function(OpencgaR, annotationSet, members, sample, samples, endpointName, params=NULL, ...) diff --git a/opencga-client/src/main/R/R/ExternalTool-methods.R b/opencga-client/src/main/R/R/ExternalTool-methods.R new file mode 100644 index 00000000000..0c00a5795a6 --- /dev/null +++ b/opencga-client/src/main/R/R/ExternalTool-methods.R @@ -0,0 +1,268 @@ + +# WARNING: AUTOGENERATED CODE +# +# This code was generated by a tool. +# +# Manual changes to this file may cause unexpected behavior in your application. +# Manual changes to this file will be overwritten if the code is regenerated. + + +# ############################################################################## +#' ExternalToolClient methods +#' @include AllClasses.R +#' @include AllGenerics.R +#' @include commons.R + +#' @description This function implements the OpenCGA calls for managing External Tools. + +#' The following table summarises the available *actions* for this client: +#' +#' | endpointName | Endpoint WS | parameters accepted | +#' | -- | :-- | --: | +#' | updateAcl | /{apiVersion}/tools/acl/{members}/update | study, members[*], action[*], body[*] | +#' | aggregationStats | /{apiVersion}/tools/aggregationStats | study, id, name, uuid, tags, draft, internal.registrationUserId, type, scope, workflowRepositoryName, dockerName, creationDate, modificationDate, acl, release, snapshot, deleted, field | +#' | buildCustom | /{apiVersion}/tools/custom/build | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | createCustom | /{apiVersion}/tools/custom/create | include, exclude, study, includeResult, body[*] | +#' | runCustomDocker | /{apiVersion}/tools/custom/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runCustom | /{apiVersion}/tools/custom/{toolId}/run | toolId[*], version, study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | updateCustom | /{apiVersion}/tools/custom/{toolId}/update | include, exclude, toolId[*], study, includeResult, body | +#' | distinct | /{apiVersion}/tools/distinct | study, id, name, uuid, tags, draft, internal.registrationUserId, type, scope, workflowRepositoryName, dockerName, creationDate, modificationDate, acl, release, snapshot, deleted, field[*] | +#' | search | /{apiVersion}/tools/search | include, exclude, limit, skip, count, study, id, name, uuid, tags, draft, internal.registrationUserId, type, scope, workflowRepositoryName, dockerName, creationDate, modificationDate, acl, release, snapshot, deleted | +#' | createWorkflow | /{apiVersion}/tools/workflow/create | include, exclude, study, includeResult, body[*] | +#' | importWorkflow | /{apiVersion}/tools/workflow/import | study, body[*] | +#' | runWorkflow | /{apiVersion}/tools/workflow/{toolId}/run | toolId[*], version, study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | updateWorkflow | /{apiVersion}/tools/workflow/{toolId}/update | include, exclude, toolId[*], study, includeResult, body | +#' | acl | /{apiVersion}/tools/{tools}/acl | tools[*], study, member, silent | +#' | delete | /{apiVersion}/tools/{tools}/delete | study, tools[*] | +#' | info | /{apiVersion}/tools/{tools}/info | include, exclude, tools[*], study, version, deleted | +#' +#' @md +#' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation +#' \url{http://bioinfo.hpc.cam.ac.uk/opencga-prod/webservices/} +#' [*]: Required parameter +#' @export + +setMethod("externaltoolClient", "OpencgaR", function(OpencgaR, members, toolId, tools, endpointName, params=NULL, ...) { + switch(endpointName, + + #' @section Endpoint /{apiVersion}/tools/acl/{members}/update: + #' Update the set of external tool permissions granted for the member. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param members Comma separated list of user or group ids. + #' @param action Action to be performed [ADD, SET, REMOVE or RESET]. Allowed values: ['SET ADD REMOVE RESET'] + #' @param data JSON containing the parameters to update the permissions. + updateAcl=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="acl", + subcategoryId=members, action="update", params=params, httpMethod="POST", as.queryParam=c("action"), + ...), + + #' @section Endpoint /{apiVersion}/tools/aggregationStats: + #' Fetch external tool stats. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param id Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param name Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param uuid Comma separated list of external tool UUIDs up to a maximum of 100. + #' @param tags Comma separated list of tags. + #' @param draft Boolean field indicating whether the workflow is a draft or not. + #' @param internal.registrationUserId UserId that created the workflow. + #' @param type External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + #' @param scope External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + #' @param workflowRepositoryName Workflow repository name. + #' @param dockerName Docker name. + #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param acl Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS permissions. Only study owners or administrators can query by this field. . + #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). + #' @param deleted Boolean to retrieve deleted entries. + #' @param field Field to apply aggregation statistics to (or a list of fields separated by semicolons), e.g.: studies;type;numSamples[0..10]:1;format:sum(size). + aggregationStats=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory=NULL, + subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, + ...), + + #' @section Endpoint /{apiVersion}/tools/custom/build: + #' Execute an analysis from a custom binary. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data body. + buildCustom=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=NULL, action="build", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/custom/create: + #' Register a new external tool of type CUSTOM_TOOL. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing workflow information. + createCustom=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=NULL, action="create", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/custom/run: + #' Execute an analysis from a custom binary. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data External tool run parameters. + runCustomDocker=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/custom/{toolId}/run: + #' Execute an analysis from a custom binary. + #' @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param version Tool version. If not provided, the latest version will be used. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data External tool run parameters. + runCustom=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=toolId, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/custom/{toolId}/update: + #' Update some custom external tool attributes. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data body. + updateCustom=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=toolId, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/distinct: + #' External tool distinct method. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param id Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param name Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param uuid Comma separated list of external tool UUIDs up to a maximum of 100. + #' @param tags Comma separated list of tags. + #' @param draft Boolean field indicating whether the workflow is a draft or not. + #' @param internal.registrationUserId UserId that created the workflow. + #' @param type External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + #' @param scope External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + #' @param workflowRepositoryName Workflow repository name. + #' @param dockerName Docker name. + #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param acl Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS permissions. Only study owners or administrators can query by this field. . + #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). + #' @param deleted Boolean to retrieve deleted entries. + #' @param field Comma separated list of fields for which to obtain the distinct values. + distinct=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, + action="distinct", params=params, httpMethod="GET", as.queryParam=c("field"), ...), + + #' @section Endpoint /{apiVersion}/tools/search: + #' External tool search method. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param limit Number of results to be returned. + #' @param skip Number of results to skip. + #' @param count Get the total number of results matching the query. Deactivated by default. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param id Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param name Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param uuid Comma separated list of external tool UUIDs up to a maximum of 100. + #' @param tags Comma separated list of tags. + #' @param draft Boolean field indicating whether the workflow is a draft or not. + #' @param internal.registrationUserId UserId that created the workflow. + #' @param type External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + #' @param scope External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + #' @param workflowRepositoryName Workflow repository name. + #' @param dockerName Docker name. + #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param acl Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS permissions. Only study owners or administrators can query by this field. . + #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). + #' @param deleted Boolean to retrieve deleted entries. + search=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, + action="search", params=params, httpMethod="GET", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/workflow/create: + #' Register a new external tool of type WORKFLOW. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing workflow information. + createWorkflow=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="workflow", + subcategoryId=NULL, action="create", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/workflow/import: + #' Import an external tool of type WORKFLOW. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param data Repository parameters. + importWorkflow=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="workflow", + subcategoryId=NULL, action="import", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/workflow/{toolId}/run: + #' Execute an external tool of type WORKFLOW. + #' @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param version Tool version. If not provided, the latest version will be used. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data External tool run parameters. + runWorkflow=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="workflow", + subcategoryId=toolId, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/workflow/{toolId}/update: + #' Update some external tool attributes. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data body. + updateWorkflow=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="workflow", + subcategoryId=toolId, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/{tools}/acl: + #' Returns the acl of the external tools. If member is provided, it will only return the acl for the member. + #' @param tools Comma separated of external tool ids. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param member User or group id. + #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. + acl=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=tools, subcategory=NULL, subcategoryId=NULL, + action="acl", params=params, httpMethod="GET", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/{tools}/delete: + #' Delete external tools. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param tools Comma separated of external tool ids. + delete=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=tools, subcategory=NULL, subcategoryId=NULL, + action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/{tools}/info: + #' Get external tool information. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param tools Comma separated of external tool ids. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param version Comma separated list of external tool versions. 'all' to get all the external tool versions. Not supported if multiple external tool ids are provided. + #' @param deleted Boolean to retrieve deleted entries. + info=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=tools, subcategory=NULL, subcategoryId=NULL, + action="info", params=params, httpMethod="GET", as.queryParam=NULL, ...), + ) +}) \ No newline at end of file diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java index 60f34aad9fe..cb0b33494b4 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java @@ -131,6 +131,10 @@ public WorkflowClient getWorkflowClient() { return getClient(WorkflowClient.class, () -> new WorkflowClient(token, clientConfiguration)); } + public ExternalToolClient getExternalToolClient() { + return getClient(ExternalToolClient.class, () -> new ExternalToolClient(token, clientConfiguration)); + } + public AlignmentClient getAlignmentClient() { return getClient(AlignmentClient.class, () -> new AlignmentClient(token, clientConfiguration)); } diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ExternalToolClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ExternalToolClient.java new file mode 100644 index 00000000000..7024628afd7 --- /dev/null +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ExternalToolClient.java @@ -0,0 +1,410 @@ +/* +* Copyright 2015-2024 OpenCB +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.opencb.opencga.client.rest.clients; + +import java.lang.Object; +import org.opencb.commons.datastore.core.FacetField; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.opencga.client.rest.*; +import org.opencb.opencga.core.client.ParentClient; +import org.opencb.opencga.core.config.client.ClientConfiguration; +import org.opencb.opencga.core.exceptions.ClientException; +import org.opencb.opencga.core.models.externalTool.Docker; +import org.opencb.opencga.core.models.externalTool.ExternalTool; +import org.opencb.opencga.core.models.externalTool.ExternalToolAclEntryList; +import org.opencb.opencga.core.models.externalTool.ExternalToolAclUpdateParams; +import org.opencb.opencga.core.models.externalTool.WorkflowRepositoryParams; +import org.opencb.opencga.core.models.externalTool.custom.CustomToolCreateParams; +import org.opencb.opencga.core.models.externalTool.custom.CustomToolRunParams; +import org.opencb.opencga.core.models.externalTool.custom.CustomToolUpdateParams; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowCreateParams; +import org.opencb.opencga.core.models.externalTool.workflow.WorkflowUpdateParams; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.job.JobToolBuildParams; +import org.opencb.opencga.core.response.RestResponse; + + +/* +* WARNING: AUTOGENERATED CODE +* +* This code was generated by a tool. +* +* Manual changes to this file may cause unexpected behavior in your application. +* Manual changes to this file will be overwritten if the code is regenerated. +*/ + + +/** + * This class contains methods for the ExternalTool webservices. + * PATH: tools + */ +public class ExternalToolClient extends ParentClient { + + public ExternalToolClient(String token, ClientConfiguration configuration) { + super(token, configuration); + } + + /** + * Update the set of external tool permissions granted for the member. + * @param members Comma separated list of user or group ids. + * @param action Action to be performed [ADD, SET, REMOVE or RESET]. + * @param data JSON containing the parameters to update the permissions. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse updateAcl(String members, String action, ExternalToolAclUpdateParams data, ObjectMap + params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.putIfNotNull("action", action); + params.put("body", data); + return execute("tools", null, "acl", members, "update", params, POST, ExternalToolAclEntryList.class); + } + + /** + * Fetch external tool stats. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * id: Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator + * '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * name: Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the + * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * uuid: Comma separated list of external tool UUIDs up to a maximum of 100. + * tags: Comma separated list of tags. + * draft: Boolean field indicating whether the workflow is a draft or not. + * internal.registrationUserId: UserId that created the workflow. + * type: External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + * scope: External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + * workflowRepositoryName: Workflow repository name. + * dockerName: Docker name. + * creationDate: Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * modificationDate: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * acl: Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: + * acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS + * permissions. Only study owners or administrators can query by this field. . + * release: Release when it was created. + * snapshot: Snapshot value (Latest version of the entry in the specified release). + * deleted: Boolean to retrieve deleted entries. + * field: Field to apply aggregation statistics to (or a list of fields separated by semicolons), e.g.: + * studies;type;numSamples[0..10]:1;format:sum(size). + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse aggregationStats(ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("tools", null, null, null, "aggregationStats", params, GET, FacetField.class); + } + + /** + * Execute an analysis from a custom binary. + * @param data body. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + * jobDescription: Job description. + * jobDependsOn: Comma separated list of existing job IDs the job will depend on. + * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse buildCustom(JobToolBuildParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("tools", null, "custom", null, "build", params, POST, Job.class); + } + + /** + * Register a new external tool of type CUSTOM_TOOL. + * @param data JSON containing workflow information. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse createCustom(CustomToolCreateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("tools", null, "custom", null, "create", params, POST, ExternalTool.class); + } + + /** + * Execute an analysis from a custom binary. + * @param data External tool run parameters. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + * jobDescription: Job description. + * jobDependsOn: Comma separated list of existing job IDs the job will depend on. + * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse runCustomDocker(Docker data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("tools", null, "custom", null, "run", params, POST, Job.class); + } + + /** + * Execute an analysis from a custom binary. + * @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the + * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param data External tool run parameters. + * @param params Map containing any of the following optional parameters. + * version: Tool version. If not provided, the latest version will be used. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + * jobDescription: Job description. + * jobDependsOn: Comma separated list of existing job IDs the job will depend on. + * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse runCustom(String toolId, CustomToolRunParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("tools", null, "custom", toolId, "run", params, POST, Job.class); + } + + /** + * Update some custom external tool attributes. + * @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the + * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param data body. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse updateCustom(String toolId, CustomToolUpdateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("tools", null, "custom", toolId, "update", params, POST, ExternalTool.class); + } + + /** + * External tool distinct method. + * @param field Comma separated list of fields for which to obtain the distinct values. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * id: Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator + * '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * name: Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the + * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * uuid: Comma separated list of external tool UUIDs up to a maximum of 100. + * tags: Comma separated list of tags. + * draft: Boolean field indicating whether the workflow is a draft or not. + * internal.registrationUserId: UserId that created the workflow. + * type: External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + * scope: External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + * workflowRepositoryName: Workflow repository name. + * dockerName: Docker name. + * creationDate: Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * modificationDate: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * acl: Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: + * acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS + * permissions. Only study owners or administrators can query by this field. . + * release: Release when it was created. + * snapshot: Snapshot value (Latest version of the entry in the specified release). + * deleted: Boolean to retrieve deleted entries. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse distinct(String field, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.putIfNotNull("field", field); + return execute("tools", null, null, null, "distinct", params, GET, Object.class); + } + + /** + * External tool search method. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * limit: Number of results to be returned. + * skip: Number of results to skip. + * count: Get the total number of results matching the query. Deactivated by default. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * id: Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator + * '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * name: Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the + * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * uuid: Comma separated list of external tool UUIDs up to a maximum of 100. + * tags: Comma separated list of tags. + * draft: Boolean field indicating whether the workflow is a draft or not. + * internal.registrationUserId: UserId that created the workflow. + * type: External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + * scope: External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + * workflowRepositoryName: Workflow repository name. + * dockerName: Docker name. + * creationDate: Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * modificationDate: Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * acl: Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: + * acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS + * permissions. Only study owners or administrators can query by this field. . + * release: Release when it was created. + * snapshot: Snapshot value (Latest version of the entry in the specified release). + * deleted: Boolean to retrieve deleted entries. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse search(ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("tools", null, null, null, "search", params, GET, ExternalTool.class); + } + + /** + * Register a new external tool of type WORKFLOW. + * @param data JSON containing workflow information. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse createWorkflow(WorkflowCreateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("tools", null, "workflow", null, "create", params, POST, ExternalTool.class); + } + + /** + * Import an external tool of type WORKFLOW. + * @param data Repository parameters. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse importWorkflow(WorkflowRepositoryParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("tools", null, "workflow", null, "import", params, POST, ExternalTool.class); + } + + /** + * Execute an external tool of type WORKFLOW. + * @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the + * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param data External tool run parameters. + * @param params Map containing any of the following optional parameters. + * version: Tool version. If not provided, the latest version will be used. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * jobId: Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + * jobDescription: Job description. + * jobDependsOn: Comma separated list of existing job IDs the job will depend on. + * jobTags: Job tags. + * jobScheduledStartTime: Time when the job is scheduled to start. + * jobPriority: Priority of the job. + * jobDryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all + * parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse runWorkflow(String toolId, ObjectMap data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("tools", null, "workflow", toolId, "run", params, POST, Job.class); + } + + /** + * Update some external tool attributes. + * @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the + * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param data body. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * includeResult: Flag indicating to include the created or updated document result in the response. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse updateWorkflow(String toolId, WorkflowUpdateParams data, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + params.put("body", data); + return execute("tools", null, "workflow", toolId, "update", params, POST, ExternalTool.class); + } + + /** + * Returns the acl of the external tools. If member is provided, it will only return the acl for the member. + * @param tools Comma separated of external tool ids. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * member: User or group id. + * silent: Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries + * looked for cannot be shown for whichever reason. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse acl(String tools, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("tools", tools, null, null, "acl", params, GET, ExternalToolAclEntryList.class); + } + + /** + * Delete external tools. + * @param tools Comma separated of external tool ids. + * @param params Map containing any of the following optional parameters. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse delete(String tools, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("tools", tools, null, null, "delete", params, DELETE, ExternalTool.class); + } + + /** + * Get external tool information. + * @param tools Comma separated of external tool ids. + * @param params Map containing any of the following optional parameters. + * include: Fields included in the response, whole JSON path must be provided. + * exclude: Fields excluded in the response, whole JSON path must be provided. + * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. + * version: Comma separated list of external tool versions. 'all' to get all the external tool versions. Not supported if + * multiple external tool ids are provided. + * deleted: Boolean to retrieve deleted entries. + * @return a RestResponse object. + * @throws ClientException ClientException if there is any server error. + */ + public RestResponse info(String tools, ObjectMap params) throws ClientException { + params = params != null ? params : new ObjectMap(); + return execute("tools", tools, null, null, "info", params, GET, ExternalTool.class); + } +} diff --git a/opencga-client/src/main/javascript/ExternalTool.js b/opencga-client/src/main/javascript/ExternalTool.js new file mode 100644 index 00000000000..257dd07ff90 --- /dev/null +++ b/opencga-client/src/main/javascript/ExternalTool.js @@ -0,0 +1,335 @@ +/** + * Copyright 2015-2024 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * WARNING: AUTOGENERATED CODE + * + * This code was generated by a tool. + * + * Manual changes to this file may cause unexpected behavior in your application. + * Manual changes to this file will be overwritten if the code is regenerated. + * +**/ + +import OpenCGAParentClass from "./../opencga-parent-class.js"; + + +/** + * This class contains the methods for the "ExternalTool" resource + */ + +export default class ExternalTool extends OpenCGAParentClass { + + constructor(config) { + super(config); + } + + /** Update the set of external tool permissions granted for the member + * @param {String} members - Comma separated list of user or group ids. + * @param {Object} data - JSON containing the parameters to update the permissions. + * @param {"SET ADD REMOVE RESET"} action = "ADD" - Action to be performed [ADD, SET, REMOVE or RESET]. The default value is ADD. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateAcl(members, action, data, params) { + return this._post("tools", null, "acl", members, "update", data, {action, ...params}); + } + + /** Fetch external tool stats + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.id] - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions + * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.name] - Comma separated list of external tool names up to a maximum of 100. Also admits basic regular + * expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.uuid] - Comma separated list of external tool UUIDs up to a maximum of 100. + * @param {String} [params.tags] - Comma separated list of tags. + * @param {Boolean} [params.draft] - Boolean field indicating whether the workflow is a draft or not. + * @param {String} [params.internal.registrationUserId] - UserId that created the workflow. + * @param {String} [params.type] - External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + * @param {String} [params.scope] - External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + * @param {String} [params.workflowRepositoryName] - Workflow repository name. + * @param {String} [params.dockerName] - Docker name. + * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.acl] - Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. + * Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS + * permissions. Only study owners or administrators can query by this field. . + * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). + * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. + * @param {String} [params.field] - Field to apply aggregation statistics to (or a list of fields separated by semicolons), e.g.: + * studies;type;numSamples[0..10]:1;format:sum(size). + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + aggregationStats(params) { + return this._get("tools", null, null, null, "aggregationStats", params); + } + + /** Execute an analysis from a custom binary. + * @param {Object} data - body. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + buildCustom(data, params) { + return this._post("tools", null, "custom", null, "build", data, params); + } + + /** Register a new external tool of type CUSTOM_TOOL + * @param {Object} data - JSON containing workflow information. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + createCustom(data, params) { + return this._post("tools", null, "custom", null, "create", data, params); + } + + /** Execute an analysis from a custom binary. + * @param {Object} data - External tool run parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + runCustomDocker(data, params) { + return this._post("tools", null, "custom", null, "run", data, params); + } + + /** Execute an analysis from a custom binary. + * @param {String} toolId - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using + * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {Object} data - External tool run parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {Number} [params.version] - Tool version. If not provided, the latest version will be used. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + runCustom(toolId, data, params) { + return this._post("tools", null, "custom", toolId, "run", data, params); + } + + /** Update some custom external tool attributes + * @param {String} toolId - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using + * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {Object} [data] - body. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateCustom(toolId, data, params) { + return this._post("tools", null, "custom", toolId, "update", data, params); + } + + /** External tool distinct method + * @param {String} field - Comma separated list of fields for which to obtain the distinct values. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.id] - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions + * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.name] - Comma separated list of external tool names up to a maximum of 100. Also admits basic regular + * expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.uuid] - Comma separated list of external tool UUIDs up to a maximum of 100. + * @param {String} [params.tags] - Comma separated list of tags. + * @param {Boolean} [params.draft] - Boolean field indicating whether the workflow is a draft or not. + * @param {String} [params.internal.registrationUserId] - UserId that created the workflow. + * @param {String} [params.type] - External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + * @param {String} [params.scope] - External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + * @param {String} [params.workflowRepositoryName] - Workflow repository name. + * @param {String} [params.dockerName] - Docker name. + * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.acl] - Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. + * Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS + * permissions. Only study owners or administrators can query by this field. . + * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). + * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + distinct(field, params) { + return this._get("tools", null, null, null, "distinct", {field, ...params}); + } + + /** External tool search method + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {Number} [params.limit] - Number of results to be returned. + * @param {Number} [params.skip] - Number of results to skip. + * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default + * value is false. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.id] - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions + * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.name] - Comma separated list of external tool names up to a maximum of 100. Also admits basic regular + * expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.uuid] - Comma separated list of external tool UUIDs up to a maximum of 100. + * @param {String} [params.tags] - Comma separated list of tags. + * @param {Boolean} [params.draft] - Boolean field indicating whether the workflow is a draft or not. + * @param {String} [params.internal.registrationUserId] - UserId that created the workflow. + * @param {String} [params.type] - External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + * @param {String} [params.scope] - External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + * @param {String} [params.workflowRepositoryName] - Workflow repository name. + * @param {String} [params.dockerName] - Docker name. + * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.acl] - Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. + * Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS + * permissions. Only study owners or administrators can query by this field. . + * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). + * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + search(params) { + return this._get("tools", null, null, null, "search", params); + } + + /** Register a new external tool of type WORKFLOW + * @param {Object} data - JSON containing workflow information. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + createWorkflow(data, params) { + return this._post("tools", null, "workflow", null, "create", data, params); + } + + /** Import an external tool of type WORKFLOW + * @param {Object} data - Repository parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + importWorkflow(data, params) { + return this._post("tools", null, "workflow", null, "import", data, params); + } + + /** Execute an external tool of type WORKFLOW + * @param {String} toolId - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using + * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {Object} data - External tool run parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {Number} [params.version] - Tool version. If not provided, the latest version will be used. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + runWorkflow(toolId, data, params) { + return this._post("tools", null, "workflow", toolId, "run", data, params); + } + + /** Update some external tool attributes + * @param {String} toolId - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using + * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {Object} [data] - body. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateWorkflow(toolId, data, params) { + return this._post("tools", null, "workflow", toolId, "update", data, params); + } + + /** Returns the acl of the external tools. If member is provided, it will only return the acl for the member. + * @param {String} tools - Comma separated of external tool ids. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.member] - User or group id. + * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an + * exception whenever one of the entries looked for cannot be shown for whichever reason. The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + acl(tools, params) { + return this._get("tools", tools, null, null, "acl", params); + } + + /** Delete external tools + * @param {String} tools - Comma separated of external tool ids. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + delete(tools, params) { + return this._delete("tools", tools, null, null, "delete", params); + } + + /** Get external tool information + * @param {String} tools - Comma separated of external tool ids. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.version] - Comma separated list of external tool versions. 'all' to get all the external tool versions. Not + * supported if multiple external tool ids are provided. + * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + info(tools, params) { + return this._get("tools", tools, null, null, "info", params); + } + +} \ No newline at end of file diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/external_tool_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/external_tool_client.py new file mode 100644 index 00000000000..7e3724cf5aa --- /dev/null +++ b/opencga-client/src/main/python/pyopencga/rest_clients/external_tool_client.py @@ -0,0 +1,441 @@ +""" +WARNING: AUTOGENERATED CODE + + This code was generated by a tool. + + Manual changes to this file may cause unexpected behavior in your application. + Manual changes to this file will be overwritten if the code is regenerated. +""" + +from pyopencga.rest_clients._parent_rest_clients import _ParentRestClient + + +class ExternalTool(_ParentRestClient): + """ + This class contains methods for the 'External Tools' webservices + PATH: /{apiVersion}/tools + """ + + def __init__(self, configuration, token=None, login_handler=None, *args, **kwargs): + super(ExternalTool, self).__init__(configuration, token, login_handler, *args, **kwargs) + + def update_acl(self, members, action, data=None, **options): + """ + Update the set of external tool permissions granted for the member. + PATH: /{apiVersion}/tools/acl/{members}/update + + :param dict data: JSON containing the parameters to update the + permissions. (REQUIRED) + :param str action: Action to be performed [ADD, SET, REMOVE or RESET]. + Allowed values: ['SET ADD REMOVE RESET'] (REQUIRED) + :param str members: Comma separated list of user or group ids. + (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + """ + + options['action'] = action + return self._post(category='tools', resource='update', subcategory='acl', second_query_id=members, data=data, **options) + + def aggregation_stats(self, **options): + """ + Fetch external tool stats. + PATH: /{apiVersion}/tools/aggregationStats + + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str name: Comma separated list of external tool names up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str uuid: Comma separated list of external tool UUIDs up to a + maximum of 100. + :param str tags: Comma separated list of tags. + :param bool draft: Boolean field indicating whether the workflow is a + draft or not. + :param str internal.registration_user_id: UserId that created the + workflow. + :param str type: External tool type. Allowed types: [CUSTOM_TOOL, + VARIANT_WALKER or WORKFLOW]. + :param str scope: External tool scope. Allowed types: + [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + :param str workflow_repository_name: Workflow repository name. + :param str docker_name: Docker name. + :param str creation_date: Creation date. Format: yyyyMMddHHmmss. + Examples: >2018, 2017-2018, <201805. + :param str modification_date: Modification date. Format: + yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + :param str acl: Filter entries for which a user has the provided + permissions. Format: acl={user}:{permissions}. Example: + acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which + user john has both WRITE and WRITE_ANNOTATIONS permissions. Only + study owners or administrators can query by this field. . + :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). + :param bool deleted: Boolean to retrieve deleted entries. + :param str field: Field to apply aggregation statistics to (or a list + of fields separated by semicolons), e.g.: + studies;type;numSamples[0..10]:1;format:sum(size). + """ + + return self._get(category='tools', resource='aggregationStats', **options) + + def build_custom(self, data=None, **options): + """ + Execute an analysis from a custom binary. + PATH: /{apiVersion}/tools/custom/build + + :param dict data: body. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='tools', resource='build', subcategory='custom', data=data, **options) + + def create_custom(self, data=None, **options): + """ + Register a new external tool of type CUSTOM_TOOL. + PATH: /{apiVersion}/tools/custom/create + + :param dict data: JSON containing workflow information. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='tools', resource='create', subcategory='custom', data=data, **options) + + def run_custom_docker(self, data=None, **options): + """ + Execute an analysis from a custom binary. + PATH: /{apiVersion}/tools/custom/run + + :param dict data: External tool run parameters. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='tools', resource='run', subcategory='custom', data=data, **options) + + def run_custom(self, tool_id, data=None, **options): + """ + Execute an analysis from a custom binary. + PATH: /{apiVersion}/tools/custom/{toolId}/run + + :param dict data: External tool run parameters. (REQUIRED) + :param str tool_id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. (REQUIRED) + :param int version: Tool version. If not provided, the latest version + will be used. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='tools', resource='run', subcategory='custom', second_query_id=tool_id, data=data, **options) + + def update_custom(self, tool_id, data=None, **options): + """ + Update some custom external tool attributes. + PATH: /{apiVersion}/tools/custom/{toolId}/update + + :param str tool_id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + :param dict data: body. + """ + + return self._post(category='tools', resource='update', subcategory='custom', second_query_id=tool_id, data=data, **options) + + def distinct(self, field, **options): + """ + External tool distinct method. + PATH: /{apiVersion}/tools/distinct + + :param str field: Comma separated list of fields for which to obtain + the distinct values. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str name: Comma separated list of external tool names up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str uuid: Comma separated list of external tool UUIDs up to a + maximum of 100. + :param str tags: Comma separated list of tags. + :param bool draft: Boolean field indicating whether the workflow is a + draft or not. + :param str internal.registration_user_id: UserId that created the + workflow. + :param str type: External tool type. Allowed types: [CUSTOM_TOOL, + VARIANT_WALKER or WORKFLOW]. + :param str scope: External tool scope. Allowed types: + [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + :param str workflow_repository_name: Workflow repository name. + :param str docker_name: Docker name. + :param str creation_date: Creation date. Format: yyyyMMddHHmmss. + Examples: >2018, 2017-2018, <201805. + :param str modification_date: Modification date. Format: + yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + :param str acl: Filter entries for which a user has the provided + permissions. Format: acl={user}:{permissions}. Example: + acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which + user john has both WRITE and WRITE_ANNOTATIONS permissions. Only + study owners or administrators can query by this field. . + :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). + :param bool deleted: Boolean to retrieve deleted entries. + """ + + options['field'] = field + return self._get(category='tools', resource='distinct', **options) + + def search(self, **options): + """ + External tool search method. + PATH: /{apiVersion}/tools/search + + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param int limit: Number of results to be returned. + :param int skip: Number of results to skip. + :param bool count: Get the total number of results matching the query. + Deactivated by default. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str name: Comma separated list of external tool names up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str uuid: Comma separated list of external tool UUIDs up to a + maximum of 100. + :param str tags: Comma separated list of tags. + :param bool draft: Boolean field indicating whether the workflow is a + draft or not. + :param str internal.registration_user_id: UserId that created the + workflow. + :param str type: External tool type. Allowed types: [CUSTOM_TOOL, + VARIANT_WALKER or WORKFLOW]. + :param str scope: External tool scope. Allowed types: + [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + :param str workflow_repository_name: Workflow repository name. + :param str docker_name: Docker name. + :param str creation_date: Creation date. Format: yyyyMMddHHmmss. + Examples: >2018, 2017-2018, <201805. + :param str modification_date: Modification date. Format: + yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + :param str acl: Filter entries for which a user has the provided + permissions. Format: acl={user}:{permissions}. Example: + acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which + user john has both WRITE and WRITE_ANNOTATIONS permissions. Only + study owners or administrators can query by this field. . + :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). + :param bool deleted: Boolean to retrieve deleted entries. + """ + + return self._get(category='tools', resource='search', **options) + + def create_workflow(self, data=None, **options): + """ + Register a new external tool of type WORKFLOW. + PATH: /{apiVersion}/tools/workflow/create + + :param dict data: JSON containing workflow information. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='tools', resource='create', subcategory='workflow', data=data, **options) + + def import_workflow(self, data=None, **options): + """ + Import an external tool of type WORKFLOW. + PATH: /{apiVersion}/tools/workflow/import + + :param dict data: Repository parameters. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + """ + + return self._post(category='tools', resource='import', subcategory='workflow', data=data, **options) + + def run_workflow(self, tool_id, data=None, **options): + """ + Execute an external tool of type WORKFLOW. + PATH: /{apiVersion}/tools/workflow/{toolId}/run + + :param dict data: External tool run parameters. (REQUIRED) + :param str tool_id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. (REQUIRED) + :param int version: Tool version. If not provided, the latest version + will be used. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='tools', resource='run', subcategory='workflow', second_query_id=tool_id, data=data, **options) + + def update_workflow(self, tool_id, data=None, **options): + """ + Update some external tool attributes. + PATH: /{apiVersion}/tools/workflow/{toolId}/update + + :param str tool_id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + :param dict data: body. + """ + + return self._post(category='tools', resource='update', subcategory='workflow', second_query_id=tool_id, data=data, **options) + + def acl(self, tools, **options): + """ + Returns the acl of the external tools. If member is provided, it will + only return the acl for the member. + PATH: /{apiVersion}/tools/{tools}/acl + + :param str tools: Comma separated of external tool ids. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str member: User or group id. + :param bool silent: Boolean to retrieve all possible entries that are + queried for, false to raise an exception whenever one of the + entries looked for cannot be shown for whichever reason. + """ + + return self._get(category='tools', resource='acl', query_id=tools, **options) + + def delete(self, tools, **options): + """ + Delete external tools. + PATH: /{apiVersion}/tools/{tools}/delete + + :param str tools: Comma separated of external tool ids. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + """ + + return self._delete(category='tools', resource='delete', query_id=tools, **options) + + def info(self, tools, **options): + """ + Get external tool information. + PATH: /{apiVersion}/tools/{tools}/info + + :param str tools: Comma separated of external tool ids. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str version: Comma separated list of external tool versions. + 'all' to get all the external tool versions. Not supported if + multiple external tool ids are provided. + :param bool deleted: Boolean to retrieve deleted entries. + """ + + return self._get(category='tools', resource='info', query_id=tools, **options) + diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/commons/ApiCommonsImpl.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/commons/ApiCommonsImpl.java index fe752e5ea7c..5ea79c33916 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/commons/ApiCommonsImpl.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/commons/ApiCommonsImpl.java @@ -22,6 +22,7 @@ public List> getApiClasses(){ classes.add(FileWSServer.class); classes.add(JobWSServer.class); classes.add(WorkflowWSServer.class); + classes.add(ExternalToolWSServer.class); classes.add(SampleWSServer.class); classes.add(IndividualWSServer.class); classes.add(FamilyWSServer.class); diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/ApiConfig.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/ApiConfig.java index b22daca2f78..f96eb3cd90f 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/ApiConfig.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/ApiConfig.java @@ -9,6 +9,7 @@ public class ApiConfig { private String executorsParentClass; private String optionsParserParentClass; private String executorsOpencgaClientPrefix; + private List restMethodParseList; public ApiConfig() { } @@ -18,12 +19,13 @@ public ApiConfig(List categoryConfigList) { } public ApiConfig(List categoryConfigList, List shortcuts, String executorsParentClass, - String optionsParserParentClass, String executorsOpencgaClientPrefix) { + String optionsParserParentClass, String executorsOpencgaClientPrefix, List restMethodParseList) { this.categoryConfigList = categoryConfigList; this.shortcuts = shortcuts; this.executorsParentClass = executorsParentClass; this.optionsParserParentClass = optionsParserParentClass; this.executorsOpencgaClientPrefix = executorsOpencgaClientPrefix; + this.restMethodParseList = restMethodParseList; } @Override @@ -34,6 +36,7 @@ public String toString() { sb.append(", executorsParentClass='").append(executorsParentClass).append('\''); sb.append(", optionsParserParentClass='").append(optionsParserParentClass).append('\''); sb.append(", executorsOpencgaClientPrefix='").append(executorsOpencgaClientPrefix).append('\''); + sb.append(", restMethodParseList=").append(restMethodParseList); sb.append('}'); return sb.toString(); } @@ -82,4 +85,13 @@ public ApiConfig setExecutorsOpencgaClientPrefix(String executorsOpencgaClientPr this.executorsOpencgaClientPrefix = executorsOpencgaClientPrefix; return this; } + + public List getRestMethodParseList() { + return restMethodParseList; + } + + public ApiConfig setRestMethodParseList(List restMethodParseList) { + this.restMethodParseList = restMethodParseList; + return this; + } } diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/RestMethodParse.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/RestMethodParse.java new file mode 100644 index 00000000000..f94e22f2a05 --- /dev/null +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/config/RestMethodParse.java @@ -0,0 +1,42 @@ +package org.opencb.opencga.server.generator.config; + +public class RestMethodParse { + + private String rest; + private String methodName; + + public RestMethodParse() { + } + + public RestMethodParse(String rest, String methodName) { + this.rest = rest; + this.methodName = methodName; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("RestMethodParse{"); + sb.append("rest='").append(rest).append('\''); + sb.append(", methodName='").append(methodName).append('\''); + sb.append('}'); + return sb.toString(); + } + + public String getRest() { + return rest; + } + + public RestMethodParse setRest(String rest) { + this.rest = rest; + return this; + } + + public String getMethodName() { + return methodName; + } + + public RestMethodParse setMethodName(String methodName) { + this.methodName = methodName; + return this; + } +} diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java index b34d5191a5f..58fa21a6a5b 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java @@ -16,10 +16,12 @@ package org.opencb.opencga.server.generator.writers; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.server.generator.config.CategoryConfig; import org.opencb.opencga.server.generator.config.Command; import org.opencb.opencga.server.generator.config.CommandLineConfiguration; +import org.opencb.opencga.server.generator.config.RestMethodParse; import org.opencb.opencga.server.generator.models.RestApi; import org.opencb.opencga.server.generator.models.RestCategory; import org.opencb.opencga.server.generator.models.RestEndpoint; @@ -186,16 +188,34 @@ private String getCleanPath(String path) { return path.replace("/{apiVersion}/", "").replace("/", "_"); } - public static String getCommandName(RestCategory restCategory, RestEndpoint restEndpoint) { + public String getCommandName(RestCategory restCategory, RestEndpoint restEndpoint) { return getMethodName(restCategory, restEndpoint).replaceAll("_", "-"); } - protected static String getMethodName(RestCategory restCategory, RestEndpoint restEndpoint) { - String subpath = restEndpoint.getPath().replace(restCategory.getPath() + "/", ""); - return getMethodName(subpath); + protected String getMethodName(RestCategory restCategory, RestEndpoint restEndpoint) { + String endpoint = restEndpoint.getPath().replace("/{apiVersion}/", ""); + String methodName = getSpecialMethodName(endpoint); + if (methodName != null) { + System.out.println("Special method name '" + methodName + "' found for endpoint '" + endpoint + "'"); + return methodName; + } else { + String subpath = restEndpoint.getPath().replace(restCategory.getPath() + "/", ""); + return getMethodName(subpath); + } + } + + private String getSpecialMethodName(String endpoint) { + if (CollectionUtils.isNotEmpty(config.getApiConfig().getRestMethodParseList())) { + for (RestMethodParse restMethodParse : config.getApiConfig().getRestMethodParseList()) { + if (restMethodParse.getRest().equalsIgnoreCase(endpoint)) { + return restMethodParse.getMethodName(); + } + } + } + return null; } - protected static String getMethodName(String subpath) { + protected String getMethodName(String subpath) { String methodName = ""; // String subpath = restEndpoint.getPath().replace(restCategory.getPath() + "/", ""); String[] items = subpath.split("/"); @@ -351,6 +371,7 @@ protected String getSubCommandOptionsVarName(CategoryConfig categoryConfig, Stri } protected String getJavaMethodName(CategoryConfig config, String commandName) { + System.out.println("Command name: " + commandName); Command command = config.getCommand(commandName); String commandMethod; if (command != null && StringUtils.isNotEmpty(command.getRename())) { diff --git a/opencga-server/src/main/resources/cli-config.yaml b/opencga-server/src/main/resources/cli-config.yaml index ffd7f5f120a..86e1a8dad76 100644 --- a/opencga-server/src/main/resources/cli-config.yaml +++ b/opencga-server/src/main/resources/cli-config.yaml @@ -19,6 +19,9 @@ options: - java.lang.Float - java.lang. apiConfig: + restMethodParseList: + - rest: "tools/custom/run" + methodName: "run-custom-docker" shortcuts: - name: user shortcut: u @@ -156,6 +159,9 @@ apiConfig: - name: import ## Rename needed as `import` is a reserved word in Java rename: importPanels + - name: tools + ignore: False + key: ExternalTool - name: workflows commandName: workflows ignore: False diff --git a/opencga-server/src/test/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriterTest.java b/opencga-server/src/test/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriterTest.java index 2e3ac00d107..e5156dac424 100644 --- a/opencga-server/src/test/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriterTest.java +++ b/opencga-server/src/test/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriterTest.java @@ -1,19 +1,41 @@ package org.opencb.opencga.server.generator.writers; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.junit.Test; import org.junit.experimental.categories.Category; import org.opencb.opencga.core.testclassification.duration.ShortTests; +import org.opencb.opencga.server.generator.RestApiParser; +import org.opencb.opencga.server.generator.commons.ApiCommonsImpl; +import org.opencb.opencga.server.generator.config.CommandLineConfiguration; +import org.opencb.opencga.server.generator.models.RestApi; import org.opencb.opencga.server.generator.models.RestCategory; import org.opencb.opencga.server.generator.models.RestEndpoint; +import org.opencb.opencga.server.generator.writers.cli.OptionsCliRestApiWriter; -import static org.junit.Assert.*; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.junit.Assert.assertEquals; @Category(ShortTests.class) public class ParentClientRestApiWriterTest { @Test - public void getCommandName() { - String commandName = ParentClientRestApiWriter.getCommandName( + public void getCommandName() throws IOException { + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + CommandLineConfiguration config; + try (InputStream inputStream = getClass().getResource("/cli-config.yaml").openStream()) { + config = objectMapper.readValue(inputStream, CommandLineConfiguration.class); + } + + ApiCommonsImpl apiCommons = new ApiCommonsImpl(); + List> classes = apiCommons.getApiClasses(); + RestApi restApi = new RestApiParser().parse(classes, true); + + OptionsCliRestApiWriter optionsCliRestApiWriter = new OptionsCliRestApiWriter(restApi, config); + String commandName = optionsCliRestApiWriter.getCommandName( new RestCategory().setPath("/{apiVersion}/operation"), new RestEndpoint().setPath("/{apiVersion}/operation/variant/secondaryIndex")); assertEquals("secondaryIndex-variant", commandName); diff --git a/opencga-server/src/test/resources/cli-config.yaml b/opencga-server/src/test/resources/cli-config.yaml new file mode 100644 index 00000000000..779b13ec810 --- /dev/null +++ b/opencga-server/src/test/resources/cli-config.yaml @@ -0,0 +1,221 @@ +options: + version: Opencga + outputDir: opencga-app/src/main/java/org/opencb/opencga/app/cli/main/ + ignoreTypes: + - string + - object + - integer + - int + - map + - boolean + - enum + - long + - java.lang.String + - java.lang.Boolean + - java.lang.Integer + - java.lang.Long + - java.lang.Short + - java.lang.Double + - java.lang.Float + - java.lang. +apiConfig: + shortcuts: + - name: user + shortcut: u + - name: uri + shortcut: input + - name: uri + shortcut: i + - name: study + shortcut: s + - name: exclude + shortcut: E + - name: include + shortcut: I + - name: project + shortcut: p + - name: creationDate + shortcut: cd + - name: modificationDate + shortcut: md + - name: name + shortcut: n + categoryConfigList: + - name: organizations + ignore: False + key: Organization + - name: users + ignore: False + key: User + executorExtended: True + optionExtended: True + commands: + - name: update-filters + ignore: True + - name: login + executorExtended: True + optionExtended: True + - name: logout + executorExtended: True + optionExtended: True + addedMethods: + - logout + - name: projects + ignore: False + key: Project + - name: studies + executorExtended: True + optionExtended: True + commands: + - name: run-templates + executorExtended: True + optionExtended: True + - name: upload-templates + executorExtended: True + optionExtended: True + ignore: False + key: Study + - name: files + ignore: False + key: File + executorExtended: True + optionExtended: True + commands: + - name: upload + executorExtended: True + optionExtended: True + - name: update + subcommands: + - name: annotationSetsAction + ignore: True + - name: relatedFilesAction + ignore: True + - name: tagsAction + ignore: True + - name: jobs + ignore: False + key: Job + executorExtended: True + optionExtended: True + commands: + - name: top + executorExtended: True + optionExtended: True + - name: log + executorExtended: True + optionExtended: True + addedMethods: + - log + - name: samples + ignore: False + key: Sample + commands: + - name: update + subcommands: + - name: phenotypesAction + ignore: True + - name: annotationSetsAction + ignore: True + - name: individuals + ignore: False + key: Individual + commands: + - name: update + subcommands: + - name: phenotypesAction + ignore: True + - name: disordersAction + ignore: True + - name: samplesAction + ignore: True + - name: annotationSetsAction + ignore: True + - name: families + ignore: False + key: Family + commands: + - name: update + subcommands: + - name: annotationSetsAction + ignore: True + - name: cohorts + ignore: False + key: Cohort + commands: + - name: update + subcommands: + - name: samplesAction + ignore: True + - name: annotationSetsAction + ignore: True + - name: panels + commandName: panels + ignore: False + key: DiseasePanel + commands: + - name: import + ## Rename needed as `import` is a reserved word in Java + rename: importPanels + - name: tools + ignore: False + key: ExternalTool + - name: workflows + commandName: workflows + ignore: False + key: Workflow + commands: + - name: import + ## Rename needed as `import` is a reserved word in Java + rename: importWorkflow + - name: analysisAlignment + commandName: alignments + ignore: False + key: Alignment + analysis: True + - name: analysisVariant + commandName: variant + ignore: False + key: Variant + analysis: True + - name: analysisClinical + commandName: clinical + ignore: False + key: ClinicalAnalysis + analysis: True + commands: + - name: update + subcommands: + - name: commentsAction + ignore: True + - name: flagsAction + ignore: True + - name: filesAction + ignore: True + - name: panelsAction + ignore: True + - name: interpretation-update + subcommands: + - name: primaryFindingsAction + ignore: True + - name: secondaryFindingsAction + ignore: True + - name: methodsAction + ignore: True + - name: operation + commandName: operations + ignore: False + key: VariantOperation + operations: True + - name: meta + commandName: meta + ignore: False + key: Meta + commands: + - name: openapi + ignore: True + - name: ga4gh + ignore: True + key: GA4GH + - name: admin + ignore: false + key: Admin From 4bab571b7328a12ad5789853550812f4758f3c72 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 21 May 2025 12:26:52 +0200 Subject: [PATCH 10/22] server: remove system out print, #TASK-7610 --- .../server/generator/writers/ParentClientRestApiWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java index 58fa21a6a5b..0f89578446c 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/generator/writers/ParentClientRestApiWriter.java @@ -371,7 +371,6 @@ protected String getSubCommandOptionsVarName(CategoryConfig categoryConfig, Stri } protected String getJavaMethodName(CategoryConfig config, String commandName) { - System.out.println("Command name: " + commandName); Command command = config.getCommand(commandName); String commandMethod; if (command != null && StringUtils.isNotEmpty(command.getRename())) { From 32d8a140b605236b6daf738723da21003f49d17f Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 21 May 2025 12:34:55 +0200 Subject: [PATCH 11/22] app: enable tools cli, #TASK-7610 --- opencga-app/src/main/resources/cli-usage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/opencga-app/src/main/resources/cli-usage.yml b/opencga-app/src/main/resources/cli-usage.yml index 4452bc2efdd..1e0128bf91f 100644 --- a/opencga-app/src/main/resources/cli-usage.yml +++ b/opencga-app/src/main/resources/cli-usage.yml @@ -7,6 +7,7 @@ categories: - projects - studies - files + - tools - workflows - jobs - individuals From 8bc76a5b63ff857e4cd40f6353c9e3f0ecfd8d93 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 22 May 2025 11:20:50 +0200 Subject: [PATCH 12/22] catalog: fix workflow creation issue, #TASK-7610 --- .../opencb/opencga/catalog/managers/ExternalToolManager.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java index 5e35f8d3b1c..d588fddb7d3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java @@ -929,8 +929,9 @@ private void validateNewWorkflow(ExternalTool externalTool, String userId) throw } externalTool.getWorkflow().setRepository(externalTool.getWorkflow().getRepository() != null ? externalTool.getWorkflow().getRepository() - : new WorkflowRepository("")); - if (externalTool.getWorkflow().getRepository() != null) { + : new WorkflowRepository()); + if (CollectionUtils.isEmpty(externalTool.getWorkflow().getScripts())) { + // If scripts are not provided, repository must be provided validateWorkflowRepository(externalTool.getWorkflow().getRepository()); } if (StringUtils.isEmpty(externalTool.getWorkflow().getRepository().getName()) From f8a7a7d17548224926160ca7f4c48bb71ea4f90c Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 25 Jul 2025 12:30:47 +0200 Subject: [PATCH 13/22] catalog: change scheduler implementation to support queues, #TASK-7442 --- .../main/executors/JobsCommandExecutor.java | 1 + .../executors/WorkflowsCommandExecutor.java | 5 + .../main/options/WorkflowsCommandOptions.java | 12 + .../catalog/JobQueueMigration_Task7442.java | 36 +++ .../opencga/catalog/managers/JobManager.java | 18 ++ .../catalog/utils/JobExecutionUtils.java | 29 ++ .../catalog/utils/JobExecutionUtilsTest.java | 146 ++++++++++ .../src/test/resources/configuration-test.yml | 13 + .../opencga/core/api/FieldConstants.java | 3 + .../opencga/core/config/Configuration.java | 10 + .../opencb/opencga/core/config/Execution.java | 79 +++++- .../opencga/core/config/ExecutionFactor.java | 52 ++++ .../opencga/core/config/ExecutionQueue.java | 127 +++++++++ .../opencga/core/config/ExecutionRequest.java | 51 ++++ .../core/models/job/MinimumRequirements.java | 33 ++- .../core/tools/result/ExecutionResult.java | 14 + .../src/main/resources/configuration.yml | 13 + opencga-master/pom.xml | 5 + .../monitor/daemons/ExecutionDaemon.java | 175 +++++++----- .../monitor/daemons/MonitorParentDaemon.java | 24 +- .../monitor/executors/ExecutorFactory.java | 92 +++++-- .../monitor/schedulers/JobScheduler.java | 130 +++++---- .../monitor/daemons/ExecutionDaemonTest.java | 29 +- .../monitor/schedulers/JobSchedulerTest.java | 255 ++++++++++++++++++ 24 files changed, 1187 insertions(+), 165 deletions(-) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/JobQueueMigration_Task7442.java create mode 100644 opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java create mode 100644 opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionFactor.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionRequest.java create mode 100644 opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java index 7cf4fd39dd2..6ad06fd9885 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java @@ -16,6 +16,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.config.ExecutionQueue; import org.opencb.opencga.core.exceptions.ClientException; import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.file.FileContent; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java index 967a6e6fcd7..f9c70e318d3 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java @@ -12,6 +12,7 @@ import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.core.common.JacksonUtils; +import org.opencb.opencga.core.config.ExecutionQueue; import org.opencb.opencga.core.exceptions.ClientException; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.job.MinimumRequirements; @@ -174,6 +175,8 @@ private RestResponse create() throws Exception { putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); + putNestedIfNotNull(beanParams, "minimumRequirements.type", commandOptions.minimumRequirementsType, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.queue", commandOptions.minimumRequirementsQueue, true); putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); @@ -360,6 +363,8 @@ private RestResponse update() throws Exception { putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); + putNestedIfNotNull(beanParams, "minimumRequirements.type", commandOptions.minimumRequirementsType, true); + putNestedIfNotEmpty(beanParams, "minimumRequirements.queue", commandOptions.minimumRequirementsQueue, true); putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); putNestedMapIfNotEmpty(beanParams, "attributes", commandOptions.attributes, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java index d2f42a95541..127b5eebf7f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java @@ -160,6 +160,12 @@ public class CreateCommandOptions { @Parameter(names = {"--minimum-requirements-disk"}, description = "Minimum disk required to execute the process.", required = false, arity = 1) public String minimumRequirementsDisk; + @Parameter(names = {"--minimum-requirements-type"}, description = "Execution type. Must be one of CPU, GPU or FPGA.", required = false, arity = 1) + public String minimumRequirementsType; + + @Parameter(names = {"--minimum-requirements-queue"}, description = "Expected queue to execute the process.", required = false, arity = 1) + public String minimumRequirementsQueue; + @Parameter(names = {"--creation-date", "--cd"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was first registered.", required = false, arity = 1) public String creationDate; @@ -434,6 +440,12 @@ public class UpdateCommandOptions { @Parameter(names = {"--minimum-requirements-disk"}, description = "Minimum disk required to execute the process.", required = false, arity = 1) public String minimumRequirementsDisk; + @Parameter(names = {"--minimum-requirements-type"}, description = "Execution type. Must be one of CPU, GPU or FPGA.", required = false, arity = 1) + public String minimumRequirementsType; + + @Parameter(names = {"--minimum-requirements-queue"}, description = "Expected queue to execute the process.", required = false, arity = 1) + public String minimumRequirementsQueue; + @Parameter(names = {"--creation-date", "--cd"}, description = "Autogenerated date following the format YYYYMMDDhhmmss containing the date when the entry was first registered.", required = false, arity = 1) public String creationDate; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/JobQueueMigration_Task7442.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/JobQueueMigration_Task7442.java new file mode 100644 index 00000000000..85badcb6853 --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/JobQueueMigration_Task7442.java @@ -0,0 +1,36 @@ +package org.opencb.opencga.app.migrations.v5.v5_0_0.catalog; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import org.bson.Document; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.config.ExecutionQueue; + +import java.util.Arrays; + +@Migration(id = "support_queues_in_job__task_5662", + description = "Support for multiple queues in Job #7442", version = "5.0.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20250725) +public class JobQueueMigration_Task7442 extends MigrationTool { + + @Override + protected void run() throws Exception { + migrateCollection(Arrays.asList(OrganizationMongoDBAdaptorFactory.JOB_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_JOB_COLLECTION), + Filters.exists("tool.minimumRequirements.type", false), + Projections.include("_id"), + (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + + updateDocument.getSet().put("tool.minimumRequirements.type", ExecutionQueue.ExecutionType.CPU.name()); + updateDocument.getSet().put("tool.minimumRequirements.queue", ""); + updateDocument.getSet().put("execution.queue", new Document()); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java index 80423098ccb..c5eac540b3e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java @@ -39,6 +39,8 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.Execution; +import org.opencb.opencga.core.config.ExecutionQueue; +import org.opencb.opencga.core.config.ExecutionRequest; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.AclParams; import org.opencb.opencga.core.models.JwtPayload; @@ -331,6 +333,20 @@ private void autoCompleteNewJob(String organizationId, Study study, Job job, Jwt job.setInternal(JobInternal.init()); job.getInternal().setWebhook(new JobInternalWebhook(study.getNotification().getWebhook(), new HashMap<>())); + if (job.getTool().getMinimumRequirements() == null) { + job.getTool().setMinimumRequirements(new MinimumRequirements()); + } + ExecutionRequest request = configuration.getAnalysis().getExecution().getDefaultRequest(); + MinimumRequirements requirements = job.getTool().getMinimumRequirements(); + requirements.setCpu(ParamUtils.defaultString(requirements.getCpu(), String.valueOf(request.getCpu()))); + requirements.setMemory(ParamUtils.defaultString(requirements.getMemory(), request.getMemory())); + requirements.setType(ParamUtils.defaultObject(requirements.getType(), ExecutionQueue.ExecutionType.CPU)); + if (StringUtils.isEmpty(requirements.getQueue())) { + ExecutionQueue executionQueue = JobExecutionUtils.findOptimalQueues(configuration.getAnalysis().getExecution().getQueues(), + requirements).get(0); + requirements.setQueue(executionQueue.getId()); + } + if (StringUtils.isNotEmpty(job.getParentId())) { // Check parent job exists try { @@ -384,6 +400,8 @@ private void autoCompleteNewJob(String organizationId, Study study, Job job, Jwt job.setInput(inputFiles); } + + job.setAttributes(ParamUtils.defaultObject(job.getAttributes(), HashMap::new)); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java new file mode 100644 index 00000000000..ce6bca06d7f --- /dev/null +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java @@ -0,0 +1,29 @@ +package org.opencb.opencga.catalog.utils; + +import org.opencb.opencga.core.common.IOUtils; +import org.opencb.opencga.core.config.ExecutionQueue; +import org.opencb.opencga.core.models.job.MinimumRequirements; + +import java.util.List; +import java.util.stream.Collectors; + +public class JobExecutionUtils { + + public static List findOptimalQueues(List queues, MinimumRequirements requirements) { + // Sort the queues based on their number of CPUs and memory + // Also filter out queues that do not meet the minimum requirements + return queues.stream() + .filter(queue -> queue.getType().equals(requirements.getType()) + && queue.getCpu() >= Integer.parseInt(requirements.getCpu()) + && IOUtils.fromHumanReadableToByte(queue.getMemory()) >= IOUtils.fromHumanReadableToByte(requirements.getMemory())) + .sorted((q1, q2) -> { + int cpuComparison = Integer.compare(q1.getCpu(), q2.getCpu()); + if (cpuComparison != 0) { + return cpuComparison; + } + return Long.compare(IOUtils.fromHumanReadableToByte(q1.getMemory()), IOUtils.fromHumanReadableToByte(q2.getMemory())); + }) + .collect(Collectors.toList()); + } + +} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java new file mode 100644 index 00000000000..ae4a374ea9a --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java @@ -0,0 +1,146 @@ +package org.opencb.opencga.catalog.utils; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.opencb.opencga.core.config.ExecutionQueue; +import org.opencb.opencga.core.models.job.MinimumRequirements; +import org.opencb.opencga.core.testclassification.duration.ShortTests; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Category(ShortTests.class) +public class JobExecutionUtilsTest { + + @Test + public void testFindOptimalQueuesBasicFiltering() { + // Create test queues + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(2).setMemory("4GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setType(ExecutionQueue.ExecutionType.GPU).setCpu(8).setMemory("16GB"); + + List queues = Arrays.asList(queue1, queue2, queue3); + + // Test filtering by type and minimum requirements + MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("2").setMemory("4GB"); + + List result = JobExecutionUtils.findOptimalQueues(queues, requirements); + + assertEquals(2, result.size()); + assertEquals("queue1", result.get(0).getId()); + assertEquals("queue2", result.get(1).getId()); + } + + @Test + public void testFindOptimalQueuesSortingByCpu() { + // Create queues with different CPU counts but same memory + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(8).setMemory("8GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(2).setMemory("8GB"); + ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + + List queues = Arrays.asList(queue1, queue2, queue3); + + MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("2").setMemory("8GB"); + + List result = JobExecutionUtils.findOptimalQueues(queues, requirements); + + assertEquals(3, result.size()); + // Should be sorted by CPU in ascending order + assertEquals("queue2", result.get(0).getId()); // 2 CPUs + assertEquals("queue3", result.get(1).getId()); // 4 CPUs + assertEquals("queue1", result.get(2).getId()); // 8 CPUs + } + + @Test + public void testFindOptimalQueuesSortingByMemory() { + // Create queues with same CPU but different memory + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("16GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("4GB"); + ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + + List queues = Arrays.asList(queue1, queue2, queue3); + + MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("4").setMemory("4GB"); + + List result = JobExecutionUtils.findOptimalQueues(queues, requirements); + + assertEquals(3, result.size()); + // Should be sorted by memory in ascending order when CPU is equal + assertEquals("queue2", result.get(0).getId()); // 4GB + assertEquals("queue3", result.get(1).getId()); // 8GB + assertEquals("queue1", result.get(2).getId()); // 16GB + } + + @Test + public void testFindOptimalQueuesFiltersByMinimumCpu() { + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(1).setMemory("8GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + + List queues = Arrays.asList(queue1, queue2); + + MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("2").setMemory("8GB"); + + List result = JobExecutionUtils.findOptimalQueues(queues, requirements); + + assertEquals(1, result.size()); + assertEquals("queue2", result.get(0).getId()); + } + + @Test + public void testFindOptimalQueuesFiltersByMinimumMemory() { + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("2GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + + List queues = Arrays.asList(queue1, queue2); + + MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("4").setMemory("4GB"); + + List result = JobExecutionUtils.findOptimalQueues(queues, requirements); + + assertEquals(1, result.size()); + assertEquals("queue2", result.get(0).getId()); + } + + @Test + public void testFindOptimalQueuesFiltersByType() { + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.GPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + + List queues = Arrays.asList(queue1, queue2); + + MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("4").setMemory("8GB"); + + List result = JobExecutionUtils.findOptimalQueues(queues, requirements); + + assertEquals(1, result.size()); + assertEquals("queue2", result.get(0).getId()); + } + + @Test + public void testFindOptimalQueuesEmptyResult() { + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(2).setMemory("4GB"); + + List queues = Collections.singletonList(queue1); + + // Requirements that no queue can meet + MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("8").setMemory("16GB"); + + List result = JobExecutionUtils.findOptimalQueues(queues, requirements); + + assertTrue(result.isEmpty()); + } + + @Test + public void testFindOptimalQueuesEmptyInput() { + List queues = Collections.emptyList(); + MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("4").setMemory("8GB"); + + List result = JobExecutionUtils.findOptimalQueues(queues, requirements); + + assertTrue(result.isEmpty()); + } +} \ No newline at end of file diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index ab9cbc37ec8..9a1a51ef1fa 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -137,6 +137,19 @@ analysis: # Accepted values are "local", "SGE", "azure-batch", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory id: "local" + queues: + - id: "default" + executor: "local" + description: "Default queue for local execution" + type: "CPU" + cpu: 1 + memory: "2G" + defaultRequest: + cpu: 1 + memory: "1G" + requestFactor: + cpu: 0.95 + memory: 0.9 maxConcurrentJobs: variant-index: 20 variant-annotation-index: 5 diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index 8b2843a67b1..61ea62616f7 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -352,6 +352,8 @@ public class FieldConstants { public static final String MIN_REQUIREMENTS_CPU_DESCRIPTION = "Minimum number of cpu cores required to execute the process."; public static final String MIN_REQUIREMENTS_MEMORY_DESCRIPTION = "Minimum memory required to execute the process."; public static final String MIN_REQUIREMENTS_DISK_DESCRIPTION = "Minimum disk required to execute the process."; + public static final String MIN_REQUIREMENTS_TYPE_DESCRIPTION = "Execution type. Must be one of CPU, GPU or FPGA."; + public static final String MIN_REQUIREMENTS_QUEUE_DESCRIPTION = "Expected queue to execute the process."; public static final String WORKFLOW_SYSTEM_ID_DESCRIPTION = "Workflow system id. Valid values: NEXTFLOW."; public static final String WORKFLOW_SYSTEM_VERSION_DESCRIPTION = "Workflow system version to use."; @@ -421,6 +423,7 @@ public class FieldConstants { public static final String EXECUTION_RESULT_EXECUTION_INFO = "Object describes execution information."; public static final String EXECUTION_RESULT_START = "Date the execution started."; public static final String EXECUTION_RESULT_END = "Date the execution was completed."; + public static final String EXECUTION_RESULT_QUEUE = "Queue where the job is scheduled or was executed."; public static final String EXECUTION_RESULT_STATUS = "Executor status can have the values PENDING, RUNNING, DONE and ERROR."; public static final String EXECUTION_RESULT_DEPENDENCIES = "List of tool dependencies used during the execution."; public static final String EXECUTION_RESULT_EXTERNAL_FILES = "List of uris to the external files."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java index 1366b121a8d..4780d573f0e 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Configuration.java @@ -247,6 +247,16 @@ public static void reportUnusedField(String field, Object value) { } } + public static void reportUnusedField(String field, Object value, String description) { + // Report only if the value is not null and not an empty string + if (value != null && !(value instanceof String && ((String) value).isEmpty())) { + if (reportedFields.add(field)) { + // Only log the first time a field is found + logger.warn("Ignored configuration option '{}' with value '{}'. {}.", field, value, description); + } + } + } + public static void reportMovedField(String previousField, String newField, Object value) { // Report only if the value is not null and not an empty string if (value != null && !(value instanceof String && ((String) value).isEmpty())) { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/Execution.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/Execution.java index 1384d0a7df1..d29212cc529 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/Execution.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/Execution.java @@ -16,12 +16,12 @@ package org.opencb.opencga.core.config; +import org.opencb.commons.annotations.DataField; import org.opencb.commons.datastore.core.ObjectMap; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; + +import static org.opencb.opencga.core.config.Configuration.reportUnusedField; /** * Created by imedina on 18/04/16. @@ -38,9 +38,31 @@ public class Execution { "variant-secondary-annotation-index", "variant-secondary-sample-index" ); + + @Deprecated + @DataField(id = "id", deprecated = true, description = "Use queues.id instead") private String id; + + @Deprecated + @DataField(id = "defaultQueue", deprecated = true, description = "Use queues instead") private String defaultQueue; + + @Deprecated + @DataField(id = "availableQueues", deprecated = true, description = "Use queues instead") private String availableQueues; + + @DataField(id = "queues", description = "List of execution queues") + private List queues; + + @DataField(id = "defaultRequest", description = "Default request values for new executions.") + private ExecutionRequest defaultRequest; + + @DataField(id = "requestFactor", description = "Execution request factor to be applied. This is to take into account that systems " + + "always reserve some resources for the system itself, so the request factor is used to scale down the number of resources " + + "requested by the user so that multiple jobs can run in parallel without overloading the system.") + private ExecutionFactor requestFactor; + + @Deprecated private Map> toolsPerQueue; private Map maxConcurrentJobs; private ObjectMap options; @@ -49,41 +71,81 @@ public Execution() { toolsPerQueue = new HashMap<>(); options = new ObjectMap(); maxConcurrentJobs = new HashMap<>(); + queues = new ArrayList<>(); } + @Deprecated public String getId() { return id; } + @Deprecated public Execution setId(String id) { - this.id = id; + reportUnusedField("configuration.yml#analysis.execution.id", availableQueues, "The option was deprecated and a list of " + + "configuration.yml#analysis.execution.queues must now be used instead."); return this; } + @Deprecated public String getDefaultQueue() { return defaultQueue; } + @Deprecated public Execution setDefaultQueue(String defaultQueue) { - this.defaultQueue = defaultQueue; + reportUnusedField("configuration.yml#analysis.execution.defaultQueue", availableQueues, "The option was deprecated and a list of " + + "configuration.yml#analysis.execution.queues must now be used instead."); return this; } + @Deprecated public String getAvailableQueues() { return availableQueues; } + @Deprecated public Execution setAvailableQueues(String availableQueues) { - this.availableQueues = availableQueues; + reportUnusedField("configuration.yml#analysis.execution.availableQueues", availableQueues, "The option was deprecated and a list of" + + " configuration.yml#analysis.execution.queues must now be used instead."); + return this; + } + + public List getQueues() { + return queues; + } + + public Execution setQueues(List queues) { + this.queues = queues; + return this; + } + + public ExecutionRequest getDefaultRequest() { + return defaultRequest; + } + + public Execution setDefaultRequest(ExecutionRequest defaultRequest) { + this.defaultRequest = defaultRequest; + return this; + } + + public ExecutionFactor getRequestFactor() { + return requestFactor; + } + + public Execution setRequestFactor(ExecutionFactor requestFactor) { + this.requestFactor = requestFactor; return this; } + @Deprecated public Map> getToolsPerQueue() { return toolsPerQueue; } + @Deprecated public Execution setToolsPerQueue(Map> toolsPerQueue) { - this.toolsPerQueue = toolsPerQueue; + reportUnusedField("configuration.yml#analysis.execution.toolsPerQueue", toolsPerQueue, "The option was deprecated and a list of " + + "configuration.yml#analysis.execution.queues must now be used instead."); return this; } @@ -111,6 +173,7 @@ public String toString() { sb.append("id='").append(id).append('\''); sb.append(", defaultQueue='").append(defaultQueue).append('\''); sb.append(", availableQueues='").append(availableQueues).append('\''); + sb.append(", queues='").append(queues).append('\''); sb.append(", toolsPerQueue=").append(toolsPerQueue); sb.append(", maxConcurrentJobs=").append(maxConcurrentJobs); sb.append(", options=").append(options); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionFactor.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionFactor.java new file mode 100644 index 00000000000..7202a68efdd --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionFactor.java @@ -0,0 +1,52 @@ +package org.opencb.opencga.core.config; + +import org.opencb.commons.annotations.DataField; + +public class ExecutionFactor { + + @DataField(id = "cpu", description = "CPU factor to be applied to the cpu requested by the job. Must be a value between 0 and 1.") + public float cpu; + + @DataField(id = "memory", description = "Memory factor to be applied to the memory requested by the job." + + " Must be a value between 0 and 1.") + public float memory; + + public ExecutionFactor() { + } + + public ExecutionFactor(float cpu, float memory) { + this.cpu = cpu; + this.memory = memory; + } + + public static ExecutionFactor defaultFactor() { + return new ExecutionFactor(0.95f, 0.9f); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ExecutionFactor{"); + sb.append("cpu=").append(cpu); + sb.append(", memory=").append(memory); + sb.append('}'); + return sb.toString(); + } + + public float getCpu() { + return cpu; + } + + public ExecutionFactor setCpu(float cpu) { + this.cpu = cpu; + return this; + } + + public float getMemory() { + return memory; + } + + public ExecutionFactor setMemory(float memory) { + this.memory = memory; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java new file mode 100644 index 00000000000..5f5fceb6341 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java @@ -0,0 +1,127 @@ +package org.opencb.opencga.core.config; + +import org.opencb.commons.annotations.DataField; +import org.opencb.commons.datastore.core.ObjectMap; + +public class ExecutionQueue { + + @DataField(id = "id", description = "Queue identifier") + private String id; + + @DataField(id = "executor", description = "Executor identifier") + private String executor; + + @DataField(id = "description", description = "Description of the queue") + private String description; + + @DataField(id = "type", description = "Execution type for the queue") + private ExecutionType type; + + @DataField(id = "cpu", description = "Number of CPUs allocated for this queue") + private int cpu; + + @DataField(id = "memory", description = "Memory allocated for this queue") + private String memory; + + @DataField(id = "options", description = "Additional options for the queue") + private ObjectMap options; + + public enum ExecutionType { + CPU, // Default execution type + GPU, // Execution type for GPU-based tasks + FPGA // Execution type for FPGA-based tasks + } + + public ExecutionQueue() { + } + + public ExecutionQueue(String id, String executor, String description, ExecutionType type, int cpu, String memory, ObjectMap options) { + this.id = id; + this.executor = executor; + this.description = description; + this.type = type; + this.cpu = cpu; + this.memory = memory; + this.options = options; + } + + public static ExecutionQueue defaultQueue() { + return new ExecutionQueue("job", "k8s", "Default queue", ExecutionType.CPU, 8, "32GB", new ObjectMap()); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ExecutionQueue{"); + sb.append("id='").append(id).append('\''); + sb.append(", executor='").append(executor).append('\''); + sb.append(", type=").append(type); + sb.append(", cpu=").append(cpu); + sb.append(", memory='").append(memory).append('\''); + sb.append(", options=").append(options); + sb.append('}'); + return sb.toString(); + } + + public String getId() { + return id; + } + + public ExecutionQueue setId(String id) { + this.id = id; + return this; + } + + public String getExecutor() { + return executor; + } + + public ExecutionQueue setExecutor(String executor) { + this.executor = executor; + return this; + } + + public String getDescription() { + return description; + } + + public ExecutionQueue setDescription(String description) { + this.description = description; + return this; + } + + public ExecutionType getType() { + return type; + } + + public ExecutionQueue setType(ExecutionType type) { + this.type = type; + return this; + } + + public int getCpu() { + return cpu; + } + + public ExecutionQueue setCpu(int cpu) { + this.cpu = cpu; + return this; + } + + public String getMemory() { + return memory; + } + + public ExecutionQueue setMemory(String memory) { + this.memory = memory; + return this; + } + + public ObjectMap getOptions() { + return options; + } + + public ExecutionQueue setOptions(ObjectMap options) { + this.options = options; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionRequest.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionRequest.java new file mode 100644 index 00000000000..23dce978c95 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionRequest.java @@ -0,0 +1,51 @@ +package org.opencb.opencga.core.config; + +import org.opencb.commons.annotations.DataField; + +public class ExecutionRequest { + + @DataField(id = "cpu", description = "Default cpu for new execution requests..") + public int cpu; + + @DataField(id = "memory", description = "Default memory for new execution requests.") + public String memory; + + public ExecutionRequest() { + } + + public ExecutionRequest(int cpu, String memory) { + this.cpu = cpu; + this.memory = memory; + } + + public static ExecutionRequest defaultRequest() { + return new ExecutionRequest(2, "8G"); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("ExecutionRequest{"); + sb.append("cpu=").append(cpu); + sb.append(", memory='").append(memory).append('\''); + sb.append('}'); + return sb.toString(); + } + + public int getCpu() { + return cpu; + } + + public ExecutionRequest setCpu(int cpu) { + this.cpu = cpu; + return this; + } + + public String getMemory() { + return memory; + } + + public ExecutionRequest setMemory(String memory) { + this.memory = memory; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/MinimumRequirements.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/MinimumRequirements.java index e0c8089e110..657f4e5abba 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/MinimumRequirements.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/MinimumRequirements.java @@ -2,6 +2,7 @@ import org.opencb.commons.annotations.DataField; import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.config.ExecutionQueue; public class MinimumRequirements { @@ -14,21 +15,31 @@ public class MinimumRequirements { @DataField(id = "disk", description = FieldConstants.MIN_REQUIREMENTS_DISK_DESCRIPTION) private String disk; + @DataField(id = "type", description = FieldConstants.MIN_REQUIREMENTS_TYPE_DESCRIPTION) + private ExecutionQueue.ExecutionType type; + + @DataField(id = "queue", description = FieldConstants.MIN_REQUIREMENTS_QUEUE_DESCRIPTION) + private String queue; + public MinimumRequirements() { } - public MinimumRequirements(String cpu, String memory, String disk) { + public MinimumRequirements(String cpu, String memory, String disk, ExecutionQueue.ExecutionType type, String queue) { this.cpu = cpu; this.memory = memory; this.disk = disk; + this.type = type; + this.queue = queue; } @Override public String toString() { - final StringBuilder sb = new StringBuilder("WorkflowMinimumRequirements{"); + final StringBuilder sb = new StringBuilder("MinimumRequirements{"); sb.append("cpu='").append(cpu).append('\''); sb.append(", memory='").append(memory).append('\''); sb.append(", disk='").append(disk).append('\''); + sb.append(", type=").append(type); + sb.append(", queue='").append(queue).append('\''); sb.append('}'); return sb.toString(); } @@ -59,4 +70,22 @@ public MinimumRequirements setDisk(String disk) { this.disk = disk; return this; } + + public ExecutionQueue.ExecutionType getType() { + return type; + } + + public MinimumRequirements setType(ExecutionQueue.ExecutionType type) { + this.type = type; + return this; + } + + public String getQueue() { + return queue; + } + + public MinimumRequirements setQueue(String queue) { + this.queue = queue; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/tools/result/ExecutionResult.java b/opencga-core/src/main/java/org/opencb/opencga/core/tools/result/ExecutionResult.java index 9426a205504..232d9185971 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/tools/result/ExecutionResult.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/tools/result/ExecutionResult.java @@ -20,6 +20,7 @@ import org.opencb.commons.datastore.core.Event; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.api.FieldConstants; +import org.opencb.opencga.core.config.ExecutionQueue; import org.opencb.opencga.core.tools.ToolDependency; import java.net.URI; @@ -38,6 +39,9 @@ public class ExecutionResult { description = FieldConstants.EXECUTION_RESULT_EXECUTION_INFO) private ExecutorInfo executor; + @DataField(id = "queue", description = FieldConstants.EXECUTION_RESULT_QUEUE) + private ExecutionQueue queue; + @DataField(id = "start", indexed = true, description = FieldConstants.EXECUTION_RESULT_START) private Date start; @@ -83,6 +87,7 @@ public ExecutionResult() { public String toString() { final StringBuilder sb = new StringBuilder("ExecutionResult{"); sb.append("executor=").append(executor); + sb.append(", queue=").append(queue); sb.append(", start=").append(start); sb.append(", end=").append(end); sb.append(", status=").append(status); @@ -113,6 +118,15 @@ public ExecutionResult setExecutor(ExecutorInfo executor) { return this; } + public ExecutionQueue getQueue() { + return queue; + } + + public ExecutionResult setQueue(ExecutionQueue queue) { + this.queue = queue; + return this; + } + public Date getStart() { return start; } diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 93cb3c058eb..3bd22b490fe 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -211,6 +211,19 @@ analysis: defaultQueue: "" # Default queue to be used to submit jobs availableQueues: # Other queues for specific applications toolsPerQueue: + queues: + - id: "default" + executor: "${OPENCGA.EXECUTION.MODE}" + description: "Default queue for execution" + type: "CPU" + cpu: 8 + memory: "32G" + defaultRequest: + cpu: 2 + memory: "8G" + requestFactor: + cpu: 0.95 + memory: 0.9 # docker: # - "circos" # - "deeptools" diff --git a/opencga-master/pom.xml b/opencga-master/pom.xml index 2af34dde7be..cbfffa1bac7 100644 --- a/opencga-master/pom.xml +++ b/opencga-master/pom.xml @@ -68,6 +68,11 @@ test + + org.mockito + mockito-core + test + org.eclipse.jetty jetty-server diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java index 287b818bd67..0210010121d 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java @@ -87,7 +87,7 @@ import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.ExceptionUtils; import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.config.Execution; +import org.opencb.opencga.core.config.ExecutionQueue; import org.opencb.opencga.core.config.storage.StorageConfiguration; import org.opencb.opencga.core.models.AclEntryList; import org.opencb.opencga.core.models.JwtPayload; @@ -147,6 +147,7 @@ public class ExecutionDaemon extends MonitorParentDaemon implements Closeable { private final FileManager fileManager; private final Map jobsCountByType = new HashMap<>(); private final Map retainedLogsTime = new HashMap<>(); + private final List executionQueues; private List packages; @@ -162,6 +163,7 @@ public class ExecutionDaemon extends MonitorParentDaemon implements Closeable { private final Query queuedJobsQuery; private final Query runningJobsQuery; private final QueryOptions queryOptions; + private final QueryOptions queuedOptions = new QueryOptions(QueryOptions.INCLUDE, JobDBAdaptor.QueryParams.EXECUTION.key()); private final ExecutorService executor = Executors.newSingleThreadExecutor(); private final JobScheduler jobScheduler; @@ -266,7 +268,8 @@ public ExecutionDaemon(int interval, String token, CatalogManager catalogManager .append(QueryOptions.ORDER, QueryOptions.ASCENDING) .append(QueryOptions.LIMIT, MAX_NUM_JOBS); - this.jobScheduler = new JobScheduler(catalogManager, token); + this.executionQueues = catalogManager.getConfiguration().getAnalysis().getExecution().getQueues(); + this.jobScheduler = new JobScheduler(catalogManager, this.executionQueues, token); if (CollectionUtils.isEmpty(packages)) { this.packages = Collections.singletonList(ToolFactory.DEFAULT_PACKAGE); @@ -283,7 +286,7 @@ public void apply() throws Exception { @Override public void close() throws IOException { - batchExecutor.close(); + super.close(); try { logger.info("Attempt to shutdown webhook executor"); @@ -313,9 +316,9 @@ protected void checkJobs() throws CatalogException { */ checkQueuedJobs(organizationIds); - long totalPendingJobs = 0; - long totalQueuedJobs = 0; - long totalRunningJobs = 0; + // Clear job counts each cycle + jobsCountByType.clear(); + for (String organizationId : organizationIds) { long pendingJobs = -1; long queuedJobs = -1; @@ -329,14 +332,10 @@ protected void checkJobs() throws CatalogException { } logger.info("----- EXECUTION DAEMON ----- Organization={} --> pending={}, queued={}, running={}", organizationId, pendingJobs, queuedJobs, runningJobs); - totalPendingJobs += pendingJobs; - totalQueuedJobs += queuedJobs; - totalRunningJobs += runningJobs; - } - if (totalQueuedJobs == 0) { - // Check PENDING jobs - checkPendingJobs(organizationIds); + if (pendingJobs > 0) { + checkPendingJobs(organizationId); + } } } @@ -365,7 +364,7 @@ protected int checkRunningJob(Job job) { logger.info("[{}] - Kill signal request received for job with status='{}'. Attempting to abort execution.", job.getId(), job.getInternal().getStatus().getId()); try { - if (batchExecutor.kill(job.getId())) { + if (getBatchExecutor(job).kill(job.getId())) { return abortKillJob(job, "Job was already in execution. Job killed by the user."); } else { logger.info("[{}] - Kill signal send. Waiting for job to finish.", job.getId()); @@ -448,7 +447,7 @@ protected int checkQueuedJob(Job job) { logger.info("[{}] - Kill signal request received for job with status='{}'. Attempting to avoid execution.", job.getId(), job.getInternal().getStatus().getId()); try { - if (batchExecutor.kill(job.getId())) { + if (getBatchExecutor(job).kill(job.getId())) { return abortKillJob(job, "Job was already queued. Job killed by the user."); } else { logger.info("[{}] - Kill signal send. Waiting for job to finish.", job.getId()); @@ -488,42 +487,84 @@ protected int checkQueuedJob(Job job) { } } - protected void checkPendingJobs(List organizationIds) { - // Clear job counts each cycle - jobsCountByType.clear(); - +// protected void checkPendingJobs(List organizationIds) { +// // Clear job counts each cycle +// jobsCountByType.clear(); +// +// // If there are no queued jobs, we can queue new jobs +// List pendingJobs = new LinkedList<>(); +// List runningJobs = new LinkedList<>(); +// +// for (String organizationId : organizationIds) { +// try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, pendingJobsQuery, queryOptions, token)) { +// while (iterator.hasNext()) { +// pendingJobs.add(iterator.next()); +// } +// } catch (Exception e) { +// logger.error("Error listing pending jobs from organization {}", organizationId, e); +// return; +// } +// +// try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, runningJobsQuery, queryOptions, token)) { +// while (iterator.hasNext()) { +// runningJobs.add(iterator.next()); +// } +// } catch (Exception e) { +// logger.error("Error listing running jobs from organization {}", organizationId, e); +// return; +// } +// } +// +// Iterator iterator = jobScheduler.schedule(pendingJobs, Collections.emptyList(), runningJobs); +// boolean success = false; +// while (iterator.hasNext() && !success) { +// Job job = iterator.next(); +// try { +// success = checkPendingJob(job) > 0; +// } catch (Exception e) { +// logger.error("{}", e.getMessage(), e); +// } +// } +// } + + protected void checkPendingJobs(String organizationId) { // If there are no queued jobs, we can queue new jobs List pendingJobs = new LinkedList<>(); - List runningJobs = new LinkedList<>(); - - for (String organizationId : organizationIds) { - try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, pendingJobsQuery, queryOptions, token)) { - while (iterator.hasNext()) { - pendingJobs.add(iterator.next()); + Set exhaustedQueues = new HashSet<>(); + + try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, queuedJobsQuery, queuedOptions, token)) { + while (iterator.hasNext()) { + Job job = iterator.next(); + if (job.getExecution() != null && job.getExecution().getQueue() != null + && StringUtils.isNotEmpty(job.getExecution().getQueue().getId())) { + exhaustedQueues.add(job.getExecution().getQueue().getId()); } - } catch (Exception e) { - logger.error("Error listing pending jobs from organization {}", organizationId, e); - return; } + } catch (Exception e) { + logger.error("Error listing queued jobs from organization {}", organizationId, e); + return; + } - try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, runningJobsQuery, queryOptions, token)) { - while (iterator.hasNext()) { - runningJobs.add(iterator.next()); - } - } catch (Exception e) { - logger.error("Error listing running jobs from organization {}", organizationId, e); - return; + try (DBIterator iterator = jobManager.iteratorInOrganization(organizationId, pendingJobsQuery, queryOptions, token)) { + while (iterator.hasNext()) { + pendingJobs.add(iterator.next()); } + } catch (Exception e) { + logger.error("Error listing pending jobs from organization {}", organizationId, e); + return; } - Iterator iterator = jobScheduler.schedule(pendingJobs, Collections.emptyList(), runningJobs); - boolean success = false; - while (iterator.hasNext() && !success) { - Job job = iterator.next(); - try { - success = checkPendingJob(job) > 0; - } catch (Exception e) { - logger.error("{}", e.getMessage(), e); + Map> jobQueueMap = jobScheduler.schedule(organizationId, pendingJobs, exhaustedQueues); + for (Map.Entry> entry : jobQueueMap.entrySet()) { + String queueId = entry.getKey(); + boolean success = false; + for (int i = 0; i < entry.getValue().size() && !success; i++) { + Job job = entry.getValue().get(i); + try { + success = checkPendingJob(job, queueId) > 0; + } catch (Exception e) { + logger.error("Error checking pending job {}: {}", job.getId(), e.getMessage(), e); + } } } } @@ -532,9 +573,10 @@ protected void checkPendingJobs(List organizationIds) { * Check everything is correct and queues the job. * * @param job Job object. + * @param queueId Queue id where the job will be queued. * @return 1 if the job has changed the status, 0 otherwise. */ - protected int checkPendingJob(Job job) { + protected int checkPendingJob(Job job, String queueId) { if (StringUtils.isEmpty(job.getStudy().getId())) { return abortJob(job, "Missing mandatory 'study' field"); } @@ -552,7 +594,7 @@ protected int checkPendingJob(Job job) { return abortJob(job, "Job killed by the user."); } - if (!canBeQueued(organizationId, job)) { + if (!canBeQueued(organizationId, job, queueId)) { return 0; } @@ -638,6 +680,18 @@ protected int checkPendingJob(Job job) { } } + // Look for queue object where the job should be queued + ExecutionQueue executionQueue = null; + for (ExecutionQueue tmpQueue : this.executionQueues) { + if (tmpQueue.getId().equals(queueId)) { + executionQueue = tmpQueue; + break; + } + } + if (executionQueue == null) { + return abortJob(job, "Internal error: Queue '" + queueId + "' not found."); + } + Path outDirPath = Paths.get(updateParams.getOutDir().getUri()); params.put(OUTDIR_PARAM, outDirPath.toAbsolutePath().toString()); params.put(JOB_PARAM, job.getId()); @@ -655,6 +709,10 @@ protected int checkPendingJob(Job job) { logger.info("Updating job {} from {} to {}", job.getId(), Enums.ExecutionStatus.PENDING, Enums.ExecutionStatus.QUEUED); updateParams.setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.QUEUED))); + ExecutionResult executionResult = new ExecutionResult() + .setQueue(executionQueue); + updateParams.setExecution(executionResult); + try { jobManager.update(job.getStudy().getId(), job.getId(), updateParams, QueryOptions.empty(), token); } catch (CatalogException e) { @@ -663,9 +721,8 @@ protected int checkPendingJob(Job job) { } try { - String queue = getQueue(tool); - logger.info("Queue job '{}' on queue '{}'", job.getId(), queue); - batchExecutor.execute(job, queue, authenticatedCommandLine, stdout, stderr); + logger.info("Queue job '{}' on queue '{}'", job.getId(), queueId); + getBatchExecutor(queueId).execute(job, queueId, authenticatedCommandLine, stdout, stderr); } catch (Exception e) { return abortJob(job, "Error executing job.", e); } @@ -821,22 +878,6 @@ protected void checkToolExecutionPermission(String organizationId, Job job) thro } - private String getQueue(Tool tool) { - String queue = "default"; - Execution execution = catalogManager.getConfiguration().getAnalysis().getExecution(); - if (StringUtils.isNotEmpty(execution.getDefaultQueue())) { - queue = execution.getDefaultQueue(); - } - if (execution.getToolsPerQueue() != null) { - for (Map.Entry> entry : execution.getToolsPerQueue().entrySet()) { - if (entry.getValue().contains(tool.id())) { - queue = entry.getKey(); - } - } - } - return queue; - } - private File getValidInternalOutDir(String study, Job job, String outDirPath, String userToken) throws CatalogException { // TODO: Remove this line when we stop passing the outdir as a query param in the URL outDirPath = outDirPath.replace(":", "/"); @@ -983,7 +1024,7 @@ public static void escapeCliArg(StringBuilder cliBuilder, String value) { } } - private boolean canBeQueued(String organizationId, Job job) { + private boolean canBeQueued(String organizationId, Job job, String queueId) { if (job.getDependsOn() != null && !job.getDependsOn().isEmpty()) { for (Job tmpJob : job.getDependsOn()) { if (!Enums.ExecutionStatus.DONE.equals(tmpJob.getInternal().getStatus().getId())) { @@ -996,7 +1037,7 @@ private boolean canBeQueued(String organizationId, Job job) { } } - if (!batchExecutor.canBeQueued()) { + if (!getBatchExecutor(queueId).canBeQueued()) { return false; } @@ -1085,7 +1126,7 @@ private Enums.ExecutionStatus getCurrentStatus(Job job) { return new Enums.ExecutionStatus(execution.getStatus().getName().name()); } - String status = batchExecutor.getStatus(job.getId()); + String status = getBatchExecutor(job).getStatus(job.getId()); if (!StringUtils.isEmpty(status) && !status.equals(Enums.ExecutionStatus.UNKNOWN)) { return new Enums.ExecutionStatus(status); } else { diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java index 393cb2afdd6..d355e164413 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java @@ -16,7 +16,9 @@ package org.opencb.opencga.master.monitor.daemons; +import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.master.monitor.executors.BatchExecutor; import org.opencb.opencga.master.monitor.executors.ExecutorFactory; import org.slf4j.Logger; @@ -32,7 +34,7 @@ public abstract class MonitorParentDaemon implements Runnable, Closeable { protected final int interval; protected final CatalogManager catalogManager; - protected BatchExecutor batchExecutor; + private final ExecutorFactory executorFactory; private volatile boolean exit = false; @@ -44,8 +46,7 @@ public MonitorParentDaemon(int interval, String token, CatalogManager catalogMan this.catalogManager = catalogManager; this.token = token; logger = LoggerFactory.getLogger(this.getClass()); - ExecutorFactory executorFactory = new ExecutorFactory(catalogManager.getConfiguration()); - this.batchExecutor = executorFactory.getExecutor(); + this.executorFactory = new ExecutorFactory(catalogManager.getConfiguration()); } public boolean isExit() { @@ -94,4 +95,21 @@ public void init() throws Exception { } public abstract void apply() throws Exception; + + protected BatchExecutor getBatchExecutor(Job job) { + if (job.getExecution() == null || job.getExecution().getQueue() == null + || StringUtils.isEmpty(job.getExecution().getQueue().getId())) { + throw new IllegalArgumentException("Job or its queue ID cannot be null"); + } + return getBatchExecutor(job.getExecution().getQueue().getId()); + } + + protected BatchExecutor getBatchExecutor(String queueId) { + return this.executorFactory.getExecutor(queueId); + } + + @Override + public void close() throws IOException { + this.executorFactory.close(); + } } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java index 6410c038e8e..71af67aa7c0 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java @@ -16,47 +16,93 @@ package org.opencb.opencga.master.monitor.executors; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.Execution; +import org.opencb.opencga.core.config.ExecutionQueue; + +import java.io.Closeable; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; /** * Created by pfurio on 22/08/16. */ -public class ExecutorFactory { +public class ExecutorFactory implements Closeable { - // TODO: Change for a map - private BatchExecutor executor; + private Map queueToExecutorMap; + private Map executor; public ExecutorFactory(Configuration configuration) { + this.queueToExecutorMap = new HashMap<>(); + this.executor = new HashMap<>(); + + Set executorIds = new HashSet<>(); + if (CollectionUtils.isNotEmpty(configuration.getAnalysis().getExecution().getQueues())) { + for (ExecutionQueue queue : configuration.getAnalysis().getExecution().getQueues()) { + if (StringUtils.isEmpty(queue.getId())) { + throw new IllegalArgumentException("Queue id cannot be null or empty"); + } + if (StringUtils.isEmpty(queue.getExecutor())) { + throw new IllegalArgumentException("Queue " + queue.getId() + " does not have an associated executor"); + } + if (queueToExecutorMap.containsKey(queue.getId())) { + throw new IllegalArgumentException("Queue " + queue.getId() + " is already defined"); + } + queueToExecutorMap.put(queue.getId(), queue.getExecutor().toLowerCase()); + executorIds.add(queue.getExecutor()); + } + } else { + queueToExecutorMap.put("default", "local"); + executorIds.add("local"); + } + Execution execution = configuration.getAnalysis().getExecution(); - String mode = StringUtils.isEmpty(execution.getId()) - ? "local" : execution.getId().toLowerCase(); - - switch (mode) { - case "local": - this.executor = new LocalExecutor(execution); - break; - case "sge": - this.executor = new SGEExecutor(execution); - break; + for (String executorId : executorIds) { + switch (executorId) { + case "local": + LocalExecutor localExecutor = new LocalExecutor(execution); + this.executor.put("local", localExecutor); + break; + case "sge": + SGEExecutor sgeExecutor = new SGEExecutor(execution); + this.executor.put("sge", sgeExecutor); + break; // case "azure": // case "azure-batch": // this.executor = new AzureBatchExecutor(execution); // break; - case "k8s": - case "kubernetes": - this.executor = new K8SExecutor(configuration); - break; - default: - throw new UnsupportedOperationException("nonsoupported execution mode { " + mode - + " }, accepted modes are : local, sge, azure"); + case "k8s": + case "kubernetes": + K8SExecutor k8SExecutor = new K8SExecutor(configuration); + this.executor.put(executorId, k8SExecutor); + break; + default: + throw new UnsupportedOperationException("Unsupported execution mode { " + executorId + + " }, accepted modes are : local, sge, k8s, kubernetes"); + } } } - public BatchExecutor getExecutor() { - return this.executor; + public BatchExecutor getExecutor(String queueId) { + if (!queueToExecutorMap.containsKey(queueId)) { + throw new IllegalArgumentException("Queue " + queueId + " does not have an associated executor"); + } + String executorId = queueToExecutorMap.get(queueId); + if (!executor.containsKey(executorId)) { + throw new IllegalArgumentException("Executor " + executorId + " does not exist"); + } + return executor.get(executorId); } - // TODO: CreateCommandLine (static method) + @Override + public void close() throws IOException { + for (BatchExecutor executor : this.executor.values()) { + executor.close(); + } + } } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java index 93e06642713..188b2bd0d56 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java @@ -10,8 +10,10 @@ import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.catalog.managers.OrganizationManager; import org.opencb.opencga.catalog.utils.CatalogFqn; +import org.opencb.opencga.catalog.utils.JobExecutionUtils; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.ExecutionQueue; import org.opencb.opencga.core.models.job.Job; import org.opencb.opencga.core.models.organizations.Organization; import org.opencb.opencga.core.models.study.Group; @@ -26,6 +28,7 @@ public class JobScheduler { private final String token; private Map userRoles; + private final Map> availableQueues; private static final float PRIORITY_WEIGHT = 0.6F; private static final float IDLE_TIME_WEIGHT = 0.4F; @@ -35,38 +38,43 @@ public class JobScheduler { private final Logger logger = LoggerFactory.getLogger(JobScheduler.class); - public JobScheduler(CatalogManager catalogManager, String token) { + public JobScheduler(CatalogManager catalogManager, List queueList, String token) { this.catalogManager = catalogManager; this.token = token; + + this.availableQueues = new HashMap<>(); + for (ExecutionQueue queue : queueList) { + if (!availableQueues.containsKey(queue.getType())) { + availableQueues.put(queue.getType(), new ArrayList<>()); + } + availableQueues.get(queue.getType()).add(queue); + } } - private void getUserRoles() throws CatalogException { + private void getUserRoles(String organizationId) throws CatalogException { StopWatch stopWatch = StopWatch.createStarted(); this.userRoles = new HashMap<>(); - List organizationIds = catalogManager.getOrganizationManager().getOrganizationIds(token); - for (String organizationId : organizationIds) { - if (ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.ID.key()); - catalogManager.getUserManager().search(organizationId, new Query(), options, token).getResults() - .forEach(user -> getUserRole(organizationId, user.getId()).setSuperAdmin(true)); - } else { - Organization organization = catalogManager.getOrganizationManager().get(organizationId, - OrganizationManager.INCLUDE_ORGANIZATION_ADMINS, token).first(); - getUserRole(organizationId, organization.getOwner()).addOrganizationOwner(organizationId); - organization.getAdmins().forEach(user -> getUserRole(organizationId, user).addOrganizationAdmin(organizationId)); - - QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.FQN.key(), - StudyDBAdaptor.QueryParams.GROUPS.key())); - catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), options, token).getResults() - .forEach(study -> { - for (Group group : study.getGroups()) { - if (ParamConstants.ADMINS_GROUP.equals(group.getId())) { - group.getUserIds().forEach(user -> getUserRole(organizationId, user).addStudyAdmin(study.getFqn())); - } + if (ParamConstants.ADMIN_ORGANIZATION.equals(organizationId)) { + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, UserDBAdaptor.QueryParams.ID.key()); + catalogManager.getUserManager().search(organizationId, new Query(), options, token).getResults() + .forEach(user -> getUserRole(organizationId, user.getId()).setSuperAdmin(true)); + } else { + Organization organization = catalogManager.getOrganizationManager().get(organizationId, + OrganizationManager.INCLUDE_ORGANIZATION_ADMINS, token).first(); + getUserRole(organizationId, organization.getOwner()).addOrganizationOwner(organizationId); + organization.getAdmins().forEach(user -> getUserRole(organizationId, user).addOrganizationAdmin(organizationId)); + + QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(StudyDBAdaptor.QueryParams.FQN.key(), + StudyDBAdaptor.QueryParams.GROUPS.key())); + catalogManager.getStudyManager().searchInOrganization(organizationId, new Query(), options, token).getResults() + .forEach(study -> { + for (Group group : study.getGroups()) { + if (ParamConstants.ADMINS_GROUP.equals(group.getId())) { + group.getUserIds().forEach(user -> getUserRole(organizationId, user).addStudyAdmin(study.getFqn())); } - }); - } + } + }); } logger.debug("Time spent fetching user roles: {}", TimeUtils.durationToString(stopWatch)); @@ -152,31 +160,52 @@ private float getPriorityWeight(Job job) { return appPriority * 0.6f + usersPriority * 0.4f; } - public Iterator schedule(List pendingJobs, List queuedJobs, List runningJobs) { + public Map> schedule(String organizationId, List pendingJobs, Set exhaustedQueues) { Date currentDate = new Date(); - TreeMap> jobTreeMap = new TreeMap<>(); + + Map>> jobTreeMap = new HashMap<>(); + for (List queues : this.availableQueues.values()) { + for (ExecutionQueue queue : queues) { + if (exhaustedQueues.contains(queue.getId())) { + logger.debug("Organization '{}' has exhausted queue '{}'. Skipping scheduling for this queue.", organizationId, + queue.getId()); + } else { + jobTreeMap.put(queue.getId(), new TreeMap<>()); + logger.debug("Organization '{}' has queue '{}' available for scheduling.", organizationId, queue.getId()); + } + } + } try { - getUserRoles(); + getUserRoles(organizationId); } catch (CatalogException e) { - throw new RuntimeException("Scheduler exception: " + e.getMessage(), e); + throw new RuntimeException("Organization '" + organizationId + "'. Scheduler exception: " + e.getMessage(), e); } - Map organizationJobStatus = new HashMap<>(); + boolean exceeds; try { - List organizationIds = catalogManager.getOrganizationManager().getOrganizationIds(token); - for (String organizationId : organizationIds) { - boolean exceeds = catalogManager.getJobManager().exceedsExecutionLimitQuota(organizationId); - organizationJobStatus.put(organizationId, exceeds); - } + exceeds = catalogManager.getJobManager().exceedsExecutionLimitQuota(organizationId); } catch (CatalogException e) { - throw new RuntimeException("Couldn't get the execution quota status: " + e.getMessage(), e); + throw new RuntimeException("Couldn't get the execution quota status for Organization '" + organizationId + "': " + + e.getMessage(), e); } Map> rescheduleJobs = new HashMap<>(); - StopWatch stopWatch = StopWatch.createStarted(); for (Job job : pendingJobs) { + String queueId = job.getTool().getMinimumRequirements().getQueue(); + if (StringUtils.isEmpty(queueId)) { + List executionQueues = this.availableQueues.get(job.getTool().getMinimumRequirements().getType()); + queueId = JobExecutionUtils.findOptimalQueues(executionQueues, job.getTool().getMinimumRequirements()).get(0).getId(); + logger.debug("Job '{}' has no queue defined. Optimal queue for scheduling is '{}' .", job.getId(), queueId); + } + + if (exhaustedQueues.contains(queueId)) { + logger.debug("Job '{}' can't be queued yet. The queue '{}' has been exhausted for the organization '{}'.", + job.getId(), queueId, organizationId); + continue; + } + if (StringUtils.isNotEmpty(job.getScheduledStartTime())) { Date date = TimeUtils.toDate(job.getScheduledStartTime()); if (date.after(currentDate)) { @@ -186,9 +215,7 @@ public Iterator schedule(List pendingJobs, List queuedJobs, List< } } - // Check if execution limit quota has been exceeded - String organizationId = CatalogFqn.extractFqnFromStudyFqn(job.getStudy().getId()).getOrganizationId(); - if (organizationJobStatus.get(organizationId)) { + if (exceeds) { logger.debug("Job '{}' can't be queued yet. The organization '{}' has exceeded the execution limit quota." + " It will be scheduled for next month.", job.getId(), organizationId); rescheduleJobs.putIfAbsent(job.getStudy().getId(), new ArrayList<>()); @@ -197,19 +224,28 @@ public Iterator schedule(List pendingJobs, List queuedJobs, List< } float priority = getPriorityWeight(job); - if (!jobTreeMap.containsKey(priority)) { - jobTreeMap.put(priority, new ArrayList<>()); + if (!jobTreeMap.containsKey(queueId)) { + logger.warn("Job '{}' has no queue defined. Skipping scheduling for this job.", job.getId()); + continue; + } + if (!jobTreeMap.get(queueId).containsKey(priority)) { + jobTreeMap.get(queueId).put(priority, new ArrayList<>()); } - jobTreeMap.get(priority).add(job); + jobTreeMap.get(queueId).get(priority).add(job); } logger.debug("Time spent scheduling jobs: {}", TimeUtils.durationToString(stopWatch)); stopWatch.reset(); stopWatch.start(); - // Obtain iterator - List allJobs = new ArrayList<>(); - for (Float priority : jobTreeMap.descendingKeySet()) { - allJobs.addAll(jobTreeMap.get(priority)); + + Map> allJobs = new HashMap<>(); + for (Map.Entry>> entry : jobTreeMap.entrySet()) { + if (!entry.getValue().isEmpty()) { + allJobs.put(entry.getKey(), new ArrayList<>()); + for (Float priority : entry.getValue().descendingKeySet()) { + allJobs.get(entry.getKey()).addAll(entry.getValue().get(priority)); + } + } } logger.debug("Time spent creating iterator: {}", TimeUtils.durationToString(stopWatch)); @@ -236,7 +272,7 @@ public Iterator schedule(List pendingJobs, List queuedJobs, List< logger.debug("Time spent rescheduling jobs for next month: {}", TimeUtils.durationToString(stopWatch)); } - return allJobs.iterator(); + return allJobs; } private static class UserRole { diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java index 919440fe6c4..97f178df61c 100644 --- a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemonTest.java @@ -20,6 +20,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.mockito.Mockito; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; @@ -74,8 +75,6 @@ public class ExecutionDaemonTest extends AbstractManagerTest { private ExecutionDaemon daemon; private DummyBatchExecutor executor; - private List organizationIds; - @Override @Before public void setUp() throws Exception { @@ -84,13 +83,13 @@ public void setUp() throws Exception { String expiringToken = this.catalogManager.getUserManager().loginAsAdmin(TestParamConstants.ADMIN_PASSWORD).first().getToken(); catalogManager.getConfiguration().getAnalysis().getExecution().getMaxConcurrentJobs().put(VariantIndexOperationTool.ID, 1); - daemon = new ExecutionDaemon(1000, expiringToken, catalogManager, - new StorageConfiguration().setMode(StorageConfiguration.Mode.READ_WRITE), catalogManagerResource.getOpencgaHome().toString()); + daemon = Mockito.spy(new ExecutionDaemon(1000, expiringToken, catalogManager, + new StorageConfiguration().setMode(StorageConfiguration.Mode.READ_WRITE), catalogManagerResource.getOpencgaHome().toString())); executor = new DummyBatchExecutor(); - daemon.batchExecutor = executor; - - this.organizationIds = Arrays.asList(organizationId, ParamConstants.ADMIN_ORGANIZATION); + // Mock the getBatchExecutor method to always return our dummy executor + Mockito.doReturn(executor).when(daemon).getBatchExecutor(Mockito.any(Job.class)); + Mockito.doReturn(executor).when(daemon).getBatchExecutor(Mockito.anyString()); } @Test @@ -141,7 +140,7 @@ public void testCreateDefaultOutDir() throws Exception { HashMap params = new HashMap<>(); String jobId = catalogManager.getJobManager().submit(studyFqn, JobType.NATIVE, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); - daemon.checkPendingJobs(organizationIds); + daemon.checkPendingJobs(organizationId); URI uri = getJob(jobId).getOutDir().getUri(); Assert.assertTrue(Files.exists(Paths.get(uri))); @@ -156,7 +155,7 @@ public void testWebhookNotification() throws Exception { HashMap params = new HashMap<>(); String jobId = catalogManager.getJobManager().submit(studyFqn, JobType.NATIVE, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); - daemon.checkPendingJobs(organizationIds); + daemon.checkPendingJobs(organizationId); // We sleep because there must be a thread sending notifying to the webhook url. Thread.sleep(1500); @@ -172,7 +171,7 @@ public void testCreateOutDir() throws Exception { params.put("outdir", "outputDir/"); String jobId = catalogManager.getJobManager().submit(studyFqn, JobType.NATIVE, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); - daemon.checkPendingJobs(organizationIds); + daemon.checkPendingJobs(organizationId); URI uri = getJob(jobId).getOutDir().getUri(); Assert.assertTrue(Files.exists(Paths.get(uri))); @@ -191,7 +190,7 @@ public void testUseEmptyDirectory() throws Exception { String jobId = catalogManager.getJobManager().submit(studyFqn, JobType.NATIVE, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); - daemon.checkPendingJobs(organizationIds); + daemon.checkPendingJobs(organizationId); URI uri = getJob(jobId).getOutDir().getUri(); Assert.assertTrue(Files.exists(Paths.get(uri))); @@ -204,7 +203,7 @@ public void testNotEmptyOutDir() throws Exception { params.put("outdir", "data/"); String jobId = catalogManager.getJobManager().submit(studyFqn, JobType.NATIVE, "files-delete", Enums.Priority.MEDIUM, params, ownerToken).first().getId(); - daemon.checkPendingJobs(organizationIds); + daemon.checkPendingJobs(organizationId); OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, jobId, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); @@ -356,7 +355,7 @@ public void testDependsOnJobs() throws Exception { String job2 = catalogManager.getJobManager().submit(studyFqn, JobType.NATIVE, "files-delete", Enums.Priority.MEDIUM, params, null, null, Collections.singletonList(job1), null, null, null, false, ownerToken).first().getId(); - daemon.checkPendingJobs(organizationIds); + daemon.checkPendingJobs(organizationId); OpenCGAResult jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job1, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); @@ -371,7 +370,7 @@ public void testDependsOnJobs() throws Exception { .setInternal(new JobInternal(new Enums.ExecutionStatus(Enums.ExecutionStatus.ERROR))), QueryOptions.empty(), ownerToken); // The job that depended on job1 should be ABORTED because job1 execution "failed" - daemon.checkPendingJobs(organizationIds); + daemon.checkPendingJobs(organizationId); jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job2, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); checkStatus(jobOpenCGAResult.first(), Enums.ExecutionStatus.ABORTED); @@ -384,7 +383,7 @@ public void testDependsOnJobs() throws Exception { // And create a new job to simulate a normal successfully dependency String job3 = catalogManager.getJobManager().submit(studyFqn, JobType.NATIVE, "files-delete", Enums.Priority.MEDIUM, params, null, null, Collections.singletonList(job1), null, null, null, false, ownerToken).first().getId(); - daemon.checkPendingJobs(organizationIds); + daemon.checkPendingJobs(organizationId); jobOpenCGAResult = catalogManager.getJobManager().get(studyFqn, job3, QueryOptions.empty(), ownerToken); assertEquals(1, jobOpenCGAResult.getNumResults()); diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java new file mode 100644 index 00000000000..f74cfcb3f0f --- /dev/null +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java @@ -0,0 +1,255 @@ +package org.opencb.opencga.master.monitor.schedulers; + +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.*; +import org.opencb.opencga.core.common.TimeUtils; +import org.opencb.opencga.core.config.ExecutionQueue; +import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.job.Job; +import org.opencb.opencga.core.models.job.JobInternal; +import org.opencb.opencga.core.models.job.JobStudyParam; +import org.opencb.opencga.core.models.job.ToolInfo; +import org.opencb.opencga.core.models.organizations.Organization; +import org.opencb.opencga.core.response.OpenCGAResult; +import org.opencb.opencga.core.testclassification.duration.ShortTests; + +import java.util.*; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +@Category(ShortTests.class) +public class JobSchedulerTest { + + @Mock + private CatalogManager catalogManager; + @Mock + private JobManager jobManager; + @Mock + private UserManager userManager; + @Mock + private OrganizationManager organizationManager; + @Mock + private StudyManager studyManager; + + private JobScheduler jobScheduler; + private List queues; + private String token = "test-token"; + private String organizationId = "test-org"; + + @Before + public void setUp() throws CatalogException { + MockitoAnnotations.initMocks(this); + + // Setup catalog manager mocks + when(catalogManager.getJobManager()).thenReturn(jobManager); + when(catalogManager.getUserManager()).thenReturn(userManager); + when(catalogManager.getOrganizationManager()).thenReturn(organizationManager); + when(catalogManager.getStudyManager()).thenReturn(studyManager); + + // Setup default mocks for user roles + when(userManager.search(anyString(), any(Query.class), any(QueryOptions.class), anyString())) + .thenReturn(new OpenCGAResult<>(0, Collections.emptyList(), 0, Collections.emptyList(), 0)); + + Organization organization = new Organization().setOwner("owner").setAdmins(Collections.emptyList()); + when(organizationManager.get(anyString(), any(QueryOptions.class), anyString())) + .thenReturn(new OpenCGAResult<>(0, Collections.emptyList(), 1, Collections.singletonList(organization), 1L)); + + when(studyManager.searchInOrganization(anyString(), any(Query.class), any(QueryOptions.class), anyString())) + .thenReturn(new OpenCGAResult<>(0, Collections.emptyList(), 0, Collections.emptyList(), 0)); + + // Create test queues + queues = Arrays.asList( + new ExecutionQueue().setId("small-queue").setType(ExecutionQueue.ExecutionType.CPU).setCpu(2).setMemory("4GB"), + new ExecutionQueue().setId("medium-queue").setType(ExecutionQueue.ExecutionType.CPU).setCpu(8).setMemory("16GB"), + new ExecutionQueue().setId("large-queue").setType(ExecutionQueue.ExecutionType.CPU).setCpu(16).setMemory("32GB"), + new ExecutionQueue().setId("local-queue").setType(ExecutionQueue.ExecutionType.GPU).setCpu(4).setMemory("8GB") + ); + + jobScheduler = new JobScheduler(catalogManager, queues, token); + } + + @Test + public void testScheduleJobsWithoutQuotaExceeded() throws CatalogException { + // Setup: quota not exceeded + when(jobManager.exceedsExecutionLimitQuota(organizationId)).thenReturn(false); + + List pendingJobs = createTestJobs(); + Set exhaustedQueues = Collections.emptySet(); + + Map> result = jobScheduler.schedule(organizationId, pendingJobs, exhaustedQueues); + + // Verify jobs are assigned to appropriate queues + assertTrue(result.containsKey("small-queue")); + assertTrue(result.containsKey("medium-queue")); + assertEquals(1, result.get("small-queue").size()); + assertEquals(1, result.get("medium-queue").size()); + } + + @Test + public void testScheduleJobsWithQuotaExceeded() throws CatalogException { + // Setup: quota exceeded + when(jobManager.exceedsExecutionLimitQuota(organizationId)).thenReturn(true); + + List pendingJobs = createTestJobs(); + Set exhaustedQueues = Collections.emptySet(); + + Map> result = jobScheduler.schedule(organizationId, pendingJobs, exhaustedQueues); + + // Verify no jobs are scheduled when quota is exceeded + assertTrue(result.isEmpty() || result.values().stream().allMatch(List::isEmpty)); + + // Verify reschedule method is called + verify(jobManager, atLeastOnce()).rescheduleJobs(anyString(), anyList(), anyString(), anyString(), eq(token)); + } + + @Test + public void testScheduleJobsWithExhaustedQueues() throws CatalogException { + // Setup: quota not exceeded but some queues are exhausted + when(jobManager.exceedsExecutionLimitQuota(organizationId)).thenReturn(false); + + List pendingJobs = createTestJobs(); + Set exhaustedQueues = new HashSet<>(Arrays.asList("small-queue", "medium-queue")); + + Map> result = jobScheduler.schedule(organizationId, pendingJobs, exhaustedQueues); + + // Verify exhausted queues are not used + assertFalse(result.containsKey("small-queue")); + assertFalse(result.containsKey("medium-queue")); + + // Jobs should be assigned to available queues or not scheduled at all + assertTrue(result.isEmpty() || result.keySet().stream().noneMatch(exhaustedQueues::contains)); + } + + @Test + public void testScheduleJobsWithAllQueuesExhausted() throws CatalogException { + // Setup: quota not exceeded but all queues are exhausted + when(jobManager.exceedsExecutionLimitQuota(organizationId)).thenReturn(false); + + List pendingJobs = createTestJobs(); + Set exhaustedQueues = new HashSet<>(Arrays.asList("small-queue", "medium-queue", "large-queue", "local-queue")); + + Map> result = jobScheduler.schedule(organizationId, pendingJobs, exhaustedQueues); + + // Verify no jobs are scheduled when all queues are exhausted + assertTrue(result.isEmpty()); + } + + @Test + public void testScheduleJobsWithMixedConditions() throws CatalogException { + // Setup: quota not exceeded, some queues exhausted + when(jobManager.exceedsExecutionLimitQuota(organizationId)).thenReturn(false); + + List pendingJobs = createTestJobs(); + Set exhaustedQueues = new HashSet<>(Arrays.asList("small-queue")); + + Map> result = jobScheduler.schedule(organizationId, pendingJobs, exhaustedQueues); + + // Verify exhausted queue is not used + assertFalse(result.containsKey("small-queue")); + + // Other queues should still be available + assertTrue(result.containsKey("medium-queue")); + assertEquals(1, result.get("medium-queue").size()); + } + + @Test + public void testScheduleJobsWithScheduledStartTime() throws CatalogException { + // Setup: quota not exceeded + when(jobManager.exceedsExecutionLimitQuota(organizationId)).thenReturn(false); + + // Create job with future scheduled start time + Date futureDate = new Date(System.currentTimeMillis() + 86400000); // Tomorrow + Job futureJob = createJobWithScheduledTime("future-job", futureDate); + + List pendingJobs = Collections.singletonList(futureJob); + Set exhaustedQueues = Collections.emptySet(); + + Map> result = jobScheduler.schedule(organizationId, pendingJobs, exhaustedQueues); + + // Verify future scheduled job is not executed yet + assertTrue(result.isEmpty() || result.values().stream().allMatch(List::isEmpty)); + } + + @Test + public void testScheduleJobsWithPastScheduledTime() throws CatalogException { + // Setup: quota not exceeded + when(jobManager.exceedsExecutionLimitQuota(organizationId)).thenReturn(false); + + // Create job with past scheduled start time + Date pastDate = new Date(System.currentTimeMillis() - 86400000); // Yesterday + Job pastJob = createJobWithScheduledTime("past-job", pastDate); + + List pendingJobs = Collections.singletonList(pastJob); + Set exhaustedQueues = Collections.emptySet(); + + Map> result = jobScheduler.schedule(organizationId, pendingJobs, exhaustedQueues); + + // Verify past scheduled job is executed + assertFalse(result.isEmpty()); + assertTrue(result.values().stream().anyMatch(jobs -> !jobs.isEmpty())); + } + + private List createTestJobs() { + List jobs = new ArrayList<>(); + + // Small job for small queue + Job smallJob = new Job() + .setId("small-job") + .setUserId("user1") + .setStudy(new JobStudyParam(organizationId + "@project:study1")) + .setPriority(Enums.Priority.MEDIUM) + .setInternal(new JobInternal().setRegistrationDate(TimeUtils.getTime())) + .setTool(new ToolInfo().setMinimumRequirements( + new org.opencb.opencga.core.models.job.MinimumRequirements() + .setType(ExecutionQueue.ExecutionType.CPU) + .setCpu("2") + .setMemory("4GB") + .setQueue("small-queue") + )); + jobs.add(smallJob); + + // Medium job for medium queue + Job mediumJob = new Job() + .setId("medium-job") + .setUserId("user2") + .setStudy(new JobStudyParam(organizationId + "@project:study1")) + .setPriority(Enums.Priority.HIGH) + .setInternal(new JobInternal().setRegistrationDate(TimeUtils.getTime())) + .setTool(new ToolInfo().setMinimumRequirements( + new org.opencb.opencga.core.models.job.MinimumRequirements() + .setType(ExecutionQueue.ExecutionType.CPU) + .setCpu("4") + .setMemory("8GB") + .setQueue("medium-queue") + )); + jobs.add(mediumJob); + + return jobs; + } + + private Job createJobWithScheduledTime(String jobId, Date scheduledTime) { + return new Job() + .setId(jobId) + .setUserId("user1") + .setStudy(new JobStudyParam(organizationId + "@project:study1")) + .setPriority(Enums.Priority.MEDIUM) + .setScheduledStartTime(TimeUtils.getTime(scheduledTime)) + .setInternal(new JobInternal().setRegistrationDate(TimeUtils.getTime())) + .setTool(new ToolInfo().setMinimumRequirements( + new org.opencb.opencga.core.models.job.MinimumRequirements() + .setType(ExecutionQueue.ExecutionType.CPU) + .setCpu("2") + .setMemory("4GB") + .setQueue("small-queue") + )); + } +} \ No newline at end of file From 11c52343ba4f43610c927efda4560ebc386873ef Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 31 Jul 2025 14:45:44 +0200 Subject: [PATCH 14/22] core: rename type to processorType, #TASK-7442 --- .../main/executors/JobsCommandExecutor.java | 9 ++++ .../executors/WorkflowsCommandExecutor.java | 4 +- .../cli/main/options/JobsCommandOptions.java | 27 +++++++++++ .../main/options/WorkflowsCommandOptions.java | 8 ++-- .../catalog/JobQueueMigration_Task7442.java | 2 +- .../opencga/catalog/db/api/JobDBAdaptor.java | 3 ++ .../catalog/db/mongodb/JobMongoDBAdaptor.java | 3 ++ .../opencga/catalog/managers/JobManager.java | 2 +- .../catalog/utils/JobExecutionUtils.java | 2 +- .../src/main/resources/catalog-indexes.txt | 3 ++ .../catalog/utils/JobExecutionUtilsTest.java | 48 +++++++++---------- opencga-client/src/main/R/R/Job-methods.R | 15 ++++-- .../client/rest/clients/JobClient.java | 9 ++++ opencga-client/src/main/javascript/Job.js | 12 +++++ .../pyopencga/rest_clients/job_client.py | 18 +++++++ .../opencga/core/api/ParamConstants.java | 7 +++ .../opencga/core/config/ExecutionQueue.java | 21 ++++---- .../core/models/job/MinimumRequirements.java | 18 +++---- .../monitor/schedulers/JobScheduler.java | 10 ++-- .../monitor/schedulers/JobSchedulerTest.java | 14 +++--- .../opencga/server/rest/JobWSServer.java | 9 ++++ 21 files changed, 177 insertions(+), 67 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java index 6ad06fd9885..ac2f7df2057 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/JobsCommandExecutor.java @@ -177,6 +177,9 @@ private RestResponse aggregationStats() throws Exception { queryParams.putIfNotEmpty("uuid", commandOptions.uuid); queryParams.putIfNotEmpty("toolId", commandOptions.toolId); queryParams.putIfNotEmpty("toolType", commandOptions.toolType); + queryParams.putIfNotEmpty("toolMinimumRequirementsQueue", commandOptions.toolMinimumRequirementsQueue); + queryParams.putIfNotEmpty("toolMinimumRequirementsProcessorType", commandOptions.toolMinimumRequirementsProcessorType); + queryParams.putIfNotEmpty("executionQueueId", commandOptions.executionQueueId); queryParams.putIfNotEmpty("userId", commandOptions.userId); queryParams.putIfNotEmpty("priority", commandOptions.priority); queryParams.putIfNotEmpty("status", commandOptions.status); @@ -262,6 +265,9 @@ private RestResponse distinct() throws Exception { queryParams.putIfNotEmpty("type", commandOptions.type); queryParams.putIfNotEmpty("toolId", commandOptions.toolId); queryParams.putIfNotEmpty("toolType", commandOptions.toolType); + queryParams.putIfNotEmpty("toolMinimumRequirementsQueue", commandOptions.toolMinimumRequirementsQueue); + queryParams.putIfNotEmpty("toolMinimumRequirementsProcessorType", commandOptions.toolMinimumRequirementsProcessorType); + queryParams.putIfNotEmpty("executionQueueId", commandOptions.executionQueueId); queryParams.putIfNotEmpty("toolExternalExecutorId", commandOptions.toolExternalExecutorId); queryParams.putIfNotEmpty("parentId", commandOptions.parentId); queryParams.putIfNotNull("dryRun", commandOptions.dryRun); @@ -343,6 +349,9 @@ private RestResponse search() throws Exception { queryParams.putIfNotEmpty("type", commandOptions.type); queryParams.putIfNotEmpty("toolId", commandOptions.toolId); queryParams.putIfNotEmpty("toolType", commandOptions.toolType); + queryParams.putIfNotEmpty("toolMinimumRequirementsQueue", commandOptions.toolMinimumRequirementsQueue); + queryParams.putIfNotEmpty("toolMinimumRequirementsProcessorType", commandOptions.toolMinimumRequirementsProcessorType); + queryParams.putIfNotEmpty("executionQueueId", commandOptions.executionQueueId); queryParams.putIfNotEmpty("toolExternalExecutorId", commandOptions.toolExternalExecutorId); queryParams.putIfNotEmpty("parentId", commandOptions.parentId); queryParams.putIfNotNull("dryRun", commandOptions.dryRun); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java index f9c70e318d3..f1b70e9426a 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/WorkflowsCommandExecutor.java @@ -175,7 +175,7 @@ private RestResponse create() throws Exception { putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); - putNestedIfNotNull(beanParams, "minimumRequirements.type", commandOptions.minimumRequirementsType, true); + putNestedIfNotNull(beanParams, "minimumRequirements.processorType", commandOptions.minimumRequirementsProcessorType, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.queue", commandOptions.minimumRequirementsQueue, true); putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); @@ -363,7 +363,7 @@ private RestResponse update() throws Exception { putNestedIfNotEmpty(beanParams, "minimumRequirements.cpu", commandOptions.minimumRequirementsCpu, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.memory", commandOptions.minimumRequirementsMemory, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.disk", commandOptions.minimumRequirementsDisk, true); - putNestedIfNotNull(beanParams, "minimumRequirements.type", commandOptions.minimumRequirementsType, true); + putNestedIfNotNull(beanParams, "minimumRequirements.processorType", commandOptions.minimumRequirementsProcessorType, true); putNestedIfNotEmpty(beanParams, "minimumRequirements.queue", commandOptions.minimumRequirementsQueue, true); putNestedIfNotEmpty(beanParams, "creationDate", commandOptions.creationDate, true); putNestedIfNotEmpty(beanParams, "modificationDate", commandOptions.modificationDate, true); diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java index bd510574728..9f112a90908 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/JobsCommandOptions.java @@ -123,6 +123,15 @@ public class AggregationStatsCommandOptions { @Parameter(names = {"--tool-type"}, description = "Tool type executed by the job [OPERATION, ANALYSIS]", required = false, arity = 1) public String toolType; + @Parameter(names = {"--tool.minimum-requirements.queue"}, description = "Queue where the job is expected to be executed.", required = false, arity = 1) + public String toolMinimumRequirementsQueue; + + @Parameter(names = {"--tool.minimum-requirements.processor-type"}, description = "Processor type required to run the job. Allowed values: [CPU, GPU, FPGA]", required = false, arity = 1) + public String toolMinimumRequirementsProcessorType; + + @Parameter(names = {"--execution.queue.id"}, description = "Queue id where the job has been submitted to be executed.", required = false, arity = 1) + public String executionQueueId; + @Parameter(names = {"--user-id"}, description = "User that created the job", required = false, arity = 1) public String userId; @@ -271,6 +280,15 @@ public class DistinctCommandOptions { @Parameter(names = {"--tool-type"}, description = "Tool type executed by the job [OPERATION, ANALYSIS]", required = false, arity = 1) public String toolType; + @Parameter(names = {"--tool.minimum-requirements.queue"}, description = "Queue where the job is expected to be executed.", required = false, arity = 1) + public String toolMinimumRequirementsQueue; + + @Parameter(names = {"--tool.minimum-requirements.processor-type"}, description = "Processor type required to run the job. Allowed values: [CPU, GPU, FPGA]", required = false, arity = 1) + public String toolMinimumRequirementsProcessorType; + + @Parameter(names = {"--execution.queue.id"}, description = "Queue id where the job has been submitted to be executed.", required = false, arity = 1) + public String executionQueueId; + @Parameter(names = {"--tool.external-executor.id"}, description = "Id of the external executor. This field is only applicable for jobs executed by an external executor.", required = false, arity = 1) public String toolExternalExecutorId; @@ -410,6 +428,15 @@ public class SearchCommandOptions { @Parameter(names = {"--tool-type"}, description = "Tool type executed by the job [OPERATION, ANALYSIS]", required = false, arity = 1) public String toolType; + @Parameter(names = {"--tool.minimum-requirements.queue"}, description = "Queue where the job is expected to be executed.", required = false, arity = 1) + public String toolMinimumRequirementsQueue; + + @Parameter(names = {"--tool.minimum-requirements.processor-type"}, description = "Processor type required to run the job. Allowed values: [CPU, GPU, FPGA]", required = false, arity = 1) + public String toolMinimumRequirementsProcessorType; + + @Parameter(names = {"--execution.queue.id"}, description = "Queue id where the job has been submitted to be executed.", required = false, arity = 1) + public String executionQueueId; + @Parameter(names = {"--tool.external-executor.id"}, description = "Id of the external executor. This field is only applicable for jobs executed by an external executor.", required = false, arity = 1) public String toolExternalExecutorId; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java index 127b5eebf7f..72b2840cd17 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/WorkflowsCommandOptions.java @@ -160,8 +160,8 @@ public class CreateCommandOptions { @Parameter(names = {"--minimum-requirements-disk"}, description = "Minimum disk required to execute the process.", required = false, arity = 1) public String minimumRequirementsDisk; - @Parameter(names = {"--minimum-requirements-type"}, description = "Execution type. Must be one of CPU, GPU or FPGA.", required = false, arity = 1) - public String minimumRequirementsType; + @Parameter(names = {"--minimum-requirements-processor-type"}, description = "Execution type. Must be one of CPU, GPU or FPGA.", required = false, arity = 1) + public String minimumRequirementsProcessorType; @Parameter(names = {"--minimum-requirements-queue"}, description = "Expected queue to execute the process.", required = false, arity = 1) public String minimumRequirementsQueue; @@ -440,8 +440,8 @@ public class UpdateCommandOptions { @Parameter(names = {"--minimum-requirements-disk"}, description = "Minimum disk required to execute the process.", required = false, arity = 1) public String minimumRequirementsDisk; - @Parameter(names = {"--minimum-requirements-type"}, description = "Execution type. Must be one of CPU, GPU or FPGA.", required = false, arity = 1) - public String minimumRequirementsType; + @Parameter(names = {"--minimum-requirements-processor-type"}, description = "Execution type. Must be one of CPU, GPU or FPGA.", required = false, arity = 1) + public String minimumRequirementsProcessorType; @Parameter(names = {"--minimum-requirements-queue"}, description = "Expected queue to execute the process.", required = false, arity = 1) public String minimumRequirementsQueue; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/JobQueueMigration_Task7442.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/JobQueueMigration_Task7442.java index 85badcb6853..13941e00b9d 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/JobQueueMigration_Task7442.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/JobQueueMigration_Task7442.java @@ -26,7 +26,7 @@ protected void run() throws Exception { (document, bulk) -> { MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); - updateDocument.getSet().put("tool.minimumRequirements.type", ExecutionQueue.ExecutionType.CPU.name()); + updateDocument.getSet().put("tool.minimumRequirements.type", ExecutionQueue.ProcessorType.CPU.name()); updateDocument.getSet().put("tool.minimumRequirements.queue", ""); updateDocument.getSet().put("execution.queue", new Document()); bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java index a172df254f3..1ee97f093c7 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/JobDBAdaptor.java @@ -119,6 +119,8 @@ enum QueryParams implements QueryParam { TOOL_EXTERNAL_EXECUTOR_ID("tool.externalExecutor.id", TEXT, ""), TOOL_TYPE("tool.type", TEXT, ""), TOOL_EXTERNAL_EXECUTOR("tool.externalExecutor", OBJECT, ""), + TOOL_REQUIREMENTS_PROCESSOR_TYPE(ParamConstants.JOB_TOOL_REQUIREMENTS_PROCESSOR_TYPE_PARAM, STRING, ""), + TOOL_REQUIREMENTS_QUEUE(ParamConstants.JOB_TOOL_REQUIREMENTS_QUEUE_PARAM, STRING, ""), PARENT_ID("parentId", TEXT, ""), PRIORITY("priority", TEXT, ""), @@ -146,6 +148,7 @@ enum QueryParams implements QueryParam { EXECUTION("execution", OBJECT, ""), EXECUTION_START("execution.start", DATE, ""), EXECUTION_END("execution.end", DATE, ""), + EXECUTION_QUEUE_ID(ParamConstants.JOB_EXECUTION_QUEUE_ID_PARAM, STRING, ""), STDOUT("stdout", OBJECT, ""), STDERR("stderr", OBJECT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java index 86c1d85d341..ead9ecc64c4 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/JobMongoDBAdaptor.java @@ -951,6 +951,8 @@ private Bson parseQuery(Query query, Document extraQuery, QueryOptions options, case TYPE: case UUID: case TOOL_TYPE: + case TOOL_REQUIREMENTS_QUEUE: + case TOOL_REQUIREMENTS_PROCESSOR_TYPE: case PRIORITY: case TOOL_EXTERNAL_EXECUTOR_ID: case PARENT_ID: @@ -961,6 +963,7 @@ private Bson parseQuery(Query query, Document extraQuery, QueryOptions options, // case OUTPUT_ERROR: // case EXECUTION_START: case EXECUTION_END: + case EXECUTION_QUEUE_ID: // case COMMAND_LINE: case VISITED: case RELEASE: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java index c5eac540b3e..881402e1e6c 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java @@ -340,7 +340,7 @@ private void autoCompleteNewJob(String organizationId, Study study, Job job, Jwt MinimumRequirements requirements = job.getTool().getMinimumRequirements(); requirements.setCpu(ParamUtils.defaultString(requirements.getCpu(), String.valueOf(request.getCpu()))); requirements.setMemory(ParamUtils.defaultString(requirements.getMemory(), request.getMemory())); - requirements.setType(ParamUtils.defaultObject(requirements.getType(), ExecutionQueue.ExecutionType.CPU)); + requirements.setProcessorType(ParamUtils.defaultObject(requirements.getProcessorType(), ExecutionQueue.ProcessorType.CPU)); if (StringUtils.isEmpty(requirements.getQueue())) { ExecutionQueue executionQueue = JobExecutionUtils.findOptimalQueues(configuration.getAnalysis().getExecution().getQueues(), requirements).get(0); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java index ce6bca06d7f..78dbb563386 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java @@ -13,7 +13,7 @@ public static List findOptimalQueues(List queues // Sort the queues based on their number of CPUs and memory // Also filter out queues that do not meet the minimum requirements return queues.stream() - .filter(queue -> queue.getType().equals(requirements.getType()) + .filter(queue -> queue.getProcessorType().equals(requirements.getProcessorType()) && queue.getCpu() >= Integer.parseInt(requirements.getCpu()) && IOUtils.fromHumanReadableToByte(queue.getMemory()) >= IOUtils.fromHumanReadableToByte(requirements.getMemory())) .sorted((q1, q2) -> { diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index eab521fc907..a96dd49ae03 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -23,6 +23,8 @@ {"collections": ["job"], "fields": {"type": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"tool.id": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"tool.type": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"tool.minimumRequirements.processorType": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"tool.minimumRequirements.queue": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"userId": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"parentId": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"outDir.uid": 1, "studyUid": 1}, "options": {}} @@ -35,6 +37,7 @@ {"collections": ["job"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"internal.status.id": 1, "_priority": 1, "_creationDate": 1}, "options": {}} {"collections": ["job"], "fields": {"internal.killJobRequested": 1, "studyUid": 1}, "options": {}} +{"collections": ["job"], "fields": {"execution.queue.id": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"_creationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"_modificationDate": 1, "studyUid": 1}, "options": {}} {"collections": ["job"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java index ae4a374ea9a..8ea183c537e 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java @@ -19,14 +19,14 @@ public class JobExecutionUtilsTest { @Test public void testFindOptimalQueuesBasicFiltering() { // Create test queues - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(2).setMemory("4GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); - ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setType(ExecutionQueue.ExecutionType.GPU).setCpu(8).setMemory("16GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(2).setMemory("4GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setProcessorType(ExecutionQueue.ProcessorType.GPU).setCpu(8).setMemory("16GB"); List queues = Arrays.asList(queue1, queue2, queue3); // Test filtering by type and minimum requirements - MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("2").setMemory("4GB"); + MinimumRequirements requirements = new MinimumRequirements().setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("2").setMemory("4GB"); List result = JobExecutionUtils.findOptimalQueues(queues, requirements); @@ -38,13 +38,13 @@ public void testFindOptimalQueuesBasicFiltering() { @Test public void testFindOptimalQueuesSortingByCpu() { // Create queues with different CPU counts but same memory - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(8).setMemory("8GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(2).setMemory("8GB"); - ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(8).setMemory("8GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(2).setMemory("8GB"); + ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); List queues = Arrays.asList(queue1, queue2, queue3); - MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("2").setMemory("8GB"); + MinimumRequirements requirements = new MinimumRequirements().setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("2").setMemory("8GB"); List result = JobExecutionUtils.findOptimalQueues(queues, requirements); @@ -58,13 +58,13 @@ public void testFindOptimalQueuesSortingByCpu() { @Test public void testFindOptimalQueuesSortingByMemory() { // Create queues with same CPU but different memory - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("16GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("4GB"); - ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("16GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("4GB"); + ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); List queues = Arrays.asList(queue1, queue2, queue3); - MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("4").setMemory("4GB"); + MinimumRequirements requirements = new MinimumRequirements().setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("4GB"); List result = JobExecutionUtils.findOptimalQueues(queues, requirements); @@ -77,12 +77,12 @@ public void testFindOptimalQueuesSortingByMemory() { @Test public void testFindOptimalQueuesFiltersByMinimumCpu() { - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(1).setMemory("8GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(1).setMemory("8GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); List queues = Arrays.asList(queue1, queue2); - MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("2").setMemory("8GB"); + MinimumRequirements requirements = new MinimumRequirements().setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("2").setMemory("8GB"); List result = JobExecutionUtils.findOptimalQueues(queues, requirements); @@ -92,12 +92,12 @@ public void testFindOptimalQueuesFiltersByMinimumCpu() { @Test public void testFindOptimalQueuesFiltersByMinimumMemory() { - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("2GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("2GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); List queues = Arrays.asList(queue1, queue2); - MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("4").setMemory("4GB"); + MinimumRequirements requirements = new MinimumRequirements().setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("4GB"); List result = JobExecutionUtils.findOptimalQueues(queues, requirements); @@ -107,12 +107,12 @@ public void testFindOptimalQueuesFiltersByMinimumMemory() { @Test public void testFindOptimalQueuesFiltersByType() { - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.GPU).setCpu(4).setMemory("8GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setType(ExecutionQueue.ExecutionType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.GPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); List queues = Arrays.asList(queue1, queue2); - MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("4").setMemory("8GB"); + MinimumRequirements requirements = new MinimumRequirements().setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("8GB"); List result = JobExecutionUtils.findOptimalQueues(queues, requirements); @@ -122,12 +122,12 @@ public void testFindOptimalQueuesFiltersByType() { @Test public void testFindOptimalQueuesEmptyResult() { - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setType(ExecutionQueue.ExecutionType.CPU).setCpu(2).setMemory("4GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(2).setMemory("4GB"); List queues = Collections.singletonList(queue1); // Requirements that no queue can meet - MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("8").setMemory("16GB"); + MinimumRequirements requirements = new MinimumRequirements().setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("8").setMemory("16GB"); List result = JobExecutionUtils.findOptimalQueues(queues, requirements); @@ -137,7 +137,7 @@ public void testFindOptimalQueuesEmptyResult() { @Test public void testFindOptimalQueuesEmptyInput() { List queues = Collections.emptyList(); - MinimumRequirements requirements = new MinimumRequirements().setType(ExecutionQueue.ExecutionType.CPU).setCpu("4").setMemory("8GB"); + MinimumRequirements requirements = new MinimumRequirements().setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("8GB"); List result = JobExecutionUtils.findOptimalQueues(queues, requirements); diff --git a/opencga-client/src/main/R/R/Job-methods.R b/opencga-client/src/main/R/R/Job-methods.R index 7f8ba86e665..ccf048cc6ef 100644 --- a/opencga-client/src/main/R/R/Job-methods.R +++ b/opencga-client/src/main/R/R/Job-methods.R @@ -20,11 +20,11 @@ #' | endpointName | Endpoint WS | parameters accepted | #' | -- | :-- | --: | #' | updateAcl | /{apiVersion}/jobs/acl/{members}/update | members[*], action[*], body[*] | -#' | aggregationStats | /{apiVersion}/jobs/aggregationStats | study, otherStudies, id, uuid, toolId, toolType, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted, field | +#' | aggregationStats | /{apiVersion}/jobs/aggregationStats | study, otherStudies, id, uuid, toolId, toolType, tool.minimumRequirements.queue, tool.minimumRequirements.processorType, execution.queue.id, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted, field | #' | create | /{apiVersion}/jobs/create | study, body[*] | -#' | distinct | /{apiVersion}/jobs/distinct | study, otherStudies, id, uuid, type, toolId, toolType, tool.externalExecutor.id, parentId, dryRun, internal.killJobRequested, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted, field[*] | +#' | distinct | /{apiVersion}/jobs/distinct | study, otherStudies, id, uuid, type, toolId, toolType, tool.minimumRequirements.queue, tool.minimumRequirements.processorType, execution.queue.id, tool.externalExecutor.id, parentId, dryRun, internal.killJobRequested, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted, field[*] | #' | retry | /{apiVersion}/jobs/retry | jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, study, body[*] | -#' | search | /{apiVersion}/jobs/search | include, exclude, limit, skip, count, study, otherStudies, id, uuid, type, toolId, toolType, tool.externalExecutor.id, parentId, dryRun, internal.killJobRequested, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted | +#' | search | /{apiVersion}/jobs/search | include, exclude, limit, skip, count, study, otherStudies, id, uuid, type, toolId, toolType, tool.minimumRequirements.queue, tool.minimumRequirements.processorType, execution.queue.id, tool.externalExecutor.id, parentId, dryRun, internal.killJobRequested, userId, priority, status, internalStatus, creationDate, modificationDate, visited, tags, input, output, acl, release, deleted | #' | buildTool | /{apiVersion}/jobs/tool/build | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | runTool | /{apiVersion}/jobs/tool/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | #' | top | /{apiVersion}/jobs/top | limit, study, internalStatus, priority, userId, toolId | @@ -62,6 +62,9 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpoi #' @param uuid Comma separated list of job UUIDs up to a maximum of 100. #' @param toolId Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param toolType Tool type executed by the job [OPERATION, ANALYSIS]. + #' @param tool.minimumRequirements.queue Queue where the job is expected to be executed. + #' @param tool.minimumRequirements.processorType Processor type required to run the job. Allowed values: [CPU, GPU, FPGA]. + #' @param execution.queue.id Queue id where the job has been submitted to be executed. #' @param userId User that created the job. #' @param priority Priority of the job. #' @param status Filter by status. @@ -96,6 +99,9 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpoi #' @param type Job type (NATIVE, WORKFLOW, CUSTOM or WALKER). #' @param toolId Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param toolType Tool type executed by the job [OPERATION, ANALYSIS]. + #' @param tool.minimumRequirements.queue Queue where the job is expected to be executed. + #' @param tool.minimumRequirements.processorType Processor type required to run the job. Allowed values: [CPU, GPU, FPGA]. + #' @param execution.queue.id Queue id where the job has been submitted to be executed. #' @param tool.externalExecutor.id Id of the external executor. This field is only applicable for jobs executed by an external executor. #' @param parentId Job id that generated this job (if any). #' @param dryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. @@ -143,6 +149,9 @@ setMethod("jobClient", "OpencgaR", function(OpencgaR, job, jobs, members, endpoi #' @param type Job type (NATIVE, WORKFLOW, CUSTOM or WALKER). #' @param toolId Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. #' @param toolType Tool type executed by the job [OPERATION, ANALYSIS]. + #' @param tool.minimumRequirements.queue Queue where the job is expected to be executed. + #' @param tool.minimumRequirements.processorType Processor type required to run the job. Allowed values: [CPU, GPU, FPGA]. + #' @param execution.queue.id Queue id where the job has been submitted to be executed. #' @param tool.externalExecutor.id Id of the external executor. This field is only applicable for jobs executed by an external executor. #' @param parentId Job id that generated this job (if any). #' @param dryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java index f3eb67bf579..00c0d96a254 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/JobClient.java @@ -82,6 +82,9 @@ public RestResponse updateAcl(String members, String action, Jo * toolId: Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. * '~value' for case sensitive, '~/value/i' for case insensitive search. * toolType: Tool type executed by the job [OPERATION, ANALYSIS]. + * tool.minimumRequirements.queue: Queue where the job is expected to be executed. + * tool.minimumRequirements.processorType: Processor type required to run the job. Allowed values: [CPU, GPU, FPGA]. + * execution.queue.id: Queue id where the job has been submitted to be executed. * userId: User that created the job. * priority: Priority of the job. * status: Filter by status. @@ -134,6 +137,9 @@ public RestResponse create(JobCreateParams data, ObjectMap params) throws C * toolId: Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. * '~value' for case sensitive, '~/value/i' for case insensitive search. * toolType: Tool type executed by the job [OPERATION, ANALYSIS]. + * tool.minimumRequirements.queue: Queue where the job is expected to be executed. + * tool.minimumRequirements.processorType: Processor type required to run the job. Allowed values: [CPU, GPU, FPGA]. + * execution.queue.id: Queue id where the job has been submitted to be executed. * tool.externalExecutor.id: Id of the external executor. This field is only applicable for jobs executed by an external executor. * parentId: Job id that generated this job (if any). * dryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters @@ -199,6 +205,9 @@ public RestResponse retry(JobRetryParams data, ObjectMap params) throws Cli * toolId: Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. * '~value' for case sensitive, '~/value/i' for case insensitive search. * toolType: Tool type executed by the job [OPERATION, ANALYSIS]. + * tool.minimumRequirements.queue: Queue where the job is expected to be executed. + * tool.minimumRequirements.processorType: Processor type required to run the job. Allowed values: [CPU, GPU, FPGA]. + * execution.queue.id: Queue id where the job has been submitted to be executed. * tool.externalExecutor.id: Id of the external executor. This field is only applicable for jobs executed by an external executor. * parentId: Job id that generated this job (if any). * dryRun: Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters diff --git a/opencga-client/src/main/javascript/Job.js b/opencga-client/src/main/javascript/Job.js index 8469a01bde6..7bd05a57258 100644 --- a/opencga-client/src/main/javascript/Job.js +++ b/opencga-client/src/main/javascript/Job.js @@ -55,6 +55,10 @@ export default class Job extends OpenCGAParentClass { * @param {String} [params.toolId] - Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.toolType] - Tool type executed by the job [OPERATION, ANALYSIS]. + * @param {String} [params.tool.minimumRequirements.queue] - Queue where the job is expected to be executed. + * @param {String} [params.tool.minimumRequirements.processorType] - Processor type required to run the job. Allowed values: [CPU, GPU, + * FPGA]. + * @param {String} [params.execution.queue.id] - Queue id where the job has been submitted to be executed. * @param {String} [params.userId] - User that created the job. * @param {String} [params.priority] - Priority of the job. * @param {String} [params.status] - Filter by status. @@ -101,6 +105,10 @@ export default class Job extends OpenCGAParentClass { * @param {String} [params.toolId] - Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.toolType] - Tool type executed by the job [OPERATION, ANALYSIS]. + * @param {String} [params.tool.minimumRequirements.queue] - Queue where the job is expected to be executed. + * @param {String} [params.tool.minimumRequirements.processorType] - Processor type required to run the job. Allowed values: [CPU, GPU, + * FPGA]. + * @param {String} [params.execution.queue.id] - Queue id where the job has been submitted to be executed. * @param {String} [params.tool.externalExecutor.id] - Id of the external executor. This field is only applicable for jobs executed by an * external executor. * @param {String} [params.parentId] - Job id that generated this job (if any). @@ -162,6 +170,10 @@ export default class Job extends OpenCGAParentClass { * @param {String} [params.toolId] - Tool ID executed by the job. Also admits basic regular expressions using the operator '~', i.e. * '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param {String} [params.toolType] - Tool type executed by the job [OPERATION, ANALYSIS]. + * @param {String} [params.tool.minimumRequirements.queue] - Queue where the job is expected to be executed. + * @param {String} [params.tool.minimumRequirements.processorType] - Processor type required to run the job. Allowed values: [CPU, GPU, + * FPGA]. + * @param {String} [params.execution.queue.id] - Queue id where the job has been submitted to be executed. * @param {String} [params.tool.externalExecutor.id] - Id of the external executor. This field is only applicable for jobs executed by an * external executor. * @param {String} [params.parentId] - Job id that generated this job (if any). diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py index 6e43d071508..6bcd8949817 100644 --- a/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py +++ b/opencga-client/src/main/python/pyopencga/rest_clients/job_client.py @@ -56,6 +56,12 @@ def aggregation_stats(self, **options): search. :param str tool_type: Tool type executed by the job [OPERATION, ANALYSIS]. + :param str tool.minimum_requirements.queue: Queue where the job is + expected to be executed. + :param str tool.minimum_requirements.processor_type: Processor type + required to run the job. Allowed values: [CPU, GPU, FPGA]. + :param str execution.queue.id: Queue id where the job has been + submitted to be executed. :param str user_id: User that created the job. :param str priority: Priority of the job. :param str status: Filter by status. @@ -118,6 +124,12 @@ def distinct(self, field, **options): search. :param str tool_type: Tool type executed by the job [OPERATION, ANALYSIS]. + :param str tool.minimum_requirements.queue: Queue where the job is + expected to be executed. + :param str tool.minimum_requirements.processor_type: Processor type + required to run the job. Allowed values: [CPU, GPU, FPGA]. + :param str execution.queue.id: Queue id where the job has been + submitted to be executed. :param str tool.external_executor.id: Id of the external executor. This field is only applicable for jobs executed by an external executor. @@ -202,6 +214,12 @@ def search(self, **options): search. :param str tool_type: Tool type executed by the job [OPERATION, ANALYSIS]. + :param str tool.minimum_requirements.queue: Queue where the job is + expected to be executed. + :param str tool.minimum_requirements.processor_type: Processor type + required to run the job. Allowed values: [CPU, GPU, FPGA]. + :param str execution.queue.id: Queue id where the job has been + submitted to be executed. :param str tool.external_executor.id: Id of the external executor. This field is only applicable for jobs executed by an external executor. diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java index 7f98cf64e69..087a2d4a9f1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/ParamConstants.java @@ -575,9 +575,16 @@ public class ParamConstants { public static final String JOB_INTERNAL_KILL_JOB_REQUESTED_PARAM = "internal.killJobRequested"; public static final String JOB_INTERNAL_KILL_JOB_REQUESTED_DESCRIPTION = "Flag indicating that the user requested to kill the job."; public static final String JOB_TOOL_TYPE_PARAM = "toolType"; + public static final String JOB_TOOL_REQUIREMENTS_PROCESSOR_TYPE_PARAM = "tool.minimumRequirements.processorType"; + public static final String JOB_TOOL_REQUIREMENTS_QUEUE_PARAM = "tool.minimumRequirements.queue"; + public static final String JOB_EXECUTION_QUEUE_ID_PARAM = "execution.queue.id"; public static final String JOB_TYPE_DESCRIPTION = "Job type (NATIVE, WORKFLOW, CUSTOM or WALKER)"; public static final String JOB_TOOL_ID_DESCRIPTION = "Tool ID executed by the job" + REGEX_SUPPORT; public static final String JOB_TOOL_TYPE_DESCRIPTION = "Tool type executed by the job [OPERATION, ANALYSIS]"; + public static final String JOB_TOOL_REQUIREMENTS_PROCESSOR_TYPE_DESCRIPTION = "Processor type required to run the job. " + + "Allowed values: [CPU, GPU, FPGA]"; + public static final String JOB_TOOL_REQUIREMENTS_QUEUE_DESCRIPTION = "Queue where the job is expected to be executed."; + public static final String JOB_EXECUTION_QUEUE_ID_DESCRIPTION = "Queue id where the job has been submitted to be executed."; public static final String JOB_USER_PARAM = "userId"; public static final String JOB_USER_DESCRIPTION = "User that created the job"; public static final String JOB_PRIORITY_PARAM = "priority"; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java index 5f5fceb6341..b5b4d41bcc1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java @@ -15,7 +15,7 @@ public class ExecutionQueue { private String description; @DataField(id = "type", description = "Execution type for the queue") - private ExecutionType type; + private ProcessorType processorType; @DataField(id = "cpu", description = "Number of CPUs allocated for this queue") private int cpu; @@ -26,7 +26,7 @@ public class ExecutionQueue { @DataField(id = "options", description = "Additional options for the queue") private ObjectMap options; - public enum ExecutionType { + public enum ProcessorType { CPU, // Default execution type GPU, // Execution type for GPU-based tasks FPGA // Execution type for FPGA-based tasks @@ -35,18 +35,19 @@ public enum ExecutionType { public ExecutionQueue() { } - public ExecutionQueue(String id, String executor, String description, ExecutionType type, int cpu, String memory, ObjectMap options) { + public ExecutionQueue(String id, String executor, String description, ProcessorType processorType, int cpu, String memory, + ObjectMap options) { this.id = id; this.executor = executor; this.description = description; - this.type = type; + this.processorType = processorType; this.cpu = cpu; this.memory = memory; this.options = options; } public static ExecutionQueue defaultQueue() { - return new ExecutionQueue("job", "k8s", "Default queue", ExecutionType.CPU, 8, "32GB", new ObjectMap()); + return new ExecutionQueue("job", "k8s", "Default queue", ProcessorType.CPU, 8, "32GB", new ObjectMap()); } @Override @@ -54,7 +55,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("ExecutionQueue{"); sb.append("id='").append(id).append('\''); sb.append(", executor='").append(executor).append('\''); - sb.append(", type=").append(type); + sb.append(", processorType=").append(processorType); sb.append(", cpu=").append(cpu); sb.append(", memory='").append(memory).append('\''); sb.append(", options=").append(options); @@ -89,12 +90,12 @@ public ExecutionQueue setDescription(String description) { return this; } - public ExecutionType getType() { - return type; + public ProcessorType getProcessorType() { + return processorType; } - public ExecutionQueue setType(ExecutionType type) { - this.type = type; + public ExecutionQueue setProcessorType(ProcessorType type) { + this.processorType = type; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/MinimumRequirements.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/MinimumRequirements.java index 657f4e5abba..535981a1a89 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/job/MinimumRequirements.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/job/MinimumRequirements.java @@ -15,8 +15,8 @@ public class MinimumRequirements { @DataField(id = "disk", description = FieldConstants.MIN_REQUIREMENTS_DISK_DESCRIPTION) private String disk; - @DataField(id = "type", description = FieldConstants.MIN_REQUIREMENTS_TYPE_DESCRIPTION) - private ExecutionQueue.ExecutionType type; + @DataField(id = "processorType", description = FieldConstants.MIN_REQUIREMENTS_TYPE_DESCRIPTION) + private ExecutionQueue.ProcessorType processorType; @DataField(id = "queue", description = FieldConstants.MIN_REQUIREMENTS_QUEUE_DESCRIPTION) private String queue; @@ -24,11 +24,11 @@ public class MinimumRequirements { public MinimumRequirements() { } - public MinimumRequirements(String cpu, String memory, String disk, ExecutionQueue.ExecutionType type, String queue) { + public MinimumRequirements(String cpu, String memory, String disk, ExecutionQueue.ProcessorType processorType, String queue) { this.cpu = cpu; this.memory = memory; this.disk = disk; - this.type = type; + this.processorType = processorType; this.queue = queue; } @@ -38,7 +38,7 @@ public String toString() { sb.append("cpu='").append(cpu).append('\''); sb.append(", memory='").append(memory).append('\''); sb.append(", disk='").append(disk).append('\''); - sb.append(", type=").append(type); + sb.append(", processorType=").append(processorType); sb.append(", queue='").append(queue).append('\''); sb.append('}'); return sb.toString(); @@ -71,12 +71,12 @@ public MinimumRequirements setDisk(String disk) { return this; } - public ExecutionQueue.ExecutionType getType() { - return type; + public ExecutionQueue.ProcessorType getProcessorType() { + return processorType; } - public MinimumRequirements setType(ExecutionQueue.ExecutionType type) { - this.type = type; + public MinimumRequirements setProcessorType(ExecutionQueue.ProcessorType processorType) { + this.processorType = processorType; return this; } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java index 188b2bd0d56..77cadc0dc3b 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/schedulers/JobScheduler.java @@ -28,7 +28,7 @@ public class JobScheduler { private final String token; private Map userRoles; - private final Map> availableQueues; + private final Map> availableQueues; private static final float PRIORITY_WEIGHT = 0.6F; private static final float IDLE_TIME_WEIGHT = 0.4F; @@ -44,10 +44,10 @@ public JobScheduler(CatalogManager catalogManager, List queueLis this.availableQueues = new HashMap<>(); for (ExecutionQueue queue : queueList) { - if (!availableQueues.containsKey(queue.getType())) { - availableQueues.put(queue.getType(), new ArrayList<>()); + if (!availableQueues.containsKey(queue.getProcessorType())) { + availableQueues.put(queue.getProcessorType(), new ArrayList<>()); } - availableQueues.get(queue.getType()).add(queue); + availableQueues.get(queue.getProcessorType()).add(queue); } } @@ -195,7 +195,7 @@ public Map> schedule(String organizationId, List pendingJ for (Job job : pendingJobs) { String queueId = job.getTool().getMinimumRequirements().getQueue(); if (StringUtils.isEmpty(queueId)) { - List executionQueues = this.availableQueues.get(job.getTool().getMinimumRequirements().getType()); + List executionQueues = this.availableQueues.get(job.getTool().getMinimumRequirements().getProcessorType()); queueId = JobExecutionUtils.findOptimalQueues(executionQueues, job.getTool().getMinimumRequirements()).get(0).getId(); logger.debug("Job '{}' has no queue defined. Optimal queue for scheduling is '{}' .", job.getId(), queueId); } diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java index f74cfcb3f0f..d8bc397afec 100644 --- a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java @@ -68,10 +68,10 @@ public void setUp() throws CatalogException { // Create test queues queues = Arrays.asList( - new ExecutionQueue().setId("small-queue").setType(ExecutionQueue.ExecutionType.CPU).setCpu(2).setMemory("4GB"), - new ExecutionQueue().setId("medium-queue").setType(ExecutionQueue.ExecutionType.CPU).setCpu(8).setMemory("16GB"), - new ExecutionQueue().setId("large-queue").setType(ExecutionQueue.ExecutionType.CPU).setCpu(16).setMemory("32GB"), - new ExecutionQueue().setId("local-queue").setType(ExecutionQueue.ExecutionType.GPU).setCpu(4).setMemory("8GB") + new ExecutionQueue().setId("small-queue").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(2).setMemory("4GB"), + new ExecutionQueue().setId("medium-queue").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(8).setMemory("16GB"), + new ExecutionQueue().setId("large-queue").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(16).setMemory("32GB"), + new ExecutionQueue().setId("local-queue").setProcessorType(ExecutionQueue.ProcessorType.GPU).setCpu(4).setMemory("8GB") ); jobScheduler = new JobScheduler(catalogManager, queues, token); @@ -210,7 +210,7 @@ private List createTestJobs() { .setInternal(new JobInternal().setRegistrationDate(TimeUtils.getTime())) .setTool(new ToolInfo().setMinimumRequirements( new org.opencb.opencga.core.models.job.MinimumRequirements() - .setType(ExecutionQueue.ExecutionType.CPU) + .setProcessorType(ExecutionQueue.ProcessorType.CPU) .setCpu("2") .setMemory("4GB") .setQueue("small-queue") @@ -226,7 +226,7 @@ private List createTestJobs() { .setInternal(new JobInternal().setRegistrationDate(TimeUtils.getTime())) .setTool(new ToolInfo().setMinimumRequirements( new org.opencb.opencga.core.models.job.MinimumRequirements() - .setType(ExecutionQueue.ExecutionType.CPU) + .setProcessorType(ExecutionQueue.ProcessorType.CPU) .setCpu("4") .setMemory("8GB") .setQueue("medium-queue") @@ -246,7 +246,7 @@ private Job createJobWithScheduledTime(String jobId, Date scheduledTime) { .setInternal(new JobInternal().setRegistrationDate(TimeUtils.getTime())) .setTool(new ToolInfo().setMinimumRequirements( new org.opencb.opencga.core.models.job.MinimumRequirements() - .setType(ExecutionQueue.ExecutionType.CPU) + .setProcessorType(ExecutionQueue.ProcessorType.CPU) .setCpu("2") .setMemory("4GB") .setQueue("small-queue") diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java index 0edeed7373b..face171cc1d 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/JobWSServer.java @@ -238,6 +238,9 @@ public Response search( @ApiParam(value = ParamConstants.JOB_TYPE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TYPE_PARAM) String type, @ApiParam(value = ParamConstants.JOB_TOOL_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_ID_PARAM) String toolId, @ApiParam(value = ParamConstants.JOB_TOOL_TYPE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_TYPE_PARAM) String toolType, + @ApiParam(value = ParamConstants.JOB_TOOL_REQUIREMENTS_QUEUE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_REQUIREMENTS_QUEUE_PARAM) String toolQueue, + @ApiParam(value = ParamConstants.JOB_TOOL_REQUIREMENTS_PROCESSOR_TYPE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_REQUIREMENTS_PROCESSOR_TYPE_PARAM) String toolProcessorType, + @ApiParam(value = ParamConstants.JOB_EXECUTION_QUEUE_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_EXECUTION_QUEUE_ID_PARAM) String executionQueueId, @ApiParam(value = ParamConstants.JOB_TOOL_EXTERNAL_EXECUTOR_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_EXTERNAL_EXECUTOR_ID_PARAM) String externalExecutorId, @ApiParam(value = ParamConstants.JOB_PARENT_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_PARENT_ID_PARAM) String parentId, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN_PARAM) Boolean dryRun, @@ -274,6 +277,9 @@ public Response distinct( @ApiParam(value = ParamConstants.JOB_TYPE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TYPE_PARAM) String type, @ApiParam(value = ParamConstants.JOB_TOOL_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_ID_PARAM) String toolId, @ApiParam(value = ParamConstants.JOB_TOOL_TYPE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_TYPE_PARAM) String toolType, + @ApiParam(value = ParamConstants.JOB_TOOL_REQUIREMENTS_QUEUE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_REQUIREMENTS_QUEUE_PARAM) String toolQueue, + @ApiParam(value = ParamConstants.JOB_TOOL_REQUIREMENTS_PROCESSOR_TYPE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_REQUIREMENTS_PROCESSOR_TYPE_PARAM) String toolProcessorType, + @ApiParam(value = ParamConstants.JOB_EXECUTION_QUEUE_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_EXECUTION_QUEUE_ID_PARAM) String executionQueueId, @ApiParam(value = ParamConstants.JOB_TOOL_EXTERNAL_EXECUTOR_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_EXTERNAL_EXECUTOR_ID_PARAM) String externalExecutorId, @ApiParam(value = ParamConstants.JOB_PARENT_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_PARENT_ID_PARAM) String parentId, @ApiParam(value = ParamConstants.JOB_DRY_RUN_DESCRIPTION) @QueryParam(ParamConstants.JOB_DRY_RUN_PARAM) Boolean dryRun, @@ -426,6 +432,9 @@ public Response getAggregationStats( @ApiParam(value = ParamConstants.JOB_UUIDS_DESCRIPTION) @QueryParam(ParamConstants.JOB_UUID_PARAM) String uuid, @ApiParam(value = ParamConstants.JOB_TOOL_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_ID_PARAM) String toolId, @ApiParam(value = ParamConstants.JOB_TOOL_TYPE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_TYPE_PARAM) String toolType, + @ApiParam(value = ParamConstants.JOB_TOOL_REQUIREMENTS_QUEUE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_REQUIREMENTS_QUEUE_PARAM) String toolQueue, + @ApiParam(value = ParamConstants.JOB_TOOL_REQUIREMENTS_PROCESSOR_TYPE_DESCRIPTION) @QueryParam(ParamConstants.JOB_TOOL_REQUIREMENTS_PROCESSOR_TYPE_PARAM) String toolProcessorType, + @ApiParam(value = ParamConstants.JOB_EXECUTION_QUEUE_ID_DESCRIPTION) @QueryParam(ParamConstants.JOB_EXECUTION_QUEUE_ID_PARAM) String executionQueueId, @ApiParam(value = ParamConstants.JOB_USER_DESCRIPTION) @QueryParam(ParamConstants.JOB_USER_PARAM) String user, @ApiParam(value = ParamConstants.JOB_PRIORITY_DESCRIPTION) @QueryParam(ParamConstants.JOB_PRIORITY_PARAM) String priority, @ApiParam(value = ParamConstants.JOB_STATUS_DESCRIPTION) @QueryParam(ParamConstants.JOB_STATUS_PARAM) String status, From e8261b232f3c559e6b40c8dfabdfb85021dc3a7e Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 12 Aug 2025 09:15:08 +0200 Subject: [PATCH 15/22] app: rename ExternalTool category to UserTool, #TASK-7610 --- .../app/misc/clients/r_client_generator.py | 2 +- .../app/misc/clients/rest_client_generator.py | 2 +- .../app/cli/main/OpencgaCliOptionsParser.java | 44 +- .../cli/main/executors/ExecutorProvider.java | 3 + ...tor.java => UserToolsCommandExecutor.java} | 114 ++--- ...ions.java => UserToolsCommandOptions.java} | 34 +- .../catalog/managers/ExternalToolManager.java | 2 +- opencga-client/src/main/R/R/AllGenerics.R | 6 +- .../src/main/R/R/UserTool-methods.R | 268 +++++++++++ .../opencga/client/rest/OpenCGAClient.java | 4 +- ...nalToolClient.java => UserToolClient.java} | 32 +- .../src/main/javascript/UserTool.js | 335 +++++++++++++ .../rest_clients/user_tool_client.py | 441 ++++++++++++++++++ .../server/rest/ExternalToolWSServer.java | 28 +- .../src/main/resources/cli-config.yaml | 2 +- 15 files changed, 1182 insertions(+), 135 deletions(-) rename opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/{ExternalToolsCommandExecutor.java => UserToolsCommandExecutor.java} (85%) rename opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/{ExternalToolsCommandOptions.java => UserToolsCommandOptions.java} (98%) create mode 100644 opencga-client/src/main/R/R/UserTool-methods.R rename opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/{ExternalToolClient.java => UserToolClient.java} (96%) create mode 100644 opencga-client/src/main/javascript/UserTool.js create mode 100644 opencga-client/src/main/python/pyopencga/rest_clients/user_tool_client.py diff --git a/opencga-app/app/misc/clients/r_client_generator.py b/opencga-app/app/misc/clients/r_client_generator.py index 1955147435b..b4510a23f93 100644 --- a/opencga-app/app/misc/clients/r_client_generator.py +++ b/opencga-app/app/misc/clients/r_client_generator.py @@ -19,7 +19,7 @@ def __init__(self, server_url, output_dir): 'Files': 'File', 'Jobs': 'Job', 'Workflows': 'Workflow', - 'External Tools': 'ExternalTool', + 'User Tools': 'UserTool', 'Samples': 'Sample', 'Individuals': 'Individual', 'Families': 'Family', diff --git a/opencga-app/app/misc/clients/rest_client_generator.py b/opencga-app/app/misc/clients/rest_client_generator.py index 73e0a8af9c5..c8a0e350289 100644 --- a/opencga-app/app/misc/clients/rest_client_generator.py +++ b/opencga-app/app/misc/clients/rest_client_generator.py @@ -51,7 +51,7 @@ def __init__(self, rest_api_file, output_dir): 'Disease Panels': 'DiseasePanel', 'Notes': 'Note', 'Workflows': 'Workflow', - 'External Tools': 'ExternalTool', + 'User Tools': 'UserTool', 'Analysis - Alignment': 'Alignment', 'Analysis - Variant': 'Variant', 'Analysis - Clinical': 'ClinicalAnalysis', diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java index d9f4f66c805..aaf8c3ff399 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java @@ -27,7 +27,7 @@ public class OpencgaCliOptionsParser extends CustomCliOptionsParser { private final WorkflowsCommandOptions workflowsCommandOptions; private final IndividualsCommandOptions individualsCommandOptions; private final FamiliesCommandOptions familiesCommandOptions; - private final ExternalToolsCommandOptions externalToolsCommandOptions; + private final UserToolsCommandOptions userToolsCommandOptions; private final UsersCommandOptions usersCommandOptions; private final SamplesCommandOptions samplesCommandOptions; private final AnalysisAlignmentCommandOptions analysisAlignmentCommandOptions; @@ -234,25 +234,25 @@ public OpencgaCliOptionsParser() { familiesSubCommands.addCommand("update", familiesCommandOptions.updateCommandOptions); familiesSubCommands.addCommand("annotation-sets-annotations-update", familiesCommandOptions.updateAnnotationSetsAnnotationsCommandOptions); - externalToolsCommandOptions = new ExternalToolsCommandOptions(commonCommandOptions, jCommander); - jCommander.addCommand("tools", externalToolsCommandOptions); - JCommander externalToolsSubCommands = jCommander.getCommands().get("tools"); - externalToolsSubCommands.addCommand("acl-update", externalToolsCommandOptions.updateAclCommandOptions); - externalToolsSubCommands.addCommand("aggregationstats", externalToolsCommandOptions.aggregationStatsCommandOptions); - externalToolsSubCommands.addCommand("custom-build", externalToolsCommandOptions.buildCustomCommandOptions); - externalToolsSubCommands.addCommand("custom-create", externalToolsCommandOptions.createCustomCommandOptions); - externalToolsSubCommands.addCommand("custom-docker-run", externalToolsCommandOptions.runCustomDockerCommandOptions); - externalToolsSubCommands.addCommand("custom-run", externalToolsCommandOptions.runCustomCommandOptions); - externalToolsSubCommands.addCommand("custom-update", externalToolsCommandOptions.updateCustomCommandOptions); - externalToolsSubCommands.addCommand("distinct", externalToolsCommandOptions.distinctCommandOptions); - externalToolsSubCommands.addCommand("search", externalToolsCommandOptions.searchCommandOptions); - externalToolsSubCommands.addCommand("workflow-create", externalToolsCommandOptions.createWorkflowCommandOptions); - externalToolsSubCommands.addCommand("workflow-import", externalToolsCommandOptions.importWorkflowCommandOptions); - externalToolsSubCommands.addCommand("workflow-run", externalToolsCommandOptions.runWorkflowCommandOptions); - externalToolsSubCommands.addCommand("workflow-update", externalToolsCommandOptions.updateWorkflowCommandOptions); - externalToolsSubCommands.addCommand("acl", externalToolsCommandOptions.aclCommandOptions); - externalToolsSubCommands.addCommand("delete", externalToolsCommandOptions.deleteCommandOptions); - externalToolsSubCommands.addCommand("info", externalToolsCommandOptions.infoCommandOptions); + userToolsCommandOptions = new UserToolsCommandOptions(commonCommandOptions, jCommander); + jCommander.addCommand("tools", userToolsCommandOptions); + JCommander userToolsSubCommands = jCommander.getCommands().get("tools"); + userToolsSubCommands.addCommand("acl-update", userToolsCommandOptions.updateAclCommandOptions); + userToolsSubCommands.addCommand("aggregationstats", userToolsCommandOptions.aggregationStatsCommandOptions); + userToolsSubCommands.addCommand("custom-build", userToolsCommandOptions.buildCustomCommandOptions); + userToolsSubCommands.addCommand("custom-create", userToolsCommandOptions.createCustomCommandOptions); + userToolsSubCommands.addCommand("custom-docker-run", userToolsCommandOptions.runCustomDockerCommandOptions); + userToolsSubCommands.addCommand("custom-run", userToolsCommandOptions.runCustomCommandOptions); + userToolsSubCommands.addCommand("custom-update", userToolsCommandOptions.updateCustomCommandOptions); + userToolsSubCommands.addCommand("distinct", userToolsCommandOptions.distinctCommandOptions); + userToolsSubCommands.addCommand("search", userToolsCommandOptions.searchCommandOptions); + userToolsSubCommands.addCommand("workflow-create", userToolsCommandOptions.createWorkflowCommandOptions); + userToolsSubCommands.addCommand("workflow-import", userToolsCommandOptions.importWorkflowCommandOptions); + userToolsSubCommands.addCommand("workflow-run", userToolsCommandOptions.runWorkflowCommandOptions); + userToolsSubCommands.addCommand("workflow-update", userToolsCommandOptions.updateWorkflowCommandOptions); + userToolsSubCommands.addCommand("acl", userToolsCommandOptions.aclCommandOptions); + userToolsSubCommands.addCommand("delete", userToolsCommandOptions.deleteCommandOptions); + userToolsSubCommands.addCommand("info", userToolsCommandOptions.infoCommandOptions); usersCommandOptions = new UsersCommandOptions(commonCommandOptions, jCommander); jCommander.addCommand("users", usersCommandOptions); @@ -480,8 +480,8 @@ public FamiliesCommandOptions getFamiliesCommandOptions() { } - public ExternalToolsCommandOptions getExternalToolsCommandOptions() { - return externalToolsCommandOptions; + public UserToolsCommandOptions getUserToolsCommandOptions() { + return userToolsCommandOptions; } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExecutorProvider.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExecutorProvider.java index 5bc40b93c2d..ec1e77864b6 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExecutorProvider.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExecutorProvider.java @@ -28,6 +28,9 @@ public static OpencgaCommandExecutor getOpencgaCommandExecutor(OpencgaCliOptions case "workflows": commandExecutor = new WorkflowsCommandExecutor(cliOptionsParser.getWorkflowsCommandOptions()); break; + case "tools": + commandExecutor = new UserToolsCommandExecutor(cliOptionsParser.getUserToolsCommandOptions()); + break; case "jobs": commandExecutor = new JobsCommandExecutor(cliOptionsParser.getJobsCommandOptions()); break; diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExternalToolsCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UserToolsCommandExecutor.java similarity index 85% rename from opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExternalToolsCommandExecutor.java rename to opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UserToolsCommandExecutor.java index 735e2439e38..b0bccec9a9f 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/ExternalToolsCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UserToolsCommandExecutor.java @@ -10,7 +10,7 @@ import org.opencb.commons.utils.PrintUtils; import org.opencb.opencga.app.cli.main.*; import org.opencb.opencga.app.cli.main.executors.OpencgaCommandExecutor; -import org.opencb.opencga.app.cli.main.options.ExternalToolsCommandOptions; +import org.opencb.opencga.app.cli.main.options.UserToolsCommandOptions; import org.opencb.opencga.catalog.exceptions.CatalogAuthenticationException; import org.opencb.opencga.catalog.utils.ParamUtils.AclAction; import org.opencb.opencga.core.common.JacksonUtils; @@ -49,25 +49,25 @@ * */ /** - * This class contains methods for the External Tools command line. + * This class contains methods for the User Tools command line. * PATH: /{apiVersion}/tools */ -public class ExternalToolsCommandExecutor extends OpencgaCommandExecutor { +public class UserToolsCommandExecutor extends OpencgaCommandExecutor { public String categoryName = "tools"; - public ExternalToolsCommandOptions externalToolsCommandOptions; + public UserToolsCommandOptions userToolsCommandOptions; - public ExternalToolsCommandExecutor(ExternalToolsCommandOptions externalToolsCommandOptions) throws CatalogAuthenticationException { - super(externalToolsCommandOptions.commonCommandOptions); - this.externalToolsCommandOptions = externalToolsCommandOptions; + public UserToolsCommandExecutor(UserToolsCommandOptions userToolsCommandOptions) throws CatalogAuthenticationException { + super(userToolsCommandOptions.commonCommandOptions); + this.userToolsCommandOptions = userToolsCommandOptions; } @Override public void execute() throws Exception { - logger.debug("Executing External Tools command line"); + logger.debug("Executing User Tools command line"); - String subCommandString = getParsedSubCommand(externalToolsCommandOptions.jCommander); + String subCommandString = getParsedSubCommand(userToolsCommandOptions.jCommander); RestResponse queryResponse = null; @@ -130,9 +130,9 @@ public void execute() throws Exception { } private RestResponse updateAcl() throws Exception { - logger.debug("Executing updateAcl in External Tools command line"); + logger.debug("Executing updateAcl in User Tools command line"); - ExternalToolsCommandOptions.UpdateAclCommandOptions commandOptions = externalToolsCommandOptions.updateAclCommandOptions; + UserToolsCommandOptions.UpdateAclCommandOptions commandOptions = userToolsCommandOptions.updateAclCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("study", commandOptions.study); @@ -159,13 +159,13 @@ private RestResponse updateAcl() throws Exception { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), ExternalToolAclUpdateParams.class); } - return openCGAClient.getExternalToolClient().updateAcl(commandOptions.members, commandOptions.action, externalToolAclUpdateParams, queryParams); + return openCGAClient.getUserToolClient().updateAcl(commandOptions.members, commandOptions.action, externalToolAclUpdateParams, queryParams); } private RestResponse aggregationStats() throws Exception { - logger.debug("Executing aggregationStats in External Tools command line"); + logger.debug("Executing aggregationStats in User Tools command line"); - ExternalToolsCommandOptions.AggregationStatsCommandOptions commandOptions = externalToolsCommandOptions.aggregationStatsCommandOptions; + UserToolsCommandOptions.AggregationStatsCommandOptions commandOptions = userToolsCommandOptions.aggregationStatsCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("study", commandOptions.study); @@ -190,13 +190,13 @@ private RestResponse aggregationStats() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } - return openCGAClient.getExternalToolClient().aggregationStats(queryParams); + return openCGAClient.getUserToolClient().aggregationStats(queryParams); } private RestResponse buildCustom() throws Exception { - logger.debug("Executing buildCustom in External Tools command line"); + logger.debug("Executing buildCustom in User Tools command line"); - ExternalToolsCommandOptions.BuildCustomCommandOptions commandOptions = externalToolsCommandOptions.buildCustomCommandOptions; + UserToolsCommandOptions.BuildCustomCommandOptions commandOptions = userToolsCommandOptions.buildCustomCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("study", commandOptions.study); @@ -236,13 +236,13 @@ private RestResponse buildCustom() throws Exception { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), JobToolBuildParams.class); } - return openCGAClient.getExternalToolClient().buildCustom(jobToolBuildParams, queryParams); + return openCGAClient.getUserToolClient().buildCustom(jobToolBuildParams, queryParams); } private RestResponse createCustom() throws Exception { - logger.debug("Executing createCustom in External Tools command line"); + logger.debug("Executing createCustom in User Tools command line"); - ExternalToolsCommandOptions.CreateCustomCommandOptions commandOptions = externalToolsCommandOptions.createCustomCommandOptions; + UserToolsCommandOptions.CreateCustomCommandOptions commandOptions = userToolsCommandOptions.createCustomCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); @@ -290,13 +290,13 @@ private RestResponse createCustom() throws Exception { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), CustomToolCreateParams.class); } - return openCGAClient.getExternalToolClient().createCustom(customToolCreateParams, queryParams); + return openCGAClient.getUserToolClient().createCustom(customToolCreateParams, queryParams); } private RestResponse runCustomDocker() throws Exception { - logger.debug("Executing runCustomDocker in External Tools command line"); + logger.debug("Executing runCustomDocker in User Tools command line"); - ExternalToolsCommandOptions.RunCustomDockerCommandOptions commandOptions = externalToolsCommandOptions.runCustomDockerCommandOptions; + UserToolsCommandOptions.RunCustomDockerCommandOptions commandOptions = userToolsCommandOptions.runCustomDockerCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("study", commandOptions.study); @@ -333,13 +333,13 @@ private RestResponse runCustomDocker() throws Exception { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), Docker.class); } - return openCGAClient.getExternalToolClient().runCustomDocker(docker, queryParams); + return openCGAClient.getUserToolClient().runCustomDocker(docker, queryParams); } private RestResponse runCustom() throws Exception { - logger.debug("Executing runCustom in External Tools command line"); + logger.debug("Executing runCustom in User Tools command line"); - ExternalToolsCommandOptions.RunCustomCommandOptions commandOptions = externalToolsCommandOptions.runCustomCommandOptions; + UserToolsCommandOptions.RunCustomCommandOptions commandOptions = userToolsCommandOptions.runCustomCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotNull("version", commandOptions.version); @@ -374,13 +374,13 @@ private RestResponse runCustom() throws Exception { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), CustomToolRunParams.class); } - return openCGAClient.getExternalToolClient().runCustom(commandOptions.toolId, customToolRunParams, queryParams); + return openCGAClient.getUserToolClient().runCustom(commandOptions.toolId, customToolRunParams, queryParams); } private RestResponse updateCustom() throws Exception { - logger.debug("Executing updateCustom in External Tools command line"); + logger.debug("Executing updateCustom in User Tools command line"); - ExternalToolsCommandOptions.UpdateCustomCommandOptions commandOptions = externalToolsCommandOptions.updateCustomCommandOptions; + UserToolsCommandOptions.UpdateCustomCommandOptions commandOptions = userToolsCommandOptions.updateCustomCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); @@ -424,13 +424,13 @@ private RestResponse updateCustom() throws Exception { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), CustomToolUpdateParams.class); } - return openCGAClient.getExternalToolClient().updateCustom(commandOptions.toolId, customToolUpdateParams, queryParams); + return openCGAClient.getUserToolClient().updateCustom(commandOptions.toolId, customToolUpdateParams, queryParams); } private RestResponse distinct() throws Exception { - logger.debug("Executing distinct in External Tools command line"); + logger.debug("Executing distinct in User Tools command line"); - ExternalToolsCommandOptions.DistinctCommandOptions commandOptions = externalToolsCommandOptions.distinctCommandOptions; + UserToolsCommandOptions.DistinctCommandOptions commandOptions = userToolsCommandOptions.distinctCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("study", commandOptions.study); @@ -454,13 +454,13 @@ private RestResponse distinct() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } - return openCGAClient.getExternalToolClient().distinct(commandOptions.field, queryParams); + return openCGAClient.getUserToolClient().distinct(commandOptions.field, queryParams); } private RestResponse search() throws Exception { - logger.debug("Executing search in External Tools command line"); + logger.debug("Executing search in User Tools command line"); - ExternalToolsCommandOptions.SearchCommandOptions commandOptions = externalToolsCommandOptions.searchCommandOptions; + UserToolsCommandOptions.SearchCommandOptions commandOptions = userToolsCommandOptions.searchCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); @@ -489,13 +489,13 @@ private RestResponse search() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } - return openCGAClient.getExternalToolClient().search(queryParams); + return openCGAClient.getUserToolClient().search(queryParams); } private RestResponse createWorkflow() throws Exception { - logger.debug("Executing createWorkflow in External Tools command line"); + logger.debug("Executing createWorkflow in User Tools command line"); - ExternalToolsCommandOptions.CreateWorkflowCommandOptions commandOptions = externalToolsCommandOptions.createWorkflowCommandOptions; + UserToolsCommandOptions.CreateWorkflowCommandOptions commandOptions = userToolsCommandOptions.createWorkflowCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); @@ -538,13 +538,13 @@ private RestResponse createWorkflow() throws Exception { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), WorkflowCreateParams.class); } - return openCGAClient.getExternalToolClient().createWorkflow(workflowCreateParams, queryParams); + return openCGAClient.getUserToolClient().createWorkflow(workflowCreateParams, queryParams); } private RestResponse importWorkflow() throws Exception { - logger.debug("Executing importWorkflow in External Tools command line"); + logger.debug("Executing importWorkflow in User Tools command line"); - ExternalToolsCommandOptions.ImportWorkflowCommandOptions commandOptions = externalToolsCommandOptions.importWorkflowCommandOptions; + UserToolsCommandOptions.ImportWorkflowCommandOptions commandOptions = userToolsCommandOptions.importWorkflowCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("study", commandOptions.study); @@ -573,13 +573,13 @@ private RestResponse importWorkflow() throws Exception { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), WorkflowRepositoryParams.class); } - return openCGAClient.getExternalToolClient().importWorkflow(workflowRepositoryParams, queryParams); + return openCGAClient.getUserToolClient().importWorkflow(workflowRepositoryParams, queryParams); } private RestResponse runWorkflow() throws Exception { - logger.debug("Executing runWorkflow in External Tools command line"); + logger.debug("Executing runWorkflow in User Tools command line"); - ExternalToolsCommandOptions.RunWorkflowCommandOptions commandOptions = externalToolsCommandOptions.runWorkflowCommandOptions; + UserToolsCommandOptions.RunWorkflowCommandOptions commandOptions = userToolsCommandOptions.runWorkflowCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotNull("version", commandOptions.version); @@ -606,13 +606,13 @@ private RestResponse runWorkflow() throws Exception { objectMap = JacksonUtils.getDefaultObjectMapper() .readValue(new java.io.File(commandOptions.jsonFile), ObjectMap.class); } - return openCGAClient.getExternalToolClient().runWorkflow(commandOptions.toolId, objectMap, queryParams); + return openCGAClient.getUserToolClient().runWorkflow(commandOptions.toolId, objectMap, queryParams); } private RestResponse updateWorkflow() throws Exception { - logger.debug("Executing updateWorkflow in External Tools command line"); + logger.debug("Executing updateWorkflow in User Tools command line"); - ExternalToolsCommandOptions.UpdateWorkflowCommandOptions commandOptions = externalToolsCommandOptions.updateWorkflowCommandOptions; + UserToolsCommandOptions.UpdateWorkflowCommandOptions commandOptions = userToolsCommandOptions.updateWorkflowCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); @@ -651,13 +651,13 @@ private RestResponse updateWorkflow() throws Exception { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) .readValue(beanParams.toJson(), WorkflowUpdateParams.class); } - return openCGAClient.getExternalToolClient().updateWorkflow(commandOptions.toolId, workflowUpdateParams, queryParams); + return openCGAClient.getUserToolClient().updateWorkflow(commandOptions.toolId, workflowUpdateParams, queryParams); } private RestResponse acl() throws Exception { - logger.debug("Executing acl in External Tools command line"); + logger.debug("Executing acl in User Tools command line"); - ExternalToolsCommandOptions.AclCommandOptions commandOptions = externalToolsCommandOptions.aclCommandOptions; + UserToolsCommandOptions.AclCommandOptions commandOptions = userToolsCommandOptions.aclCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("study", commandOptions.study); @@ -667,13 +667,13 @@ private RestResponse acl() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } - return openCGAClient.getExternalToolClient().acl(commandOptions.tools, queryParams); + return openCGAClient.getUserToolClient().acl(commandOptions.tools, queryParams); } private RestResponse delete() throws Exception { - logger.debug("Executing delete in External Tools command line"); + logger.debug("Executing delete in User Tools command line"); - ExternalToolsCommandOptions.DeleteCommandOptions commandOptions = externalToolsCommandOptions.deleteCommandOptions; + UserToolsCommandOptions.DeleteCommandOptions commandOptions = userToolsCommandOptions.deleteCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("study", commandOptions.study); @@ -681,13 +681,13 @@ private RestResponse delete() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } - return openCGAClient.getExternalToolClient().delete(commandOptions.tools, queryParams); + return openCGAClient.getUserToolClient().delete(commandOptions.tools, queryParams); } private RestResponse info() throws Exception { - logger.debug("Executing info in External Tools command line"); + logger.debug("Executing info in User Tools command line"); - ExternalToolsCommandOptions.InfoCommandOptions commandOptions = externalToolsCommandOptions.infoCommandOptions; + UserToolsCommandOptions.InfoCommandOptions commandOptions = userToolsCommandOptions.infoCommandOptions; ObjectMap queryParams = new ObjectMap(); queryParams.putIfNotEmpty("include", commandOptions.include); @@ -699,6 +699,6 @@ private RestResponse info() throws Exception { queryParams.putIfNotEmpty("study", sessionManager.getSession().getCurrentStudy()); } - return openCGAClient.getExternalToolClient().info(commandOptions.tools, queryParams); + return openCGAClient.getUserToolClient().info(commandOptions.tools, queryParams); } } \ No newline at end of file diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ExternalToolsCommandOptions.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UserToolsCommandOptions.java similarity index 98% rename from opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ExternalToolsCommandOptions.java rename to opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UserToolsCommandOptions.java index 2ab26242efd..50fdf68d6e4 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/ExternalToolsCommandOptions.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UserToolsCommandOptions.java @@ -24,11 +24,11 @@ */ /** - * This class contains methods for the External Tools command line. + * This class contains methods for the User Tools command line. * PATH: /{apiVersion}/tools */ -@Parameters(commandNames = {"tools"}, commandDescription = "External Tools commands") -public class ExternalToolsCommandOptions { +@Parameters(commandNames = {"tools"}, commandDescription = "User Tools commands") +public class UserToolsCommandOptions { public JCommander jCommander; public CommonCommandOptions commonCommandOptions; @@ -51,7 +51,7 @@ public class ExternalToolsCommandOptions { public InfoCommandOptions infoCommandOptions; - public ExternalToolsCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) { + public UserToolsCommandOptions(CommonCommandOptions commonCommandOptions, JCommander jCommander) { this.jCommander = jCommander; this.commonCommandOptions = commonCommandOptions; @@ -74,7 +74,7 @@ public ExternalToolsCommandOptions(CommonCommandOptions commonCommandOptions, JC } - @Parameters(commandNames = {"acl-update"}, commandDescription ="Update the set of external tool permissions granted for the member") + @Parameters(commandNames = {"acl-update"}, commandDescription ="Update the set of user tool permissions granted for the member") public class UpdateAclCommandOptions { @ParametersDelegate @@ -103,7 +103,7 @@ public class UpdateAclCommandOptions { } - @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch external tool stats") + @Parameters(commandNames = {"aggregationstats"}, commandDescription ="Fetch user tool stats") public class AggregationStatsCommandOptions { @ParametersDelegate @@ -227,7 +227,7 @@ public class BuildCustomCommandOptions { } - @Parameters(commandNames = {"custom-create"}, commandDescription ="Register a new external tool of type CUSTOM_TOOL") + @Parameters(commandNames = {"custom-create"}, commandDescription ="Register a new user tool of type CUSTOM_TOOL") public class CreateCustomCommandOptions { @ParametersDelegate @@ -416,7 +416,7 @@ public class RunCustomCommandOptions { } - @Parameters(commandNames = {"custom-update"}, commandDescription ="Update some custom external tool attributes") + @Parameters(commandNames = {"custom-update"}, commandDescription ="Update some custom user tool attributes") public class UpdateCustomCommandOptions { @ParametersDelegate @@ -493,7 +493,7 @@ public class UpdateCustomCommandOptions { } - @Parameters(commandNames = {"distinct"}, commandDescription ="External tool distinct method") + @Parameters(commandNames = {"distinct"}, commandDescription ="User tool distinct method") public class DistinctCommandOptions { @ParametersDelegate @@ -555,7 +555,7 @@ public class DistinctCommandOptions { } - @Parameters(commandNames = {"search"}, commandDescription ="External tool search method") + @Parameters(commandNames = {"search"}, commandDescription ="User tool search method") public class SearchCommandOptions { @ParametersDelegate @@ -629,7 +629,7 @@ public class SearchCommandOptions { } - @Parameters(commandNames = {"workflow-create"}, commandDescription ="Register a new external tool of type WORKFLOW") + @Parameters(commandNames = {"workflow-create"}, commandDescription ="Register a new user tool of type WORKFLOW") public class CreateWorkflowCommandOptions { @ParametersDelegate @@ -700,7 +700,7 @@ public class CreateWorkflowCommandOptions { } - @Parameters(commandNames = {"workflow-import"}, commandDescription ="Import an external tool of type WORKFLOW") + @Parameters(commandNames = {"workflow-import"}, commandDescription ="Import a user tool of type WORKFLOW") public class ImportWorkflowCommandOptions { @ParametersDelegate @@ -729,7 +729,7 @@ public class ImportWorkflowCommandOptions { } - @Parameters(commandNames = {"workflow-run"}, commandDescription ="Execute an external tool of type WORKFLOW") + @Parameters(commandNames = {"workflow-run"}, commandDescription ="Execute a user tool of type WORKFLOW") public class RunWorkflowCommandOptions { @ParametersDelegate @@ -773,7 +773,7 @@ public class RunWorkflowCommandOptions { } - @Parameters(commandNames = {"workflow-update"}, commandDescription ="Update some external tool attributes") + @Parameters(commandNames = {"workflow-update"}, commandDescription ="Update some user tool attributes") public class UpdateWorkflowCommandOptions { @ParametersDelegate @@ -835,7 +835,7 @@ public class UpdateWorkflowCommandOptions { } - @Parameters(commandNames = {"acl"}, commandDescription ="Returns the acl of the external tools. If member is provided, it will only return the acl for the member.") + @Parameters(commandNames = {"acl"}, commandDescription ="Returns the acl of the user tools. If member is provided, it will only return the acl for the member.") public class AclCommandOptions { @ParametersDelegate @@ -855,7 +855,7 @@ public class AclCommandOptions { } - @Parameters(commandNames = {"delete"}, commandDescription ="Delete external tools") + @Parameters(commandNames = {"delete"}, commandDescription ="Delete user tools") public class DeleteCommandOptions { @ParametersDelegate @@ -869,7 +869,7 @@ public class DeleteCommandOptions { } - @Parameters(commandNames = {"info"}, commandDescription ="Get external tool information") + @Parameters(commandNames = {"info"}, commandDescription ="Get user tool information") public class InfoCommandOptions { @ParametersDelegate diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java index d588fddb7d3..d8987bfc473 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/ExternalToolManager.java @@ -106,7 +106,7 @@ InternalGetDataResult internalGet(String organizationId, long stud OpenCGAResult resultsNoCheck = getWorkflowDBAdaptor(organizationId).get(queryCopy, queryOptions); if (resultsNoCheck.getNumResults() == externalToolResult.getNumResults()) { - throw CatalogException.notFound("external tools", getMissingFields(uniqueList, externalToolResult.getResults(), + throw CatalogException.notFound("user tools", getMissingFields(uniqueList, externalToolResult.getResults(), externalToolStringFunction)); } else { throw new CatalogAuthorizationException("Permission denied. " + user + " is not allowed to see some or none of the external" diff --git a/opencga-client/src/main/R/R/AllGenerics.R b/opencga-client/src/main/R/R/AllGenerics.R index f8ee99409c4..4774aa2ccee 100644 --- a/opencga-client/src/main/R/R/AllGenerics.R +++ b/opencga-client/src/main/R/R/AllGenerics.R @@ -34,9 +34,9 @@ setGeneric("workflowClient", function(OpencgaR, members, workflowId, workflows, standardGeneric("workflowClient")) # ############################################################################## -## ExternalToolClient -setGeneric("externaltoolClient", function(OpencgaR, members, toolId, tools, endpointName, params=NULL, ...) - standardGeneric("externaltoolClient")) +## UserToolClient +setGeneric("usertoolClient", function(OpencgaR, members, toolId, tools, endpointName, params=NULL, ...) + standardGeneric("usertoolClient")) # ############################################################################## ## SampleClient diff --git a/opencga-client/src/main/R/R/UserTool-methods.R b/opencga-client/src/main/R/R/UserTool-methods.R new file mode 100644 index 00000000000..a808ea6fda4 --- /dev/null +++ b/opencga-client/src/main/R/R/UserTool-methods.R @@ -0,0 +1,268 @@ + +# WARNING: AUTOGENERATED CODE +# +# This code was generated by a tool. +# +# Manual changes to this file may cause unexpected behavior in your application. +# Manual changes to this file will be overwritten if the code is regenerated. + + +# ############################################################################## +#' UserToolClient methods +#' @include AllClasses.R +#' @include AllGenerics.R +#' @include commons.R + +#' @description This function implements the OpenCGA calls for managing User Tools. + +#' The following table summarises the available *actions* for this client: +#' +#' | endpointName | Endpoint WS | parameters accepted | +#' | -- | :-- | --: | +#' | updateAcl | /{apiVersion}/tools/acl/{members}/update | study, members[*], action[*], body[*] | +#' | aggregationStats | /{apiVersion}/tools/aggregationStats | study, id, name, uuid, tags, draft, internal.registrationUserId, type, scope, workflowRepositoryName, dockerName, creationDate, modificationDate, acl, release, snapshot, deleted, field | +#' | buildCustom | /{apiVersion}/tools/custom/build | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | createCustom | /{apiVersion}/tools/custom/create | include, exclude, study, includeResult, body[*] | +#' | runCustomDocker | /{apiVersion}/tools/custom/run | study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | runCustom | /{apiVersion}/tools/custom/{toolId}/run | toolId[*], version, study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | updateCustom | /{apiVersion}/tools/custom/{toolId}/update | include, exclude, toolId[*], study, includeResult, body | +#' | distinct | /{apiVersion}/tools/distinct | study, id, name, uuid, tags, draft, internal.registrationUserId, type, scope, workflowRepositoryName, dockerName, creationDate, modificationDate, acl, release, snapshot, deleted, field[*] | +#' | search | /{apiVersion}/tools/search | include, exclude, limit, skip, count, study, id, name, uuid, tags, draft, internal.registrationUserId, type, scope, workflowRepositoryName, dockerName, creationDate, modificationDate, acl, release, snapshot, deleted | +#' | createWorkflow | /{apiVersion}/tools/workflow/create | include, exclude, study, includeResult, body[*] | +#' | importWorkflow | /{apiVersion}/tools/workflow/import | study, body[*] | +#' | runWorkflow | /{apiVersion}/tools/workflow/{toolId}/run | toolId[*], version, study, jobId, jobDescription, jobDependsOn, jobTags, jobScheduledStartTime, jobPriority, jobDryRun, body[*] | +#' | updateWorkflow | /{apiVersion}/tools/workflow/{toolId}/update | include, exclude, toolId[*], study, includeResult, body | +#' | acl | /{apiVersion}/tools/{tools}/acl | tools[*], study, member, silent | +#' | delete | /{apiVersion}/tools/{tools}/delete | study, tools[*] | +#' | info | /{apiVersion}/tools/{tools}/info | include, exclude, tools[*], study, version, deleted | +#' +#' @md +#' @seealso \url{http://docs.opencb.org/display/opencga/Using+OpenCGA} and the RESTful API documentation +#' \url{http://bioinfo.hpc.cam.ac.uk/opencga-prod/webservices/} +#' [*]: Required parameter +#' @export + +setMethod("usertoolClient", "OpencgaR", function(OpencgaR, members, toolId, tools, endpointName, params=NULL, ...) { + switch(endpointName, + + #' @section Endpoint /{apiVersion}/tools/acl/{members}/update: + #' Update the set of user tool permissions granted for the member. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param members Comma separated list of user or group ids. + #' @param action Action to be performed [ADD, SET, REMOVE or RESET]. Allowed values: ['SET ADD REMOVE RESET'] + #' @param data JSON containing the parameters to update the permissions. + updateAcl=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="acl", + subcategoryId=members, action="update", params=params, httpMethod="POST", as.queryParam=c("action"), + ...), + + #' @section Endpoint /{apiVersion}/tools/aggregationStats: + #' Fetch user tool stats. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param id Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param name Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param uuid Comma separated list of external tool UUIDs up to a maximum of 100. + #' @param tags Comma separated list of tags. + #' @param draft Boolean field indicating whether the workflow is a draft or not. + #' @param internal.registrationUserId UserId that created the workflow. + #' @param type External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + #' @param scope External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + #' @param workflowRepositoryName Workflow repository name. + #' @param dockerName Docker name. + #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param acl Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS permissions. Only study owners or administrators can query by this field. . + #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). + #' @param deleted Boolean to retrieve deleted entries. + #' @param field Field to apply aggregation statistics to (or a list of fields separated by semicolons), e.g.: studies;type;numSamples[0..10]:1;format:sum(size). + aggregationStats=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory=NULL, + subcategoryId=NULL, action="aggregationStats", params=params, httpMethod="GET", as.queryParam=NULL, + ...), + + #' @section Endpoint /{apiVersion}/tools/custom/build: + #' Execute an analysis from a custom binary. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data body. + buildCustom=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=NULL, action="build", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/custom/create: + #' Register a new user tool of type CUSTOM_TOOL. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing workflow information. + createCustom=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=NULL, action="create", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/custom/run: + #' Execute an analysis from a custom binary. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data External tool run parameters. + runCustomDocker=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=NULL, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/custom/{toolId}/run: + #' Execute an analysis from a custom binary. + #' @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param version Tool version. If not provided, the latest version will be used. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data External tool run parameters. + runCustom=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=toolId, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/custom/{toolId}/update: + #' Update some custom user tool attributes. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data body. + updateCustom=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="custom", + subcategoryId=toolId, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/distinct: + #' User tool distinct method. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param id Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param name Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param uuid Comma separated list of external tool UUIDs up to a maximum of 100. + #' @param tags Comma separated list of tags. + #' @param draft Boolean field indicating whether the workflow is a draft or not. + #' @param internal.registrationUserId UserId that created the workflow. + #' @param type External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + #' @param scope External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + #' @param workflowRepositoryName Workflow repository name. + #' @param dockerName Docker name. + #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param acl Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS permissions. Only study owners or administrators can query by this field. . + #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). + #' @param deleted Boolean to retrieve deleted entries. + #' @param field Comma separated list of fields for which to obtain the distinct values. + distinct=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, + action="distinct", params=params, httpMethod="GET", as.queryParam=c("field"), ...), + + #' @section Endpoint /{apiVersion}/tools/search: + #' User tool search method. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param limit Number of results to be returned. + #' @param skip Number of results to skip. + #' @param count Get the total number of results matching the query. Deactivated by default. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param id Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param name Comma separated list of external tool names up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param uuid Comma separated list of external tool UUIDs up to a maximum of 100. + #' @param tags Comma separated list of tags. + #' @param draft Boolean field indicating whether the workflow is a draft or not. + #' @param internal.registrationUserId UserId that created the workflow. + #' @param type External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + #' @param scope External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + #' @param workflowRepositoryName Workflow repository name. + #' @param dockerName Docker name. + #' @param creationDate Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param modificationDate Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + #' @param acl Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS permissions. Only study owners or administrators can query by this field. . + #' @param release Release when it was created. + #' @param snapshot Snapshot value (Latest version of the entry in the specified release). + #' @param deleted Boolean to retrieve deleted entries. + search=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory=NULL, subcategoryId=NULL, + action="search", params=params, httpMethod="GET", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/workflow/create: + #' Register a new user tool of type WORKFLOW. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data JSON containing workflow information. + createWorkflow=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="workflow", + subcategoryId=NULL, action="create", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/workflow/import: + #' Import a user tool of type WORKFLOW. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param data Repository parameters. + importWorkflow=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="workflow", + subcategoryId=NULL, action="import", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/workflow/{toolId}/run: + #' Execute a user tool of type WORKFLOW. + #' @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param version Tool version. If not provided, the latest version will be used. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param jobId Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not provided. + #' @param jobDescription Job description. + #' @param jobDependsOn Comma separated list of existing job IDs the job will depend on. + #' @param jobTags Job tags. + #' @param jobScheduledStartTime Time when the job is scheduled to start. + #' @param jobPriority Priority of the job. + #' @param jobDryRun Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + #' @param data External tool run parameters. + runWorkflow=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="workflow", + subcategoryId=toolId, action="run", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/workflow/{toolId}/update: + #' Update some user tool attributes. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param includeResult Flag indicating to include the created or updated document result in the response. + #' @param data body. + updateWorkflow=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=NULL, subcategory="workflow", + subcategoryId=toolId, action="update", params=params, httpMethod="POST", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/{tools}/acl: + #' Returns the acl of the user tools. If member is provided, it will only return the acl for the member. + #' @param tools Comma separated of external tool ids. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param member User or group id. + #' @param silent Boolean to retrieve all possible entries that are queried for, false to raise an exception whenever one of the entries looked for cannot be shown for whichever reason. + acl=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=tools, subcategory=NULL, subcategoryId=NULL, + action="acl", params=params, httpMethod="GET", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/{tools}/delete: + #' Delete user tools. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param tools Comma separated of external tool ids. + delete=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=tools, subcategory=NULL, subcategoryId=NULL, + action="delete", params=params, httpMethod="DELETE", as.queryParam=NULL, ...), + + #' @section Endpoint /{apiVersion}/tools/{tools}/info: + #' Get user tool information. + #' @param include Fields included in the response, whole JSON path must be provided. + #' @param exclude Fields excluded in the response, whole JSON path must be provided. + #' @param tools Comma separated of external tool ids. + #' @param study Study [[organization@]project:]study where study and project can be either the ID or UUID. + #' @param version Comma separated list of external tool versions. 'all' to get all the external tool versions. Not supported if multiple external tool ids are provided. + #' @param deleted Boolean to retrieve deleted entries. + info=fetchOpenCGA(object=OpencgaR, category="tools", categoryId=tools, subcategory=NULL, subcategoryId=NULL, + action="info", params=params, httpMethod="GET", as.queryParam=NULL, ...), + ) +}) \ No newline at end of file diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java index cb0b33494b4..fc8d928c46c 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/OpenCGAClient.java @@ -131,8 +131,8 @@ public WorkflowClient getWorkflowClient() { return getClient(WorkflowClient.class, () -> new WorkflowClient(token, clientConfiguration)); } - public ExternalToolClient getExternalToolClient() { - return getClient(ExternalToolClient.class, () -> new ExternalToolClient(token, clientConfiguration)); + public UserToolClient getUserToolClient() { + return getClient(UserToolClient.class, () -> new UserToolClient(token, clientConfiguration)); } public AlignmentClient getAlignmentClient() { diff --git a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ExternalToolClient.java b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserToolClient.java similarity index 96% rename from opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ExternalToolClient.java rename to opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserToolClient.java index 7024628afd7..19fa64c7372 100644 --- a/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/ExternalToolClient.java +++ b/opencga-client/src/main/java/org/opencb/opencga/client/rest/clients/UserToolClient.java @@ -49,17 +49,17 @@ /** - * This class contains methods for the ExternalTool webservices. + * This class contains methods for the UserTool webservices. * PATH: tools */ -public class ExternalToolClient extends ParentClient { +public class UserToolClient extends ParentClient { - public ExternalToolClient(String token, ClientConfiguration configuration) { + public UserToolClient(String token, ClientConfiguration configuration) { super(token, configuration); } /** - * Update the set of external tool permissions granted for the member. + * Update the set of user tool permissions granted for the member. * @param members Comma separated list of user or group ids. * @param action Action to be performed [ADD, SET, REMOVE or RESET]. * @param data JSON containing the parameters to update the permissions. @@ -77,7 +77,7 @@ public RestResponse updateAcl(String members, String a } /** - * Fetch external tool stats. + * Fetch user tool stats. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. * id: Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the operator @@ -133,7 +133,7 @@ public RestResponse buildCustom(JobToolBuildParams data, ObjectMap params) } /** - * Register a new external tool of type CUSTOM_TOOL. + * Register a new user tool of type CUSTOM_TOOL. * @param data JSON containing workflow information. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. @@ -197,7 +197,7 @@ public RestResponse runCustom(String toolId, CustomToolRunParams data, Obje } /** - * Update some custom external tool attributes. + * Update some custom user tool attributes. * @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param data body. @@ -216,7 +216,7 @@ public RestResponse updateCustom(String toolId, CustomToolUpdatePa } /** - * External tool distinct method. + * User tool distinct method. * @param field Comma separated list of fields for which to obtain the distinct values. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. @@ -250,7 +250,7 @@ public RestResponse distinct(String field, ObjectMap params) throws Clie } /** - * External tool search method. + * User tool search method. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. * exclude: Fields excluded in the response, whole JSON path must be provided. @@ -287,7 +287,7 @@ public RestResponse search(ObjectMap params) throws ClientExceptio } /** - * Register a new external tool of type WORKFLOW. + * Register a new user tool of type WORKFLOW. * @param data JSON containing workflow information. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. @@ -304,7 +304,7 @@ public RestResponse createWorkflow(WorkflowCreateParams data, Obje } /** - * Import an external tool of type WORKFLOW. + * Import a user tool of type WORKFLOW. * @param data Repository parameters. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. @@ -318,7 +318,7 @@ public RestResponse importWorkflow(WorkflowRepositoryParams data, } /** - * Execute an external tool of type WORKFLOW. + * Execute a user tool of type WORKFLOW. * @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param data External tool run parameters. @@ -343,7 +343,7 @@ public RestResponse runWorkflow(String toolId, ObjectMap data, ObjectMap pa } /** - * Update some external tool attributes. + * Update some user tool attributes. * @param toolId Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using the * operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. * @param data body. @@ -362,7 +362,7 @@ public RestResponse updateWorkflow(String toolId, WorkflowUpdatePa } /** - * Returns the acl of the external tools. If member is provided, it will only return the acl for the member. + * Returns the acl of the user tools. If member is provided, it will only return the acl for the member. * @param tools Comma separated of external tool ids. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. @@ -378,7 +378,7 @@ public RestResponse acl(String tools, ObjectMap params } /** - * Delete external tools. + * Delete user tools. * @param tools Comma separated of external tool ids. * @param params Map containing any of the following optional parameters. * study: Study [[organization@]project:]study where study and project can be either the ID or UUID. @@ -391,7 +391,7 @@ public RestResponse delete(String tools, ObjectMap params) throws } /** - * Get external tool information. + * Get user tool information. * @param tools Comma separated of external tool ids. * @param params Map containing any of the following optional parameters. * include: Fields included in the response, whole JSON path must be provided. diff --git a/opencga-client/src/main/javascript/UserTool.js b/opencga-client/src/main/javascript/UserTool.js new file mode 100644 index 00000000000..257a278ed78 --- /dev/null +++ b/opencga-client/src/main/javascript/UserTool.js @@ -0,0 +1,335 @@ +/** + * Copyright 2015-2024 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * WARNING: AUTOGENERATED CODE + * + * This code was generated by a tool. + * + * Manual changes to this file may cause unexpected behavior in your application. + * Manual changes to this file will be overwritten if the code is regenerated. + * +**/ + +import OpenCGAParentClass from "./../opencga-parent-class.js"; + + +/** + * This class contains the methods for the "UserTool" resource + */ + +export default class UserTool extends OpenCGAParentClass { + + constructor(config) { + super(config); + } + + /** Update the set of user tool permissions granted for the member + * @param {String} members - Comma separated list of user or group ids. + * @param {Object} data - JSON containing the parameters to update the permissions. + * @param {"SET ADD REMOVE RESET"} action = "ADD" - Action to be performed [ADD, SET, REMOVE or RESET]. The default value is ADD. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateAcl(members, action, data, params) { + return this._post("tools", null, "acl", members, "update", data, {action, ...params}); + } + + /** Fetch user tool stats + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.id] - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions + * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.name] - Comma separated list of external tool names up to a maximum of 100. Also admits basic regular + * expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.uuid] - Comma separated list of external tool UUIDs up to a maximum of 100. + * @param {String} [params.tags] - Comma separated list of tags. + * @param {Boolean} [params.draft] - Boolean field indicating whether the workflow is a draft or not. + * @param {String} [params.internal.registrationUserId] - UserId that created the workflow. + * @param {String} [params.type] - External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + * @param {String} [params.scope] - External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + * @param {String} [params.workflowRepositoryName] - Workflow repository name. + * @param {String} [params.dockerName] - Docker name. + * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.acl] - Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. + * Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS + * permissions. Only study owners or administrators can query by this field. . + * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). + * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. + * @param {String} [params.field] - Field to apply aggregation statistics to (or a list of fields separated by semicolons), e.g.: + * studies;type;numSamples[0..10]:1;format:sum(size). + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + aggregationStats(params) { + return this._get("tools", null, null, null, "aggregationStats", params); + } + + /** Execute an analysis from a custom binary. + * @param {Object} data - body. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + buildCustom(data, params) { + return this._post("tools", null, "custom", null, "build", data, params); + } + + /** Register a new user tool of type CUSTOM_TOOL + * @param {Object} data - JSON containing workflow information. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + createCustom(data, params) { + return this._post("tools", null, "custom", null, "create", data, params); + } + + /** Execute an analysis from a custom binary. + * @param {Object} data - External tool run parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + runCustomDocker(data, params) { + return this._post("tools", null, "custom", null, "run", data, params); + } + + /** Execute an analysis from a custom binary. + * @param {String} toolId - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using + * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {Object} data - External tool run parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {Number} [params.version] - Tool version. If not provided, the latest version will be used. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + runCustom(toolId, data, params) { + return this._post("tools", null, "custom", toolId, "run", data, params); + } + + /** Update some custom user tool attributes + * @param {String} toolId - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using + * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {Object} [data] - body. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateCustom(toolId, data, params) { + return this._post("tools", null, "custom", toolId, "update", data, params); + } + + /** User tool distinct method + * @param {String} field - Comma separated list of fields for which to obtain the distinct values. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.id] - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions + * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.name] - Comma separated list of external tool names up to a maximum of 100. Also admits basic regular + * expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.uuid] - Comma separated list of external tool UUIDs up to a maximum of 100. + * @param {String} [params.tags] - Comma separated list of tags. + * @param {Boolean} [params.draft] - Boolean field indicating whether the workflow is a draft or not. + * @param {String} [params.internal.registrationUserId] - UserId that created the workflow. + * @param {String} [params.type] - External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + * @param {String} [params.scope] - External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + * @param {String} [params.workflowRepositoryName] - Workflow repository name. + * @param {String} [params.dockerName] - Docker name. + * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.acl] - Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. + * Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS + * permissions. Only study owners or administrators can query by this field. . + * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). + * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + distinct(field, params) { + return this._get("tools", null, null, null, "distinct", {field, ...params}); + } + + /** User tool search method + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {Number} [params.limit] - Number of results to be returned. + * @param {Number} [params.skip] - Number of results to skip. + * @param {Boolean} [params.count = "false"] - Get the total number of results matching the query. Deactivated by default. The default + * value is false. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.id] - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions + * using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.name] - Comma separated list of external tool names up to a maximum of 100. Also admits basic regular + * expressions using the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {String} [params.uuid] - Comma separated list of external tool UUIDs up to a maximum of 100. + * @param {String} [params.tags] - Comma separated list of tags. + * @param {Boolean} [params.draft] - Boolean field indicating whether the workflow is a draft or not. + * @param {String} [params.internal.registrationUserId] - UserId that created the workflow. + * @param {String} [params.type] - External tool type. Allowed types: [CUSTOM_TOOL, VARIANT_WALKER or WORKFLOW]. + * @param {String} [params.scope] - External tool scope. Allowed types: [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + * @param {String} [params.workflowRepositoryName] - Workflow repository name. + * @param {String} [params.dockerName] - Docker name. + * @param {String} [params.creationDate] - Creation date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.modificationDate] - Modification date. Format: yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + * @param {String} [params.acl] - Filter entries for which a user has the provided permissions. Format: acl={user}:{permissions}. + * Example: acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which user john has both WRITE and WRITE_ANNOTATIONS + * permissions. Only study owners or administrators can query by this field. . + * @param {String} [params.release] - Release when it was created. + * @param {Number} [params.snapshot] - Snapshot value (Latest version of the entry in the specified release). + * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + search(params) { + return this._get("tools", null, null, null, "search", params); + } + + /** Register a new user tool of type WORKFLOW + * @param {Object} data - JSON containing workflow information. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + createWorkflow(data, params) { + return this._post("tools", null, "workflow", null, "create", data, params); + } + + /** Import a user tool of type WORKFLOW + * @param {Object} data - Repository parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + importWorkflow(data, params) { + return this._post("tools", null, "workflow", null, "import", data, params); + } + + /** Execute a user tool of type WORKFLOW + * @param {String} toolId - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using + * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {Object} data - External tool run parameters. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {Number} [params.version] - Tool version. If not provided, the latest version will be used. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.jobId] - Job ID. It must be a unique string within the study. An ID will be autogenerated automatically if not + * provided. + * @param {String} [params.jobDescription] - Job description. + * @param {String} [params.jobDependsOn] - Comma separated list of existing job IDs the job will depend on. + * @param {String} [params.jobTags] - Job tags. + * @param {String} [params.jobScheduledStartTime] - Time when the job is scheduled to start. + * @param {String} [params.jobPriority] - Priority of the job. + * @param {Boolean} [params.jobDryRun] - Flag indicating that the job will be executed in dry-run mode. In this mode, OpenCGA will + * validate that all parameters and prerequisites are correctly set for successful execution, but the job will not actually run. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + runWorkflow(toolId, data, params) { + return this._post("tools", null, "workflow", toolId, "run", data, params); + } + + /** Update some user tool attributes + * @param {String} toolId - Comma separated list of external tool IDs up to a maximum of 100. Also admits basic regular expressions using + * the operator '~', i.e. '~{perl-regex}' e.g. '~value' for case sensitive, '~/value/i' for case insensitive search. + * @param {Object} [data] - body. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {Boolean} [params.includeResult = "false"] - Flag indicating to include the created or updated document result in the response. + * The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + updateWorkflow(toolId, data, params) { + return this._post("tools", null, "workflow", toolId, "update", data, params); + } + + /** Returns the acl of the user tools. If member is provided, it will only return the acl for the member. + * @param {String} tools - Comma separated of external tool ids. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.member] - User or group id. + * @param {Boolean} [params.silent = "false"] - Boolean to retrieve all possible entries that are queried for, false to raise an + * exception whenever one of the entries looked for cannot be shown for whichever reason. The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + acl(tools, params) { + return this._get("tools", tools, null, null, "acl", params); + } + + /** Delete user tools + * @param {String} tools - Comma separated of external tool ids. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + delete(tools, params) { + return this._delete("tools", tools, null, null, "delete", params); + } + + /** Get user tool information + * @param {String} tools - Comma separated of external tool ids. + * @param {Object} [params] - The Object containing the following optional parameters: + * @param {String} [params.include] - Fields included in the response, whole JSON path must be provided. + * @param {String} [params.exclude] - Fields excluded in the response, whole JSON path must be provided. + * @param {String} [params.study] - Study [[organization@]project:]study where study and project can be either the ID or UUID. + * @param {String} [params.version] - Comma separated list of external tool versions. 'all' to get all the external tool versions. Not + * supported if multiple external tool ids are provided. + * @param {Boolean} [params.deleted = "false"] - Boolean to retrieve deleted entries. The default value is false. + * @returns {Promise} Promise object in the form of RestResponse instance. + */ + info(tools, params) { + return this._get("tools", tools, null, null, "info", params); + } + +} \ No newline at end of file diff --git a/opencga-client/src/main/python/pyopencga/rest_clients/user_tool_client.py b/opencga-client/src/main/python/pyopencga/rest_clients/user_tool_client.py new file mode 100644 index 00000000000..ab13f1e18fb --- /dev/null +++ b/opencga-client/src/main/python/pyopencga/rest_clients/user_tool_client.py @@ -0,0 +1,441 @@ +""" +WARNING: AUTOGENERATED CODE + + This code was generated by a tool. + + Manual changes to this file may cause unexpected behavior in your application. + Manual changes to this file will be overwritten if the code is regenerated. +""" + +from pyopencga.rest_clients._parent_rest_clients import _ParentRestClient + + +class UserTool(_ParentRestClient): + """ + This class contains methods for the 'User Tools' webservices + PATH: /{apiVersion}/tools + """ + + def __init__(self, configuration, token=None, login_handler=None, *args, **kwargs): + super(UserTool, self).__init__(configuration, token, login_handler, *args, **kwargs) + + def update_acl(self, members, action, data=None, **options): + """ + Update the set of user tool permissions granted for the member. + PATH: /{apiVersion}/tools/acl/{members}/update + + :param dict data: JSON containing the parameters to update the + permissions. (REQUIRED) + :param str action: Action to be performed [ADD, SET, REMOVE or RESET]. + Allowed values: ['SET ADD REMOVE RESET'] (REQUIRED) + :param str members: Comma separated list of user or group ids. + (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + """ + + options['action'] = action + return self._post(category='tools', resource='update', subcategory='acl', second_query_id=members, data=data, **options) + + def aggregation_stats(self, **options): + """ + Fetch user tool stats. + PATH: /{apiVersion}/tools/aggregationStats + + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str name: Comma separated list of external tool names up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str uuid: Comma separated list of external tool UUIDs up to a + maximum of 100. + :param str tags: Comma separated list of tags. + :param bool draft: Boolean field indicating whether the workflow is a + draft or not. + :param str internal.registration_user_id: UserId that created the + workflow. + :param str type: External tool type. Allowed types: [CUSTOM_TOOL, + VARIANT_WALKER or WORKFLOW]. + :param str scope: External tool scope. Allowed types: + [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + :param str workflow_repository_name: Workflow repository name. + :param str docker_name: Docker name. + :param str creation_date: Creation date. Format: yyyyMMddHHmmss. + Examples: >2018, 2017-2018, <201805. + :param str modification_date: Modification date. Format: + yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + :param str acl: Filter entries for which a user has the provided + permissions. Format: acl={user}:{permissions}. Example: + acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which + user john has both WRITE and WRITE_ANNOTATIONS permissions. Only + study owners or administrators can query by this field. . + :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). + :param bool deleted: Boolean to retrieve deleted entries. + :param str field: Field to apply aggregation statistics to (or a list + of fields separated by semicolons), e.g.: + studies;type;numSamples[0..10]:1;format:sum(size). + """ + + return self._get(category='tools', resource='aggregationStats', **options) + + def build_custom(self, data=None, **options): + """ + Execute an analysis from a custom binary. + PATH: /{apiVersion}/tools/custom/build + + :param dict data: body. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='tools', resource='build', subcategory='custom', data=data, **options) + + def create_custom(self, data=None, **options): + """ + Register a new user tool of type CUSTOM_TOOL. + PATH: /{apiVersion}/tools/custom/create + + :param dict data: JSON containing workflow information. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='tools', resource='create', subcategory='custom', data=data, **options) + + def run_custom_docker(self, data=None, **options): + """ + Execute an analysis from a custom binary. + PATH: /{apiVersion}/tools/custom/run + + :param dict data: External tool run parameters. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='tools', resource='run', subcategory='custom', data=data, **options) + + def run_custom(self, tool_id, data=None, **options): + """ + Execute an analysis from a custom binary. + PATH: /{apiVersion}/tools/custom/{toolId}/run + + :param dict data: External tool run parameters. (REQUIRED) + :param str tool_id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. (REQUIRED) + :param int version: Tool version. If not provided, the latest version + will be used. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='tools', resource='run', subcategory='custom', second_query_id=tool_id, data=data, **options) + + def update_custom(self, tool_id, data=None, **options): + """ + Update some custom user tool attributes. + PATH: /{apiVersion}/tools/custom/{toolId}/update + + :param str tool_id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + :param dict data: body. + """ + + return self._post(category='tools', resource='update', subcategory='custom', second_query_id=tool_id, data=data, **options) + + def distinct(self, field, **options): + """ + User tool distinct method. + PATH: /{apiVersion}/tools/distinct + + :param str field: Comma separated list of fields for which to obtain + the distinct values. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str name: Comma separated list of external tool names up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str uuid: Comma separated list of external tool UUIDs up to a + maximum of 100. + :param str tags: Comma separated list of tags. + :param bool draft: Boolean field indicating whether the workflow is a + draft or not. + :param str internal.registration_user_id: UserId that created the + workflow. + :param str type: External tool type. Allowed types: [CUSTOM_TOOL, + VARIANT_WALKER or WORKFLOW]. + :param str scope: External tool scope. Allowed types: + [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + :param str workflow_repository_name: Workflow repository name. + :param str docker_name: Docker name. + :param str creation_date: Creation date. Format: yyyyMMddHHmmss. + Examples: >2018, 2017-2018, <201805. + :param str modification_date: Modification date. Format: + yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + :param str acl: Filter entries for which a user has the provided + permissions. Format: acl={user}:{permissions}. Example: + acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which + user john has both WRITE and WRITE_ANNOTATIONS permissions. Only + study owners or administrators can query by this field. . + :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). + :param bool deleted: Boolean to retrieve deleted entries. + """ + + options['field'] = field + return self._get(category='tools', resource='distinct', **options) + + def search(self, **options): + """ + User tool search method. + PATH: /{apiVersion}/tools/search + + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param int limit: Number of results to be returned. + :param int skip: Number of results to skip. + :param bool count: Get the total number of results matching the query. + Deactivated by default. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str name: Comma separated list of external tool names up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. + :param str uuid: Comma separated list of external tool UUIDs up to a + maximum of 100. + :param str tags: Comma separated list of tags. + :param bool draft: Boolean field indicating whether the workflow is a + draft or not. + :param str internal.registration_user_id: UserId that created the + workflow. + :param str type: External tool type. Allowed types: [CUSTOM_TOOL, + VARIANT_WALKER or WORKFLOW]. + :param str scope: External tool scope. Allowed types: + [CLINICAL_INTERPRETATION, SECONDARY_ANALYSIS, RESEARCH or OTHER]. + :param str workflow_repository_name: Workflow repository name. + :param str docker_name: Docker name. + :param str creation_date: Creation date. Format: yyyyMMddHHmmss. + Examples: >2018, 2017-2018, <201805. + :param str modification_date: Modification date. Format: + yyyyMMddHHmmss. Examples: >2018, 2017-2018, <201805. + :param str acl: Filter entries for which a user has the provided + permissions. Format: acl={user}:{permissions}. Example: + acl=john:WRITE,WRITE_ANNOTATIONS will return all entries for which + user john has both WRITE and WRITE_ANNOTATIONS permissions. Only + study owners or administrators can query by this field. . + :param str release: Release when it was created. + :param int snapshot: Snapshot value (Latest version of the entry in + the specified release). + :param bool deleted: Boolean to retrieve deleted entries. + """ + + return self._get(category='tools', resource='search', **options) + + def create_workflow(self, data=None, **options): + """ + Register a new user tool of type WORKFLOW. + PATH: /{apiVersion}/tools/workflow/create + + :param dict data: JSON containing workflow information. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + """ + + return self._post(category='tools', resource='create', subcategory='workflow', data=data, **options) + + def import_workflow(self, data=None, **options): + """ + Import a user tool of type WORKFLOW. + PATH: /{apiVersion}/tools/workflow/import + + :param dict data: Repository parameters. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + """ + + return self._post(category='tools', resource='import', subcategory='workflow', data=data, **options) + + def run_workflow(self, tool_id, data=None, **options): + """ + Execute a user tool of type WORKFLOW. + PATH: /{apiVersion}/tools/workflow/{toolId}/run + + :param dict data: External tool run parameters. (REQUIRED) + :param str tool_id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. (REQUIRED) + :param int version: Tool version. If not provided, the latest version + will be used. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str job_id: Job ID. It must be a unique string within the + study. An ID will be autogenerated automatically if not provided. + :param str job_description: Job description. + :param str job_depends_on: Comma separated list of existing job IDs + the job will depend on. + :param str job_tags: Job tags. + :param str job_scheduled_start_time: Time when the job is scheduled to + start. + :param str job_priority: Priority of the job. + :param bool job_dry_run: Flag indicating that the job will be executed + in dry-run mode. In this mode, OpenCGA will validate that all + parameters and prerequisites are correctly set for successful + execution, but the job will not actually run. + """ + + return self._post(category='tools', resource='run', subcategory='workflow', second_query_id=tool_id, data=data, **options) + + def update_workflow(self, tool_id, data=None, **options): + """ + Update some user tool attributes. + PATH: /{apiVersion}/tools/workflow/{toolId}/update + + :param str tool_id: Comma separated list of external tool IDs up to a + maximum of 100. Also admits basic regular expressions using the + operator '~', i.e. '~{perl-regex}' e.g. '~value' for case + sensitive, '~/value/i' for case insensitive search. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param bool include_result: Flag indicating to include the created or + updated document result in the response. + :param dict data: body. + """ + + return self._post(category='tools', resource='update', subcategory='workflow', second_query_id=tool_id, data=data, **options) + + def acl(self, tools, **options): + """ + Returns the acl of the user tools. If member is provided, it will only + return the acl for the member. + PATH: /{apiVersion}/tools/{tools}/acl + + :param str tools: Comma separated of external tool ids. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str member: User or group id. + :param bool silent: Boolean to retrieve all possible entries that are + queried for, false to raise an exception whenever one of the + entries looked for cannot be shown for whichever reason. + """ + + return self._get(category='tools', resource='acl', query_id=tools, **options) + + def delete(self, tools, **options): + """ + Delete user tools. + PATH: /{apiVersion}/tools/{tools}/delete + + :param str tools: Comma separated of external tool ids. (REQUIRED) + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + """ + + return self._delete(category='tools', resource='delete', query_id=tools, **options) + + def info(self, tools, **options): + """ + Get user tool information. + PATH: /{apiVersion}/tools/{tools}/info + + :param str tools: Comma separated of external tool ids. (REQUIRED) + :param str include: Fields included in the response, whole JSON path + must be provided. + :param str exclude: Fields excluded in the response, whole JSON path + must be provided. + :param str study: Study [[organization@]project:]study where study and + project can be either the ID or UUID. + :param str version: Comma separated list of external tool versions. + 'all' to get all the external tool versions. Not supported if + multiple external tool ids are provided. + :param bool deleted: Boolean to retrieve deleted entries. + """ + + return self._get(category='tools', resource='info', query_id=tools, **options) + diff --git a/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java b/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java index 7fc804cea6f..f4c8f51739a 100644 --- a/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java +++ b/opencga-server/src/main/java/org/opencb/opencga/server/rest/ExternalToolWSServer.java @@ -32,7 +32,7 @@ @Path("/{apiVersion}/tools") @Produces(MediaType.APPLICATION_JSON) -@Api(value = "External Tools", description = "Methods for working with 'tools' endpoint") +@Api(value = "User Tools", description = "Methods for working with 'tools' endpoint") public class ExternalToolWSServer extends OpenCGAWSServer { private ExternalToolManager externalToolManager; @@ -45,7 +45,7 @@ public ExternalToolWSServer(@Context UriInfo uriInfo, @Context HttpServletReques @GET @Path("/{tools}/info") - @ApiOperation(value = "Get external tool information", response = ExternalTool.class) + @ApiOperation(value = "Get user tool information", response = ExternalTool.class) @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, format = "", example = "name,attributes", dataType = "string", paramType = "query"), @@ -66,7 +66,7 @@ public Response info( @GET @Path("/search") - @ApiOperation(value = "External tool search method", response = ExternalTool.class) + @ApiOperation(value = "User tool search method", response = ExternalTool.class) @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, example = "name,attributes", dataType = "string", paramType = "query"), @@ -105,7 +105,7 @@ public Response search( @GET @Path("/aggregationStats") - @ApiOperation(value = "Fetch external tool stats", response = FacetField.class) + @ApiOperation(value = "Fetch user tool stats", response = FacetField.class) public Response getAggregationStats( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_ID_PARAM) String id, @@ -136,7 +136,7 @@ public Response getAggregationStats( @GET @Path("/distinct") - @ApiOperation(value = "External tool distinct method", response = Object.class) + @ApiOperation(value = "User tool distinct method", response = Object.class) public Response distinct( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION) @QueryParam(ParamConstants.EXTERNAL_TOOL_ID_PARAM) String id, @@ -166,7 +166,7 @@ public Response distinct( @GET @Path("/{tools}/acl") - @ApiOperation(value = "Returns the acl of the external tools. If member is provided, it will only return the acl for the member.", + @ApiOperation(value = "Returns the acl of the user tools. If member is provided, it will only return the acl for the member.", response = ExternalToolAclEntryList.class) public Response getAcls(@ApiParam(value = ParamConstants.EXTERNAL_TOOLS_DESCRIPTION, required = true) @PathParam("tools") String toolsStr, @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @@ -180,7 +180,7 @@ public Response getAcls(@ApiParam(value = ParamConstants.EXTERNAL_TOOLS_DESCRIPT @POST @Path("/acl/{members}/update") - @ApiOperation(value = "Update the set of external tool permissions granted for the member", response = ExternalToolAclEntryList.class) + @ApiOperation(value = "Update the set of user tool permissions granted for the member", response = ExternalToolAclEntryList.class) public Response updateAcl( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = "Comma separated list of user or group ids", required = true) @PathParam("members") String memberIds, @@ -192,7 +192,7 @@ public Response updateAcl( @DELETE @Path("/{tools}/delete") - @ApiOperation(value = "Delete external tools", response = ExternalTool.class) + @ApiOperation(value = "Delete user tools", response = ExternalTool.class) public Response delete( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String studyStr, @ApiParam(value = ParamConstants.EXTERNAL_TOOLS_DESCRIPTION) @PathParam("tools") String toolsStr) { @@ -220,7 +220,7 @@ public Response dockerBuildByPost( @POST @Path("/custom/create") - @ApiOperation(value = "Register a new external tool of type CUSTOM_TOOL", response = ExternalTool.class) + @ApiOperation(value = "Register a new user tool of type CUSTOM_TOOL", response = ExternalTool.class) @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), @@ -237,7 +237,7 @@ public Response createCustomTool( @POST @Path("/custom/{toolId}/update") @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation(value = "Update some custom external tool attributes", response = ExternalTool.class) + @ApiOperation(value = "Update some custom user tool attributes", response = ExternalTool.class) @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), @@ -297,7 +297,7 @@ public Response runCustomToolByToolId( @POST @Path("/workflow/create") - @ApiOperation(value = "Register a new external tool of type WORKFLOW", response = ExternalTool.class) + @ApiOperation(value = "Register a new user tool of type WORKFLOW", response = ExternalTool.class) @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), @@ -314,7 +314,7 @@ public Response createWorkflow( @POST @Path("/workflow/{toolId}/update") @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation(value = "Update some external tool attributes", response = ExternalTool.class) + @ApiOperation(value = "Update some user tool attributes", response = ExternalTool.class) @ApiImplicitParams({ @ApiImplicitParam(name = QueryOptions.INCLUDE, value = ParamConstants.INCLUDE_DESCRIPTION, dataType = "string", paramType = "query"), @@ -336,7 +336,7 @@ public Response updateWorkflow( @POST @Path("/workflow/import") @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation(value = "Import an external tool of type WORKFLOW", response = ExternalTool.class) + @ApiOperation(value = "Import a user tool of type WORKFLOW", response = ExternalTool.class) public Response importWorkflow( @ApiParam(value = ParamConstants.STUDY_DESCRIPTION) @QueryParam(ParamConstants.STUDY_PARAM) String study, @ApiParam(value = "Repository parameters", required = true) WorkflowRepositoryParams params) { @@ -346,7 +346,7 @@ public Response importWorkflow( @POST @Path("/workflow/{toolId}/run") @Consumes(MediaType.APPLICATION_JSON) - @ApiOperation(value = "Execute an external tool of type WORKFLOW", response = Job.class) + @ApiOperation(value = "Execute a user tool of type WORKFLOW", response = Job.class) public Response executeWorkflow( @ApiParam(value = ParamConstants.EXTERNAL_TOOL_ID_DESCRIPTION, required = true) @PathParam("toolId") String toolId, @ApiParam(value = "Tool version. If not provided, the latest version will be used.") @QueryParam(ParamConstants.EXTERNAL_TOOL_VERSION_PARAM) Integer version, diff --git a/opencga-server/src/main/resources/cli-config.yaml b/opencga-server/src/main/resources/cli-config.yaml index 86e1a8dad76..108913c5346 100644 --- a/opencga-server/src/main/resources/cli-config.yaml +++ b/opencga-server/src/main/resources/cli-config.yaml @@ -161,7 +161,7 @@ apiConfig: rename: importPanels - name: tools ignore: False - key: ExternalTool + key: UserTool - name: workflows commandName: workflows ignore: False From 7028e7cdcfe6c1bf722f3077d6bf69d596786b3b Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 3 Sep 2025 12:21:14 +0200 Subject: [PATCH 16/22] master: edit k8sExecutor to use queues, #TASK-7442 --- .../src/main/resources/configuration.yml | 32 +++++++++++++++---- .../monitor/executors/ExecutorFactory.java | 16 +++++----- .../master/monitor/executors/K8SExecutor.java | 32 ++++++++++++------- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 3bd22b490fe..4ed7d604bce 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -207,17 +207,35 @@ analysis: execution: # Accepted values are "local", "SGE", "k8s" # see org.opencb.opencga.master.monitor.executors.ExecutorFactory - id: "${OPENCGA.EXECUTION.MODE}" - defaultQueue: "" # Default queue to be used to submit jobs - availableQueues: # Other queues for specific applications - toolsPerQueue: + id: "${OPENCGA.EXECUTION.MODE}" # DEPRECATED Use queues.executor instead + defaultQueue: "" # DEPRECATED + availableQueues: # DEPRECATED Use queues + toolsPerQueue: # DEPRECATED Never used queues: - - id: "default" - executor: "${OPENCGA.EXECUTION.MODE}" - description: "Default queue for execution" +# - id: "default" +# executor: "local" +# description: "Default queue for execution" +# type: "CPU" +# cpu: 8 +# memory: "32G" + - id: "scpu" + executor: "k8s" + description: "Small queue for execution" type: "CPU" cpu: 8 memory: "32G" + - id: "mcpu" + executor: "k8s" + description: "Medium queue for execution" + type: "CPU" + cpu: 16 + memory: "64G" + - id: "sgpu" + executor: "k8s" + description: "Small queue for GPU executions" + type: "GPU" + cpu: 6 + memory: "112G" defaultRequest: cpu: 2 memory: "8G" diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java index 71af67aa7c0..a32c2041a1c 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java @@ -41,7 +41,7 @@ public ExecutorFactory(Configuration configuration) { this.queueToExecutorMap = new HashMap<>(); this.executor = new HashMap<>(); - Set executorIds = new HashSet<>(); + Set executorQueues = new HashSet<>(); if (CollectionUtils.isNotEmpty(configuration.getAnalysis().getExecution().getQueues())) { for (ExecutionQueue queue : configuration.getAnalysis().getExecution().getQueues()) { if (StringUtils.isEmpty(queue.getId())) { @@ -54,16 +54,16 @@ public ExecutorFactory(Configuration configuration) { throw new IllegalArgumentException("Queue " + queue.getId() + " is already defined"); } queueToExecutorMap.put(queue.getId(), queue.getExecutor().toLowerCase()); - executorIds.add(queue.getExecutor()); + executorQueues.add(queue); } } else { queueToExecutorMap.put("default", "local"); - executorIds.add("local"); + executorQueues.add(new ExecutionQueue().setId("local")); } Execution execution = configuration.getAnalysis().getExecution(); - for (String executorId : executorIds) { - switch (executorId) { + for (ExecutionQueue executorQueue : executorQueues) { + switch (executorQueue.getId()) { case "local": LocalExecutor localExecutor = new LocalExecutor(execution); this.executor.put("local", localExecutor); @@ -78,11 +78,11 @@ public ExecutorFactory(Configuration configuration) { // break; case "k8s": case "kubernetes": - K8SExecutor k8SExecutor = new K8SExecutor(configuration); - this.executor.put(executorId, k8SExecutor); + K8SExecutor k8SExecutor = new K8SExecutor(configuration, executorQueue); + this.executor.put(executorQueue.getId(), k8SExecutor); break; default: - throw new UnsupportedOperationException("Unsupported execution mode { " + executorId + throw new UnsupportedOperationException("Unsupported execution mode { " + executorQueue.getId() + " }, accepted modes are : local, sge, k8s, kubernetes"); } } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java index 47f0ff50833..d3e77de7fa3 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java @@ -32,6 +32,8 @@ import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.config.Configuration; import org.opencb.opencga.core.config.Execution; +import org.opencb.opencga.core.config.ExecutionFactor; +import org.opencb.opencga.core.config.ExecutionQueue; import org.opencb.opencga.core.models.common.Enums; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,8 +117,9 @@ public class K8SExecutor implements BatchExecutor { private final int ttlSecondsAfterFinished; private final boolean logToStdout; private long terminationGracePeriodSeconds; + private final ExecutionFactor requestFactor; - public K8SExecutor(Configuration configuration) { + public K8SExecutor(Configuration configuration, ExecutionQueue executionQueue) { Execution execution = configuration.getAnalysis().getExecution(); String k8sClusterMaster = execution.getOptions().getString(K8S_MASTER_NODE); this.namespace = execution.getOptions().getString(K8S_NAMESPACE); @@ -131,12 +134,13 @@ public K8SExecutor(Configuration configuration) { // Read timeout in ms .withRequestTimeout(execution.getOptions().getInt(K8S_CLIENT_TIMEOUT, 30000)) .build(); - this.kubernetesClient = new DefaultKubernetesClient(k8sConfig).inNamespace(namespace); + this.kubernetesClient = new DefaultKubernetesClient(k8sConfig).inNamespace(executionQueue.getId()); this.imagePullPolicy = execution.getOptions().getString(K8S_IMAGE_PULL_POLICY, "IfNotPresent"); this.imagePullSecrets = buildLocalObjectReference(execution.getOptions().get(K8S_IMAGE_PULL_SECRETS)); this.ttlSecondsAfterFinished = execution.getOptions().getInt(K8S_TTL_SECONDS_AFTER_FINISHED, 3600); this.terminationGracePeriodSeconds = execution.getOptions().getInt(K8S_TERMINATION_GRACE_PERIOD_SECONDS, 5 * 60); this.logToStdout = execution.getOptions().getBoolean(K8S_LOG_TO_STDOUT, true); + this.requestFactor = execution.getRequestFactor(); nodeSelector = getMap(execution.getOptions(), K8S_NODE_SELECTOR); if (execution.getOptions().containsKey(K8S_SECURITY_CONTEXT)) { securityContext = buildObject(execution.getOptions().get(K8S_SECURITY_CONTEXT), SecurityContext.class); @@ -156,13 +160,19 @@ public K8SExecutor(Configuration configuration) { } HashMap requests = new HashMap<>(); - for (Map.Entry entry : getMap(execution.getOptions(), K8S_REQUESTS).entrySet()) { - requests.put(entry.getKey(), new Quantity(entry.getValue())); - } + requests.put("cpu", new Quantity(String.valueOf(execution.getDefaultRequest().getCpu() * this.requestFactor.getCpu()))); + long memoryBytes = IOUtils.fromHumanReadableToByte(execution.getDefaultRequest().getMemory()); + Quantity memory = new Quantity(String.valueOf(memoryBytes * this.requestFactor.getMemory())); + requests.put("memory", memory); +// for (Map.Entry entry : getMap(execution.getOptions(), K8S_REQUESTS).entrySet()) { +// requests.put(entry.getKey(), new Quantity(entry.getValue())); +// } HashMap limits = new HashMap<>(); - for (Map.Entry entry : getMap(execution.getOptions(), K8S_LIMITS).entrySet()) { - limits.put(entry.getKey(), new Quantity(entry.getValue())); - } + limits.put("cpu", new Quantity(String.valueOf(executionQueue.getCpu()))); + limits.put("memory", new Quantity(String.valueOf(executionQueue.getMemory()))); +// for (Map.Entry entry : getMap(execution.getOptions(), K8S_LIMITS).entrySet()) { +// limits.put(entry.getKey(), new Quantity(entry.getValue())); +// } resources = new ResourceRequirementsBuilder() .withLimits(limits) .withRequests(requests) @@ -172,7 +182,7 @@ public K8SExecutor(Configuration configuration) { String javaHeap = execution.getOptions().getString(K8S_JAVA_HEAP); if (StringUtils.isEmpty(javaHeap) && requests.containsKey("memory")) { - Quantity memory = requests.get("memory"); + memory = requests.get("memory"); String amount = memory.getAmount(); long bytes = IOUtils.fromHumanReadableToByte(amount); bytes -= IOUtils.fromHumanReadableToByte("300Mi"); @@ -348,13 +358,13 @@ private ResourceRequirements getResources(org.opencb.opencga.core.models.job.Job ResourceRequirementsBuilder resources = new ResourceRequirementsBuilder(this.resources); if (StringUtils.isNotEmpty(job.getTool().getMinimumRequirements().getMemory())) { long memoryBytes = IOUtils.fromHumanReadableToByte(job.getTool().getMinimumRequirements().getMemory()); - Quantity memory = new Quantity(String.valueOf(memoryBytes)); + Quantity memory = new Quantity(String.valueOf(memoryBytes * this.requestFactor.getMemory())); resources.addToRequests("memory", memory); resources.addToLimits("memory", memory); } if (StringUtils.isNotEmpty(job.getTool().getMinimumRequirements().getCpu())) { double cpuUnits = Double.parseDouble(job.getTool().getMinimumRequirements().getCpu()); - Quantity cpu = new Quantity(Double.toString(cpuUnits)); + Quantity cpu = new Quantity(Double.toString(cpuUnits * this.requestFactor.getCpu())); resources.addToRequests("cpu", cpu); resources.addToLimits("cpu", cpu); } From a46805fba8869423a228426b5a07ff6508192d90 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 3 Sep 2025 15:35:17 +0200 Subject: [PATCH 17/22] master: add correct namespace to k8s, #TASK-7442 --- .../opencb/opencga/master/monitor/executors/K8SExecutor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java index d3e77de7fa3..f8cec09105f 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java @@ -122,7 +122,8 @@ public class K8SExecutor implements BatchExecutor { public K8SExecutor(Configuration configuration, ExecutionQueue executionQueue) { Execution execution = configuration.getAnalysis().getExecution(); String k8sClusterMaster = execution.getOptions().getString(K8S_MASTER_NODE); - this.namespace = execution.getOptions().getString(K8S_NAMESPACE); +// this.namespace = execution.getOptions().getString(K8S_NAMESPACE); + this.namespace = executionQueue.getId(); this.imageName = execution.getOptions().getString(K8S_IMAGE_NAME); this.volumeMounts = buildVolumeMounts(execution.getOptions().getList(K8S_VOLUME_MOUNTS)); this.volumes = buildVolumes(execution.getOptions().getList(K8S_VOLUMES)); @@ -134,7 +135,7 @@ public K8SExecutor(Configuration configuration, ExecutionQueue executionQueue) { // Read timeout in ms .withRequestTimeout(execution.getOptions().getInt(K8S_CLIENT_TIMEOUT, 30000)) .build(); - this.kubernetesClient = new DefaultKubernetesClient(k8sConfig).inNamespace(executionQueue.getId()); + this.kubernetesClient = new DefaultKubernetesClient(k8sConfig).inNamespace(namespace); this.imagePullPolicy = execution.getOptions().getString(K8S_IMAGE_PULL_POLICY, "IfNotPresent"); this.imagePullSecrets = buildLocalObjectReference(execution.getOptions().get(K8S_IMAGE_PULL_SECRETS)); this.ttlSecondsAfterFinished = execution.getOptions().getInt(K8S_TTL_SECONDS_AFTER_FINISHED, 3600); From 0799dfa201dafe6e8c60e575d080f7455a5ea992 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 3 Sep 2025 16:17:22 +0200 Subject: [PATCH 18/22] core: fix type from configuration file, #TASK-7442 --- .../java/org/opencb/opencga/core/config/ExecutionQueue.java | 2 +- opencga-core/src/main/resources/configuration.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java index b5b4d41bcc1..435fcd9ec26 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java @@ -14,7 +14,7 @@ public class ExecutionQueue { @DataField(id = "description", description = "Description of the queue") private String description; - @DataField(id = "type", description = "Execution type for the queue") + @DataField(id = "processorType", description = "Execution type for the queue") private ProcessorType processorType; @DataField(id = "cpu", description = "Number of CPUs allocated for this queue") diff --git a/opencga-core/src/main/resources/configuration.yml b/opencga-core/src/main/resources/configuration.yml index 4ed7d604bce..1e3b542bcf4 100644 --- a/opencga-core/src/main/resources/configuration.yml +++ b/opencga-core/src/main/resources/configuration.yml @@ -221,19 +221,19 @@ analysis: - id: "scpu" executor: "k8s" description: "Small queue for execution" - type: "CPU" + processorType: "CPU" cpu: 8 memory: "32G" - id: "mcpu" executor: "k8s" description: "Medium queue for execution" - type: "CPU" + processorType: "CPU" cpu: 16 memory: "64G" - id: "sgpu" executor: "k8s" description: "Small queue for GPU executions" - type: "GPU" + processorType: "GPU" cpu: 6 memory: "112G" defaultRequest: From e807e83582074bd541fabad26d4431e1f68965cf Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 4 Sep 2025 10:59:03 +0200 Subject: [PATCH 19/22] master: fix factory to process queues, #TASK-7442 --- .../opencga/core/config/ExecutionQueue.java | 3 +- .../monitor/executors/ExecutorFactory.java | 65 +++++++------------ .../master/monitor/executors/K8SExecutor.java | 6 +- 3 files changed, 30 insertions(+), 44 deletions(-) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java index 435fcd9ec26..b33494431a7 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java @@ -47,7 +47,8 @@ public ExecutionQueue(String id, String executor, String description, ProcessorT } public static ExecutionQueue defaultQueue() { - return new ExecutionQueue("job", "k8s", "Default queue", ProcessorType.CPU, 8, "32GB", new ObjectMap()); + return new ExecutionQueue("default", "local", "Default local executor", ExecutionQueue.ProcessorType.CPU, 8, "16G", + new ObjectMap()); } @Override diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java index a32c2041a1c..e8d8ac1b7dc 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/ExecutorFactory.java @@ -19,29 +19,23 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.core.config.Configuration; -import org.opencb.opencga.core.config.Execution; import org.opencb.opencga.core.config.ExecutionQueue; import java.io.Closeable; import java.io.IOException; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; /** * Created by pfurio on 22/08/16. */ public class ExecutorFactory implements Closeable { - private Map queueToExecutorMap; - private Map executor; + private Map queueToExecutor; public ExecutorFactory(Configuration configuration) { - this.queueToExecutorMap = new HashMap<>(); - this.executor = new HashMap<>(); + this.queueToExecutor = new HashMap<>(); - Set executorQueues = new HashSet<>(); if (CollectionUtils.isNotEmpty(configuration.getAnalysis().getExecution().getQueues())) { for (ExecutionQueue queue : configuration.getAnalysis().getExecution().getQueues()) { if (StringUtils.isEmpty(queue.getId())) { @@ -50,58 +44,49 @@ public ExecutorFactory(Configuration configuration) { if (StringUtils.isEmpty(queue.getExecutor())) { throw new IllegalArgumentException("Queue " + queue.getId() + " does not have an associated executor"); } - if (queueToExecutorMap.containsKey(queue.getId())) { + if (queueToExecutor.containsKey(queue.getId())) { throw new IllegalArgumentException("Queue " + queue.getId() + " is already defined"); } - queueToExecutorMap.put(queue.getId(), queue.getExecutor().toLowerCase()); - executorQueues.add(queue); + BatchExecutor executor = createNewExecutor(configuration, queue); + queueToExecutor.put(queue.getId(), executor); } } else { - queueToExecutorMap.put("default", "local"); - executorQueues.add(new ExecutionQueue().setId("local")); + // Default executor + ExecutionQueue executionQueue = ExecutionQueue.defaultQueue(); + BatchExecutor executor = createNewExecutor(configuration, executionQueue); + queueToExecutor.put(executionQueue.getId(), executor); } + } - Execution execution = configuration.getAnalysis().getExecution(); - for (ExecutionQueue executorQueue : executorQueues) { - switch (executorQueue.getId()) { - case "local": - LocalExecutor localExecutor = new LocalExecutor(execution); - this.executor.put("local", localExecutor); - break; - case "sge": - SGEExecutor sgeExecutor = new SGEExecutor(execution); - this.executor.put("sge", sgeExecutor); - break; + private BatchExecutor createNewExecutor(Configuration configuration, ExecutionQueue queue) { + switch (queue.getExecutor()) { + case "local": + return new LocalExecutor(configuration.getAnalysis().getExecution()); + case "sge": + return new SGEExecutor(configuration.getAnalysis().getExecution()); // case "azure": // case "azure-batch": // this.executor = new AzureBatchExecutor(execution); // break; - case "k8s": - case "kubernetes": - K8SExecutor k8SExecutor = new K8SExecutor(configuration, executorQueue); - this.executor.put(executorQueue.getId(), k8SExecutor); - break; - default: - throw new UnsupportedOperationException("Unsupported execution mode { " + executorQueue.getId() - + " }, accepted modes are : local, sge, k8s, kubernetes"); - } + case "k8s": + case "kubernetes": + return new K8SExecutor(configuration, queue); + default: + throw new UnsupportedOperationException("Unsupported execution mode { " + queue.getExecutor() + + " }, accepted modes are : local, sge, k8s, kubernetes"); } } public BatchExecutor getExecutor(String queueId) { - if (!queueToExecutorMap.containsKey(queueId)) { + if (!queueToExecutor.containsKey(queueId)) { throw new IllegalArgumentException("Queue " + queueId + " does not have an associated executor"); } - String executorId = queueToExecutorMap.get(queueId); - if (!executor.containsKey(executorId)) { - throw new IllegalArgumentException("Executor " + executorId + " does not exist"); - } - return executor.get(executorId); + return queueToExecutor.get(queueId); } @Override public void close() throws IOException { - for (BatchExecutor executor : this.executor.values()) { + for (BatchExecutor executor : this.queueToExecutor.values()) { executor.close(); } } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java index f8cec09105f..5a62f192ed2 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java @@ -122,8 +122,7 @@ public class K8SExecutor implements BatchExecutor { public K8SExecutor(Configuration configuration, ExecutionQueue executionQueue) { Execution execution = configuration.getAnalysis().getExecution(); String k8sClusterMaster = execution.getOptions().getString(K8S_MASTER_NODE); -// this.namespace = execution.getOptions().getString(K8S_NAMESPACE); - this.namespace = executionQueue.getId(); + this.namespace = execution.getOptions().getString(K8S_NAMESPACE); this.imageName = execution.getOptions().getString(K8S_IMAGE_NAME); this.volumeMounts = buildVolumeMounts(execution.getOptions().getList(K8S_VOLUME_MOUNTS)); this.volumes = buildVolumes(execution.getOptions().getList(K8S_VOLUMES)); @@ -142,7 +141,8 @@ public K8SExecutor(Configuration configuration, ExecutionQueue executionQueue) { this.terminationGracePeriodSeconds = execution.getOptions().getInt(K8S_TERMINATION_GRACE_PERIOD_SECONDS, 5 * 60); this.logToStdout = execution.getOptions().getBoolean(K8S_LOG_TO_STDOUT, true); this.requestFactor = execution.getRequestFactor(); - nodeSelector = getMap(execution.getOptions(), K8S_NODE_SELECTOR); + this.nodeSelector = new HashMap<>(); + this.nodeSelector.put("agentpool", executionQueue.getId()); if (execution.getOptions().containsKey(K8S_SECURITY_CONTEXT)) { securityContext = buildObject(execution.getOptions().get(K8S_SECURITY_CONTEXT), SecurityContext.class); } else { From bdc3ed26f7bb6a9ade3b40ad85c93a99132d8728 Mon Sep 17 00:00:00 2001 From: pfurio Date: Fri, 5 Sep 2025 12:35:58 +0200 Subject: [PATCH 20/22] catalog: change cpu type from int to String, #TASK-7442 --- .../catalog/utils/JobExecutionUtils.java | 4 +-- .../catalog/utils/JobExecutionUtilsTest.java | 32 +++++++++---------- .../opencga/core/config/ExecutionQueue.java | 10 +++--- .../monitor/schedulers/JobSchedulerTest.java | 8 ++--- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java index 78dbb563386..763d00ff35e 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/utils/JobExecutionUtils.java @@ -14,10 +14,10 @@ public static List findOptimalQueues(List queues // Also filter out queues that do not meet the minimum requirements return queues.stream() .filter(queue -> queue.getProcessorType().equals(requirements.getProcessorType()) - && queue.getCpu() >= Integer.parseInt(requirements.getCpu()) + && Integer.parseInt(queue.getCpu()) >= Integer.parseInt(requirements.getCpu()) && IOUtils.fromHumanReadableToByte(queue.getMemory()) >= IOUtils.fromHumanReadableToByte(requirements.getMemory())) .sorted((q1, q2) -> { - int cpuComparison = Integer.compare(q1.getCpu(), q2.getCpu()); + int cpuComparison = Integer.compare(Integer.parseInt(q1.getCpu()), Integer.parseInt(q2.getCpu())); if (cpuComparison != 0) { return cpuComparison; } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java index 8ea183c537e..1c4745abeb6 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/utils/JobExecutionUtilsTest.java @@ -19,9 +19,9 @@ public class JobExecutionUtilsTest { @Test public void testFindOptimalQueuesBasicFiltering() { // Create test queues - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(2).setMemory("4GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); - ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setProcessorType(ExecutionQueue.ProcessorType.GPU).setCpu(8).setMemory("16GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("2").setMemory("4GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("8GB"); + ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setProcessorType(ExecutionQueue.ProcessorType.GPU).setCpu("8").setMemory("16GB"); List queues = Arrays.asList(queue1, queue2, queue3); @@ -38,9 +38,9 @@ public void testFindOptimalQueuesBasicFiltering() { @Test public void testFindOptimalQueuesSortingByCpu() { // Create queues with different CPU counts but same memory - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(8).setMemory("8GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(2).setMemory("8GB"); - ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("8").setMemory("8GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("2").setMemory("8GB"); + ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("8GB"); List queues = Arrays.asList(queue1, queue2, queue3); @@ -58,9 +58,9 @@ public void testFindOptimalQueuesSortingByCpu() { @Test public void testFindOptimalQueuesSortingByMemory() { // Create queues with same CPU but different memory - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("16GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("4GB"); - ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("16GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("4GB"); + ExecutionQueue queue3 = new ExecutionQueue().setId("queue3").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("8GB"); List queues = Arrays.asList(queue1, queue2, queue3); @@ -77,8 +77,8 @@ public void testFindOptimalQueuesSortingByMemory() { @Test public void testFindOptimalQueuesFiltersByMinimumCpu() { - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(1).setMemory("8GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("1").setMemory("8GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("8GB"); List queues = Arrays.asList(queue1, queue2); @@ -92,8 +92,8 @@ public void testFindOptimalQueuesFiltersByMinimumCpu() { @Test public void testFindOptimalQueuesFiltersByMinimumMemory() { - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("2GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("2GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("8GB"); List queues = Arrays.asList(queue1, queue2); @@ -107,8 +107,8 @@ public void testFindOptimalQueuesFiltersByMinimumMemory() { @Test public void testFindOptimalQueuesFiltersByType() { - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.GPU).setCpu(4).setMemory("8GB"); - ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(4).setMemory("8GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.GPU).setCpu("4").setMemory("8GB"); + ExecutionQueue queue2 = new ExecutionQueue().setId("queue2").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("4").setMemory("8GB"); List queues = Arrays.asList(queue1, queue2); @@ -122,7 +122,7 @@ public void testFindOptimalQueuesFiltersByType() { @Test public void testFindOptimalQueuesEmptyResult() { - ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(2).setMemory("4GB"); + ExecutionQueue queue1 = new ExecutionQueue().setId("queue1").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("2").setMemory("4GB"); List queues = Collections.singletonList(queue1); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java index b33494431a7..06f5f000c29 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/config/ExecutionQueue.java @@ -18,7 +18,7 @@ public class ExecutionQueue { private ProcessorType processorType; @DataField(id = "cpu", description = "Number of CPUs allocated for this queue") - private int cpu; + private String cpu; @DataField(id = "memory", description = "Memory allocated for this queue") private String memory; @@ -35,7 +35,7 @@ public enum ProcessorType { public ExecutionQueue() { } - public ExecutionQueue(String id, String executor, String description, ProcessorType processorType, int cpu, String memory, + public ExecutionQueue(String id, String executor, String description, ProcessorType processorType, String cpu, String memory, ObjectMap options) { this.id = id; this.executor = executor; @@ -47,7 +47,7 @@ public ExecutionQueue(String id, String executor, String description, ProcessorT } public static ExecutionQueue defaultQueue() { - return new ExecutionQueue("default", "local", "Default local executor", ExecutionQueue.ProcessorType.CPU, 8, "16G", + return new ExecutionQueue("default", "local", "Default local executor", ExecutionQueue.ProcessorType.CPU, "8", "16G", new ObjectMap()); } @@ -100,11 +100,11 @@ public ExecutionQueue setProcessorType(ProcessorType type) { return this; } - public int getCpu() { + public String getCpu() { return cpu; } - public ExecutionQueue setCpu(int cpu) { + public ExecutionQueue setCpu(String cpu) { this.cpu = cpu; return this; } diff --git a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java index d8bc397afec..acea553cc87 100644 --- a/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java +++ b/opencga-master/src/test/java/org/opencb/opencga/master/monitor/schedulers/JobSchedulerTest.java @@ -68,10 +68,10 @@ public void setUp() throws CatalogException { // Create test queues queues = Arrays.asList( - new ExecutionQueue().setId("small-queue").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(2).setMemory("4GB"), - new ExecutionQueue().setId("medium-queue").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(8).setMemory("16GB"), - new ExecutionQueue().setId("large-queue").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu(16).setMemory("32GB"), - new ExecutionQueue().setId("local-queue").setProcessorType(ExecutionQueue.ProcessorType.GPU).setCpu(4).setMemory("8GB") + new ExecutionQueue().setId("small-queue").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("2").setMemory("4GB"), + new ExecutionQueue().setId("medium-queue").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("8").setMemory("16GB"), + new ExecutionQueue().setId("large-queue").setProcessorType(ExecutionQueue.ProcessorType.CPU).setCpu("16").setMemory("32GB"), + new ExecutionQueue().setId("local-queue").setProcessorType(ExecutionQueue.ProcessorType.GPU).setCpu("4").setMemory("8GB") ); jobScheduler = new JobScheduler(catalogManager, queues, token); From 8f90dd12137f1d124567284854b2f158a3df5f8d Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 22 Oct 2025 10:15:22 +0200 Subject: [PATCH 21/22] catalog: improve catalog junit performance tests, #TASK-7442 --- opencga-catalog/pom.xml | 15 + .../catalog/managers/AbstractManagerTest.java | 6 + .../CatalogManagerExternalResource.java | 55 +-- .../managers/EmbeddedMongoDBManager.java | 357 ++++++++++++++++++ .../src/test/resources/log4j2-test.xml | 34 ++ pom.xml | 21 ++ 6 files changed, 453 insertions(+), 35 deletions(-) create mode 100644 opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/EmbeddedMongoDBManager.java create mode 100644 opencga-catalog/src/test/resources/log4j2-test.xml diff --git a/opencga-catalog/pom.xml b/opencga-catalog/pom.xml index 19a8553f3e7..d98b4e28a35 100644 --- a/opencga-catalog/pom.xml +++ b/opencga-catalog/pom.xml @@ -195,6 +195,21 @@ junit test + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + test + + + de.flapdoodle.embed + de.flapdoodle.embed.process + test + + + de.flapdoodle.reverse + de.flapdoodle.reverse + test + diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java index 05ce5abf628..77934596f93 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/AbstractManagerTest.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; import org.junit.experimental.categories.Category; @@ -180,6 +181,11 @@ public class AbstractManagerTest extends GenericTest { protected static final QueryOptions INCLUDE_RESULT = new QueryOptions(ParamConstants.INCLUDE_RESULT_PARAM, true); + @BeforeClass + public static void configureFirstExecution() { + firstExecutionFinished = false; + } + @Before public void setUp() throws Exception { catalogManager = catalogManagerResource.getCatalogManager(); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java index 0d272fb93ec..adbf307c6a2 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java @@ -22,19 +22,14 @@ import org.junit.rules.ExternalResource; import org.opencb.opencga.TestParamConstants; import org.opencb.opencga.catalog.auth.authentication.JwtManager; -import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.io.CatalogIOManager; -import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; -import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.config.Configuration; import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -77,15 +72,20 @@ public void before() throws Exception { System.out.println("-------------------------------------------------------------------------------"); System.out.println("Initializing CatalogManagerExternalResource"); System.out.println("-------------------------------------------------------------------------------"); + + // Start embedded MongoDB if enabled + EmbeddedMongoDBManager embeddedMongo = EmbeddedMongoDBManager.getInstance(); + if (!embeddedMongo.isRunning()) { + embeddedMongo.start(); + System.out.println("Embedded MongoDB started on port: " + embeddedMongo.getPort()); + } + if (catalogManager != null) { catalogManager.close(); catalogManager = null; } clearOpenCGAHome("static"); - //Catalog database might be already installed. Need to delete it before installing it again. - clearCatalog(configuration); - catalogManager = new CatalogManager(configuration); String secretKey = PasswordUtils.getStrongRandomPassword(JwtManager.SECRET_KEY_MIN_LENGTH); catalogManager.installCatalogDB("HS256", secretKey, TestParamConstants.ADMIN_PASSWORD, "opencga@admin.com", true); @@ -101,6 +101,14 @@ public Path clearOpenCGAHome(String testName) throws IOException { } while (opencgaHome.toFile().exists()); Files.createDirectories(opencgaHome); configuration = Configuration.load(getClass().getResource("/configuration-test.yml").openStream()); + + // Update MongoDB connection string to use embedded MongoDB if enabled + EmbeddedMongoDBManager embeddedMongo = EmbeddedMongoDBManager.getInstance(); + if (embeddedMongo.isEnabled()) { + configuration.getCatalog().getDatabase().setHosts(java.util.Collections.singletonList(embeddedMongo.getConnectionString())); + System.out.println("Using embedded MongoDB at: " + embeddedMongo.getConnectionString()); + } + Path confPath = Files.createDirectories(opencgaHome.resolve("conf")); Files.copy(getClass().getResource("/configuration-test.yml").openStream(), confPath.resolve("configuration.yml"), StandardCopyOption.REPLACE_EXISTING); Path scratch = opencgaHome.resolve("scratch"); @@ -130,6 +138,10 @@ public void after() { } catch (CatalogException e) { throw new RuntimeException(e); } + + // Note: We don't stop embedded MongoDB here as it's shared across test classes + // It will be stopped when the JVM exits or via shutdown hook + System.out.println("-------------------------------------------------------------------------------"); System.out.println("Shutting down CatalogManagerExternalResource"); System.out.println("-------------------------------------------------------------------------------"); @@ -158,33 +170,6 @@ public Path getOpencgaHome() { return opencgaHome; } - public static void clearCatalog(Configuration configuration) throws CatalogException, URISyntaxException { - CatalogIOManager catalogIOManager = new CatalogIOManager(configuration); - try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(configuration, new IOManagerFactory(), catalogIOManager)) { - dbAdaptorFactory.deleteCatalogDB(); - } - - Path rootdir = Paths.get(UriUtils.createDirectoryUri(configuration.getWorkspace())); - deleteFolderTree(rootdir.toFile()); - - Path jobdir = Paths.get(UriUtils.createDirectoryUri(configuration.getJobDir())); - deleteFolderTree(jobdir.toFile()); - } - - public static void deleteFolderTree(java.io.File folder) { - java.io.File[] files = folder.listFiles(); - if (files != null) { - for (java.io.File f : files) { - if (f.isDirectory()) { - deleteFolderTree(f); - } else { - f.delete(); - } - } - } - folder.delete(); - } - public URI getResourceUri(String resourceName) throws IOException { return getResourceUri(resourceName, resourceName); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/EmbeddedMongoDBManager.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/EmbeddedMongoDBManager.java new file mode 100644 index 00000000000..44f02d00898 --- /dev/null +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/EmbeddedMongoDBManager.java @@ -0,0 +1,357 @@ +/* + * Copyright 2015-2020 OpenCB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opencb.opencga.catalog.managers; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; +import de.flapdoodle.embed.mongo.commands.MongodArguments; +import de.flapdoodle.embed.mongo.config.Net; +import de.flapdoodle.embed.mongo.config.Storage; +import de.flapdoodle.embed.mongo.distribution.Version; +import de.flapdoodle.embed.mongo.transitions.Mongod; +import de.flapdoodle.embed.mongo.transitions.RunningMongodProcess; +import de.flapdoodle.reverse.TransitionWalker; +import de.flapdoodle.reverse.transitions.Start; +import org.bson.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.PrintStream; +import java.net.ServerSocket; +import java.util.Collections; + +/** + * Manages an embedded MongoDB instance for testing purposes. + * This provides faster test execution and better isolation compared to external MongoDB. + * + *

Configuration via system properties:

+ *
    + *
  • opencga.test.embeddedMongo - Enable/disable embedded MongoDB (default: true)
    • + *
  • opencga.test.mongodb.version - MongoDB version to use (default: 7.0)
    • + *
  • opencga.test.mongo.log.level - MongoDB log level: error, warn, info, debug (default: error)
    • + *
  • opencga.test.mongo.verbose - Enable verbose logging for debugging (default: false)
    • + *
+ * + *

Example usage:

+ * 
+ * # Run tests with default (minimal) MongoDB logging
+ * mvn test
+ *
+ * # Run tests with specific MongoDB version
+ * mvn test -Dopencga.test.mongodb.version=5.0
+ *
+ * # Run tests with verbose MongoDB logging for debugging
+ * mvn test -Dopencga.test.mongo.log.level=debug -Dopencga.test.mongo.verbose=true
+ *
+ * # Run tests with external MongoDB
+ * mvn test -Dopencga.test.embeddedMongo=false
+ *
+ */ +public class EmbeddedMongoDBManager { + private static final Logger logger = LoggerFactory.getLogger(EmbeddedMongoDBManager.class); + private static final String DEFAULT_MONGODB_VERSION = "7.0"; + + // Store original streams so logger output can bypass the filter if needed + private static final PrintStream ORIGINAL_OUT = System.out; + private static final PrintStream ORIGINAL_ERR = System.err; + + private static EmbeddedMongoDBManager instance; + private TransitionWalker.ReachedState runningMongod; + private int port; + private final boolean enabled; + private final String mongoVersion; + private final boolean verbose; + + private EmbeddedMongoDBManager() { + // Check if embedded MongoDB should be used (can be disabled via system property) + this.enabled = Boolean.parseBoolean(System.getProperty("opencga.test.embeddedMongo", "true")); + this.mongoVersion = System.getProperty("opencga.test.mongodb.version", DEFAULT_MONGODB_VERSION); + this.verbose = Boolean.parseBoolean(System.getProperty("opencga.test.mongo.verbose", "false")); + } + + public static synchronized EmbeddedMongoDBManager getInstance() { + if (instance == null) { + instance = new EmbeddedMongoDBManager(); + } + return instance; + } + + /** + * Converts the configured version string to a Flapdoodle Version enum. + */ + private Version.Main getMongoVersion() { + switch (mongoVersion) { + case "3.6": + return Version.Main.V3_6; + case "4.0": + return Version.Main.V4_0; + case "4.2": + return Version.Main.V4_2; + case "4.4": + return Version.Main.V4_4; + case "5.0": + return Version.Main.V5_0; + case "6.0": + return Version.Main.V6_0; + case "7.0": + return Version.Main.V7_0; + default: + logger.warn("Unknown MongoDB version '{}', using default {}", mongoVersion, DEFAULT_MONGODB_VERSION); + return Version.Main.V4_4; + } + } + + public synchronized void start() throws IOException { + if (!enabled) { + logger.info("Embedded MongoDB is disabled. Using external MongoDB instance."); + return; + } + + if (runningMongod != null) { + logger.debug("Embedded MongoDB is already running on port {}", port); + return; + } + + try { + // Output these logs BEFORE installing the filter so they're visible + logger.info("Starting embedded MongoDB {} with replica set support", mongoVersion); + + // Find an available port before starting MongoDB + port = findAvailablePort(); + logger.info("Found available port: {}", port); + + // NOW install the filtering streams (after initial logs are output) + if (!verbose) { + // Install filtering PrintStreams that suppress mongod JSON logs permanently + System.setOut(new FilteringPrintStream(System.out)); + System.setErr(new FilteringPrintStream(System.err)); + logger.info("MongoDB output filtering enabled (use -Dopencga.test.mongo.verbose=true to see all logs)"); + } + + // Configure and start MongoDB with replica set enabled using the pre-allocated port + runningMongod = Mongod.instance() + .withNet(Start.to(Net.class).initializedWith(Net.defaults().withPort(port))) + .withMongodArguments(Start.to(MongodArguments.class) + .initializedWith(MongodArguments.defaults() + .withReplication(Storage.of("rs0", 10)))) + .start(getMongoVersion()); + + logger.info("Embedded MongoDB {} started on port {}, initializing replica set...", mongoVersion, port); + + // Wait a bit for MongoDB to be fully ready before initializing replica set + Thread.sleep(500); + + // Initialize the replica set + initializeReplicaSet(); + + logger.info("Embedded MongoDB {} with replica set 'rs0' ready on port {}", mongoVersion, port); + + // Add shutdown hook to ensure cleanup + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + if (verbose) { + logger.info("Shutting down embedded MongoDB via shutdown hook"); + } + stop(); + })); + + } catch (Exception e) { + logger.error("Failed to start embedded MongoDB version {}", mongoVersion, e); + throw new IOException("Failed to start embedded MongoDB", e); + } + } + + /** + * Initializes the replica set configuration for the embedded MongoDB instance. + * This is required for tests that use transactions or other replica set features. + */ + private void initializeReplicaSet() { + MongoClient mongoClient = null; + try { + // Connect without replica set parameter for initial setup + String connectionString = String.format("mongodb://localhost:%d", port); + mongoClient = MongoClients.create(connectionString); + + MongoDatabase adminDb = mongoClient.getDatabase("admin"); + + // Configure replica set with a single member + Document config = new Document("_id", "rs0") + .append("members", Collections.singletonList( + new Document("_id", 0) + .append("host", "localhost:" + port) + )); + + // Initiate the replica set + adminDb.runCommand(new Document("replSetInitiate", config)); + + logger.debug("Replica set initiation command sent"); + + // Wait for replica set to become PRIMARY (ready to accept operations) + long timeout = System.currentTimeMillis() + 30000; // 30 seconds timeout + while (System.currentTimeMillis() < timeout) { + try { + Document result = adminDb.runCommand(new Document("replSetGetStatus", 1)); + Integer myState = result.getInteger("myState"); + if (myState != null && myState == 1) { // 1 = PRIMARY + logger.info("Replica set initialized successfully and is PRIMARY"); + return; + } + logger.debug("Replica set state: {}, waiting for PRIMARY...", myState); + } catch (Exception e) { + // Replica set might not be ready yet, continue waiting + logger.trace("Waiting for replica set to be ready: {}", e.getMessage()); + } + Thread.sleep(100); + } + + throw new RuntimeException("Replica set initialization timed out after 30 seconds"); + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Replica set initialization was interrupted", e); + } catch (Exception e) { + logger.error("Failed to initialize replica set", e); + throw new RuntimeException("Could not initialize replica set", e); + } finally { + if (mongoClient != null) { + try { + mongoClient.close(); + } catch (Exception e) { + logger.warn("Error closing MongoDB client", e); + } + } + } + } + + public synchronized void stop() { + if (!enabled) { + return; + } + + if (runningMongod != null) { + logger.info("Stopping embedded MongoDB on port {}", port); + try { + runningMongod.close(); + } catch (Exception e) { + logger.warn("Error stopping embedded MongoDB", e); + } finally { + runningMongod = null; + } + } + } + + public int getPort() { + return port; + } + + public String getConnectionString() { + if (!enabled) { + // Fall back to default MongoDB connection + return "localhost:27017"; + } + return "localhost:" + port; + } + + public boolean isEnabled() { + return enabled; + } + + public boolean isRunning() { + return enabled && runningMongod != null; + } + + /** + * Finds an available port on the local machine. + * This is used to work around issues with Flapdoodle's dynamic port allocation. + * + * @return an available port number + * @throws IOException if unable to find an available port + */ + private int findAvailablePort() throws IOException { + try (ServerSocket socket = new ServerSocket(0)) { + socket.setReuseAddress(true); + return socket.getLocalPort(); + } catch (IOException e) { + throw new IOException("Failed to find an available port", e); + } + } + + /** + * Custom PrintStream that filters out mongod JSON log lines while allowing other output through. + * This prevents the noisy mongod logs from cluttering test output while preserving all other logs. + */ + private static class FilteringPrintStream extends PrintStream { + private final PrintStream original; + private final StringBuilder lineBuffer = new StringBuilder(); + private boolean inLine = false; + + public FilteringPrintStream(PrintStream original) { + super(original); + this.original = original; + } + + @Override + public void write(int b) { + if (b == '\n') { + // End of line - check if we should filter it + String line = lineBuffer.toString(); + boolean isMongodLog = line.contains("{\"t\":{\"$date\":") || line.startsWith("[mongod output]") || line.startsWith("[mongod error]"); + + if (!isMongodLog) { + // Not a mongod log - write the buffered line and newline + original.print(lineBuffer.toString()); + original.write(b); + original.flush(); + } + // Clear buffer for next line + lineBuffer.setLength(0); + inLine = false; + } else if (b == '\r') { + // Carriage return - just pass through for now + // Don't filter yet, wait for \n + } else { + // Regular character - add to buffer + lineBuffer.append((char) b); + inLine = true; + } + } + + @Override + public void write(byte[] buf, int off, int len) { + // Process byte by byte to maintain line buffering + for (int i = 0; i < len; i++) { + write(buf[off + i]); + } + } + + @Override + public void flush() { + // Flush any remaining content in buffer (partial line) + if (lineBuffer.length() > 0) { + String line = lineBuffer.toString(); + boolean isMongodLog = line.contains("{\"t\":{\"$date\":") || line.startsWith("[mongod output]") || line.startsWith("[mongod error]"); + + if (!isMongodLog) { + original.print(lineBuffer.toString()); + } + lineBuffer.setLength(0); + } + original.flush(); + } + } +} diff --git a/opencga-catalog/src/test/resources/log4j2-test.xml b/opencga-catalog/src/test/resources/log4j2-test.xml new file mode 100644 index 00000000000..f03c44c952a --- /dev/null +++ b/opencga-catalog/src/test/resources/log4j2-test.xml @@ -0,0 +1,34 @@ + + + + ${sys:opencga.log.level:-info} + ${sys:opencga.test.mongo.log.level:-info} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 43494b67f9a..9b40cbc4cd9 100644 --- a/pom.xml +++ b/pom.xml @@ -115,6 +115,9 @@ 1.1.10.4 ${parquet-common.version} 5.0 + 4.21.0 + 1.9.1 + 4.17.0 opencga opencga-env.sh @@ -1139,6 +1142,24 @@ kryo ${kryo.version} + + de.flapdoodle.embed + de.flapdoodle.embed.mongo + ${flapdoodle.version} + test + + + de.flapdoodle.embed + de.flapdoodle.embed.process + ${flapdoodle.process.version} + test + + + de.flapdoodle.reverse + de.flapdoodle.reverse + ${flapdoodle.reverse.version} + test + From 546c47bae7a469176ffc3fdb2f829ae634a48599 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 22 Oct 2025 11:04:49 +0200 Subject: [PATCH 22/22] catalog: fix configuration test yml, #TASK-7442 --- opencga-catalog/src/test/resources/configuration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencga-catalog/src/test/resources/configuration-test.yml b/opencga-catalog/src/test/resources/configuration-test.yml index 9a1a51ef1fa..d8dabdcf46e 100644 --- a/opencga-catalog/src/test/resources/configuration-test.yml +++ b/opencga-catalog/src/test/resources/configuration-test.yml @@ -141,7 +141,7 @@ analysis: - id: "default" executor: "local" description: "Default queue for local execution" - type: "CPU" + processorType: "CPU" cpu: 1 memory: "2G" defaultRequest: