Skip to content

Commit

Permalink
catalog: add basic tests for workflows, #TASK-6445
Browse files Browse the repository at this point in the history
  • Loading branch information
pfurio committed Sep 13, 2024
1 parent aa3edc0 commit 1b1f4be
Show file tree
Hide file tree
Showing 12 changed files with 171 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ enum QueryParams implements QueryParam {
ID("id", TEXT, ""),
UID("uid", LONG, ""),
UUID("uuid", TEXT, ""),
NAME("name", TEXT, ""),
DESCRIPTION("description", TEXT, ""),
DRAFT("draft", BOOLEAN, ""),
TYPE("type", TEXT, ""),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ private Family insert(ClientSession clientSession, long studyUid, Family family,
familyDocument.put(PERMISSION_RULES_APPLIED, Collections.emptyList());

logger.debug("Inserting family '{}' ({})...", family.getId(), family.getUid());
versionedMongoDBAdaptor.insert(clientSession, familyDocument);
versionedMongoDBAdaptor.insert(clientSession, familyDocument, family.getRelease());
logger.debug("Family '{}' successfully inserted", family.getId());

// Add family reference to the members
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ Individual insert(ClientSession clientSession, long studyId, Individual individu
individualDocument.put(PERMISSION_RULES_APPLIED, Collections.emptyList());

logger.debug("Inserting individual '{}' ({})...", individual.getId(), individual.getUid());
versionedMongoDBAdaptor.insert(clientSession, individualDocument);
versionedMongoDBAdaptor.insert(clientSession, individualDocument, individual.getRelease());
logger.debug("Individual '{}' successfully inserted", individual.getId());

if (individual.getSamples() != null && !individual.getSamples().isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ Interpretation insert(ClientSession clientSession, long studyId, Interpretation
interpretationObject.put(LAST_OF_VERSION, true);
interpretationObject.put(LAST_OF_RELEASE, true);

versionedMongoDBAdaptor.insert(clientSession, interpretationObject);
versionedMongoDBAdaptor.insert(clientSession, interpretationObject, interpretation.getRelease());

return interpretation;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ void insert(ClientSession clientSession, long studyUid, Panel panel) throws Cata
logger.debug("Inserting panel '{}' ({})", panel.getId(), panel.getUid());

Document panelDocument = getPanelDocumentForInsertion(clientSession, panel, studyUid);
versionedMongoDBAdaptor.insert(clientSession, panelDocument);
versionedMongoDBAdaptor.insert(clientSession, panelDocument, panel.getRelease());
logger.info("Panel '" + panel.getId() + "(" + panel.getUid() + ")' successfully created");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ Sample insert(ClientSession clientSession, long studyUid, Sample sample, List<Va
sampleObject.put(PRIVATE_INDIVIDUAL_UID, individualUid);

logger.debug("Inserting sample '{}' ({})...", sample.getId(), sample.getUid());
versionedMongoDBAdaptor.insert(clientSession, sampleObject);
versionedMongoDBAdaptor.insert(clientSession, sampleObject, sample.getRelease());
logger.debug("Sample '{}' successfully inserted", sample.getId());

if (individualUid > 0) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,12 @@ DBIterator<T> iterator(ClientSession session, Query query, QueryOptions options)
throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException;
}

protected void insert(ClientSession session, Document document) {
protected void insert(ClientSession session, Document document, int release) {
String uuid = getClientSessionUuid(session);
document.put(VERSION, 1);
document.put(LAST_OF_VERSION, true);
document.put(LAST_OF_RELEASE, true);
document.put(RELEASE_FROM_VERSION, Collections.singletonList(release));
document.put(PRIVATE_TRANSACTION_ID, uuid);
collection.insert(session, document, QueryOptions.empty());
archiveCollection.insert(session, document, QueryOptions.empty());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ Workflow insert(ClientSession clientSession, long studyUid, Workflow workflow) t
long uid = getNewUid(clientSession);
workflow.setUid(uid);
workflow.setStudyUid(studyUid);
workflow.setRelease(dbAdaptorFactory.getCatalogStudyDBAdaptor().getCurrentRelease(clientSession, studyUid));

Document workflowObject = workflowConverter.convertToStorageType(workflow);

Expand All @@ -92,7 +93,7 @@ Workflow insert(ClientSession clientSession, long studyUid, Workflow workflow) t
? TimeUtils.toDate(workflow.getModificationDate()) : TimeUtils.getDate());

logger.debug("Inserting workflow '{}' ({})...", workflow.getId(), workflow.getUid());
versionedMongoDBAdaptor.insert(clientSession, workflowObject);
versionedMongoDBAdaptor.insert(clientSession, workflowObject, workflow.getRelease());
logger.debug("Workflow '{}' successfully inserted", workflow.getId());

return workflow;
Expand Down Expand Up @@ -323,7 +324,7 @@ private UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, QueryO
final String[] acceptedBooleanParams = {QueryParams.DRAFT.key()};
filterBooleanParams(parameters, document.getSet(), acceptedBooleanParams);

final String[] acceptedParams = {QueryParams.DESCRIPTION.key(), QueryParams.COMMAND_LINE.key()};
final String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.DESCRIPTION.key(), QueryParams.COMMAND_LINE.key()};
filterStringParams(parameters, document.getSet(), acceptedParams);

final String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key()};
Expand Down Expand Up @@ -513,6 +514,7 @@ protected Bson parseQuery(Query query, Document extraQuery, String user)
case RELEASE:
case VERSION:
case TYPE:
case DRAFT:
addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList);
break;
default:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ public OpenCGAResult<Workflow> update(String studyStr, String workflowId, Workfl
ObjectMap updateMap = null;
try {
if (updateParams != null) {
updateMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(this));
updateMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(updateParams));
}
} catch (JsonProcessingException e) {
throw new CatalogException("Could not parse WorkflowUpdateParams object: " + e.getMessage(), e);
Expand Down Expand Up @@ -548,7 +548,13 @@ private WorkflowDBAdaptor.QueryParams getFieldFilter(List<String> idList) throws

private void validateNewWorkflow(Workflow workflow) throws CatalogParameterException {
ParamUtils.checkIdentifier(workflow.getId(), ID.key());
ParamUtils.checkObj(workflow.getType(), TYPE.key());
if (Workflow.Type.values().length > 1) {
ParamUtils.checkObj(workflow.getType(), TYPE.key());
} else if (workflow.getType() == null) {
// TODO: Remove this condition in the future once we know we support more than one type.
// If there is only one valid type, we set it
workflow.setType(Workflow.Type.NEXTFLOW);
}
workflow.setScripts(workflow.getScripts() != null ? workflow.getScripts() : Collections.emptyList());
boolean main = false;
for (WorkflowScript script : workflow.getScripts()) {
Expand All @@ -566,7 +572,11 @@ private void validateNewWorkflow(Workflow workflow) throws CatalogParameterExcep
}
workflow.setRepository(workflow.getRepository() != null ? workflow.getRepository() : new WorkflowRepository(""));
if (StringUtils.isEmpty(workflow.getRepository().getImage()) && CollectionUtils.isEmpty(workflow.getScripts())) {
throw new CatalogParameterException("No docker image or scripts found.");
throw new CatalogParameterException("No repository image or scripts found.");
}
if (StringUtils.isNotEmpty(workflow.getRepository().getImage()) && CollectionUtils.isNotEmpty(workflow.getScripts())) {
throw new CatalogParameterException("Both repository image and scripts found. Please, either add scripts or a repository"
+ " image.");
}

workflow.setName(ParamUtils.defaultString(workflow.getName(), workflow.getId()));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package org.opencb.opencga.catalog.managers;

import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.catalog.db.api.WorkflowDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.core.models.workflow.Workflow;
import org.opencb.opencga.core.models.workflow.WorkflowRepository;
import org.opencb.opencga.core.models.workflow.WorkflowScript;
import org.opencb.opencga.core.models.workflow.WorkflowUpdateParams;
import org.opencb.opencga.core.response.OpenCGAResult;
import org.opencb.opencga.core.testclassification.duration.MediumTests;

import java.util.Arrays;
import java.util.Collections;

import static org.junit.Assert.*;

@Category(MediumTests.class)
public class WorkflowManagerTest extends AbstractManagerTest {

private WorkflowManager workflowManager;

@Before
public void setUp() throws Exception {
super.setUp();
workflowManager = catalogManager.getWorkflowManager();
}

@Test
public void createWorkflowTest() throws CatalogException {
Workflow workflow = new Workflow()
.setId("workflow")
.setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)));
OpenCGAResult<Workflow> result = workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken);
assertEquals(1, result.getNumResults());
assertNotNull(result.first());
assertEquals(workflow.getId(), result.first().getId());

// Add repository to workflow
workflow.setId("workflow2");
workflow.setRepository(new WorkflowRepository("blabla"));
CatalogException catalogException = assertThrows(CatalogException.class,
() -> workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken));
assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("repository"));

// Remove script from workflow
workflow.setScripts(Collections.emptyList());
result = workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken);
assertEquals(1, result.getNumResults());
assertNotNull(result.first());
assertEquals(workflow.getId(), result.first().getId());

// Remove script and add two scripts with 2 mains
workflow.setId("workflow3");
workflow.setRepository(null);
workflow.setScripts(Arrays.asList(
new WorkflowScript("script1", "echo 'Hello'", true),
new WorkflowScript("script2", "echo 'World'", true)
));
catalogException = assertThrows(CatalogException.class,
() -> workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken));
assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main"));

// Add one single script without main
workflow.setScripts(Collections.singletonList(
new WorkflowScript("script1", "echo 'Hello'", false)
));
catalogException = assertThrows(CatalogException.class,
() -> workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken));
assertTrue(catalogException.getMessage().contains("script") && catalogException.getMessage().contains("main"));
}

@Test
public void workflowSearchTest() throws CatalogException {
Workflow workflow = new Workflow()
.setId("workflow")
.setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)));
workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken);

workflow = new Workflow()
.setId("workflow2")
.setDraft(true)
.setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)));
workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken);

OpenCGAResult<Workflow> search = workflowManager.search(studyFqn, new Query(), QueryOptions.empty(), ownerToken);
assertEquals(2, search.getNumResults());

Query query = new Query(WorkflowDBAdaptor.QueryParams.DRAFT.key(), true);
search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken);
assertEquals(1, search.getNumResults());
assertEquals("workflow2", search.first().getId());
assertTrue(search.first().isDraft());

query = new Query(WorkflowDBAdaptor.QueryParams.DRAFT.key(), false);
search = workflowManager.search(studyFqn, query, QueryOptions.empty(), ownerToken);
assertEquals(1, search.getNumResults());
assertEquals("workflow", search.first().getId());
assertFalse(search.first().isDraft());
}

@Test
public void updateWorkflowTest() throws CatalogException {
Workflow workflow = new Workflow()
.setId("workflow")
.setScripts(Collections.singletonList(new WorkflowScript("pipeline.nf", "echo 'Hello world!'", true)));
workflowManager.create(studyFqn, workflow, INCLUDE_RESULT, ownerToken);

WorkflowUpdateParams updateParams = new WorkflowUpdateParams()
.setName("newName")
.setDraft(true)
.setCreationDate("20240101000000")
.setModificationDate("20240201000000")
.setDescription("description");

OpenCGAResult<Workflow> update = workflowManager.update(studyFqn, workflow.getId(), updateParams, INCLUDE_RESULT, ownerToken);
assertEquals(1, update.getNumUpdated());
Workflow updatedWorkflow = update.first();
assertEquals(updateParams.getName(), updatedWorkflow.getName());
assertEquals(updateParams.isDraft(), updatedWorkflow.isDraft());
assertEquals(updateParams.getCreationDate(), updatedWorkflow.getCreationDate());
assertEquals(updateParams.getModificationDate(), updatedWorkflow.getModificationDate());
assertEquals(updateParams.getDescription(), updatedWorkflow.getDescription());
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.opencb.opencga.core.models.sample.Sample;
import org.opencb.opencga.core.models.study.Study;
import org.opencb.opencga.core.models.study.VariableSet;
import org.opencb.opencga.core.models.workflow.Workflow;

import javax.ws.rs.ext.ContextResolver;
import java.io.IOException;
Expand Down Expand Up @@ -109,6 +110,7 @@ private static ObjectMapper generateUpdateObjectMapper(JsonFactory jf) {
objectMapper.addMixIn(VariableSet.class, PrivateUidMixin.class);
objectMapper.addMixIn(ClinicalAnalysis.class, PrivateUidMixin.class);
objectMapper.addMixIn(Interpretation.class, PrivateUidMixin.class);
objectMapper.addMixIn(Workflow.class, PrivateUidMixin.class);

objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);

Expand Down Expand Up @@ -143,6 +145,7 @@ private static ObjectMapper generateOpenCGAObjectMapper(JsonFactory jf) {
objectMapper.addMixIn(VariableSet.class, PrivateUidMixin.class);
objectMapper.addMixIn(ClinicalAnalysis.class, PrivateUidMixin.class);
objectMapper.addMixIn(Interpretation.class, PrivateUidMixin.class);
objectMapper.addMixIn(Workflow.class, PrivateUidMixin.class);
objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);

return objectMapper;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ public class Workflow extends PrivateStudyUid {
@DataField(id = "version", managed = true, indexed = true, description = FieldConstants.GENERIC_VERSION_DESCRIPTION)
private int version;

@DataField(id = "release", managed = true, indexed = true, description = FieldConstants.GENERIC_RELEASE_DESCRIPTION)
private int release;

@DataField(id = "type", description = FieldConstants.WORKFLOW_TYPE_DESCRIPTION)
private Type type;

Expand Down Expand Up @@ -83,6 +86,7 @@ public String toString() {
sb.append(", description='").append(description).append('\'');
sb.append(", draft=").append(draft);
sb.append(", version=").append(version);
sb.append(", release=").append(release);
sb.append(", type=").append(type);
sb.append(", repository=").append(repository);
sb.append(", scripts=").append(scripts);
Expand Down Expand Up @@ -139,6 +143,15 @@ public Workflow setVersion(int version) {
return this;
}

public int getRelease() {
return release;
}

public Workflow setRelease(int release) {
this.release = release;
return this;
}

public Type getType() {
return type;
}
Expand Down

0 comments on commit 1b1f4be

Please sign in to comment.