From faea2787db111b3d06515d46485f9c3c8723c46d Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 12 Jun 2025 16:18:56 +0200 Subject: [PATCH 01/13] wip --- .../monitor/daemons/CatalogService.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/CatalogService.java diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/CatalogService.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/CatalogService.java new file mode 100644 index 00000000000..de2f4d75004 --- /dev/null +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/CatalogService.java @@ -0,0 +1,27 @@ +package org.opencb.opencga.master.monitor.daemons; + +import org.opencb.opencga.catalog.managers.CatalogManager; + +import java.io.Closeable; +import java.io.IOException; + +public class CatalogService extends MonitorParentDaemon implements Closeable { + + + public CatalogService(int interval, String token, CatalogManager catalogManager) { + super(interval, token, catalogManager); + } + + @Override + public void apply() throws Exception { + + } + + private void checkCVDB() { + } + + @Override + public void close() throws IOException { + + } +} From 32735cde9cc67a0a7bf4eef128f2a5f2b527b0e3 Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 16 Jun 2025 14:41:30 +0200 Subject: [PATCH 02/13] core: add new CatalogService configuration, #TASK-7756 --- .../study/CatalogServiceConfiguration.java | 30 ++++++++++++ .../study/CatalogStudyConfiguration.java | 49 +++++++++++++++++++ .../configuration/StudyConfiguration.java | 20 +++++++- 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java new file mode 100644 index 00000000000..7d289692a45 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java @@ -0,0 +1,30 @@ +package org.opencb.opencga.core.models.study; + +public class CatalogServiceConfiguration { + + private boolean active; + + public CatalogServiceConfiguration() { + } + + public CatalogServiceConfiguration(boolean active) { + this.active = active; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CatalogServiceConfiguration{"); + sb.append("active=").append(active); + sb.append('}'); + return sb.toString(); + } + + public boolean isActive() { + return active; + } + + public CatalogServiceConfiguration setActive(boolean active) { + this.active = active; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java new file mode 100644 index 00000000000..5232175834f --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java @@ -0,0 +1,49 @@ +package org.opencb.opencga.core.models.study; + +public class CatalogStudyConfiguration { + + private CatalogServiceConfiguration cvdb; + private CatalogServiceConfiguration variantQualityControl; + + public CatalogStudyConfiguration() { + } + + public CatalogStudyConfiguration(CatalogServiceConfiguration cvdb, CatalogServiceConfiguration variantQualityControl) { + this.cvdb = cvdb; + this.variantQualityControl = variantQualityControl; + } + + public static CatalogStudyConfiguration defaultConfiguration() { + return new CatalogStudyConfiguration( + new CatalogServiceConfiguration(false), // cvdb + new CatalogServiceConfiguration(false) // variantQualityControl + ); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("StudyCatalogConfiguration{"); + sb.append("cvdb=").append(cvdb); + sb.append(", variantQualityControl=").append(variantQualityControl); + sb.append('}'); + return sb.toString(); + } + + public CatalogServiceConfiguration getCvdb() { + return cvdb; + } + + public CatalogStudyConfiguration setCvdb(CatalogServiceConfiguration cvdb) { + this.cvdb = cvdb; + return this; + } + + public CatalogServiceConfiguration getVariantQualityControl() { + return variantQualityControl; + } + + public CatalogStudyConfiguration setVariantQualityControl(CatalogServiceConfiguration variantQualityControl) { + this.variantQualityControl = variantQualityControl; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/StudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/StudyConfiguration.java index 4fd3db9326f..6e2c2b025b4 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/StudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/StudyConfiguration.java @@ -2,25 +2,31 @@ import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; +import org.opencb.opencga.core.models.study.CatalogStudyConfiguration; import org.opencb.opencga.core.models.study.StudyVariantEngineConfiguration; public class StudyConfiguration { private ClinicalAnalysisStudyConfiguration clinical; private StudyVariantEngineConfiguration variantEngine; + private CatalogStudyConfiguration catalog; + public StudyConfiguration() { } - public StudyConfiguration(ClinicalAnalysisStudyConfiguration clinical, StudyVariantEngineConfiguration variantEngine) { + public StudyConfiguration(ClinicalAnalysisStudyConfiguration clinical, StudyVariantEngineConfiguration variantEngine, + CatalogStudyConfiguration catalog) { this.clinical = clinical; this.variantEngine = variantEngine; + this.catalog = catalog; } public static StudyConfiguration init(String cellbaseVersion) { return new StudyConfiguration(ClinicalAnalysisStudyConfiguration.defaultConfiguration(), new StudyVariantEngineConfiguration(new ObjectMap(), - cellbaseVersion == null ? null : SampleIndexConfiguration.defaultConfiguration(cellbaseVersion))); + cellbaseVersion == null ? null : SampleIndexConfiguration.defaultConfiguration(cellbaseVersion)), + CatalogStudyConfiguration.defaultConfiguration()); } @Override @@ -28,6 +34,7 @@ public String toString() { final StringBuilder sb = new StringBuilder("StudyConfiguration{"); sb.append("clinical=").append(clinical); sb.append(", variantEngine=").append(variantEngine); + sb.append(", catalog=").append(catalog); sb.append('}'); return sb.toString(); } @@ -49,4 +56,13 @@ public StudyConfiguration setVariantEngine(StudyVariantEngineConfiguration varia this.variantEngine = variantEngine; return this; } + + public CatalogStudyConfiguration getCatalog() { + return catalog; + } + + public StudyConfiguration setCatalog(CatalogStudyConfiguration catalog) { + this.catalog = catalog; + return this; + } } From a7775d7b32a5725a08dac33bf1c6cc23fcb0843f Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 18 Jun 2025 10:09:51 +0200 Subject: [PATCH 03/13] catalog: add quality control statuses, #TASK-7756 --- .../catalog/db/api/FamilyDBAdaptor.java | 1 + .../catalog/db/api/IndividualDBAdaptor.java | 1 + .../catalog/db/api/SampleDBAdaptor.java | 1 + .../db/mongodb/FamilyMongoDBAdaptor.java | 1 + .../db/mongodb/IndividualMongoDBAdaptor.java | 1 + .../db/mongodb/SampleMongoDBAdaptor.java | 1 + .../models/common/QualityControlStatus.java | 49 +++++++++++++++++++ .../core/models/family/FamilyInternal.java | 24 +++++++-- .../models/individual/IndividualInternal.java | 24 +++++++-- .../core/models/sample/SampleInternal.java | 17 ++++++- .../study/CatalogServiceConfiguration.java | 18 ++++++- .../study/CatalogStudyConfiguration.java | 4 +- 12 files changed, 129 insertions(+), 13 deletions(-) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java index fe9fd976e64..6fa44461c01 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java @@ -64,6 +64,7 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_MSG("internal.status.msg", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), + INTERNAL_QUALITY_CONTROL_STATUS_ID("internal.qualityControlStatus.id", TEXT, ""), RELEASE("release", INTEGER, ""), SNAPSHOT("snapshot", INTEGER, ""), // Last version of individual at release = snapshot VERSION("version", INTEGER, ""), // Version of the individual diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java index b6dbc94b861..73b1a084835 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java @@ -68,6 +68,7 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), INTERNAL_RGA("internal.rga", OBJECT, ""), + INTERNAL_QUALITY_CONTROL_STATUS_ID("internal.qualityControlStatus.id", TEXT, ""), POPULATION_NAME("population.name", TEXT, ""), POPULATION_SUBPOPULATION("population.subpopulation", TEXT, ""), POPULATION_DESCRIPTION("population.description", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java index 6dc973eed70..e88a2fc7910 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java @@ -133,6 +133,7 @@ enum QueryParams implements QueryParam { INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX("internal.variant.secondarySampleIndex", TEXT_ARRAY, ""), INTERNAL_VARIANT_ANNOTATION_INDEX("internal.variant.annotationIndex", TEXT_ARRAY, ""), INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX("internal.variant.secondaryAnnotationIndex", TEXT_ARRAY, ""), + INTERNAL_QUALITY_CONTROL_STATUS_ID("internal.qualityControlStatus.id", TEXT, ""), RELEASE("release", INTEGER, ""), // Release where the sample was created SNAPSHOT("snapshot", INTEGER, ""), // Last version of sample at release = snapshot VERSION("version", INTEGER, ""), // Version of the sample diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java index 8e37fe511c1..ecb9589f0bc 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FamilyMongoDBAdaptor.java @@ -1298,6 +1298,7 @@ protected Bson parseQuery(Query query, Document extraQuery, String user) case EXPECTED_SIZE: case RELEASE: case VERSION: + case INTERNAL_QUALITY_CONTROL_STATUS_ID: case PHENOTYPES_ID: case PHENOTYPES_NAME: addAutoOrQuery(queryParam.key(), queryParam.key(), queryCopy, queryParam.type(), andBsonList); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java index 797015ce62b..f59234ec8d8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/IndividualMongoDBAdaptor.java @@ -1523,6 +1523,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case RELEASE: case VERSION: case SAMPLE_UIDS: + case INTERNAL_QUALITY_CONTROL_STATUS_ID: case PHENOTYPES_ID: case PHENOTYPES_NAME: case DISORDERS_ID: diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java index eb9b17cf5e9..b15c65b2ea6 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/SampleMongoDBAdaptor.java @@ -1455,6 +1455,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case ID: case UUID: case INTERNAL_VARIANT_INDEX_STATUS_ID: + case INTERNAL_QUALITY_CONTROL_STATUS_ID: case PROCESSING_PRODUCT_ID: case PROCESSING_PREPARATION_METHOD: case PROCESSING_EXTRACTION_METHOD: diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java new file mode 100644 index 00000000000..15791e3f89b --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java @@ -0,0 +1,49 @@ +package org.opencb.opencga.core.models.common; + +import java.util.Arrays; +import java.util.List; + +public class QualityControlStatus extends InternalStatus { + + public static final String ERROR = "ERROR"; + public static final String PENDING = "PENDING"; + public static final String PENDING_REMOVE = "PENDING_REMOVE"; + public static final String PENDING_OVERWRITE = "PENDING_OVERWRITE"; + + public static final List STATUS_LIST = Arrays.asList(PENDING, PENDING_REMOVE, PENDING_OVERWRITE, READY, ERROR); + + public QualityControlStatus(String status, String message) { + if (isValid(status)) { + init(status, message); + } else { + throw new IllegalArgumentException("Unknown status " + status); + } + } + + public QualityControlStatus(String status) { + this(status, ""); + } + + public QualityControlStatus() { + this(PENDING, ""); + } + + @Override + public String getId() { + return super.getId(); + } + + public static QualityControlStatus init() { + return new QualityControlStatus(); + } + + public static boolean isValid(String status) { + return status != null + && (status.equals(PENDING) + || status.equals(PENDING_REMOVE) + || status.equals(PENDING_OVERWRITE) + || status.equals(READY) + || status.equals(ERROR)); + } + +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyInternal.java index 93cc5ceea98..5c5606ee5c2 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyInternal.java @@ -18,29 +18,36 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; +import org.opencb.opencga.core.models.common.QualityControlStatus; public class FamilyInternal extends Internal { private FamilyStatus status; + private QualityControlStatus qualityControlStatus; public FamilyInternal() { } - public FamilyInternal(String registrationDate, String modificationDate, FamilyStatus status) { + public FamilyInternal(String registrationDate, String modificationDate, FamilyStatus status, + QualityControlStatus qualityControlStatus) { super(null, registrationDate, modificationDate); this.status = status; + this.qualityControlStatus = qualityControlStatus; } public static FamilyInternal init() { - return new FamilyInternal(TimeUtils.getTime(), TimeUtils.getTime(), new FamilyStatus()); + return new FamilyInternal(TimeUtils.getTime(), TimeUtils.getTime(), new FamilyStatus(), + QualityControlStatus.init()); } @Override public String toString() { final StringBuilder sb = new StringBuilder("FamilyInternal{"); - sb.append("registrationDate='").append(registrationDate).append('\''); - sb.append(", modificationDate='").append(lastModified).append('\''); + sb.append("status=").append(status); + sb.append(", qualityControlStatus=").append(qualityControlStatus); sb.append(", status=").append(status); + sb.append(", registrationDate='").append(registrationDate).append('\''); + sb.append(", lastModified='").append(lastModified).append('\''); sb.append('}'); return sb.toString(); } @@ -54,6 +61,15 @@ public FamilyInternal setStatus(FamilyStatus status) { return this; } + public QualityControlStatus getQualityControlStatus() { + return qualityControlStatus; + } + + public FamilyInternal setQualityControlStatus(QualityControlStatus qualityControlStatus) { + this.qualityControlStatus = qualityControlStatus; + return this; + } + public String getRegistrationDate() { return registrationDate; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualInternal.java index 4a309e42517..2c4450bfbf4 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/individual/IndividualInternal.java @@ -19,30 +19,46 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.common.QualityControlStatus; public class IndividualInternal extends Internal { + private QualityControlStatus qualityControlStatus; + public IndividualInternal() { } - public IndividualInternal(InternalStatus status, String registrationDate, String modificationDate) { + public IndividualInternal(QualityControlStatus qualityControlStatus, InternalStatus status, String registrationDate, + String modificationDate) { super(status, registrationDate, modificationDate); + this.qualityControlStatus = qualityControlStatus; } public static IndividualInternal init() { - return new IndividualInternal(new InternalStatus(InternalStatus.READY), TimeUtils.getTime(), TimeUtils.getTime()); + return new IndividualInternal(QualityControlStatus.init(), new InternalStatus(InternalStatus.READY), TimeUtils.getTime(), + TimeUtils.getTime()); } @Override public String toString() { final StringBuilder sb = new StringBuilder("IndividualInternal{"); - sb.append("status=").append(status); + sb.append("qualityControlStatus=").append(qualityControlStatus); + sb.append(", status=").append(status); sb.append(", registrationDate='").append(registrationDate).append('\''); - sb.append(", modificationDate='").append(lastModified).append('\''); + sb.append(", lastModified='").append(lastModified).append('\''); sb.append('}'); return sb.toString(); } + public QualityControlStatus getQualityControlStatus() { + return qualityControlStatus; + } + + public IndividualInternal setQualityControlStatus(QualityControlStatus qualityControlStatus) { + this.qualityControlStatus = qualityControlStatus; + return this; + } + public InternalStatus getStatus() { return status; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java index 2beb13a1547..b209ee43e6a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java @@ -21,6 +21,7 @@ import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.models.common.Internal; import org.opencb.opencga.core.models.common.InternalStatus; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.common.RgaIndex; import java.util.Objects; @@ -35,20 +36,22 @@ public class SampleInternal extends Internal { description = FieldConstants.SAMPLE_INTERNAL_RGA_DESCRIPTION) private RgaIndex rga; private SampleInternalVariant variant; + private QualityControlStatus qualityControlStatus; public SampleInternal() { } public SampleInternal(String registrationDate, String modificationDate, InternalStatus status, RgaIndex rga, - SampleInternalVariant variant) { + SampleInternalVariant variant, QualityControlStatus qualityControlStatus) { super(status, registrationDate, modificationDate); this.rga = rga; this.variant = variant; + this.qualityControlStatus = qualityControlStatus; } public static SampleInternal init() { return new SampleInternal(TimeUtils.getTime(), TimeUtils.getTime(), new InternalStatus(InternalStatus.READY), RgaIndex.init(), - SampleInternalVariant.init()); + SampleInternalVariant.init(), QualityControlStatus.init()); } @Override @@ -59,6 +62,7 @@ public String toString() { sb.append(", lastModified='").append(lastModified).append('\''); sb.append(", rga=").append(rga); sb.append(", variant=").append(variant); + sb.append(", qualityControlStatus=").append(qualityControlStatus); sb.append('}'); return sb.toString(); } @@ -108,6 +112,15 @@ public SampleInternal setLastModified(String lastModified) { return this; } + public QualityControlStatus getQualityControlStatus() { + return qualityControlStatus; + } + + public SampleInternal setQualityControlStatus(QualityControlStatus qualityControlStatus) { + this.qualityControlStatus = qualityControlStatus; + return this; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java index 7d289692a45..87fbd03f522 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java @@ -3,18 +3,25 @@ public class CatalogServiceConfiguration { private boolean active; + private int concurrentJobs; public CatalogServiceConfiguration() { } - public CatalogServiceConfiguration(boolean active) { + public CatalogServiceConfiguration(boolean active, int concurrentJobs) { this.active = active; + this.concurrentJobs = concurrentJobs; + } + + public static CatalogServiceConfiguration defaultConfiguration() { + return new CatalogServiceConfiguration(false, 5); } @Override public String toString() { final StringBuilder sb = new StringBuilder("CatalogServiceConfiguration{"); sb.append("active=").append(active); + sb.append(", concurrentJobs=").append(concurrentJobs); sb.append('}'); return sb.toString(); } @@ -27,4 +34,13 @@ public CatalogServiceConfiguration setActive(boolean active) { this.active = active; return this; } + + public int getConcurrentJobs() { + return concurrentJobs; + } + + public CatalogServiceConfiguration setConcurrentJobs(int concurrentJobs) { + this.concurrentJobs = concurrentJobs; + return this; + } } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java index 5232175834f..472c4a6461c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java @@ -15,8 +15,8 @@ public CatalogStudyConfiguration(CatalogServiceConfiguration cvdb, CatalogServic public static CatalogStudyConfiguration defaultConfiguration() { return new CatalogStudyConfiguration( - new CatalogServiceConfiguration(false), // cvdb - new CatalogServiceConfiguration(false) // variantQualityControl + CatalogServiceConfiguration.defaultConfiguration(), // cvdb + CatalogServiceConfiguration.defaultConfiguration() // variantQualityControl ); } From c1740f21e48cf07f8c731c94e88330660a8c4b83 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 26 Jun 2025 10:27:51 +0200 Subject: [PATCH 04/13] core: add new fields and cvdb migration, #TAKS-7756 --- .../executors/CatalogCommandExecutor.java | 2 +- .../catalog/ClinicalCvdbIndexMigration.java | 35 +--- .../catalog/ClinicalMigrationTask7756.java | 149 ++++++++++++++++++ .../db/api/ClinicalAnalysisDBAdaptor.java | 3 + .../ClinicalAnalysisMongoDBAdaptor.java | 5 +- .../managers/ClinicalAnalysisManager.java | 103 ++++++++++-- .../clinical/ClinicalAnalysisInternal.java | 10 +- .../core/models/clinical/CvdbIndex.java | 46 ++++++ .../core/models/clinical/CvdbIndexStatus.java | 14 +- 9 files changed, 308 insertions(+), 59 deletions(-) create mode 100644 opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/ClinicalMigrationTask7756.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndex.java diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java index 5879161b3d4..8ce6558ba57 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java @@ -241,7 +241,7 @@ private void daemons() throws Exception { } } - private void validateConfiguration(AdminCliOptionsParser.CatalogDatabaseCommandOptions catalogOptions) { + protected void validateConfiguration(AdminCliOptionsParser.CatalogDatabaseCommandOptions catalogOptions) { if (catalogOptions.databaseUser != null) { configuration.getCatalog().getDatabase().setUser(catalogOptions.databaseUser); } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ClinicalCvdbIndexMigration.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ClinicalCvdbIndexMigration.java index 9cb73ade19e..f9f6a1c488c 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ClinicalCvdbIndexMigration.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v4/v4_0_0/catalog/ClinicalCvdbIndexMigration.java @@ -1,47 +1,16 @@ package org.opencb.opencga.app.migrations.v4.v4_0_0.catalog; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Updates; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; -import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; import org.opencb.opencga.catalog.migration.Migration; import org.opencb.opencga.catalog.migration.MigrationTool; -import org.opencb.opencga.core.models.clinical.ClinicalStatusValue; -import org.opencb.opencga.core.models.clinical.CvdbIndexStatus; - -import java.util.Arrays; @Migration(id = "add_cvdb_index_to_clinical_analysis", description = "Add CVDB index status to Clinical Analysis #TASK-5610", version = "4.0.0", - language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241118) + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20241118, + deprecatedSince = "5.0.0") public class ClinicalCvdbIndexMigration extends MigrationTool { @Override protected void run() throws Exception { - Bson closedCaseQuery = Filters.and( - Filters.exists(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), false), - Filters.eq("status.type", ClinicalStatusValue.ClinicalStatusType.CLOSED) - ); - Bson openCaseQuery = Filters.and( - Filters.exists(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), false), - Filters.ne("status.type", ClinicalStatusValue.ClinicalStatusType.CLOSED) - ); - CvdbIndexStatus pendingCvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING); - Document pendingCvdbIndexDoc = convertToDocument(pendingCvdbIndexStatus); - Bson updateClosedCase = Updates.set(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), pendingCvdbIndexDoc); - - CvdbIndexStatus noneCvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.NONE); - Document noneCvdbIndexDoc = convertToDocument(noneCvdbIndexStatus); - Bson updateOpenCase = Updates.set(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), noneCvdbIndexDoc); - - for (String collection : Arrays.asList(OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, - OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, - OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) { - getMongoCollection(collection).updateMany(closedCaseQuery, updateClosedCase); - getMongoCollection(collection).updateMany(openCaseQuery, updateOpenCase); - } } } diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/ClinicalMigrationTask7756.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/ClinicalMigrationTask7756.java new file mode 100644 index 00000000000..847b6acf44a --- /dev/null +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/ClinicalMigrationTask7756.java @@ -0,0 +1,149 @@ +package org.opencb.opencga.app.migrations.v5.v5_0_0.catalog; + +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.UpdateOneModel; +import com.mongodb.client.model.Updates; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor; +import org.opencb.opencga.catalog.db.mongodb.OrganizationMongoDBAdaptorFactory; +import org.opencb.opencga.catalog.exceptions.CatalogDBException; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysisInternal; +import org.opencb.opencga.core.models.clinical.CvdbIndex; +import org.opencb.opencga.core.models.common.QualityControlStatus; +import org.opencb.opencga.core.models.study.CatalogStudyConfiguration; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@Migration(id = "add_clinical_configuration_migration_7756", + description = "Add CVDB index status to Clinical Analysis #TASK-5610", version = "5.0.0", + language = Migration.MigrationLanguage.JAVA, domain = Migration.MigrationDomain.CATALOG, date = 20250625) +public class ClinicalMigrationTask7756 extends MigrationTool { + + @Override + protected void run() throws Exception { + addNewQualityControlStatusField(Arrays.asList(OrganizationMongoDBAdaptorFactory.SAMPLE_COLLECTION, + OrganizationMongoDBAdaptorFactory.SAMPLE_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_SAMPLE_COLLECTION), + Filters.or( + Filters.ne("qualityControl.files", Collections.emptyList()), + Filters.ne("qualityControl.variant.variantStats", Collections.emptyList()), + Filters.ne("qualityControl.variant.signatures", Collections.emptyList()), + Filters.ne("qualityControl.variant.genomePlot", null) + )); + + addNewQualityControlStatusField(Arrays.asList(OrganizationMongoDBAdaptorFactory.INDIVIDUAL_COLLECTION, + OrganizationMongoDBAdaptorFactory.INDIVIDUAL_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_INDIVIDUAL_COLLECTION), + Filters.or( + Filters.ne("qualityControl.inferredSexReports", Collections.emptyList()), + Filters.ne("qualityControl.mendelianErrorReports", Collections.emptyList()) + )); + + addNewQualityControlStatusField(Arrays.asList(OrganizationMongoDBAdaptorFactory.FAMILY_COLLECTION, + OrganizationMongoDBAdaptorFactory.FAMILY_ARCHIVE_COLLECTION, OrganizationMongoDBAdaptorFactory.DELETED_FAMILY_COLLECTION), + Filters.ne("qualityControl.relatedness", Collections.emptyList())); + + migrateClinicalAnalysisCvdbIndex(); + + migrateStudyConfiguration(); + } + + + private void migrateStudyConfiguration() throws CatalogDBException { + // Study: + // + internal.configuration.catalog: { + // cvdb: {}, + // variantQualityControl: {} + // } + logger.info("Starting migration of Study configuration catalog"); + + List studyCollections = Arrays.asList( + OrganizationMongoDBAdaptorFactory.STUDY_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_STUDY_COLLECTION + ); + logger.debug("Processing study collections: {}", studyCollections); + + CatalogStudyConfiguration catalogStudyConfiguration = CatalogStudyConfiguration.defaultConfiguration(); + Document studyConfigurationDoc = convertToDocument(catalogStudyConfiguration); + + migrateCollection(studyCollections, Filters.exists("internal.configuration.catalog", false), + Projections.include("_id"), (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + updateDocument.getSet().put("internal.configuration.catalog", studyConfigurationDoc); + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + }); + + logger.info("Finished migration of Study configuration catalog"); + } + + private void migrateClinicalAnalysisCvdbIndex() throws CatalogDBException { + // ClinicalAnalysis + // - internal.cvdbIndex + // + internal.cvdb: { + // status: {}, + // job: "" + // } + + logger.info("Starting migration of clinical analysis CVDB index"); + + List clinicalCollections = Arrays.asList( + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, + OrganizationMongoDBAdaptorFactory.CLINICAL_ANALYSIS_ARCHIVE_COLLECTION, + OrganizationMongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION + ); + logger.debug("Processing clinical collections: {}", clinicalCollections); + + migrateCollection(clinicalCollections, Filters.exists("internal.cvdb", false), Projections.include("_id", "internal"), (document, bulk) -> { + MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); + + Document internalDoc = document.get("internal", Document.class); + if (internalDoc == null) { + updateDocument.getSet().put("internal", convertToDocument(ClinicalAnalysisInternal.init())); + } else { + Document cvdbIndexStatus = internalDoc.get("cvdbIndex", Document.class); + if (cvdbIndexStatus == null) { + updateDocument.getSet().put("internal.cvdb", convertToDocument(CvdbIndex.init())); + } else { + Document cvdbIndex = new Document() + .append("jobId", "") + .append("status", cvdbIndexStatus); + updateDocument.getSet().put("internal.cvdb", cvdbIndex); + } + } + + bulk.add(new UpdateOneModel<>(Filters.eq("_id", document.get("_id")), updateDocument.toFinalUpdateDocument())); + logger.trace("Added update for document with id: {}", document.get("_id")); + }); + + logger.info("Finished migration of clinical analysis CVDB index"); + } + + private void addNewQualityControlStatusField(List collections, Bson qualityControlCalculatedFilter) throws CatalogDBException { + // Sample | Individual | Family + // + internal.qualityControlStatus + + Bson query = Filters.and( + Filters.exists("internal.qualityControlStatus", false), + qualityControlCalculatedFilter); + QualityControlStatus controlStatus = QualityControlStatus.init(); + controlStatus.setId(QualityControlStatus.READY); + Bson update = Updates.set("internal.qualityControlStatus", convertToDocument(controlStatus)); + + Bson initQuery = Filters.exists("internal.qualityControlStatus", false); + QualityControlStatus initControlStatus = QualityControlStatus.init(); + Bson initUpdate = Updates.set("internal.qualityControlStatus", convertToDocument(initControlStatus)); + + for (String collection : collections) { + long updatedDocs = getMongoCollection(collection).updateMany(query, update).getModifiedCount(); + logger.info("Updated {} documents in {} with qualityControlStatus=READY", updatedDocs, collection); + + long initUpdatedDocs = getMongoCollection(collection).updateMany(initQuery, initUpdate).getModifiedCount(); + logger.info("Updated {} documents in {} with initial qualityControlStatus", initUpdatedDocs, collection); + } + } + +} diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java index 018d23fa8f8..6b6d1813a0b 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java @@ -63,6 +63,9 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), INTERNAL_CVDB_INDEX("internal.cvdbIndex", OBJECT, ""), + INTERNAL_CVDB_INDEX_STATUS("internal.cvdbIndex.status", OBJECT, ""), + INTERNAL_CVDB_INDEX_JOB_ID("internal.cvdbIndex.jobId", STRING, ""), + INTERNAL_CVDB_INDEX_STATUS_ID("internal.cvdbIndex.status.id", TEXT, ""), QUALITY_CONTROL("qualityControl", OBJECT, ""), QUALITY_CONTROL_SUMMARY("qualityControl.summary", TEXT, ""), CONSENT("consent", OBJECT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index 2dd04863d16..db520d4fa14 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -439,7 +439,7 @@ UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, List create(String studyStr, ClinicalAnalysis logger.info(msg); clinicalAnalysis.setLocked(true); - CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING, "User '" + userId + CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING_INDEX, "User '" + userId + "' created case with status '" + clinicalAnalysis.getStatus().getId() + "', which is of type" - + " CLOSED. Automatically setting CVDB index status to " + CvdbIndexStatus.PENDING); - clinicalAnalysis.getInternal().setCvdbIndex(cvdbIndexStatus); + + " CLOSED. Automatically setting CVDB index status to " + CvdbIndexStatus.PENDING_INDEX); + clinicalAnalysis.getInternal().setCvdbIndex(new CvdbIndex("", cvdbIndexStatus)); events.add(new Event(Event.Type.INFO, clinicalAnalysis.getId(), msg)); } @@ -1473,7 +1473,7 @@ private OpenCGAResult update(String organizationId, Study stud CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING_REMOVE, "User '" + userId + "' requested to remove the status '" + clinicalAnalysis.getStatus().getId() + "' of type " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " to set it to '" + updateParams.getStatus().getId() + "'"); - parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), cvdbIndexStatus); + parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_STATUS.key(), cvdbIndexStatus); } } @@ -1764,22 +1764,23 @@ private OpenCGAResult update(String organizationId, Study stud parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), true); events.add(new Event(Event.Type.INFO, clinicalAnalysis.getId(), msg)); - if (StringUtils.isEmpty(clinicalAnalysis.getInternal().getCvdbIndex().getId()) - || clinicalAnalysis.getInternal().getCvdbIndex().getId().equals(CvdbIndexStatus.NONE)) { - CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING, "User '" + userId + if (StringUtils.isEmpty(clinicalAnalysis.getInternal().getCvdbIndex().getStatus().getId()) + || clinicalAnalysis.getInternal().getCvdbIndex().getStatus().getId().equals(CvdbIndexStatus.NONE)) { + CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING_INDEX, "User '" + userId + "' changed case to status '" + updateParamsClone.getStatus().getId() + "', which is of type" - + " CLOSED. Automatically changing CVDB index status to " + CvdbIndexStatus.PENDING); - parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), cvdbIndexStatus); - } else if (clinicalAnalysis.getInternal().getCvdbIndex().getId().equals(CvdbIndexStatus.PENDING_REMOVE)) { + + " CLOSED. Automatically changing CVDB index status to " + CvdbIndexStatus.PENDING_INDEX); + parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_STATUS.key(), cvdbIndexStatus); + } else if (clinicalAnalysis.getInternal().getCvdbIndex().getStatus().getId() + .equals(CvdbIndexStatus.PENDING_REMOVE)) { CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING_OVERWRITE, "User '" + userId + "' changed case to status '" + updateParamsClone.getStatus().getId() + "', which is of type" + " CLOSED. CVDB index was already in " + CvdbIndexStatus.PENDING_REMOVE + ", so automatically" + " changing CVDB index status to " + CvdbIndexStatus.PENDING_OVERWRITE); - parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), cvdbIndexStatus); + parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_STATUS.key(), cvdbIndexStatus); } else { logger.warn("CVDB index status is unexpectedly set to '{}'. Although the user is closing the case, OpenCGA" + " cannot automatically infer which should be the new CVDB index status.", - clinicalAnalysis.getInternal().getCvdbIndex().getId()); + clinicalAnalysis.getInternal().getCvdbIndex().getStatus().getId()); } } } @@ -1891,7 +1892,7 @@ public OpenCGAResult updateReport(String studyStr, String clinic } } - public OpenCGAResult updateCvdbIndex(String studyFqn, ClinicalAnalysis clinical, CvdbIndexStatus index, String token) + public OpenCGAResult updateCvdbIndex(String studyFqn, ClinicalAnalysis clinical, CvdbIndex index, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, tokenPayload); @@ -1912,8 +1913,15 @@ public OpenCGAResult updateCvdbIndex(String studyFqn, ClinicalAnalysis clinic studyId = study.getFqn(); studyUuid = study.getUuid(); - ObjectMap valueAsMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(index)); - ObjectMap params = new ObjectMap(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX.key(), valueAsMap); + ObjectMap params = new ObjectMap(); + if (index.getStatus() != null) { + validateCvdbStatusTransition(clinical, index.getStatus().getId()); + ObjectMap valueAsMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(index.getStatus())); + params.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_STATUS.key(), valueAsMap); + } + if (StringUtils.isNotEmpty(index.getJobId())) { + params.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_JOB_ID.key(), index.getJobId()); + } OpenCGAResult update = getClinicalAnalysisDBAdaptor(organizationId).update(clinical.getUid(), params, Collections.emptyList(), Collections.emptyList(), QueryOptions.empty()); @@ -1928,6 +1936,71 @@ public OpenCGAResult updateCvdbIndex(String studyFqn, ClinicalAnalysis clinic } } + private void validateCvdbStatusTransition(ClinicalAnalysis clinical, String newStatus) throws CatalogException { + if (clinical == null || clinical.getInternal() == null || clinical.getInternal().getCvdbIndex() == null + || clinical.getInternal().getCvdbIndex().getStatus() == null + || StringUtils.isEmpty(clinical.getInternal().getCvdbIndex().getStatus().getId())) { + throw new CatalogException("Cannot validate CVDB Index status transition. CVDB Index status is null."); + } + switch (clinical.getInternal().getCvdbIndex().getStatus().getId()) { + case CvdbIndexStatus.NONE: + if (!Arrays.asList(CvdbIndexStatus.PENDING_INDEX, CvdbIndexStatus.READY, CvdbIndexStatus.ERROR).contains(newStatus)) { + throw new CatalogException("CVDB Index status cannot be changed from NONE to " + newStatus); + } + break; + case CvdbIndexStatus.PENDING_INDEX: + if (!Arrays.asList(CvdbIndexStatus.QUEUED_INDEX, CvdbIndexStatus.ERROR, CvdbIndexStatus.READY, CvdbIndexStatus.NONE, + CvdbIndexStatus.PENDING_REMOVE).contains(newStatus)) { + throw new CatalogException("CVDB Index status cannot be changed from PENDING_INDEX to " + newStatus); + } + break; + case CvdbIndexStatus.PENDING_OVERWRITE: + if (!Arrays.asList(CvdbIndexStatus.QUEUED_OVERWRITE, CvdbIndexStatus.ERROR, CvdbIndexStatus.READY, + CvdbIndexStatus.PENDING_REMOVE).contains(newStatus)) { + throw new CatalogException("CVDB Index status cannot be changed from PENDING_OVERWRITE to " + newStatus); + } + break; + case CvdbIndexStatus.PENDING_REMOVE: + if (!Arrays.asList(CvdbIndexStatus.QUEUED_REMOVE, CvdbIndexStatus.ERROR, CvdbIndexStatus.READY, CvdbIndexStatus.NONE, + CvdbIndexStatus.PENDING_INDEX).contains(newStatus)) { + throw new CatalogException("CVDB Index status cannot be changed from PENDING_REMOVE to " + newStatus); + } + break; + case CvdbIndexStatus.QUEUED_INDEX: + if (!Arrays.asList(CvdbIndexStatus.ERROR, CvdbIndexStatus.READY, CvdbIndexStatus.NONE, CvdbIndexStatus.PENDING_REMOVE) + .contains(newStatus)) { + throw new CatalogException("CVDB Index status cannot be changed from QUEUED_INDEX to " + newStatus); + } + break; + case CvdbIndexStatus.QUEUED_OVERWRITE: + if (!Arrays.asList(CvdbIndexStatus.ERROR, CvdbIndexStatus.READY, CvdbIndexStatus.PENDING_REMOVE).contains(newStatus)) { + throw new CatalogException("CVDB Index status cannot be changed from QUEUED_OVERWRITE to " + newStatus); + } + break; + case CvdbIndexStatus.QUEUED_REMOVE: + if (!Arrays.asList(CvdbIndexStatus.NONE, CvdbIndexStatus.ERROR, CvdbIndexStatus.READY, CvdbIndexStatus.PENDING_INDEX) + .contains(newStatus)) { + throw new CatalogException("CVDB Index status cannot be changed from QUEUED_REMOVE to " + newStatus); + } + break; + case CvdbIndexStatus.READY: + if (!Arrays.asList(CvdbIndexStatus.ERROR, CvdbIndexStatus.PENDING_REMOVE, CvdbIndexStatus.PENDING_OVERWRITE) + .contains(newStatus)) { + throw new CatalogException("CVDB Index status cannot be changed from READY to " + newStatus); + } + break; + case CvdbIndexStatus.ERROR: + if (!Arrays.asList(CvdbIndexStatus.PENDING_REMOVE, CvdbIndexStatus.PENDING_OVERWRITE, CvdbIndexStatus.PENDING_INDEX) + .contains(newStatus)) { + throw new CatalogException("CVDB Index status cannot be changed from ERROR to " + newStatus); + } + break; + default: + throw new CatalogException("CVDB Index status cannot be changed from " + + clinical.getInternal().getCvdbIndex().getStatus().getId() + " to " + newStatus); + } + } + /** * Sort the family members in the following order: proband, father, mother, others. diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java index 9c6f0506748..50e17b40cf3 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/ClinicalAnalysisInternal.java @@ -22,18 +22,18 @@ public class ClinicalAnalysisInternal extends Internal { - private CvdbIndexStatus cvdbIndex; + private CvdbIndex cvdbIndex; public ClinicalAnalysisInternal() { } - public ClinicalAnalysisInternal(InternalStatus status, String registrationDate, String lastModified, CvdbIndexStatus cvdbIndex) { + public ClinicalAnalysisInternal(InternalStatus status, String registrationDate, String lastModified, CvdbIndex cvdbIndex) { super(status, registrationDate, lastModified); this.cvdbIndex = cvdbIndex; } public static ClinicalAnalysisInternal init() { - return new ClinicalAnalysisInternal(new InternalStatus(), TimeUtils.getTime(), TimeUtils.getTime(), new CvdbIndexStatus()); + return new ClinicalAnalysisInternal(new InternalStatus(), TimeUtils.getTime(), TimeUtils.getTime(), CvdbIndex.init()); } @Override @@ -74,11 +74,11 @@ public ClinicalAnalysisInternal setLastModified(String lastModified) { return this; } - public CvdbIndexStatus getCvdbIndex() { + public CvdbIndex getCvdbIndex() { return cvdbIndex; } - public ClinicalAnalysisInternal setCvdbIndex(CvdbIndexStatus cvdbIndex) { + public ClinicalAnalysisInternal setCvdbIndex(CvdbIndex cvdbIndex) { this.cvdbIndex = cvdbIndex; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndex.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndex.java new file mode 100644 index 00000000000..4e2af0fc6d3 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndex.java @@ -0,0 +1,46 @@ +package org.opencb.opencga.core.models.clinical; + +public class CvdbIndex { + + private String jobId; + private CvdbIndexStatus status; + + public CvdbIndex() { + } + + public CvdbIndex(String jobId, CvdbIndexStatus status) { + this.jobId = jobId; + this.status = status; + } + + public static CvdbIndex init() { + return new CvdbIndex("", CvdbIndexStatus.init()); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CvdbIndex{"); + sb.append("jobId='").append(jobId).append('\''); + sb.append(", status=").append(status); + sb.append('}'); + return sb.toString(); + } + + public String getJobId() { + return jobId; + } + + public CvdbIndex setJobId(String jobId) { + this.jobId = jobId; + return this; + } + + public CvdbIndexStatus getStatus() { + return status; + } + + public CvdbIndex setStatus(CvdbIndexStatus status) { + this.status = status; + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndexStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndexStatus.java index 27338848ee9..2e6fcd53e2a 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndexStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndexStatus.java @@ -8,10 +8,15 @@ public class CvdbIndexStatus extends OperationIndexStatus { public static final String ERROR = "ERROR"; - public static final String PENDING_REMOVE = "PENDING_REMOVE"; + public static final String PENDING_INDEX = "PENDING_INDEX"; public static final String PENDING_OVERWRITE = "PENDING_OVERWRITE"; + public static final String PENDING_REMOVE = "PENDING_REMOVE"; + public static final String QUEUED_INDEX = "QUEUED_INDEX"; + public static final String QUEUED_OVERWRITE = "QUEUED_OVERWRITE"; + public static final String QUEUED_REMOVE = "QUEUED_REMOVE"; - public static final List STATUS_LIST = Arrays.asList(NONE, PENDING, PENDING_REMOVE, PENDING_OVERWRITE, READY, ERROR); + public static final List STATUS_LIST = Arrays.asList(NONE, PENDING_INDEX, PENDING_REMOVE, PENDING_OVERWRITE, QUEUED_INDEX, + QUEUED_REMOVE, QUEUED_OVERWRITE, READY, ERROR); public CvdbIndexStatus(String status, String message) { if (isValid(status)) { @@ -46,9 +51,12 @@ public static CvdbIndexStatus init() { public static boolean isValid(String status) { return status != null && (status.equals(NONE) - || status.equals(PENDING) + || status.equals(PENDING_INDEX) || status.equals(PENDING_REMOVE) || status.equals(PENDING_OVERWRITE) + || status.equals(QUEUED_INDEX) + || status.equals(QUEUED_REMOVE) + || status.equals(QUEUED_OVERWRITE) || status.equals(READY) || status.equals(ERROR)); } From e738c69217bd5ece2d9757540f7577804924ac30 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 26 Jun 2025 12:00:23 +0200 Subject: [PATCH 05/13] app: add database dump to test migration, #TASK-7756 --- .../datasets/opencga/mongo_backup.sh | 86 ++++++++++++++++++ .../mongodb/opencga/audit.json.gz | Bin 0 -> 4547 bytes .../mongodb/opencga/clinical.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/clinical_archive.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/clinical_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/cohort.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/cohort_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/family.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/family_archive.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/family_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/file.json.gz | Bin 0 -> 1075 bytes .../mongodb/opencga/file_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/individual.json.gz | Bin 0 -> 20 bytes .../opencga/individual_archive.json.gz | Bin 0 -> 20 bytes .../opencga/individual_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/interpretation.json.gz | Bin 0 -> 20 bytes .../opencga/interpretation_archive.json.gz | Bin 0 -> 20 bytes .../opencga/interpretation_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/job.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/job_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/metadata.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/migration.json.gz | Bin 0 -> 6115 bytes .../mongodb/opencga/note.json.gz | Bin 0 -> 491 bytes .../mongodb/opencga/note_archive.json.gz | Bin 0 -> 491 bytes .../mongodb/opencga/note_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/organization.json.gz | Bin 0 -> 569 bytes .../mongodb/opencga/panel.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/panel_archive.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/panel_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/project.json.gz | Bin 0 -> 518 bytes .../mongodb/opencga/project_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/sample.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/sample_archive.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/sample_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/study.json.gz | Bin 0 -> 3732 bytes .../mongodb/opencga/study_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/user.json.gz | Bin 0 -> 582 bytes .../mongodb/opencga/user_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/workflow.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/workflow_archive.json.gz | Bin 0 -> 20 bytes .../mongodb/opencga/workflow_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/audit.json.gz | Bin 0 -> 10917 bytes .../mongodb/test/clinical.json.gz | Bin 0 -> 1122 bytes .../mongodb/test/clinical_archive.json.gz | Bin 0 -> 1122 bytes .../mongodb/test/clinical_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/cohort.json.gz | Bin 0 -> 20 bytes .../mongodb/test/cohort_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/family.json.gz | Bin 0 -> 2851 bytes .../mongodb/test/family_archive.json.gz | Bin 0 -> 2967 bytes .../mongodb/test/family_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/file.json.gz | Bin 0 -> 1077 bytes .../mongodb/test/file_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/individual.json.gz | Bin 0 -> 1179 bytes .../mongodb/test/individual_archive.json.gz | Bin 0 -> 1582 bytes .../mongodb/test/individual_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/interpretation.json.gz | Bin 0 -> 689 bytes .../test/interpretation_archive.json.gz | Bin 0 -> 689 bytes .../test/interpretation_deleted.json.gz | Bin 0 -> 20 bytes .../v5.0.0-task-7756/mongodb/test/job.json.gz | Bin 0 -> 20 bytes .../mongodb/test/job_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/metadata.json.gz | Bin 0 -> 20 bytes .../mongodb/test/migration.json.gz | Bin 0 -> 6137 bytes .../mongodb/test/note.json.gz | Bin 0 -> 20 bytes .../mongodb/test/note_archive.json.gz | Bin 0 -> 20 bytes .../mongodb/test/note_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/organization.json.gz | Bin 0 -> 568 bytes .../mongodb/test/panel.json.gz | Bin 0 -> 20 bytes .../mongodb/test/panel_archive.json.gz | Bin 0 -> 20 bytes .../mongodb/test/panel_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/project.json.gz | Bin 0 -> 520 bytes .../mongodb/test/project_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/sample.json.gz | Bin 0 -> 961 bytes .../mongodb/test/sample_archive.json.gz | Bin 0 -> 1047 bytes .../mongodb/test/sample_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/study.json.gz | Bin 0 -> 3745 bytes .../mongodb/test/study_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/user.json.gz | Bin 0 -> 526 bytes .../mongodb/test/user_deleted.json.gz | Bin 0 -> 20 bytes .../mongodb/test/workflow.json.gz | Bin 0 -> 20 bytes .../mongodb/test/workflow_archive.json.gz | Bin 0 -> 20 bytes .../mongodb/test/workflow_deleted.json.gz | Bin 0 -> 20 bytes 81 files changed, 86 insertions(+) create mode 100755 opencga-app/src/test/resources/datasets/opencga/mongo_backup.sh create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/audit.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/clinical.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/clinical_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/clinical_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/cohort.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/cohort_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/family.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/family_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/family_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/file.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/file_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/individual.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/individual_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/individual_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/interpretation.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/interpretation_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/interpretation_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/job.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/job_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/metadata.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/migration.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/note.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/note_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/note_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/organization.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/panel.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/panel_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/panel_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/project.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/project_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/sample.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/sample_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/sample_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/study.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/study_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/user.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/user_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/workflow.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/workflow_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/workflow_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/audit.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/clinical.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/clinical_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/clinical_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/cohort.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/cohort_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/family.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/family_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/family_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/file.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/file_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/individual.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/individual_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/individual_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/interpretation.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/interpretation_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/interpretation_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/job.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/job_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/metadata.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/migration.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/note.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/note_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/note_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/organization.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/panel.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/panel_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/panel_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/project.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/project_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/sample.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/sample_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/sample_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/study.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/study_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/user.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/user_deleted.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/workflow.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/workflow_archive.json.gz create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/workflow_deleted.json.gz diff --git a/opencga-app/src/test/resources/datasets/opencga/mongo_backup.sh b/opencga-app/src/test/resources/datasets/opencga/mongo_backup.sh new file mode 100755 index 00000000000..9deab966747 --- /dev/null +++ b/opencga-app/src/test/resources/datasets/opencga/mongo_backup.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# MongoDB backup script that creates dumps compatible with MongoBackupUtils.restore() + +# Check if required arguments are provided +if [ "$#" -lt 3 ]; then + echo "Usage: $0 [mongodb-host] [mongodb-port]" + echo "Example: $0 task-112 opencga_catalog_opencga opencga_catalog_test localhost 27017" + exit 1 +fi + +# Input parameters +BACKUP_NAME=$1 +FIRST_DB=$2 +SECOND_DB=$3 +MONGODB_HOST=${4:-"localhost"} +MONGODB_PORT=${5:-"27017"} + +# Docker command +MONGO_CMD="docker exec mongo7 mongosh" + +# Extract suffixes +extract_suffix() { + local db_name=$1 + # Extract the part after the last underscore + echo "$db_name" | rev | cut -d '_' -f 1 | rev +} + +FIRST_SUFFIX=$(extract_suffix "$FIRST_DB") +SECOND_SUFFIX=$(extract_suffix "$SECOND_DB") + +# Create directory structure +BASE_DIR="${BACKUP_NAME}/mongodb" +mkdir -p "${BASE_DIR}/${FIRST_SUFFIX}" +mkdir -p "${BASE_DIR}/${SECOND_SUFFIX}" + +echo "Creating backup '${BACKUP_NAME}' for databases:" +echo " - ${FIRST_DB} -> ${FIRST_SUFFIX}" +echo " - ${SECOND_DB} -> ${SECOND_SUFFIX}" + +# Function to dump collections from a database +dump_collections() { + local db_name=$1 + local suffix=$2 + + echo "Dumping collections from ${db_name}..." + + # Get list of collections directly (pipe output instead of capturing in variable) + $MONGO_CMD --host "${MONGODB_HOST}" --port "${MONGODB_PORT}" "${db_name}" --quiet \ + --eval 'db.getCollectionNames().forEach(function(c) { print(c) })' | \ + while read collection; do + echo " - Exporting collection: ${collection}" + target="${BASE_DIR}/${suffix}/${collection}.json.gz" + + # Format the export exactly as expected by MongoBackupUtils.restore() + # Note the careful escaping of special characters + $MONGO_CMD --host "${MONGODB_HOST}" --port "${MONGODB_PORT}" "${db_name}" --quiet \ + --eval "db.getCollection('${collection}') + .find() + .forEach(function(d){ + print(EJSON.stringify(d, { relaxed: false}) + .replace(/\\{\"\\\$oid\":\"(\\w{24})\"\\\}/g, \"ObjectId(\\\"\$1\\\")\") + .replace(/\\{\"\\\$date\":\\{\"\\\$numberLong\":\"([0-9]*)\"\\\}\\}/g, \"{\\\"\\\$date\\\":\$1}\") + .replace(/\\{\"\\\$numberLong\":\"([0-9]*)\"\\\}/g, \"NumberLong(\$1)\")); + })" | gzip > "${target}" + + # Check if file was created successfully + if [ -s "${target}" ]; then + echo " Successfully created ${target}" + else + echo " Warning: ${target} is empty or was not created" + fi + done +} + +# Dump collections from both databases +dump_collections "${FIRST_DB}" "${FIRST_SUFFIX}" +dump_collections "${SECOND_DB}" "${SECOND_SUFFIX}" + +echo "Backup completed: ${BACKUP_NAME}" +echo "Directory structure:" +find "${BACKUP_NAME}" -type d | sort + +# Show summary of files created +echo "" +echo "Files created:" +find "${BACKUP_NAME}" -name "*.json.gz" | wc -l \ No newline at end of file diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/audit.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/audit.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..d8a7a14511abaf181d8f4de174cb5923276f7111 GIT binary patch literal 4547 zcmV;!5j^f6iwFP!000001MOVvmfE_K{y$H_Vx9Td>G*!h{5b^ZZYRwpTykAm9$AvX ziLu$XNrzZ?*!N$|aGd!+sBd;1?D``gwG zl?bXt9Mf?nMOHMIN}?bu?fZ*1?|>W?%gDh`1ku!=2=I@+h);BkG)uyq8+Yj zgc2_dtcQpMB2l7%*E zq!|LC0#L=thHN>f{6j_$AJ-^CtC%fk7%foXed9~ejVMa!xOnDO#R$-fRvIwt4|F}C z$`@l563_K;tsRGf>n%3vS~KQolPqbFWL-y*x9|cO^+Pu7XxtxmuRAbGvR%_k!!nGM zoNc&f)N%Up{^LcPP#10dWDXkYe_eY~Nsu-G1lCDlukf#(Qm?Yf^BKq9YVCXOc1)v{ z7sqL{i~N`Ztdj6EO-1(?VdN#t74W>7N~*>dn8Io#FX{xA366X;J1;W(&+&B_5x1u(I;tbuU%_b?(#lE`y53C$;1n(+6X zMA@S_LCJ zoC#`sCPLHEPi6lu?Kt8G&h@PYoYj`#K1p(hr1#^2WcX0!_dF*cpNb{i` z-UYBl!4!cPUrBhIzUO;T+y;B};!(8du#fF$Nwg_NLT$Qzv2?DnH}YOs^KX_o^j?^k ztdYE(gfscY`n}|DtWTjg6Q5trJ`ZnRO|94phT=QV(yWzv_x_Fg>AlqVZo5G9<`rY~ z1z*43ypzb%^6otJcKD6c%g=o$r(gL?(zUd~42y%_=k4&*yKlqoo%jCj@(%ib9e$hN zjb?+p!KZ=jzPhw-B2m~Xy7x&Rb^2=H;oFY$YWdP%iPRm+uU>b@?>up0IuS?WuS2$Z zN)To4AzKsnK`QH%7^bdCir^eTcAZE8&m>?Xo2H6RLiT*z`|gD$lvui z5L%xh{s6>H=`q_W-GjJOg*Y{9A-@0tp-2h*Q4gf*4Mfxxu%Nmo!m9Jj^Mvta^rGLL z?W3DJ%VuoLRNqw-#PkJgL}bh78piS1&cdf649v$&b(jY5)#$vJroye9@^J}!YD@8o zCj;1Hfhcqvrii%-S79rLg`{9FX+$3e*)z6=aZFc^&vM~KgqP_)jj8WDC@uzgD7j60 zoNe%jHfZR|DK?0!Hs~69ZO70hjY^6u8dz?&!3H8%aAaK*b!0g5Ie1@u6EJyDxt|}D*N~wS< zbsQ|fo~d%v+rd8PQ^x+0Olq4@ogpuGWqPU9GIwgox^EsebcUgv82KKGOP?Pqf*D5? z!Hil-E8jHJS;6eQZ}nmbce(CgrGi{qvwN1yRj$lr`*l>41+K!gk@xR)LDk% z=fYqXQMg%`O631sQ6^Gc;4tPMDMKPSo_HHzO0J@CmGj6cGv{5oT~ zS>uRSg(Vx#1rg#UB`p+?mY1-mBy4f-L|X}vM06cS$!_pYcV@lr@5b$ohW+DbN5X*a zcyZc-Q}=c><$5`i7xu%Lk{!2OHR{zG+K{_lPtri-L8e~7GNuLWxA7zOwa4foEj4?NJq8@aKcSf#n+Np4-FlF8 zk?$83`0NbzidW@Rr_Mv()OCIM$cyX}gq%liFImxyax-A%u zxq1}Tj}o|_&m|c+ZC~hod)j4>}J%2av=RV_ddo+-rd=i@E3W!@Gv%ygVd} z+9(T8djMIkj}|4w3@q3}?Cwo}U`=Net6#Uxc6o8^*?cgXPwer0V$B-%-O^;Oac2Tx z?A+Scbku|Y2gmlm>BEk549R>thwd*slXs)pyK&tc?D|gU(=oSHdoY@Itgbb!gQ+^v zSHK#kXiqzX@of|9^aL%Y>G5UuwzRV2i8TPp>?_!(t>N`_7c78MRw&xCEsJ=;B7R8j zes?tNHM2>r?MO!5&IGhOX<*YuQRLC+7$)Gm`<+Smrh&_Z#uP<(`4F3rEDFiIT|ZcO zX-re8`Nn9q_NkQzKrDpiAG)>JWfLdi>dK=&N!Pqc;jS#MuFw1)2z;3F^S9jXLdJJJ znpoY=Eo)rD7PAaFi!_G)g8U74sO@nF_Nk9~%K)v{9?Oo?ZtRss;(0aU(C`dIyTK=& z;dS4>o$ZbMVc-1JsU!V(SSSpdhd^iRNV_Vd{x^beWI5YR6KxWN)W`ukJN*Y z4ig=uIY^>1H*?W;yf9Bk3#5035SQjjOp5lrTa$mV2lLyRWxwv+&ig;`6#;ktF}1T$ zgxG{)#DWC~&hcc(Zi=kD^e>AE!SfLt!tf~M)texF5(i=WCWK8tvVp0+j_HPk_JZ;- z7GmOrdV!5Y=3yJ0)AwRL5X#R-W{fhDia-cQppX~8TFiH4iQUN_&G=z{l``K6* zZ5iT`Ri)7mFrd86R-br~yE0%Q-v+T)Ab6CI_3_XQL1^g9QeIdd$f1LD`^csM{(fDo z>&lCHUk+``7vV*<{xr-uFuHUzFY#{fC2INlrW4 z+89`}Mh7Oc33ii8XPZ%@l>;UpZelx(*tHWL1r}=vI01DtKI<$yyq#R912cf{??7n) z!W{*j9fAD)?k8FRt3h#$?AE}IATo%To?BiRFW}9fN1t-PvupBzJj&X5Mrr%(mMQxH zieGS}$v@>!*i9>HBaG=f38T_D*%QeBD$ONj!Q!E8v~nxIe41APnQ*XgAjTUk2tZgH zfz^8|?O9wy<=Yj*!>%W=XC69?w)K%_SGJP| zTQ=-&<)vSx?77P+HE-5&0-42%qo~wQ0C&w#WIHwTIiP;msy4Q!?xJXD@J@}e&B_@Z+%_|wcm%Zk}TO25mMn$a@3HL{srKUgP#&^vu z&8k@v2i0~VS8yF&QynBWURY`*k`xUY7$Zd$r9a**&sO-`t?*~}Btts|{HO|kD$2Dz zNpv)X1Q|)L)(rjzA_*G~>|wHEx~JXNJA?lW{s+JhA=&ZZ*Udff(+c=C$*p~-!O=}g zl(0w))R?q15*e84meapj0 zMuI{FS=NMRb8R4!i9;l!$dW0k=e_C-{@(?DgCAcX34TMYf?ul?*oYxJweS8Q(a{YU zL?Z{vIZZ@5#VS!DDin3)4E{6te-HR20No=3HN)8hzghu5GD+>119U|;OsZ<8s5YnQ zO+-jniLROK_i#@8w%i%~XYd~czbxwVBT{s8FGbg?^(kUFPA%IOV7rh_2xC;z2>vD_ zQ*%W^h%OWHoT8t>|C_+C$a?yd8pnWNG}(_1Pz<0qTnW3_P|Dy(NTl^_Th(>&;6%|y zsp&V(4iE_nl1!DIES?tg|6!+C;ENPVQKb_!f~p#EmD&*@)ePO#6%**yW{oru39{?T z2E|0?zvK01yQA|dR^2~+gTKPStNAB!@a@$W{FCtZ?S{O%8!RWcnzx#L{qo|k$#d8O zr6(l#^5%>8)v@}9Id&B{{y%q$#m!Qck3L*(FiWRmmJ+Y-?U|*-zSv4!hsaI8$kj}Q z4b5>3;s~mC+V2+q9nA7bje|AsO!;6H=^cY)s!9uXp|cn|!f41PzHh**oQ5LIPER20Q*sth(0nL3O@u#1p;T8P}e z;a@0YVX@7=UiW4+>HSF&vmq#kadhbx6h!tEM5_vt%~~M}OCnNDNtRWuF^6d+5_HLt h6cZ~lJ?$pm**l%R(_!$dLi2C;{tveV>o^!-008;{1JD2f literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/clinical.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/clinical.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/clinical_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/clinical_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/clinical_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/clinical_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/cohort.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/cohort.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/cohort_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/cohort_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/family.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/family.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/family_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/family_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/family_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/family_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/file.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/file.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..456b6e62c1e1e0c4b60e61791b632ed2029471db GIT binary patch literal 1075 zcmV-31kC#%iwFP!000001MOH@Z`(K!e&1ga2>aN?k|irq-crZKrn|Abc7mcP3WK6X z3cD1kC8->1_`fenT|SZ^1&X!-^kADD&dr(O2d|tZNu1M_^E)@cPIxj(+|e+Nqc|M~ zeh{WG^jsgD^&6+GIa!&5;kJJV8*m(Ld61*(#p<}GM*3J;m8e#p-!diM_M@N2}uclmL~y;1t|=R zKDN0%5S*)nl+y%+=74(zf+6ahrie-~K1p5*3@t&D|Mop}jfaw_>JABeTBW+m&n*aP zEmm6r1(!r|@z78(&vT-h-5tdGxRoxDK)I}TYvW)6g5+S#t{esUhH=WXR#y({h9}LA zm6!s-XP6{c_Qq=_#7bQ*suVit#tucwTz0dkBo?UiF+rV6SX=J|DT=$$WTCH{#I)tz zN_j;>N(mB|3c9Gi^7x{zc!e@e|NO<4gEpGH(}h)2{w)zD>S4^85}cX@+#*3xm5X}0 z0Cfr40G6#isjFg(rY&kk8Eev*BB}nyJ7N;$igPL#3aLHfJBqTcgB!VP0dsxvD6{2| z0fTY9uf;xpSywl&eW=4rn@CMDRa~ojS(P~qdfd(s*3|N9SI2yEcK+!Z zMSGpo*03`=a*y1@#q{jM;*Z&t(^an%Ksby6rBQ-W82d>OhY^kfFG*uBI!OYIP!z=o zcV1R*tj$$tL@M!Y^HP0Dm$u0vdDO+O6bMV;{*kb!v+3mLM{T`WKRxaa?Yz_3V6tmS zQ?iV4Wm9zV3gR<`3z)M+50Tp6x`i6*+3SjDcWCStSrwjX;%1gv7BeYrKW+|$7s=?i zK=1h7;)y56<{0G#g-cXavmz(@#bw#TQf(sPUUwETkim=?#@S?pV1wEi$412x)#GS+ zD_gxw(o)GLSvH33kpOc84aQjNm&*FX*}T+DXGKA6?^@pN_gn{;G>a{m;~kIOXc&fm zq!Ep+s^$Jm|6=L&Z>yKVv-C-LcHF|V_p`T)({IFRG3;Wr$n`Nqqu34caaVIj_m78Z z`Wco3V|vixvS#~(!Pz;asHEz^xYWb59-j5^tcPbkJnP|E56`{_JX^e-y?@;ZEpVSi zXeTW~n@<+A>-l){%`goIyD%*XhH&h8UWkK}%?Vi>_=Wyb8*DmNgSeLT6Omj4)m{eY tlqh>N*Q2=}&Gl%mM{_-z>(ShEXl^o|U(Xhk`L7D;{s)b-^Kuj&007~my-VX#GiZV z%yVYx%#C9wSvju1a{XX+@}IMlCtTH8S&&iog?%wQIXhufl5(@naf72>jjO0`;$~Mz zn=-|DO@^enPO5C%@QIgVEc!~uqB}MgrEwD@J^{V6=59!);evmwVz+s~)u*ypli60_ z%|hGv-I?Xg=r_VoxM=E=pNd_clYf0ruzV4D#P8(;`RLc*&Q7**lRWkww=5d^H2MVc zb*o0>P5ft(Pj0WTzTR9dZthS1^FM!Tfj75wz+1(eEPv63?*khR0njxDpbG}jIRwz@ z0IYBkZ*cS!S6N&%1i!M1*Enj*GAGZjqx;3(KhA@hr(k)@CL%EriHV3#L}Vg**PA&B zrfUpL7Ye2~vKr$wrAsecWl2oG`c>ReD5F)D*42tH+DF(77P4LcJnTMiKuzw7LhDOoRdvW){^Ha2;SglViCy!-em7 z-P6UlO`+-AKr};zH(L99ILWhufRRUawk{YhMS_Ki9K?}KLUI!7Pm-39T1mo4AF_f$ zOm=@+DGZ4Ys>{66LED(c^r03rR7hr@r9Z~Sn%V%e4!0F<7&_4gH;-jHKrA=Z3glVc zpmK%yC}BvKdr#nOUB?Lc#z>J5B}Jh^Fs%bC5En%$AgpmyQzOtWWnv)nmpU6>zuPK% zqPxC(fLyr54qS>yEW{H~S|&ScY9*ETtZ=RYbzsaf3ZVQcRQTs?AL+7M$3^yUzSxeM zgwMTED34!S#_2k99al0iW>W*OO;s2tS)AV95tY?-R;;6>%y*lDuFdUZiHq!eBuMyx zD@lPA^U&hcr|VDGcZkoY3scN*S-h4<8Fn2U`wV;cZzpJ=y5E zF_fec-Lg47Yxp|S$bB^vXoeOP&0xYJ4ms)FJ_dN|%l0AK?s{mBd)TBy5Q;>Wj-fjGZFLBs|)%7=6}=qd()Yx-Wyf_`dd42 z|3$z4rh~gN%Nl|$t3p3DPIwK`Y3O`!wI-d@-JY59B*mK`AdWFjVhve%x7gJ<<&ieR z!l~=70C&ZPB0d!TB8J|Pp=V5H*ErSf8GO&{1u(>$Ah1obOBS1C8UoQsKI71t=QIq&Xm_E-UNW|LE)2*BEA3qOuQPLRJe)0;TP_J&u4*x-B&go z{rz9}h!00(xH!Q1_(|YQu_5s$2yo*}Qmi=QYyUwG+N^n@-gZNdgk!?6h=bD@?LYa9 zOFp3&9&>ffon?vxi8nzoR2Ze(VaPQKeq!@DB42rKpoScmOoHHNG#@J+m_Q?df4XH7G3=&i00-GjFw=HAV(*j#hg)O=l?tvfvkd-uPARc;2#L$Ww|FNTz2I7C@kD}+2hy+O*g(!~_Mng}2vu#s)yLb}>ZiQD` zeM6fyQ4$MN%ft-zcv_M{89A0};R@BQ>d`sxZOG>x`R-Z>8skYem?tR|vxnfrE+fpc zTE|Bvqst1@JLUv6{PyDl@oz;yqrp!N(BQ(kE@)8UJQ))!B}d|VW42>T77}lQKsc3e zIV4Z$%56g$#lFg>B~&=3)E@CB2%JjDySE}AvlRCnMg_9JkB*QT*{_!18ZIY1$M}v^a9elfj+o9?oopXNanv%R zp0S5G_T-VQK zmDb!%AKdm^F$O(mRD35MlJAB(o832NGkvhlOh@oyi@gtU!tw1r5 zF10omA^)ZgpD}s~PVY|-=%ss@bDlh<4};MWXLJEAVyP z#K4$6450Q9E8J0SPQVkWQmO&vgo==XBA)1YL*YcWam5_enAtfW6u4KDI0YhwPW6NsBfvS+EcNCUFGvhlNc;5~WXqN^#R z&(n!Q7|Y(b!P)zGfF$l&O7ca&#ALZP%QT|J%IPl0=^g!I_t{2GPNgRDv^Pf-*sC*7 zsl9ByAPR>(jI6+F&m}Dpb8akXJBJ2s@tBBt^k8DSevo5HCJX8vN^@gIb`CbOV?pq~ z%6=>i-dEJSF;fVI#%yQ^wV|siwQ@MCzaRGjW%lKsEQ=9$eJB z3Xjw{Mp;crz$z=Ssl|Dm+>&Z4OCi*fuGHF}{dLV$>Erh8!z#(dMPPIfgNCL$GXFkI6k7vxntK}I6@j7juN zIho>3Z$~efO>t1VwPjMZNS8jkwPnV%OshU!?`U?{?aktpmy&6OavjSQ(GhQgfH$RO z4xQQL%fJ4({(JeLeNtPdd_?gk2!I1MJ#A)x{6wZ+64S*`ie-93e#*HOI&&2^Iuy~L z%G2}t(UPS{GS2y4<`0@e(>BI?Y@j6!W(tMuSwiA`l;k5Ct9_u=%-r!|4IdJMaow6k zHF$f=r=q;98^aS`-n?6izw2@H=c-43bsw^?GTr zcsa-GSiXT&jOo-~OAS#=C7`CRxrU4A*v1&84P=zTY^nf!Nczxf@iMA;^m@JMh#iix z@(Bl8J`v0m9y#w8=q9edMZ4l#Q9c(@9#?BzH*E@Bj*E5ks0PZ(92D0`OhaN?+gh-l zHU*+%Dt}PC2?F9&ZTNuGIPB@{4`e42-^ez2zNfZ~4GFV2yja?h2=vb7vk*tfQ!j z>u*tDg{rgnp&g8PyU-RiF?R4K zvs|@Ag|QYM5<|~f^Vp5xe_YCzs{HvEIsn5tjixDtRzU>T0Yy|)vV+5erVI>joUpcdW-XHUC>%3 zCizH)>G}RZI6+QA?-xHanWyH|=>1~J;#}{SAOE%dC_bToxV+sTNdCPP|9&T4F@^XK zs6x?199o4E*aMnno4Q)lV;mPELnswZ+1d)mbIY0gs#)ceM9Pq;BIO8BoYI7#HLG9uXY1Ba0ohddx^v>p=q?!5S_*v;b9hlBs??DOP#C>V?FvEXpjED z8S646yrDl59ugBdmPfiYiAJFRyp-m!rw}nKDQ+SR935UhhdY$69)xqKL+QNbR>#Tg zfl((%q!DO&ICEnj;~eTSI%D-b@vbSeBB|IFZtYIzRXMg=f^v)b1w1l+ifmwZ1y`@6 z=YRJeDb2uNWvP!&Qw_)1H3$y$8ic?ZC=}msC{`aye?F&xc%GV3(CaRCG<7lbO|7#ZvF??8L}2hNmAI+$3kQP(q=R^68K=)q{JJj#J4^#{%X5w-TMz++4p&(Y4o zy*FPZX}fDIgWrW_@CWXgWZ7ODj+@7ChZ5VR*8tYL~I_?-_lwBxa;*3rOKnK;)m`0z8-jX$I2G{HP&i}#;zlc+^55|WRc#1`)C=q1Ux&|dz5J5?3V z`?IN)&PQ$+=I!N`a%SxkQP)^weixjB?u}i)^Oja^&cK3wQ-?N3Ynd9$Pj|uj>E4*U zDLOE-cF9vs{1pMb!@)TFh zD5iKz5W_{1wUJxjb+z`jDFMiZcI*$lx0HVCB)kkM6kVQ+jzZx&#>}GwHjiVRa-`2g zf2kg+UA{Ydc{PfAOIW1H$XiHtmZ`Be{w}nQf8dP;$NMEvX_<)Pr-M#{LrNueAN!S0 z<%$P<1}pcdNhrz&tq4wh7f4{Ysw8-wKR?M1tob&ukfO^f5ObtB}`d9@j6+^L&7hXH7n_T*L(UD(#Ea?aG_TM z1m0BPPisE}m69S9Vd&P;3>q2R;B!RlaK`E_9r#RX?BY!T`1}KAx_83|4~*l8Y}^3@ zz-tU>=jEn7;=Q?}kq_g###~Al>Qee+mEx`5!K3ARY6r7^0IkDVjerZT5%3jNGgMT~ z*7Iq~Z@7r;DBfm+_sIw?^OB5zPd%J_I-U}v#Wwo~8$X4dvHK8Q=zR!*KagG7*4Jmn zQ=DfkQ$t!a(RRA-a$wGVXXpk^IkT16<;>nic~p4k!A0~Kr)9a7hlisPAa{)=th?ZZ zb$_%MCeIq|U8{UU)*mghy6P|;xPOaJ>tDyrQm<>_7VS-)#L}3u?m?Axe~e^ySIIDK zQ`1s%MH65ujU}+V-~@JGVT`j2uOFr55mGtvWvL>^rFM%z*1R1G{tBI|6pRTy1=r+n&ZcgfaaZ`Ix-CLuNE^?`HK zgQ1G@?zG8syyk(&WLH(PT8iKtQ>8&&msjMlroB@6Jk-tO-ErhRQ91FQ>J%IF@ItL7 ztD`*SKZSL#K3(4YviP*TIz?rLPH(RlSO0ga1HLge4xHH@jAdnZO;%gMf~`%KkQoJz z8unq$!R25EAuki?_>xT1a#ehTfXl7OqsWyHq z0~Q)(QtsRsUz$VtQZS`UZmaBx`jzPpvv>u=VLnqmNfyK9gOMCP-0`>fh)E=kxsKq> z^%pfn^o0+nd;!O}d73N4> z#x^2BuN)Pk?yw&_@a0d0N8MVOtw8pWb}IPUd&{(mY3az z11Xduq}}&p$c*+jTb=7gJ!8rI9ypmld`lbj*W6lCqx*Gn@aFjxL^MDh?su1c7plPYNc~-RpFa{wEq_ pae6q&g-+WGpSXV8{*&tJuVI&-ZAwQIZyLn({{bY|aC@V-002+ZxhDVs literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/note.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/note.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..5db7d931a3dafbe8819779759838998ad52a90c0 GIT binary patch literal 491 zcmV z_ORtZ3HjeAIq}+bX}7ZO#WK=;@9ul|*`vHRXt_skyKtg75rzn6w9+MucodhGQNlSo zEYNt0Y{y$|`?PgFo`#1>xw-j}cRy@fRDc#}aHF~VazTJ6#W>|cphdxA08FTom=I#Z zhhi#7LA8;VSP_yHKJl@{>N@?p%F@mCTmvoJv1t~_-CYWCPz`AL4Xv)Sbh}31edWP- zVrJE@9%@}Twexw<=bLo9%GO@Cv;9za?Pa$`Z^8N3_uxWBMwlQmi~W=cXI^~UnYyft zXJdBIj5a&9{O;rWDw{7u&1p9Gvozp9!(i_(bDtfWt~whyxY2u^t@=Lds-K!Tn)RLM z=dF8lX8t+4{M5Bouqg#mERH$z+dzk){N2<`)UJnifFg*)>=Ds^v5AhcEC|ODrxq(q zDV9bW!MOqn+&jn{7~H1Z&f9SPXb*4p7m<|-Elor;15sKHM3|?LRcFXLJGhtdY9gii zV_v0SYZ-x1G%5cRuMc>5U1(_}{|N0d<}aWVag{{=1FjF%k8w4eCRV5t hv#3-!(jvm3w7{yc30IUEmZ&Fj{Rwj+lTcy>001Vj^_l z_ORtZ3HjeAIq}+bX}7ZO#WK=;@9ul|*`vHRXt_skyKtg75rzn6w9+MucodhGQNlSo zEYNt0Y{y$|`?PgFo`#1>xw-j}cRy@fRDc#}aHF~VazTJ6#W>|cphdxA08FTom=I#Z zhhi#7LA8;VSP_yHKJl@{>N@?p%F@mCTmvoJv1t~_-CYWCPz`AL4Xv)Sbh}31edWP- zVrJE@9%@}Twexw<=bLo9%GO@Cv;9za?Pa$`Z^8N3_uxWBMwlQmi~W=cXI^~UnYyft zXJdBIj5a&9{O;rWDw{7u&1p9Gvozp9!(i_(bDtfWt~whyxY2u^t@=Lds-K!Tn)RLM z=dF8lX8t+4{M5Bouqg#mERH$z+dzk){N2<`)UJnifFg*)>=Ds^v5AhcEC|ODrxq(q zDV9bW!MOqn+&jn{7~H1Z&f9SPXb*4p7m<|-Elor;15sKHM3|?LRcFXLJGhtdY9gii zV_v0SYZ-x1G%5cRuMc>5U1(_}{|N0d<}aWVag{{=1FjF%k8w4eCRV5t hv#3-!(jvm3w7{yc30IUEmZ&Fj{Rwj+lTcy>001Vj^_l=FxiwFP!000001BFx3ZqqOnedjYQcuOnCZQN!&Rcl2j##T|#7}Hc7`#SYV z?8tH0O4X0ySNJmQq-9jFk%z>-Ip_M^dylthr35)QcV=MrBv}Pq7(apUEdlX@! zRJukdH@6Yu7MN2;Pnioef=PlYpyG-aHanU7Ws|_|x5{f}_CN z)b5bhRDr!eW#=UbkgO@(M=xdeP$_#3^SVYbQlYn#pzfK zIb&&>G5%=7PKK9iAy>7t&w8z3wR_k44aC2E_Kd_*cFIE^u literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/panel.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/panel.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/panel_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/panel_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/panel_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/panel_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/project.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/project.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..b3550a101d0ce012d339dbe590b99663735219b4 GIT binary patch literal 518 zcmV+h0{Q(PiwFP!000001C3KrZ__Xke&<(A7LRM|Bu$$1iAhBo0tVW6K?qG}`;r+Y zcCnozsQTY`PSTFCY2qcx`M&S+-FN3byVZ)V_w2QeZ&sAK;(3aq5_P)FmPL(e!ZTop zbJkBt({}>h2WuK4q%kv4L3P?Ng7sr4a+%>u&UjL-W}Ne6e8@s2MYgKyvgCv|(BWU? z`lq4DZqdjF-W*S*LkQZM4TK0zd7ASgEl3#8R+R6o(zTW^VibDm^fp2$y}=s#Hk@sp z{ed!2-ns^i_FZhzON}PPl2@a=OWWBq4_i9+QIG99Yo;8<;5~Mt{Y_Eh;g|PMO#p0Kr|G^NTL1Q~D{haD8PVjTKL+3}CyL?_o?GWG^TO6vf*bPZ} zkMuv&w{84a?;I&UVNO$eX96*jsHaAS4#xiX!^k4}A7jR&lYptb553ap+4|de8ny}O z477t$0YdMO_1N{>i_O<3UXKktJCuh`^EsdM+0E6(=bOtr-%)w6p;-uqx34sGthX^5Uq z&P=98Y7MPNI)V_K7JV296yHAHUK))$emuHK$tvenTF~8~d%_Np`E;-Un|*k750cPl IOxpth09O0}`v3p{ literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/project_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/project_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/sample.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/sample.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/sample_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/sample_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/sample_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/sample_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/study.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/study.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a9c6238a63576a5b46a0c08b9561240f7685433c GIT binary patch literal 3732 zcmV;F4r}oriwFP!000001LYj;ZW>AQ|2@SBowU0rjKLTu&grCO0o!c&+5u;86h9|L>ZcyVX4A{*?CiXLeleqGwQAifeJYnCGc$;mQ!08qn8SiPj;N)?Te6@8(<1$I z+HmW$#@R`=cKPC@TCJYpNA09Rr&G7;RnHsKI?N8pl4ceYpAnh)?Fvdl#)CEqaoSn+ ztWmu+S;E<#5w4?;(VB$TOQhcK&k-L&*9%gnOkSy8~=M=&oG5e04YV8%QegpBy5 zS2N;E0J0#YBFF$XrTowal~T%9My~H8Ftczy4!t(LPf6VdgQDo1i?vo|xL)C;_VD(H zUPtL#+0h%~BmF4Ea!Jtl5fju)NL|7VW~LYr@qPY*dJ2d%-6f#cbS)E1BzTvz$`>em zj{?J<90o-uHx5bQ5#d?D0aJEn@tGAasAZVj6o}r7grQ*5C`8D|Pl*l9DcCD8-Qp4K zo=9mMTqvb}bL4aflg@R={yL(n?-kWR%|`LW97$^iJbCabEZE1AWVr${SuBXHN?MYJ zM~F~k*b?9-uuyzoRdlr@M3)f}8d-Aqj*^lw&F1|)cNR75$sNRty&~#U5?DWJCK7|n zh)7x=Sh#q^shlw%+~Jli9FjYvk8bNNKA{nZX^VR7fl@)vs_REM3s=hT4Qz?Ep>jkocCT`9NiDShaBm zDZ>>|L4>2HS)fmYdAK-2xCO!J=?9{hdxiNXmYoukrIJ9jKIEWc(aOAHjR&K$)g81u zy|UHE=kef8*{W13k21|I8RiF`rX*{}1g69wSMWRK@CbQLSyA@01$~;5x_q?F8TO3X ztppD#3r<|PWCioGwPg07S+@MZRo`HLv9DEVK++6T0lc$G@Sr3Fmn%c%ZCA?Gs!c0a zcV;PuA$71gT#kb9~e2qYIYrbw187J_$yD5^;&67e2&0C*rl?if)f zdByG+eVPicRUfV444+vK0~bObtd~5J@kI{REhV^knnGmASB?)lJ=XbzY;teoeI(<$ zGoUAz;5GJ#z?gjNADFaMtPl`EID}D*b1BG_0OWBw9+mkNFNHn5OT+mQzF8-2pLJz0 ztF~mc1$}x_+J9^dBG{t@1w>&PC1i@kQfqiKv?o~k5!6TTTzu=>KvE+2g13Aku^#8* zx5dT0qRKwg3ixG@gru_M9`$7<_U%g=5(PK@XH3JfDfI{{c_iFKQAj-v<*6>*oEQuz zs&I3XnNj&wf@qZP3oBBMTt!OXO7~Ty^sJo~DWmN@B`ICEDy~QsZLp&v^-S&gQ5o5X z1rKx$zrSx}9DB8Ft#Ozh9Vc@UhJ8N1U zR27>oL#t7h&5;TG=Ox(WB1I9h<{XN43<&9r%FCsUZr5< ztNP?eyW{=dxQnb>y6~m1WFUL+C4g=YVm`)kJo|Fa{wm|%YR+*2`x9ON_sj0c>wm#j zzn~zndHw&gwI3A{6a(88ply;rzY_HP8czGG$bVSF>A5z`_ZLCGWV8GkSBq==d*%O+ z<>Dp9U=0*>gy7o^*W$rK{SJTlvD2D7gO*9clSPp!nItH2#;M^TP{5EKY^6QDCjp!O^jI z)L%PWKUXANaBx-3EnC^=2oMNVr;|l#eHk|y*Bl|MV)=xsh?mcIn9s63|M=MVh1>J+ zk=}^CUGf@F?C#+DsWvahx}K}c>_D7I2{tsIfn(Vk?v8P+1Yu5X_ z6+l0nPE~#3fgSmp*>WioOvgF;>h$!KufkK)wK$bj&6TG>x=f&uRQ*a_*!b+UUfLrQ zO|emHz{6DJ+I)7tsEliYkw`xHGP9@OuTJe$HC~cd$j5CQcBk3?HyL{=LyM$a`PKldR zsmb_+vYRs*iU;2n2ipnz>YZ{tNBMCqR{e$?3hJW1?&&pTp;_ijgW<$MXYkKXyL8A4 z>fi%(muViQZeYT7 zmIPgK5vH3rTa)gaPP%V%aol!i2W;2x zw7a+cVlgkN$MBHH+wIz?e!xDx;m^5Edi=v)+~)`2z8$D(b6cS1z#dSYn%CGp0SvI7ea{By6N=0nD=$^XS0+Qjx)LK4{vQ} zbZd7f1#vgSYDE;pCT5-W#NI$G78= zI;l>7IBs@Z-ErQSHW~d1Py>sONfq;x@5;KMk23!lKV~F5vb%kt%(;R%4f*sqO%^~Y zOCr`7vapCHAELX{8V=gUX!L125K*gXgLdr#G%X?orqKr=0pHze+O3-cEH>n&M5sc- zPBeFzg&1$w3Fb`aG<>DA0_Eg%=G%=EAWUJ^k5(>siDbx^SB(0emU)s%Uv-tb-qE%G zF;UmP*zGpq9yqkSt!58_3kWd_u2AS4_CCI|ltxX6Q+F|>ugih)H2lO|+#IX&uIq_Xx)Or(atX};GQxtf$idXh@95bd~ zPl+;~bUhgMn{5YDgkfJf`uyy>U8-*LNJ+I+qbD!23Q7jxGF7eCb`Ldn4((pFbIr~r z4RkZ*xGAEN+t|{Z1ISC`ON_Smcv_KvaQe5sN!NMP?A>X45tH-Jvy3iYztlz9y9 zu4q`qcUD-rhXWArb+?N{AR6Z4l^4X5_G%3asxchPb;+db7=h}qe(0o6p?>OKJFQrF z@Z8Qk|Co7x6F*PvbUr+{GtWO}o|E~5emg^F+`~p=F~BJ>5(Oz!np?A)ET?efvH8u# z3{4WVz(j^%C!lV`vog2Co_(!1Mg!lDK-(`D?6dss>7RK5wtM?9eI_6?kP9};3Wfy= z40`mb@;fPr!E#|>>PD&F0#H9d@pE-(;$QK|%4aThi0jf-$VFx!GXnR&4RA(TAUtAr z%R(XjNdNmh5(jaEAZ|E8V1&6Hu->x)hx5fThY6{JcM}W^xwSJ$?;`;dL=+@jSW7-=#~}zQlnQcCj*XFGK8DOQdv<3?t$GEkJKG~$Zx^wQ)r_Q zs@+8bDDX~&W*awnjIHjm+RfRYX|#w9$SvXFkxOuN`_ny>KNa?+4kkXk=F6~P`az{~ z0@&yiW#Gjfq5Q(MiB$L`A09&m#W{guJ zm8>8Z!Dr{XcjMd;x*P@woQ#K!YW3htQ-+}yhgcXGDX3Ar3(_!JnLi!!Dfw$CN$Uxd zXwPFh$>;_%ZaPgHjh}MKG0lynW03p_nWF#ap=wdtKQ2RGxXB-vsp${*!mC}>UtG>! yzEtfhXY=pJ07@@@*uY(T(WqXYU0l>(0vc73ck+e%gI9n0%YOmCm*@IdM*si?q(-*@ literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/study_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/study_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/user.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/opencga/user.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..2ee5f490f3d583ae7e1bfc1ab2f402041bc1d73a GIT binary patch literal 582 zcmV-M0=fMkiwFP!000001GQ4yZqzUkedjYm0fE{jabhQd3c9o_3ZfTLKtiihjqS;< z+uUk9r5Co}j-6e#6xw~`Dbbwcb7s!WRdA$a@aig9Y5X@}8I>U=lIOf2Nu00(NfgF3 zn5_mqp=q%e69`s=8Y;X~zzS4(TiF0vsd^xq%3+P3K&>vps;0j4NNDVfTVTB!2?-4u zVa_v*Q-@cLRD}{tIjXj(YbgC3#)9pP|LfcRJQ{9&cNB?6Xw@$4ec=JEGt_2IqJwY{ zt{vZRk9VC+smHp=^O5Tfa7>u1FCR{br;?vPtB=4YejLu?x6B#9!*l4ObX9C!T}ZpA2v z)?(G#KIWCWt2pWpzM&xW9OYr9z5%_{cxN4uNvht8!yiFyGRP#Sb^+P1uT(AQ=& zCw1C27QBK~=H#n^3I1cb7I{z6cFW)j&bOSx!{}z)-3vOj)y**qKkWF>A3eP!2Bc1|FA;uh{JdS_^i(*!A!6g!mkYmoI z%gaFN`}ub9lzyVuzjmGs9>1u^Z?bD-7vuHgkFT>s{`Pd*XRpOnow*psH%2l{2`SQ4 zVph1j#L(R?iIXsk(k!Qt0*#1_skk+2eryz8`kbTlNXPEC?$ROMm*2Yo^V!d?W~f9^4Nj_-|1 zLP}K#%QCh!stS^9gHRmOfR~SG7{Pb6ao*A(gjyg?Jra(D=krPLe11Azuv{D^IsMtN zw-`c7Ktn$cFom>>`EXWBDc72%UBrAqkP=2Q*^bIzio6X9&^lx@#CpemI;FZiv}Uah z=-P*Gbd4Mrz7qKQ2h4znSLg`)Yc?Oj3VF^~ly*54#{uOj0~E1pBtcRIP0~a`mIM*< z*FJG<2j5r*VeH3V)P3p2t_zFgMKth`%X%6{D2l_p&oE7c_aH%Nhy!Pnc7-MEsrvK& z$%j#`9cZ+xP))KZvQ2`KX30dcG!vVONNh_q4WeL#Ow&TRn5?85-`PT}oG=P*9G#28 zXv}AUk{d_=txLaboN8s-PT>s-!)rev`%0^VDJcgw4gI%awJGUE5nXR0KF$C0;~hU zD2f8diUH;Okps;==X>7Ia8Y;@Ph=u-!m}g&7b++3QCJQTO3W7L|vBo#O}86 zf8Ux7nN=}l@{S=x1|c<@WMZP2#4?eL8x0u<;i`~EB~6x0tYX6`8?w|0yi9){Zqr7S zD z?Vxv)uUuY!`pXn_!AR{i$AVKRjRhs@gZ>5AVq6{eb%2$HTTPDu((1Y%3LP z(;BvAO+z|Si6~q9uw54d^9}xhW~;VA9tGPM=Y!gQeRp)T?kXlKd0%#4Py8|Ah=a@N$3qndq{*?hAaU5D?h zm27UBlL_RFrJDA#+ZznHVQ#hScI{G=ZTm3oGWCjfX*=3QbVI8uijHmDM5;ijB{gbS zqmU^Ju=u(z+0y-8;Irw;Xwv<`9`NI>g(54m@PM_@D{Fzx>a^OXge{ZVGj-ou)P=BN zgNKI&OV;uI*J3{G&HAUii*Y`iz8>}$)%AG9Jv;IK|A}d^y`!NDubfS@^LkHXC9%v$ zvq|2}#qyi7JrX{=n+S*7A}oxz4Yug(o3CJt;_2pHymeo{4(#o6J^O3%bxyxndQXbv z-q-7u{fdfTzQp52Kvs>mXgHfqX9fDdflo~8jyIpIRH8<}U?q?E9%>!soDr7C^;bvq z3#Q4!9rcr>!K-1G_OjDQp65p$zUUom+u`V-;DvXR<^O(&BAWUG6w%mKL_;^KIz>cH z5H-!Vv0yeUVuO%jfa;2(WU0mjjN;Frh^^}VFx9*7)&nH+V8m-;)yYO>#M^{OsJ^2U zaIrO2QcbN1@r^>NWK)%>wr#`4_^2mgdl1hmK2O;E1Hhl22kab+6}6@dwSdG-1?I^L?YMjkAd z4?A79>K(_k{a~FOl-;`58MG3`|-Neu=37mjm&pLeThS*`puuyE0G<)i={Z0t8OTic< z=h?oOy$;AJ)NDuza|VacXM8xh|mB-*wq zBUO+^^8uS*ds*@y0SDTqI>`;S-_rbq}Yp|H*fcmT%h4Z~x?9%ieAr2g<-*#O3^2Li6U^-u< zN1rc3%FBi?z|Mv*SYt1;po16U3yI_1rQ8K?x+6QqV0xeQ=FviPV&U> zloC4VGjd!umR;MmcY)FvngWX))sGU4lfdptgK?5B?2m{^@*R)RFKmPX#cUZr(-Z2^ z^5DCZ>0&vW42JK9gM-wKd38ycNnB_Nb{!9-cBb|3^1LpOr!#0~F`S*xhKtM#*q!Rd zCGGUniaI@5q;2Sg9rz0-f^0jsA9VQgZ36IKldU}i5g)~T(VHz`6uGt(EzgGM)7hf2 z#nDcctT`LH5MEN!K_2Ni5lf2v4mVV^6M;xTH+~QmjraPC(c59s?qE6@9zHtad$g#F z%jPxp&!%&}E(g|yT|cCxs4bN*%?KkMcd0R;Ow*g{{FrJ}K6g!vUtX7Lo0XYo7bhSd z>@?Q6uA|8k=+nt8@l|fD)bToCtX8lihj(1BN<}xVQnTf#RKP)ekCmxabHjRYYwtyU zvWYwgS_*cj5eyjye90nLqvgVDvfd79UhI84d2 z=JvHCAJER(^sibYXa4oAbt?zdeKGlSGX1M=K`*>35BN>q#!+X%trnQFkXtU-ziyb$ zqkVl*8~ef;5g^NU(W-=)@d&n%W&dB{#cX+gF&ix!_TAE8 zN9Vm6fU$SB9L=W#_%}YZ{OJ&OoI^w|<`+=>&))3)bn*VYZVigE^Naa8*VN^BI`578 zqj?=ny90d#Rl^XM^WOOUtf}fGa5JPy)NJ~(G_&Wk(HMHRJcfOGG&z|U&H^mT8ULNskBTckZuR z?^@Fk#D;SFhE+{d?=jTe-~oot{m@~^dF3<&nDc$XcD?`$<8V550-$Eu5fT#zpBmK= zn4V5e0|$_dT(*5zE+}#Z?YW%4a2AzT%R^3*^N>b-Z~RT{@^8lh{U-)ex7|%nQJoQ?c18MJFE3D@0c1+x zk%4=%)V*2zh4#wq6;YX)dDgWYKO?F+(t8u|OEVIapgk+rWFMB}i?hXO`L=g+I5eP&@BjMKNgJ?NC1wah9?MRq!baNsZVQecxEJ_yzmKA5C~ z4$c#N;LdK^*0Zh*{>XYKap_?|d6&&Tae%urU?F!2-Cje-gRHHO`$h2qFwG9HxrxlFe<29`9@f`P zW+zF*4B*urC~-izp`eQckY6qSKn-9=Qye439M~*~F1&Q?^29hEPlk1LDYrXCk^|yF zn#LLB5@>O9>In>Dgxf@u(du+{=Q@7YNQ{qmU)6VWB(CTwe{$=cxF#{m=wVlQth+yctCYlerSBvj8V zv>$BiEiI0hR^n_eVRtJ}{W__ho19W|ya^*<7Hf{8QaSGBlO_0(gDi}_R)go8;_X+V7+ z_Chb61)j^eAdTWpdMWZfzu{p==GlxQLY&0O7%dQXh>8eWEqx+Q zrZ6Z0E2g4!e@OgAdR&sd2ZA`C9TF$Eq)l-b#j`uENqAfT= zz#iPZd+hi(`FMqbEjy%f@VrQ*(Iq%0{9YrCjwqdErBMl>epCCE6GWF;(!PiXKIS|w zpege%;nKh#={z+OY3DAf$IqGa6yk(_K=^H6I$(b8T}YsSdJ%GAT0-QlVn#!^g@dwF zdl4M5Hm#}j@7pU-2+nh%CpQGE;87=WP)wpNOdLI z;kgaFB_*(MGai?bwx0}kMqTFUd(e84n>sC*et}Z#2W5m)pi{1cydm3zcdBFu`1y-g z*6Z|=?&J3qO4&VVk6(i9dwQn(te|FWUku#jLyc`zFj>0vEdjHLe|m6xWpsbr(Y-LX z8lYcprR1m_s1D*y>LAwDp&W*!O1DP_CS}TMQkuZBuJ9O%LQ@Lpq+brxlBbD7O9uI* zacJp0?~OxK8-3IKdN#8bEw%pXiH0c|s($#R>2;!^?-C75q&3b!32dmRh%ne^kV?=SB}kJg&T?Q7^xYftf?Q_PC8gRYg}NNtZd{t05kQX#G_x zRL8cUnu=uU_eXrwOP_vuYMq9}^WJ`}y)BBZ#>xYCFMnl6rd&_<+NcT%;;)a|aY)jX zvdfYt$;$bf_S)9&?QM!RNVxp%aRn+Gw~(h!!gVOXZ5Jm57v zt#{udz5(%?c=$UfbQxC=zvJC2ilo<`K^ARAP!)?(*sz)q-zbD^6Cp{Gpe^xVyf~8@ z_i^ZdchnTd&TNGQauNSu)5cy*f+hdGFPk*%-*!UY-4dR6|n)`S4fW*QthC zp&E9LCsV_~mMCDXDUGkUYZM}~W~!2iv2C^Fx<&lsB3@MQa7eD$6~tRRh&Khbc6CM# zFvmKWOOS0ue4~(H8nP)1M3o-!9`n1A>(m2pq}Qe(lPc2`HHj#c=pyVlKLx!Ob9z#Y z<$(dUYvmSdt+dqoHz69} z&Z~F0#Z!d}(ZCg=Vbx!WjU-dlF;*-|-9t3$Lb57a7L^qnAly>x7V(dXctyAiwN}Ln z;^{8pHKF#jvaDJJi`bMUi!>v?K}fU>s)-uXHI>{S@pq%vsmI++t<}=!wJv^-dp?$jQtR8tox*SIx751jPc5}>q0v(7_TKs@C)er$3jn+%qcq#`Ug{*tOjA=?H36QU`IC@r~e5&x)& zSML(3p_#jgm&6^!+Y+vU*Ajp<%1Bi$vGKQAH3|`#65CV=Hcac^BG>rRaY-|`wk1>x zkQOxsTU2Mi*N*%g+`9NZR@qP||)zqU;jfDYR-k{VxM+9-rIm0GeS zArn9H>t2p~(WZrI(bH3K)xuP; z1zXn9|Fd^p&5i5G`map+RdtV;!&dFBOG#ADqT<%6EszAMNXnF`z^VD~w;7ZvTVs31 zGjr$%St1CayV27;e1DWGG#E}XVX5Y_K@#Aqa97#Xy;vNdeKf>_~PA-{|PC~=KSx_>WHAqzB2h1A$ z*ii59KK<}gNcBfFWip$c7@yzF_#}QQQH>wxmwwpxqQofm zA;+J(bSEU?{?Z*X4B$jdcfOvv{4kvJucuC<{`Bn}CSD~(KTlboY7M0*`zuHed7ko8 z@FZB8f4MH`>yrvC%Q5iMq?c*5w?w62+7kTLJKeF|HQ^ife-_QGRD=JSN_NIs@FlB| znOaZo`iVJdTgzu!-C|~|m9wo`C2QK%>~>XygnF(;G;$`>%x#61oRQjcTkgo?k|OVF zy?ndd&yR`>BFGh=zHKOz-yyzfm zy#fKOA;T7gu+s+ly=Ij?jjj_Q5K#ik89C7Goxh`IqH z?Sm*0h!2-0r3r{*4`QHzbq?4~0gG8+r=rZ*29VwXa{E9g0unKhvj7wMtS(p=s@|I* zXeg>Mv7amON*|Jtwd64TTRg*`^US%R{?x|dQd?+<9Wbz8ITdfkH;8q$LqQ_F<}vH4 z3(j?(d6#zuXsq;xPlMXzKxK-cDpgPi1=L_rg2sd9wV+uYXwnTdqX*4LLzD4MPe?=4 z=1k>7%XK=oCqvI*5FQ1k$bRG~NnikLz+3tn~jRC*)$3gkC{ z&r~`sq_v^chJH~Rb?0)c*Ji8+jVA26tL^rZpnJWFN$1CNVlsDBKG!RDa)`3oYPsE- znS*w=FkhBBtIl?^?qsI}l%4fdWuO8@9tK(W`=9|9M&5L@Nu@jw=SH#U@NUoqswlrWrNxd1#bT-5+&5}9FlD)CJPMPg-kCpTZsV_W6UD%%1Zb*#S zCwr7T8D#Z>n$7GTb;k4CB5iN5xz3c>USAlJRt_@cx5fU})STTgt(B`=k?oPegW3-1 zvZfN#e$GfacDhw9I^9&Uxf$ueRGJ=dy>g16=hU2(tf?-Jno7Yj1u;$Z?6H|0Gb+vR z$o)R0?6w5()?~_Vb>5J&dRndEsf$w^xz=sF)gosYtG!Xc?X;NJ2PN4H^`lwM8i2Ew zr2*xz4YiR%yOEa~nv_QLt!CDkj%8Y42p3nCl1Q}2qXARfH8H_UGMQd-@_3i(4^d|A zHIOFcas8RybeHKxG0kRlUCbAdu6M2Fewi5#)|^;z2A;c`4d#2cQ5;OiX0xQ}bGI}! z3tGRZ&Qrs*&~QD0+NKJt!n)Qm7D$y@UYQ%)RK3;PRBhKqReH0h)AO$CrqueXr;Pjk z0d2F~24{zho!GX7y+Mq+%3f-x(&7p!dv4pv&24o?8nvd%b7tM%DAu&T9dZM;Z*xRt z*k;LoUKOcU$s;v#Gu)MhR@Tk-iyE_+SI_3NFHb^z0~D&8o5q!zSOMoW*~MsVL+D$d5~8R7xXf+3=yr$ zb9Kme7v7jMWO-NV^ODokx9qbf_EHj9KB7yzY+t5Wh7mgLf;mj#NtPWl8)?wns!Ty) zyM&f<2N@?1d028y=*D6#q@BUI*b_S>B8n5MOOI-{+g~W#JgF5@x$TxS@d&MR`Da5P zkb~^cwJ(pCRiUe9+V0_2opH29cJnLH^!t>x-x!=cm^E>wST~*DpP!e@YHgoIv$`9w zwd^2o*%^$|B~vByA6a*aU3oU&P3_o}*lUdQELu!56`ol+-R9I2)PWm4EAX4$Y%lg( zE3MY4&4Q1b)UFYwY+KmE$97lR2Oq6%M)q1}x#&sok&~z4M^2KJdRCelHTbwHqnyw7 zwrts)S=n{xT27uHcyp*pMiyPQNVsX#+&w$Rr7MlFvSw_I8$Cms)SXO=%?$G> zt)whnqwNYV&RLOro;s>w7Zsh_q{x)nnvuPlhX%aBW|ZCDW5xM1`7F&yOB=9=|L5z~ssbJ9r&#vFUYrHS!!X%x zZ~_@d5DJNpP@TdKw`oDl<}CVkTS4&J{!UP2X}f+j;q}vjEl>_SkmT|B|ZJ{{>n4r%WHt zbGZo1e|sTD@2i1nK_+>p;V3p8_VpLcK*YplU^X%a@g^e7(GG*g(X%%ihtSY!N8+w0 zil3xhF0mdjSr3+`Xzt~K6!XEyRNwWNRII-Q z>U8GXU$Pv7MNLNpN#M`l4tz2?5f|@U_4WwI7u$iUU*pURmU;cmi}0O)(!_bl@uyCl zha}veIQP@fCeB}+**-=Mf??YWC`)hYYE^=G(JY-!IwU)@rb6cG+sQ{ctb~|eyzL=i7?Moq-L;GoW{py z0=T7%I%2`{qmKn#9jVr=%GU}XuHB1|Q5d>-cIp-rafLLR)_8$ZY4#^zu8_XFn2SBO z!ikah!LEmxj%51Q^Id({>ew2@ivp-Z z&IDP3@S?B&>ea*A)QM?u}6GbE3u4G+XV zG;{a{2p540U&Fu{UIV&A>ck8a&(U3!wuB#n8NHA5-ZaG+?qWJUTTCsD9K7oR8?L<|i zvy3j%yvqIpaV_pDuJ!Ke#K)LU;#x^u>#cDuhGAZfYlZLp@o}w?%}d5*X6Z zDasN>57jZ4>`{G)`mmn7Q4kVYvss4E;g5*ds_A1VMCW5*$5#*JL&~FCd z0>TE#swzh_u}(2Ildg(_$nZ#H#dny9#3?2kf2;8zqQ|E<0m@xzJbj_@h%;{HZ7{82 zgVkvG;A1sDE)yatTGTWGX~DVNKPgT{$2vtIP)FBJQDk*Y@FL3+6#D!)6@L|{qTd~- zdWh*HPL;%|-WsRkUO(6`dglY;RMamWr{X^SG{1<1`*Er;{d?n7TzCdx5~oT4%9{ZS zPw^uE4uHa30u-LrZq|BZjn`-aQ8?&0F#shdlcv#xrc;Pv>30B>1d=BjAEWVt`1qL( zDiW?V{(KV<&kDkA$Vt~Vj?#4&(eWA|n@ON(QWtnmP|*ipNt>;f>ZPny{jXG)s>4pH zbFKRtfREUPSU#ncc%)Q3Y*#`CL-V=_F7YQ|b8SAc0K&{A00R5fWOWy%pZ2O87NqLE8a+VM~huS^i-wvJH;Oi68 zbtl-(e~{xUs8&C+b1r4S;!}d{LYwW6mD~mdCYAx{0_)IlZM=k0?dtM4psWo-M<#Cj z)I6Q$S3=W$$X&Tq06%Gi=I+6wUz`yXme4+(QV`Y~!N;K1h?D4)p+n!nFdm!3P*C>K z&^fW73c<5gykdkEN~Q^e*hr=?pH}o!u8?@@)+_Z!zy5U5Q$8|ub=KX!UKoRwtH$)j z^IFc3=7at7#b6HiVlanhDdF{CPB5W=-fzzY_~_LZ|4A0=B>~V6oaOxqaZu|0>-EB} zldRYK{PlWaOZ;K$^^#5fiA{aqj<6W;aSSk?K^Gvnj(lGP>rwFVWL%?pf~eFSUiiTXyg(D*$?EX) z|Li5UG!y*=&GQr%N|O^HY|*E6gSy|Z1C+yBi22rAVJom1mvk>fz`|6&~MMU1)xk?E&X{P1`+M@P;!>N%-B zj48Ntp!RJ=H`}FJe8J7o4ZN*b|H@E*4MTjS>8jd>u(s#G20G^~PdyXOG9#cjIott?8qE5Uxk7(fy8GYk*|)hYvo?)Yxh zIG{N|gp7p`P+2%(M*8=l<8y)(m$^b}!FARhns$bMFBNsW60BZ}4@;Nedlu%D``FuJj+PRIS79)UU*iZq!TE9_n<$K-4;hN2M}SGI!Kfv~qr%YvuXI9qdBwoIbFPya1s>-g5=>In{;D#eG#;C^z!#UV$f zsFhw#J0f-dI-Ue53o9i7|pOv5@b*3 z=_pB)`FLu*^sTbyWMv)`vF22Rf(T&yO>9s7cs^aQk1iZC~19U zWtM?*quoOBY%-sZ+=1=5vw>~fuKqZK=@m-HS2OIoa||Xhk%^#%RsupmP)`McRpbZ1 zu2-6YN-i)F+6SW;Xbk0~TAhhYUW%kXEJao=kWzxir6S^XpVF*a z_pL(k7%>xZCra}cB+9VH=#dC%thby#QyPEGC{bQis%z&f!pORm1CV;@pYS>tc_+YY zyhl0Bo*MWI#7Nczqt|PmJP?piQ6@>>%1*bGDIy9%syto{+c9xpiq+9xTaqN4VI-dX z6P>j$a3+c7er}dWuq?bV^jG2T+7FhH_~?-11rmZ1(Mtht%J#A==Z`2WHScZkTd?^f zuv!G8)nXGYz3{F>anyMGKGip&|G8W7jiC9v)rPBR<%Nq+9eP6|2@=Wuk0}P^F;bc| zvPTnpI+{B6WUgyR_bR=NE$P^hro6o3dS^6T>=C1X zpL#+HBvfuHw>FM37;DE}Le-)7xKs(pr1h#0lncX9yVT_6b?R=}1LNOWuMrsbwt@f% zxQAp<-IHF%ze6NAI@09#V&c;b?VGmLX26GKCvV@%Ic~Fai3gYD z_-J-?BuGZ!GU=KaZ;q3`^{qr11iIi%L6vD9S?0R2THtw}S5u_g*F(HU{&1?JL?x!m zlsPKZJr_qco75Rtgez~m+WEnD72brat+(|zI!`&|8tu&}G8yTePR!m1b6F0Z57uSC z|K03j?FatCTkpKUTi-=~)Ma^*Q3k2VJkz&S*0$fU{IwqNAYc##pG(;mBbZ*^bxo0O zZAS^SCPRB@52C=ki9T(%R)x2Q*cy}RC?+U%ae}7t6&}Y^H*v=fPGbkI{BeSi>&7TK z8f?Z5)5Cizm3Xt(G9=Y!Q1SRc-#q09>3xJ5+b)?oob5 zMFq!>wp4mdKAlrAFmSUteW%POwmX7n=L8;`2JhYZ1p5)`f{2_-jYn9*eDR7RGnHpN zYzN<48m}f0`jN(ZBwh=i|K1?|<$MBa{kqsUY0ITVNm^>yOxBtTeZyomR($jRo{?lc obH{_p_{trO-IxsAcoq+w)OO3o9i7|pOv5@b*3 z=_pB)`FLu*^sTbyWMv)`vF22Rf(T&yO>9s7cs^aQk1iZC~19U zWtM?*quoOBY%-sZ+=1=5vw>~fuKqZK=@m-HS2OIoa||Xhk%^#%RsupmP)`McRpbZ1 zu2-6YN-i)F+6SW;Xbk0~TAhhYUW%kXEJao=kWzxir6S^XpVF*a z_pL(k7%>xZCra}cB+9VH=#dC%thby#QyPEGC{bQis%z&f!pORm1CV;@pYS>tc_+YY zyhl0Bo*MWI#7Nczqt|PmJP?piQ6@>>%1*bGDIy9%syto{+c9xpiq+9xTaqN4VI-dX z6P>j$a3+c7er}dWuq?bV^jG2T+7FhH_~?-11rmZ1(Mtht%J#A==Z`2WHScZkTd?^f zuv!G8)nXGYz3{F>anyMGKGip&|G8W7jiC9v)rPBR<%Nq+9eP6|2@=Wuk0}P^F;bc| zvPTnpI+{B6WUgyR_bR=NE$P^hro6o3dS^6T>=C1X zpL#+HBvfuHw>FM37;DE}Le-)7xKs(pr1h#0lncX9yVT_6b?R=}1LNOWuMrsbwt@f% zxQAp<-IHF%ze6NAI@09#V&c;b?VGmLX26GKCvV@%Ic~Fai3gYD z_-J-?BuGZ!GU=KaZ;q3`^{qr11iIi%L6vD9S?0R2THtw}S5u_g*F(HU{&1?JL?x!m zlsPKZJr_qco75Rtgez~m+WEnD72brat+(|zI!`&|8tu&}G8yTePR!m1b6F0Z57uSC z|K03j?FatCTkpKUTi-=~)Ma^*Q3k2VJkz&S*0$fU{IwqNAYc##pG(;mBbZ*^bxo0O zZAS^SCPRB@52C=ki9T(%R)x2Q*cy}RC?+U%ae}7t6&}Y^H*v=fPGbkI{BeSi>&7TK z8f?Z5)5Cizm3Xt(G9=Y!Q1SRc-#q09>3xJ5+b)?oob5 zMFq!>wp4mdKAlrAFmSUteW%POwmX7n=L8;`2JhYZ1p5)`f{2_-jYn9*eDR7RGnHpN zYzN<48m}f0`jN(ZBwh=i|K1?|<$MBa{kqsUY0ITVNm^>yOxBtTeZyomR($jRo{?lc obH{_p_{trO-IxsAcoq+w)OOucOGe{r-N2^}f+PVHZ(?1W6Dw0tSy z{hRN3(tooj^Lw6uelxmh5!`zz^wTIk$2Bc+o}wqqU&Pq!%Gmxv zps!f@Sm22N3AlfI*eJ$fwlzmD{6<5u92OV{BVYDhBsi?cIfCrU9PRlIElVEfA|%TW zaz9o1QpWp%oFFR7{D+N+QBkg+&B5N+O&Nk@&z{2wWo3x-;m!@CcrZR}H-m+bzH{)p z{Ny@XZ_Sy$AARx}O_b*jd_P8a>_tJ2@ia&GaBy>=vjPW3(MjFz8)`WXG6nuNhhwN8 z1GvM(au#UH8a~yfb`w&#(M(!hpY-E!I98hVMBN={YU3`n-EFA_*-C4QH5_clJ8C-) zIAh#MJ6%=w`d&Zi;oQ{Q^=Yy{o+qJgOZAq|wrf-G*4KHz?$;C~XEs-x-SjR8Metkm zCTy>nxa|uG3X%b^QqC4_G`=8*bEZpd#k_CPa$rx@5Dp?cSzK*qlU$NZ zdSSdf80Ls63m~8bhXzP!U2D?0>7sYp4F!bIo^@ip^ul^DqCk#_*;0m}grD-H&k{;7 z*MRCRP4h$T!rHJ{; zNrWnYh?Q7L&@8eRNf?_kjLZy1Wd-A4!WdX{XdbjTg|_C<(i+;>L;F~0nP~J939T9L zfedKH3R++Q?Iuud0mTl`>jRZ3(3m@nTmwz4&&niFG6pJU5m&7Mz5%dJfNB9mC*=u0 zgP{OODL}~qMhzv34GajRUqMO((wGq1f{+e`^C83)+)^;jMS`rQVSt7P44_C-B3)bA z!}nr-ImY_#wCB<-iz2=}x^1UOUVmB!>?Dk-D9#;me3~7`h{^cXV!yiu)?S;&_xYhX zG*Nlbt`^d6C&#oLsHn%l_4y1FkYdzd>zW|?9{6Ds%h=|maZfj?cM*%pPR1prR1K-w zjWM5v+O9Qh_zE}e+D?mcqt5MgJ?>O)cUzl`$|=R#oGwH~QV!EauSfyb7h_!xu(BHK zZG;;`(Zk6717bOrhBX<<^<5TtC#mV$XFJKOjC0c_a=AAYMDXyNh}IV;t^0nNi$$#7 zc;1%EzR>ZSomDl*UMSTGa9>|y`m$qI;e$3dSJE>qpORaKun3hqnPi;Uwwbf`27ci( zg_~VzAlhUfyib_t@zvgKCrFmu;2y}#_)6{Txuv;>*v7`}Oz|5lGk1sWk+0MHoF!)A zR*qb?6{?t}7%=lOkx@i7xBJ{X%4ExIvx$#IB{{{kBPgdE%^hm$ zM)LHSN%cOgN+Q^YCtmR)+Dm%QNwVJEZxv2H(gO%$6Mo7Ffd)j5AU zOE#U3!C}(O&Q0LvtxdX%RU`8!`nj=gk*B*3bv=)LBG1U(YL*{~nNeRn07bH4(=P~$ zjhUEM(yJ$540DkJjiCSMD_tk19 z_rYpAmlM^$)b&Sh#J zUMz0i8kiClohI*s9=62UMrzq_ktb_c1eA7wA zV^Is<%}Iggdz8As{x1ITiR9kH}~lJCv)wNs9Q$D%5q#dkLiYfC-~lEp5p3Gj8o zQ*VJ%Uq;0uc5MEb^daNxdb>P}N`CSh{^80eLV+hAb!034tTQ~vNr(M3un8P#yv@~$ zb!r@LYVt?QB`QnJC*Z@=CFS`^DUrDq&Q9`yM${}ykL+cgRp-QPCbT;x#!Vvx_=3vu zSL8?I+DndAz3xrLy1jMs zc~`jW!}|8Ttd;$mnrnNhflJfQyXR%ssCvH(7PDS~%UHj@T*iuSZZzfQui^4VjKxGd zHT)uH9=k|-nC&rk$9nt$Th2tPE?3Tssz_wCk9>BolnXXXF2i_x@|9?lt2mFI6#H(i8^}HNE=!bHnT{)3k>RMM<#BpOrK#S9hN6QpNUbrMYY$>dN63=D*L}{kO$pQ}WoSIl5eD z;dOZTRMf@($m{uS5d^`X{sb?z*J5~=|K;5WN$dHFJG}jS!$w-g@gRgGpB4pfzpY## z-+xLWU!79Fl`g(MdkQ~KP2nZQ*9o3x-M8}=_pBP!ln-z0gKG5z^Sb_P!7%@9F!Wyn z=Cu^068F0MTSXA! zc^__mD}RbSB{L%YQxx@-fqwm3EBrlgo@GCJ-*KKq#ZchBigBUb@9|UGdIEFK}3PJw(wI{!-~?exr2%{o?W8sQdC9_4~T-)8-G;ed&L_?)!rH zU#k1Ql<))Hm!m${eSav-e*DpU_x``)F$CEEp2yH<-v8*wkOz;U{|5j7c|vg;004); BuwMWG literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/family_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/family_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..fac4e715e31f3be3e4c4e274a1d6c9096212f5b1 GIT binary patch literal 2967 zcmV;I3uyEoiwFP!000001MOOAucJm5{XRd#ns21TzD3GNfbqUy8{6P$6ctclY!;i{ zwEDl##qaf!e%+m9Ch06=KM7M+cRS~n@v9@h?)?~gz3*Q4{+V{~Jwf6E%V~-L{{yW$sb#VS30Bcq|L_ zD~guIR}@9vFEC_%fkuueF+2sq5)@;w9-ugkZe8>kcLDR>cR#%TOhHtar2?BUO+Hv2 zC&g02<|-<`hFMO zcUn^5EJ1gIK8mr{mA?IjKrdK%S749-mim@@wOJ^}X0$YWPyB{mu^bi{2P0qhTqHQG z$2o%R${g+a4lPR_=OQG_4szdD`BcW6j+`LOi|mVyiBMk5@6Ey9%cgWe(tFQeLko)| z&bmAI7)G7(X1VAreDs-x=iz(T-h63}^mXsOPj8^yXJBjhF4zl$4C7&jF2~NzfzApX z7==4^xvr_{Fh~`6EjHVuz760Gcf(PjDRX#Nr`lOa;6^iPb-vTL-C|#<=L2=M9jUcD z(N^cF7Nj$+&gZbO=&z_{Kj4ghEv#lv+SZ_P55x5t4Uefa zY;qR@d80LjM@P;dHPUgs(a#V#nryc5B(#&s%}pW6Wx6?M(_5ZHvTAmdYDn13P5m}M zjoW!Uryx1AIN9u?bvY=4Uz%rOc}m4)n@doTY?u^boH){GWd(`@2m>g80wph?%r%s^ zgR%~g{s^fqvkg^1_-0{lX>gUnTm_TCW*bdlY5}+F9lZ9t=wR(=&jxU4A6=Y1U0^HX zeT$X@d#D~^A++Pk$z~SuAwDDr#=C%FhL|D;0t#@bfrQ4j1~)gEv@W}#fDqcVc8r%C zSP!-+kRxKYkRd4HCp_u1gc6K3pn3_E?tw}K^x<(h4uOsv(7*t60pJn|$dxmIZvbo)pjrUYNqE9fVN(F41fXO9qkQ1oNyeuY?$rD07*a&?gf-ia%^^4Xi@mBzkq zV!2ow3L<#;PDJyAlT>Xx&BQ!X&pdBSMVsq*&W_58V-J+-1h}mZ5q(%Ov*VRE7AMj( zET57ahOh{g+nHn(+qRjp_5!})QiU6xXdv2Tue?u~`~JyZEeA-J+~5+(O#ei!tFfiI zo5;q-XixDAD>YY}<(99K%a|ob$E6&)Y9o{pOELH)gmYHd&w_#ER;PhwY9gbEYG${Y zx0T6;S*8OYi%PtUXh%?XXPVno)S2YzK9lI}u`GySecbVi7t&tba!#DK?s}b9rkhC z-zTz^T+K$=mKYh;!2?hvd#u|DL9r1N(Modi_ob%k84tRw2mK%xcU0Y(+PIIkHa2z!c(Iwj#O@|}6y?=3dvg$iqS+{o)hy7|;$ zYu=PKLCQHYZ$iE7N2znB{kh?YLa<0HGdV44Td4!=81;}!n6*0A2l5frZ z>85N2k40rRiY{()tSoseNESP>2Ef-254{OWd>Q4F$g%lt+#VTU*PH2HRI;5{^H*2i z5ehu{s4ZLad!6AqPTH)8flc60<87`|tX=JJLzCZ9E>T))HUJ-iSfDfqX53asv(U$ggpD({W)|8zEWV*wv1(&(8R{a@j z`s>8HRlcBGX9G`1iy-^!=PThNQ&+rd4-jb*iRZHE6ptca7Otnv@zas%F%>}*!Rk!!<}I>B ziC4|$*79<6rLW4FcI8-Zs8g=#D5o8fsN!Z-K9NFfvlZQ9@kpRgX!4`?R20xxDb~Em3U0 zQtHFW!$) z8g)P7cBg-@n@Gwi>V%N^{UFE9r^N~6`*$hi#VYkv>Eh+uU9-7s3Qs9sCb*m3J|l`; z(y~)i-k#Vy!RZd>dHid^Fuyk#`j>!t?!-GVPrAfAEblw@jORVtuiT)1S~^izG4(C~ zX<*EUz`6i$I`s_|MHV_`aTjV}{3Ci+>3*#8Il4SghVN#>52v3cv_Y1}ySIAO4=*uy zew|pf^xs+eD9Z7>>GfMF?(OtXH9?5zy}kLVb}90d%!u$uQPf=s`thY&_;cPYE4}l+ z;4BXFuEcd2;as^~qPw_thYm;nBu8!+d3=|FMeK5-%)R0{R({3F4t^y&qVtOODca#g z3e#BXz4^!MPs!90{5MZ=2(hlua)QtK^h2ip!)5B%0KGV-9e2iTyqJ<>wcqi9{O*&V z9@DJ&PMOXAMSV;0T^HOm!yn7rZ+=Iun`hatsdY0Ct@{^f-Ca{2TK7ZierVkft^1*M zKeX;pB?uXX>2Wj0!R(?Y2exCiBN;l84 zznjvHKC5&)+($pi=0C4=a~~?*%&$?p5k-02$4WOZa9DOdM9~!fSm|bdqICaZzj5X{ z>gRReyUqX8Z=9bH|Dn3?Qwd+teL3nw-S>;a?7J_$*Kht7k0HSRzdVLM@cykILmoVa N{sSc|e3=+B006fAaMXhi%!;Tk5#jbT@X_PEZs@VNlda zVV5GcB$Z(>bc|*_Hym5+(lNDJ6;xEopePLiu>{aKDBUKSUTtCJZSOHdogQC#u4Ul*2`QzcS z=XpKz+J2ezBNNuI7tRnv`&nA*G4evp;9^k26h* z2!Di%Zvg)zi_AG?1*O^uLG*b-D4e>k%X6haz;KFO2~rZyq{~GrT?JCPx!_*`E9v@f zpk?Jq-ISgRpnxQ7IeNj2Uf=kM1t6s*og-z;4!mICg+b_hz5ZaNFK0X^3Bk|u#2~RC zxna@AHn2y6b9Iz(8iUXraIZixM4i(FQ3=K-$qRv@AxQGyp@lY;k|*j8342<`xeU%N z2x&EzTLC$jL~-#@Q!vjmqUzlpMEbb7og;xVS?)H*z#Ih0z^Gk23h)i%l&6iX4Ac#e z>m4gF1%l5oiLY$BD<;H>1YuPO?R2e&B4sjN?M$lqs=x8xHHk9CIh9L=)B*7wMd{ALjojCOxlTMv zZ8@aCU{dX?YoEVt%FJsUD)*AwQ&UVO*ScC3X5Ea0-8y#a>^NUH`M1`(iS9Agwly|^ERV?6HraT59CQQXG} zjmHtft(B!28g&O03q zCfkNICCeCBHbfV%z&}&CfH{lx5UK2~TBxR;y)J+DhsHsXmF}4)u4kELF_Y5v&_wuFPIU-IGb(}tW#U#*oau6 zY8d^uhGs(9{y@do+34jljhtM3wZ literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/file_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/file_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/individual.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/individual.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..14e202034ab0e84b76052a5f80d9cfb95320c6c2 GIT binary patch literal 1179 zcmV;M1Z4XkiwFP!000001MQewZ`(E$$KUrch&|c`Nt2SxQ8y0!ln` zh{=*hNx_1^-@T-6zD0HdH%pfKC6Re}E~l09pt9MSl(PWLr!<*bhq1F%t~yZ!4NzW`>hAMHTWJ-fih^X|VJW2;Vj}z@ z#@KUz$S@iK@dXZ!@M{=g)yEB!e_g}sa zP@IOc&}rfdkxCiJtsyDnc!07rFJye2Dri?iUEHbSGX_*wJvDqa5*dpxSo@7ae50zPGRDzaSf8QEQT{@vY%S-kJ8+w#a*UD z8>cqi%D9BQIh&55I@kPb8({*mty3lTTy@^2BakSFwM@c;!2iA z6UxrvGWtcE;-=kZAO@MkMyHAEwADl2!;xb=pV0HtEYi&DCrSsG%FwMtW0JT;gyz9iTXppFE_G1HRe!&5LM@JOv=hnP(HDWTgd9P4gMGcxeo;W$Q(LQ z=-Da6c2cO>voD2SJ51Qi!-Q>xNo9WeUR&c`i!U=x?oOogt zClvSNgmyRyZ*;7lM3achjbF%!$HD-7dv>7~syc3BcQEPyrnHQiegqx>boe&&-{Oa1 tEOz0i+4HvX(;V_8@zeIX4?mUtP2h)Ou?K#Zy`KX=-vB2ja8A)6002)jPFnx~ literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/individual_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/individual_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..863734106b25012884f3be5ef8ab11b15559bea4 GIT binary patch literal 1582 zcmV+}2GRK+iwFP!000001MQq!PvkZffZzKwqCS$9?bu1YZ4m*Un|Ggn`b4z^q>&_;QYO&appy;^e`)j)e8`!TLGXzb)4%0=NXgP+$-s zMnV}d;1O$jU{!6)q?!aW|B@y*s)Mg<>aI~DpmkssWhL&vKGqb~LTV`%3tYe$qc(ul zvjG6E{sHaKk@>b?bz7jV{=hq>dMDhKpD6Eg) z{ywncl(RBQ6Fm?nL@oMnaJ`s6*GN2RfC&pn8-xP zIAY0ho~QYe$S{g!d&$#@YS>h$PKEqlk44+|?C{I++;-|p$b#ol))+$<6ls<7wzA6O z=7N=F9tBmYH9Sm}%zs#ZpO*Jo#ILk;>+AXF^NT;vRZ}(mBx+37>fD9CEX}HT=CVdy zYoRJwtZWLGw6wMp%GMX*FHv6JcFl4bOIFB*a6C&5R@6Q+^2D3&e?BnzqF%nclY1DA4&p$G#=eSx`J3~0dke3Z>FrrLuy zQBmek+nE*RS?v!g*6XLM(RN^6v(~)Vcf}dY+QMj78Z8#LF?5orlRvt9{k9%5x}P4% zML28KdZ{hG>|(JiUJfxOQFIqeD<|{_pva}jSBs>j=gReZ@V%sWCi6*D=pkNKD$owI zEUpb#-?}ojiZrezpVr2t4x+ArwxnvU#tsN=#$3ntJQfHSIfz4PO@DfL3?&{y0$|1* z9y-_!F_*4E$;(hO$Axv2s;S0q z*dFr(8+#1dzGU3S*b@MS4x~`@0?gqM6OSs72MAN@L$4QLbP-_aB>=-1k#H4YK*bn# zAd^6Nj{yw56M&7m7dkmMFd2F`M3J)*CQX|?he>mjqr#-Mb1zJE{eHm2gRZj;Ocr|Y z7$ykx!UTsaFk!tgfnY|Hc9DMxPbhH2Do>z`1eU^sD(2>abjlDbRJk| zsR4=Ele`&}kY}KT{0Jx^8X3CVjfpVt?U^{`gcf+i0Q|lfsfh6ycT*e0CW?2+BW-*p7tikL{Deu zKJ?V}dx9PW=oaW%?7eIB&|dUNwsbf^68yf>Vfqhf8JrlL4ELzsB*jcp?0GmT^Xd_r zw59JNCB{6-u}85@utRN318LL13v3ty7+?=bDE;Tdr2nMZn@ourn3%-aYv9Br#!O=D z)p0UPT5PvC@%KF~W&mXHGcbNka?B*hULQXrrN*|w4|AMF{3w^0S1`?cGX`HnLU(V0pZ-_lz_P#4`CpWXNa8XUI1& z`SAAS!w8N=K4i$Z1M+DH8uIC`${ItFulGxP+znTDCWVWzu1GhHy`Zt+QGBE!s4 znWO0zV}zGqCIYq6Y}h-Xkk^PcG&{EJ*+C^*b{0s zv616cLWuv~aez=DY*nepUcjEm=b3LF**FWqo%b8(R@r|iN+|T&EhageCiS}ja@=WW`8_4XMLV0{WKv_AFU-V;l!Oo#k5!% zp(M&tzzQJ+qs2hdVqJ-9+&+g&+6aw}w0YG4URu;4lnQT9Qc>WQZ}hzN?0ZJt7+Y1YI42Q)Ma4?#uz9Iid^e|N0I6S6Y-*uOEcu zLM|KWol^l*l3$1|6i9P0GSnV7mk=0BUA|B)sN7lH;ynXBmNpJzD`EubdsSP(X1l|x ztAoIKoXi)~(aU)Bbu{euAC71DgX!>^dgqK7U!F&^`8BJ-bUuC<+|RFJJWZxoecuWX z)k~1j=C|!+^gl~yETN!K><3^s=d`E*8%_hYV>YD5wzg0FkVJmukudQ>?B`N(k>ljO zWz!DBSp=Sq?{u6j!!=(&@At~i`je7Ni)+p6E{&I1)rv|e_LCu`0gYpCZFeEe)Ok_P|)(V11n literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/interpretation_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/interpretation_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..6508f0836e1f50464d660c81a9591d8448498ad6 GIT binary patch literal 689 zcmV;i0#5xOiwFP!000001MO8ykJ>O0zV}zGqCIYq6Y}h-Xkk^PcG&{EJ*+C^*b{0s zv616cLWuv~aez=DY*nepUcjEm=b3LF**FWqo%b8(R@r|iN+|T&EhageCiS}ja@=WW`8_4XMLV0{WKv_AFU-V;l!Oo#k5!% zp(M&tzzQJ+qs2hdVqJ-9+&+g&+6aw}w0YG4URu;4lnQT9Qc>WQZ}hzN?0ZJt7+Y1YI42Q)Ma4?#uz9Iid^e|N0I6S6Y-*uOEcu zLM|KWol^l*l3$1|6i9P0GSnV7mk=0BUA|B)sN7lH;ynXBmNpJzD`EubdsSP(X1l|x ztAoIKoXi)~(aU)Bbu{euAC71DgX!>^dgqK7U!F&^`8BJ-bUuC<+|RFJJWZxoecuWX z)k~1j=C|!+^gl~yETN!K><3^s=d`E*8%_hYV>YD5wzg0FkVJmukudQ>?B`N(k>ljO zWz!DBSp=Sq?{u6j!!=(&@At~i`je7Ni)+p6E{&I1)rv|e_LCu`0gYpCZFeEe)Ok_P|)(V11n literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/interpretation_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/interpretation_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/job.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/job.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/job_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/job_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/metadata.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/metadata.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/migration.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/migration.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..11c5d01b788ed713b5d1a7f26eeb315dc9d6bb0d GIT binary patch literal 6137 zcmVAB4%o!`O|Nt~XnHA(@{1`|ZgCDXX-|>8R=X^n7yi_Jo_@B#D-IlV(p>Nfo6z z{Vz!$(qxA-IwT=gT&CMv7%q|o<>V1%X;q_Qi5feU=IE-iLvNfQ^rszxo*y^*;KD--~gbX>iok2|@!eE;qG{o?xW80Yf|P% z=G>tjsta_>?dVjSPdpPCeQPlKP#EU}FqS088xk$kj6{FXwU*XTQC#G8S!DE^Bun&( zcf(E42Ws*UVR0c0kT75vdPW!q)-Vj97-DU?PR~6B;xhdiiH9Futv6(w;h3OPG47l} zc&e5yS{7yVNt&FRU>91$E`(wik1XR&k)+Es-do1>haXO*vB%@*_DihDK-orwXG zj%y1^@g@k8`^>IO54L5wrz50~kR&B~)Q@-sC2T~?=O zRlFt}j5avOE3%=;oYMs_9x?{!bYcrm@g@kI2Q1M&aGf78-Gq}*#VXAgbxk(gy84{1 znu(<^^(8k@W7(|yR*@3Dxm(=+{cJX&9?Aee4~*Z`5I+x!Uk`(H6n=D>)={;_CH0MT zp8ZbZTEpy$j!7xUspur+BfpA)m4qyqOid88wt%PL1w2F+k8ReA{3BYH#irfPqZ(H~B0rpY z$Bs6q-1ZYSJ!L?)Uy|fhE_`V|is{T6rZXs}8o8Y9&(fG=**&gEM0asa&)ylKU-VAP zge)=LrIE5=tP&$>fzqX7oF)LmLpp|=jSCFv;EBd7z4nNRAE@aC{q=A83$BuD`m3Bd zA$_ZcARCInQ|+H^Sr19O|M9en72TD1c#ppo}`C6Oz^gLD)NA* zB0SLok{y@I0vsYZOV|C9P!AiEvEc59^e6hjZBT3TR?b&%9-6>sjW>Bf-h?L_({!fi ze&5kNF0$Pw=Ue=`AUQ==lJAK>csgHv`gHl}@)ikuHYKHf!`>$bG4iQzjW_u~-h?M( zTr?+-5zeyWkuv!Ew5Xr9{reHiG01U9jzg!bU7C>S=6Z4ZmQf^sW&)nIqmK_f`rwI1 zIw$9|mLhQv%6(yHL35rY4%JtH-Rk(qYZwQ!oklm~zvqq%5}9r-|ZgePM` z%)VR%1L^sh2)5H0=OJd+mh%j}oHc^k+0ofjZWwx%u5x~yp;PNv7DBCSWrVs$DieY8 zg!;?PeVTKGMW9`7{*+i1C0-Fes4BA31Z`_aq#5vtgdL5Ib|x&vWXsmE+SFwfj-Db( z){|+Jqt$jUD6Am$!i1-MOm_5{H9|Cl5+Y}afOa?ZV=15R=53yzo!QD^%Qry~(kP?b zA|#?~d_%Zmzx-Y}{ERUc-=7S{+is88vSW~lv^UWYc_jBuq8x!WhZ#T}rt^~gcgx71 zzw_)#F3k3yz?w%4z#dVflk<*8#96${7=~Mtq^pvUXjS6vT6czOAo?l?qK|AK-&!ci zMD>~oAxmns-UDkMF#vnSp*UuI1_!xddHvl3*tehCg91B zLx&-Jer8ue`R)_Pu%BDgY|Vk1Ej%5foyT$=f`tN79g>)uW;*8ZW4tMz&|~~`Ol#SM zm4s%H!?Z1lSlui{*Z5;DCmdv{7dMxSyUXb6`tI`P>&@le;_m9pHR5(vH~OrC&n8e? z^UQOoXPyp4K%4DNc?84VO(o#~n_FmY^@rfqKgL*lSN~{~WO(h%aEZ~%53SMF5K334 z8eMgc7PlBv^VVY;SD%;Z-S(gU}@A99kn@A(VJchq!IyL8bo7(1A*{UhY5LiLEs+ z-vj|rW3DDqW&`%r8)t4vlgUZnwWfG?p^A6x9*|J)eA2>5e@HRuqL?3=s%YkUdg_fy zh}+qQnt}T-`t>&h+#OpnKKUjH?n9(grs=l0#92oF1&$XN z@Z6jlhj3Zqr>3~z&BeE~>E!p=tACncXGbCdi; zL`h2}kMv%kAL(y|Das1Ed#ju#-ME`nQ(RD(V-Y^-K4uhKI1MsOAImpEusvX(hsxu_ zgC^6>_3k`Zd*cw-*F8r<=Q_3$BJxcT&@}Go9dO5k*eHw6DfvgB!?&x84TX&w(0w|l z(~+`vHg{1%wiGsYWM`!XYjThY69thIyMqQVKfS%t;muK)y89YjBEBPz4<`~qVwhtFg>rkQ(FoV`6d9K-b*5% z`(6})>w%j!#n+mO-i4~@u{Rb8?azjq;! z!GjpBL&uNnCdD$YYweurLeH7rP`I;t%Hwre_kfC+rWs=Iyq% zLei=cDzQ}Eg#T$!=H|YFa^4h2C~xwHLY-Vl5@T7Fzb)Y5_?knis4bR+Qy}LgYwqycX8PVwvSYFd3VAD^6t=VUVDt&7~ zav!=-{pkQqOOg;l;Vxu!SCsn&PB_&d`_K$l@m*D)B8QKnX6<+a7O?w6fku^*DRyN{ zog0%9u$uactBv$BNx2({;crnp{w>QA#zd$Ig!sCNlr<4_`N%PBUqRV!xNyPBc7KQm zvV|)eO7OZ#k$MnL?1-3)tz@BaJrxjFr*F|I3+J42P){i>PuWdCEC|Wf|(N+Rnz6pY&#vwa2{C<~aNmQ=z zSW7q2=Y*xigkMHC9~WvKzHw%u$r=x=#c~5^EY}~Vk8Pz|QpD9kEkEWnst3wjdS>1$ zP%D<=EkiUHa;fm zsaf#RET}~z3l2>j zqcusk3zlThv_?jg{TWwKznG{08i*{i9s5gX6w2^Z)4IrWO$y3gYeH%lETo?GB)7c- zuPAGon;PgJ6hI2fM_v zTJyf_f>Dp2zO+K8HIGp~VQ!5`&Y?tdHl&}F!5}rI=!_C4p<@A{?`rNTi(A&8gIO>X zo5wI5rzWh(T7zI0+8`LubSgPmZB>$OQP!O^=5a}7!+&nLO;_e7iMbORbdzyIYgw)k zn&p}e)t)Q80EL02T6zT4#kPN6lG9X^MmLYD&Skave$z)bd)sG@;%qG=7D6*(!4R`# zbyqnBl>DR)k=3khic}JMk+eCsV^O0sWrRL|xsp56S7kH`psglS@=Xu`Uy|h>ZI;`& z2gx3}Ci{@n%uHbS*zr&K+#YUW~8U1Vnms5jM2+x|H+5_vU z13jhP=?9f&8Ci+W!tt)8`;Y8mmKB((`gBM5p6=U}FG@Sq+KEOy%1(m%lW77s=>Cau zcJ4G~b8*qm@ap>e;?vdpQ^dNC-&`)<|F1#Eac^RaJIOae!0+LpPS*#hqDgra8t&bp z2C3lhY-*5D^Q>vmJ+KBn80x0kw8|s(Zrh6!M#*lod20LWu#qBsS?`jR((}i(UUQad zf}1T;>4Ej!U^7w--ZBWe^+5Zw8PdU?tu)*<>7B%2W9!0B-Zi zzpU*m5={JKS$zomPBZFW|R1=XRTej2W^**=R;ZRw!dO(Ug^&B&VYTy zn&8_53%)}wOz3K|p5Gtq&OB2)iC!fq1+4zT}o3)6SqHb%xy*p+?6; z-x~Arq0A>7D&Q6umI+=imC{0c66nVp-nLeDzYt@UP$e+oL9ftf-V`!uFelXV)E#A1 z#fp7NE(MQ?*nise|Cz(|2AXtF7|CJ=nIz~Y`$g_Wn2sW8E!N{hV?Ck9DCcd*{tYgF zM7#V)UOeVehRYSH>SlAwNWQA|R<)n04Z;3(*W(49Rgv6og(G|>0 z_6=+8^*wNVeSE+LwSt5F8zIDtjxAFaHI(eHFOJAEqx-D;dBq&w%-T*egYG1Fh#tx; z{`Xce))EDae0fDMV8PBD|Q*y_Kf6l+U>+?5+s{j^O$p}$Gjg3Nh%2*&nx`@ zi14g6&-cL1^Y44o)lRc{vkSxMqf3S+SIPCU@@7*m8^VO%M!+a@U>g zpN67Q2Jj%C`qD}1TPxP^!NnT!0aMf{cl?mt%k5>aHhrjX$Hn*e@8k|rY0)&TgU8Gy z1L#}3I=}~C9T4mHcx&3^GH$w|@Ka-ZIa|5wEfVB%<(muD6Gg^$8#Bbrc%YmlywAAk z`)C5JEf4C0YcgVuQ+Ddv)BJ%ROt|bvQAu3J>-2$CExKnjw=dDEESrpjvh0+vck&Qx zoD{`YogQ6-sJd^hl*0#?a>RNc$OGz*o2&m=Nk?Vw7?fJKeSkVS_J!L!{ZpP%QtUg{ zysrcHzC&bGQUZCs^7CX^BIg5=VEM0b(|{?CP*G6ab6aCOV|R6t<}r1=tVT|GT@gW> z>fYdnh(00}5qm#%ro1zLr$nv^MqE4lnG~nB$9cvS z#x-rYQMKLU_{SqI`6cW0_e^=45K%lLD8%8z#*lr5D+mHJWWiSuBtvxeIHFNE7J4MD zd3zK>ggY@($~LS>3s2Lc~lnUd%Xc-rcaSXXnAe`6VA zfn4=jQ8axf)8Voecd=~l(&5BRPu0Soy9Qx<-&*~N53WB+#;#!OEqY`UYDmo^mtY%eT77Gm6!_pv3X-9# za`tWzF0$Pw=O=>gx*&P_Q;NLidMmMRs?M2ACLfWs=9x}mt(0CSo!NZXR(3?b2?FR) z7*$*#6z%d|MUrl#tG=ENJai?7c6BwZ(||KKxuU>-g)0gS-5&Ug0y5TgvN?vT5CC~d zS+bWzCH)(D994hG%*drQyWv)H6X%cyCPydFnxpifj*`5jS&)IF>}wa)_(zk5LB6%3 z6dzoaLdHsfnvT6uwcBiP`P86zTQ^vek+#|v?JSfnY!ZLSatO&VkP*2+(OaQO)ty$6TsysnaHRTjIg z!Cg2w%aAaU7ve}>&F%R`IIhX*#kaT% zRi60Z$`dja$~?A@sO|dsTS=2mGTlvPwhDCRn;=lWcrLW!Pb*dEt^8b=QkOoAvA^$_Tw%eCA!#WM8Uxiz}|Bd3W{kpI0B{C+r&+ zH~RzW*H`lEH}Vyuh+Fdk;F^JDPlxl4R&L-jJMq%K9{fFb*)?ZyWhFKw`K-51ai7tI zwpK>qgUbk(S_AZhUSTq8`IS!Y$D&LsvAg$Y;^Y{#P8n`brga0pwYmWxTsN@PSmdPB zKCxjqf(Z?BAjN;^7xf|g2@RBozO`ZjA6zW3G${NT=ntX6_#aUa`_}F)@WJ;MEQdmo z;zp}O#nOh<>!MHc#Z@4D78Xk+MoI6tuU-;aUXz(=?K%S=e4W8^sK+GpfAk0aidjcf zdc3LPHA!|ENp9H>c9$NxTK=?SBX7piEqODnrg=1jLuGE9z-c-x8pSBBDc}78<@^5u La-xH}qqhJ6}LyiwFP!000001BFxDZqq;zedjY4UXYS%?f4Q$l~9z17L_)El!pkFYA@qt z!QKt)HEoFU@66gRA%Ye^tUcp%X3ku8NA8p)qaAr={F@}hSkgEh2DywV4_TZ8XEYXM z-y;pcLFGt~)OBMq9?K|WQJ)4`+NYEXe+2zhq%zNF6!QTfTC)-kTyw9%9!w@nr7JS} zeA^?!0&~jfGv+)Yq+v{xFbU`&h*HFsMygziCoo|UAZBCDR}ij?n1ck{;C+>%P@o@w zRbYRBZ49hV8`MKax-JUTVRWum^?}H0w?<}{dS=zC)x2xFGMi7YC+B0#?zS5^a;yzj z*@ZP5&+fA9QG$e=)rI?nQl<|ZWseFAd@HFx@gXg{R6ki4V@E1gZVHue=U_`!RjoT` zMFm)oGvA<&leZ78{$V(mda<2APG6JJd=VYd>|v9WFm_Kne!kU%1AJl zWt@q_{C>J(JKraWM1ms@s&W3eEc{@r!mdKB&8{B27=7LFUx<~gP>*aU;F|y)cBzp G1ONcw*a)To literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/panel.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/panel.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/panel_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/panel_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/panel_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/panel_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/project.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/project.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..d18526586e7958bba53ba8ed0f171040c064c6aa GIT binary patch literal 520 zcmV+j0{8tNiwFP!000001C3KnZ`&{oz3*29!jAcno!GH^TCoo6u(d1FUWQ>PD2Yxr z*it6R8QLQMeU$Aa2wD`o_=9|WdVHjw*}Ycm?3ul{@y+rgQ#?;)QmF+ODP$E2;JIMC zF>42;X&Zs=n>A}HNMfcd4d}y-v%gUWM&J(3K;%5Zw4CsyTuwOWN&iTAzEnjnQl6EB zHqaoU?}%b_t5%jZS2UyhQdC|u}x!5pEGS;j~RH6ji?_9`+4>8 zVO~U+B-XV6kCC!c*!Z*A?603bVF<8BV;e2~lo)3R_?T_c`CjKXn-)<#1Xy8> z9UEIxgv5Ns$Nv0yR_h#TK41=G`(na=Y85K%n~wQ7u3r3!GkxeFV4(M*RT@1z`}vEe zZvr|4b!Sz8(E2?eTYWme`0>K*zJXV#a_lsl@+qI(UZ3CGUVgtDT-H(i0`pWLRHc%T zi@92eyp-i4QI$x_LMo zdOA2Wm>Nqfs6A2$LU3BNp(~*O`_uKM(Ww2$>D5|hyiD_=OzFz7U95b$*#BL#Ets>`z>*Y6TWlx_gCdU{ z0=kW&f(AzZdr3-mY{z!sCRjEsFE+`;b9i_z{GbJzDTz)N=-1l)qd1Ku=3yu?2#(`4 zCn6VVn4V}lLp2fLjK*l&= z$rxiy-A^#4T!f)WQj&#{Ggfz6fHUQ_>`KK;=tmM-p;cp4UG)pg&2HT3!VT zt#U3b+)Q@D{Jg~x;>Ku|bp~2%<=&qF4cFYMorAfl2fOgyXfC!_3!Yv3SWSn8zdK$T zW$p$SxPd*#h`K01^v*3%uImz=eEl{;=B|NVrpmQb516$|h1L13u8gh=Z(0>j>THX; zETJ;P?3Qa~PjGvN;kDsjI7G9x(*WJut*x3@+;ke&rE1;_=TOxpd@SmRJT!I^xOCm< z2^f3L;mDOQ&Q%Qs@D|o8YwEl~Uhx)mJH%!~=&Md>@cEhaaUhsPI3{s`BZ5O0t6HjD zi9LHL33gXUX%z%I$$8O&t%DnTFPDwMcU4JstE7#u&$XR8Gww!Hxr*;it<7rc;@7q7 z#rf#ahfFT(*0o?|Kv&#nZ_CEj+3Do31BM=jViiXey~FSD`1<1XRYNx|bRlr)Q(U^I|<7O+%O!8v0Y7Y|g(fvaC$C`m-%X!(H*Q-ua>Pe ztZp`lwI`Q@%@;q3Fe3>gp?$59yUgry>pY>IVw;ntb_2Rpt#k0Ivz>n0G=)!kfALiV z?c|9E)6CXV)%NeKZ!?~Npb4Ynh=y!TSq5X4CD}O0agfmjLy+5Q_+_yZev)p%Pwt0b zyahjU1pE?0^EBOtU&dubB@2@*dV}99!_N-_bOZypsrT)lhZ&{xNtng>e*v>T6t4=i z-i7@z>oajI%$(IXn7tg#Qk>Ffz$|!en6Z5@%ZS*7872E+mTbW+JO*YVk0sxR8H*&~ zghdf1{0(ORnJ>8=6kGpX>z{=e=UQ->YjHrH=9=I00$d~iYOeLyd{wUXE*#*RmmkYD jr}f6Q7vx%ykmqo1P5U`qV?WC^VY&7L{COC`q7(oCrgZ0} literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/sample_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/sample_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..ac89867a34ef8086d8450f1a9d33c0b63d6f29da GIT binary patch literal 1047 zcmV+y1nB!8iwFP!000001MQdHZ`(Eyz~A?05PP%>ilV;q7R=dZU`dLk8&(vBL6MY0 zKwqP%ph1xTeI!M>{>~f+b-L=srg*#~?~cbG?h(uc180xm=gRzpFb)_bzRw`11cq^j zXhvf{22Ufmj^W%*%p}tR}7^Qw-j8(0q+?cXjdZr`=??#L_REk=Os_Z8E#b(^f+zj-FwX7&_ z8exv`XueoW)ypl808vVcv{k&(R-Wt*ye5(uwM(wz8P ztD@kga$omE3cZ6_Gfrs=Ob_@CtPU{+=9h6<`k0W zoHTst(oD@pz%wbl=dy8bZwjg8HuRIuMFdd*Llk;2K+rc~sF=tE-MeN>;tQ=n!C6M~ zhU*mHX>UxR-PuLS#JylTbT*Cp(_DSy(UdJhcUMWZ_-u5f+GhFQZEX5AN&S^H)m17k zOJWnVX5;4KeDZORp$#_|aRk9T_zsS5ug>4!{(k)dbcq%wrkIB)DU!qET{hq;t}!OPdkSw#nt3EPD#Bu0PDCSCfmsFD5%3 zds?}fIv4y)f91Mo={J|W>57Qcs$@hyxN=;Uky0yIKenr!ZeJ?xH+m+F*V$r~yQOaWrmv-bdbLEfMb-~ipYtey35pWr z>!%nPcV?T#bBK1$Wg!}K#G6*v`gvXFHtn73ixS!j3rn4uzD;G*J)*YINW>|Q5DbGlm*NO=JpfNXJ#Hmi${56i_%!ozm<5nW1K9?$JwrAZP4ggiwYS2# zUMl9Da2a||R_hN7s#nJ!Iv#%*c?1651pc~A4ETEq{*-LD(9g|&L-@r|=j5dD^LN6J z;kCAo?S@~t20wHF{33+2I9`WeN?3qd;zwyP!0-Ek-yg}ca>B5$Rw^WtD2+mA^$JMh49fcw^M+JC`W z2!Dt+?K{9(mx%#qr)|@Ygqe5hHZ9o=GrHQQVYC}&(HhMBLty5UkdbwmB?03wN`e3) zGQezT(~fD=LJvL9HM`~&xQ5;|*E%;(%C+tYd$?xhhjPtm4P1Lgu6YqUhHFdO$8arq RDc7jx+P_hsE6r9S003;i5%&N9 literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/sample_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/sample_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/study.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/study.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..0db3d10a40c768d7f05de4599b0bec33c07bae64 GIT binary patch literal 3745 zcmV;S4qoveiwFP!000001MM8`ZX!qW|2;(uos{H+F<|3e@9Crvz&2UFGT_ZcD{Iuu zG|<^$nw{?PhOAb)ceppaC%LNb`GOe?c$2Ih<;UWoy1Kfmy1KePX8BmMxm$YmvGjL= ze@hoH8*cUDtX`YB=hbPQHfC%}s~5AapyrCmq{n!D@1`|07(8T%ZSzW6~L<~)$F|= zG4L&Zi~{j9bJVLWM>FQKfEr>SjT(y9xMzWkz5rccsDw?N_JK_oLcA-z@Ate1qDrQ_Is~lNb(0t@5@C@TEA;vM5b#tGGT33^&$!Ec z#i>_%HKU#cAompu{0v}IzYi@?DWzg%H2OXS^HS^P{XFG!0|uMJ<|0^|?NjR&Vzh_1 z-}O3re`hrG)Env{=cvANDPZqI9xyjyatRnbLWw}s^TY?{Y9MlRmw;ZgwM-~c|6R@? zFHq|)15sT$^oz`_&!|s+FhY=-2ndiSYQbWKNb-TJ1&PGm)Qee|Di!c)s1Wk;Q({MR z1_lcnXo(P5N%XZ1E>u~+IkLNhN$0v_eHp!-?bX{rV-qnWu|yKo7y53ZL`jz`P!r*P zs!Pe#5h_%Ca&O^zPt}(NJvnP@?A6*NW}p0vz|0C!KuB^NmE<%7id>&OLR&KqI9t}X z0V~wk7d5lKj^4zl%Rxbiz6Z&4O;CagVU>A>j0dAK z=?>bRUYYdqc|3SiCY4I%2<@5kyrsWQ;M23P+7T9(aafPHv1rzonzo=%Pfyh&^ptq> z&r(!$qmGr^X7{;xA^|@+C;wuvrpAi;E)84)5^x=^qSjDT$43zm*-4NGB1SgJd#@d% zPt%dr_S@JlPnMB0>(w9{7yf!FLK#QVWwPW_GQdtDQi+xA38{{CcvD^!z->XFrmim^ zt?Ed;NW-f@D8~I02U0|Tj@q>b%`)+PNB;$VRtV`(XJN>If8gjAFyoZWB~+0mD4qzP zkrfO^dr~3Y8PNiRza;`<9gnV7umHxA;D8tOp6T-f9+|})$ui*zf^mo}4a-y_-D5V? zSwP|@q*&(8QEuRAZfotB>nw58sQFaF<04Af(tJpn0X|JV)%OSDU|oKqD&aU`5TJXJ zI(=bi(O2Exq+@{^VST*nOaM9qJ%Vj=oy#QFk7LAvOcFemqJX><0h%yC(DseaB2r(6 z60QMn4Fpy+0R6dm?!NlxSKk1+M_F^5VGgk_vgdSY=T3de%d~dDP6>3mE zC8O5HGl&M=4SEPy-*in&s)K6od$LjLjR&uboaVSZS! zAPGRw6hU*_`XpcT3MjRDS}T(pepJj;T}MUV9G?h1qpIrulEm*;Ref$H!Ts6d?^zOb zJnnOcdSf{9x_1H*&UWbg0Ls&Jo3;IZb7vkI2}@)~+9=hTG9Q4D+wgbHTiaV-S5z$o zcv$X~N%lDe0s%&Rc9tahZ8D?y288YBYCA0|l;!htq-PuR?{D?|31a>wNXI-dicS!S z?OiG!EZnW(ci(qflV^~`CNdF*LP>L$jrvBt?%=h7!yW*I>|iSm{*TZv$JKf|S=dr; z!N?ap2X9NBs#jqFPc`6ku2}S~7X4}HoaJ#%sxFF3Ov;p~6zm(w%|Xe>M2!muXF&YA zo(s(0EPS~j(P-h=le@qVs9N4^2tx39qvHi;xu3DrSLEx#|Nise;Wwv)3IQiFz#Bnr z_#|KWCs@FYN?|6P2h?GVDyEVJ_dqnAvj8*4wL`hliG`4h%=ta@Vdq?a&t338N4XmK zNR8j!wKe03)g4?v)t<%JgzqK~opDguF~<8W+lBi(_kU2k@VWMP_Lt88>hCBe*mFz*dEcuHuHpo^FMyr0(xl0RLc!ItUiN z6?#V4aw!bBDMYYWr>Cc4rB0)+#i?W(SDpgv@)Y0HJzbf1j^%0Xv|iey5W{Y?;}Acm z`ZnfeOw&NNyqO>w0HODkdnk>j3i5^W!MB@T^Qv`f>AD}2bVxpK)37?t_D>HG%DuCl zzCPWkR4dh!@t`>x-wZMBcf@kZbq|7F)TfNjE?tLSOwZi&>BXgUd0um8)7s@r_uQfM z@^VU@luN_LC)DbkOBFo$+Ble$>g(6b@$luxF{}OsB@CE@+j!5e!NFvOFAath8$;25 zciN>x-c<*0Vwz9$Fnz^5E|z5}zs>5^z8YHEiC816Gigq`!$D#Fs1;Y;ny^W0iOy6K zaHg&)bL;I}1#m)CRH}`96iP9kG_46pk+{2bWves7K4Z~}-3^ks9Hf<8F3@}XF1lRI z4F%C}`5Uhsu$2f@x_GlS>AvZt>$Zo3&f(Iz;KZ|``5;57)ffyo*j#qPwEa&+bdU4zi(n@D13D3^o=k+}G~F9$CX{t1}+&Bg%ce z!sp;M<&pcfMd+bjD)t;(1hf%9VrOz>9@vk#U5G8TAac3UeSM|NrN-AoeBy=1b-lfV z@2h}|Ain4#er~Pn&LDqcZuoqG{~sorqF)$*)k+Vy2Q>$yXi5|jB$D5BuW#}>AOj!c zlErb$)el&%-)VPm`^9QrGMD2akC)rE&e;L$^oBp?7U_u(dug8^K>K#^dNBMk??G?< zcfR=GCu>y5M0+ilL=9GqxAN+&?BbRfz>Rarl-uzwpucWfKMf~8 zjq=_g#U0;{NBX4N{o%OTZFR?aW7-t-2Ote1+EE0VN4_hYN(AcsWBi!W?8xf&fiwFG z?&=CR?~vrwF=0zXb@l#@Bkv>+RgSIk0a5+;_tj}x8Js0sJfT@2})OfN>m zPaV{nV^!X@JQ`8DxYZn7ckJF|Yvs2^#r16;SpYk$M%Ra7IV-?Ub`y`3@SE7T;kZ3I zn%_pAwIUH}1$eu}+J-eAA!_3Az=lS`ygTq6SF-qCnDJkjYs~422U5j5tEWRuO>-8g z?_-w@0}hr81}WJ&tu>1+Um9lwN^_tcJd;-LCF9lQ`Ks1 z_fTW!(C$e)=j@!)KsQs4>s>myjV--7fVebHVvM!NXhr_P?%(z%UHeV5ciY+HD}3$z zC9|VY#I;@KPz)9zILDe3=4jmf#h4H~U^ax|0Vp1Po^M8^FU*60OFyuIsJ)UbbcK!e zCEPAk*0N1mtH3#ZOxD9*XuL8Kil%HZM4dp`2^RQdu7yBOv@nK2?j(Rr$qQg-5RWW3 zfVS@hx*Rm+E{EI|4U2mA3hS6~0OGaoc1Z|K11_e#pq{kI9EqSS+Ob}jTsn57Q1hp+ z?DR>Qr|z}gij@b??acG{nddk0^TbN=;kli8{yy`Z;1Bxk44-lF9gQUdPC<~+PlYnl zMziU13P&ER_9B>}Nn#Y3=n(7#wj1fJqU^9|U7Ll`z_)Lp?XJi^%il)-%oDKPlmPTu z0Fi-Q@L5(d+}B{(M~`X0lY%%*4q8TNi$_)-cbH8bhpm(dGV4Sm zkpFFfGn)mL`pHuHE{?W>IeKUjtn({PuI#A^wrsfFvZ`Mi^4jHOz~M?N$SjuHiZXB)?5=p^ z%r=Mo7OXynHVUD}T_k`4?^I~E@tby$t?sefP3+G!TBHW-mhkY%B{;hM;a(I!6}}*N zF!R|tzl{jdi+QDT0@!$AF9R=rH7`FgeVs2eRtPjp*t)uix%e(73Neb=d5BobLm}3# zVZFf_b(rs)0%dWEW*5>v+-6cYa|JngaoG2e{cO(vY|Ej(Fx6kUF`I)98G}o9RQzq^ zg)*rv0 zKRZ)1AGPYm3-{&4v|erKmm}=WACv)-Zv3!G`$eOAS-*IB`4Y(Jy1bn)-yb~t)8GCN LF93*mbw>aIWj|)N literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/study_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/study_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/user.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/user.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..76ecec203e3c86e42950de1cbe7cbf534f18b6b6 GIT binary patch literal 526 zcmV+p0`dJHiwFP!000001GQ4yZrd;necxwz-LQu=kmXD40Rt8^#V$ZESk?_iQ3RAk zCrWM0k>s{$=8xI0?8_!OLAy&E>}5|jd8os~L#~6F6v69tu&4YtNV7zcBn!hr#Dq~u z3S>Z15_E&0FR*BAFbHa>&_xrhl#8$eS-McT@U3dF=5rWwRXIrMIn?qFtW@>GbFQ(^ zOu%{?N@zk-nua8zBy@PCge)ZAStDzUx`xuLHx_Jb{9iBf)8Tk>?n(q2u4S|B?GsOM zU86ReqGU*hWPf@*KApZl`Rt{ts*=_rF{YFu6uID#vPi@%&3PV&qF`a3iI@ZAIRpOS z&PIlFPUBn}t9Oo5g0V;a*eD*ufpgW?*3XEyjr@}={jInGV{VidPgN%*r}00wcKQpq zUKC?%v1)8@^IqLn4D|;uDCn;PTD!p7ZkKkyij&jB literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/user_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/user_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/workflow.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/workflow.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/workflow_archive.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/workflow_archive.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/workflow_deleted.json.gz b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/mongodb/test/workflow_deleted.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..229151a5a27ab0cc4661f529cc0eda27e3c03e10 GIT binary patch literal 20 Rcmb2|=3oE=W@ZQtBmoVe0J#7F literal 0 HcmV?d00001 From f56cf9ee0353b86331e7373db093ddf90ebaf7be Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 26 Jun 2025 12:57:09 +0200 Subject: [PATCH 06/13] app: add migration tests, #TASK-7756 --- .../catalog/ClinicalMigrationTask7756.java | 14 +- .../app/migrations/MigrationTask7756Test.java | 182 ++++++++++++++++++ .../opencga/v5.0.0-task-7756/README.txt | 59 ++++++ 3 files changed, 248 insertions(+), 7 deletions(-) create mode 100644 opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationTask7756Test.java create mode 100644 opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/README.txt diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/ClinicalMigrationTask7756.java b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/ClinicalMigrationTask7756.java index 847b6acf44a..7872e748be9 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/ClinicalMigrationTask7756.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/migrations/v5/v5_0_0/catalog/ClinicalMigrationTask7756.java @@ -83,10 +83,10 @@ private void migrateStudyConfiguration() throws CatalogDBException { private void migrateClinicalAnalysisCvdbIndex() throws CatalogDBException { // ClinicalAnalysis // - internal.cvdbIndex - // + internal.cvdb: { - // status: {}, - // job: "" - // } + // + internal.cvdbIndex: { + // status: // old internal.cvdbIndex + // jobId: "" + // } logger.info("Starting migration of clinical analysis CVDB index"); @@ -97,7 +97,7 @@ private void migrateClinicalAnalysisCvdbIndex() throws CatalogDBException { ); logger.debug("Processing clinical collections: {}", clinicalCollections); - migrateCollection(clinicalCollections, Filters.exists("internal.cvdb", false), Projections.include("_id", "internal"), (document, bulk) -> { + migrateCollection(clinicalCollections, Filters.exists("internal.cvdbIndex.jobId", false), Projections.include("_id", "internal"), (document, bulk) -> { MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument(); Document internalDoc = document.get("internal", Document.class); @@ -106,12 +106,12 @@ private void migrateClinicalAnalysisCvdbIndex() throws CatalogDBException { } else { Document cvdbIndexStatus = internalDoc.get("cvdbIndex", Document.class); if (cvdbIndexStatus == null) { - updateDocument.getSet().put("internal.cvdb", convertToDocument(CvdbIndex.init())); + updateDocument.getSet().put("internal.cvdbIndex", convertToDocument(CvdbIndex.init())); } else { Document cvdbIndex = new Document() .append("jobId", "") .append("status", cvdbIndexStatus); - updateDocument.getSet().put("internal.cvdb", cvdbIndex); + updateDocument.getSet().put("internal.cvdbIndex", cvdbIndex); } } diff --git a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationTask7756Test.java b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationTask7756Test.java new file mode 100644 index 00000000000..a7688d2ff2c --- /dev/null +++ b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationTask7756Test.java @@ -0,0 +1,182 @@ +package org.opencb.opencga.app.migrations; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Before; +import org.junit.Test; +import org.opencb.commons.datastore.core.ObjectMap; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.opencga.app.migrations.v5.v5_0_0.catalog.ClinicalMigrationTask7756; +import org.opencb.opencga.catalog.db.mongodb.MongoBackupUtils; +import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.managers.CatalogManager; +import org.opencb.opencga.catalog.managers.CatalogManagerExternalResource; +import org.opencb.opencga.catalog.migration.Migration; +import org.opencb.opencga.catalog.migration.MigrationTool; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; +import org.opencb.opencga.core.models.family.Family; +import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.sample.Sample; +import org.opencb.opencga.core.models.study.CatalogStudyConfiguration; + +import java.net.URL; +import java.nio.file.Paths; + +import static org.junit.Assert.*; + +public class MigrationTask7756Test { + + private static boolean initialized = false; + private static CatalogManagerExternalResource externalResource; + private static CatalogManager catalogManager; + private static String token; + + @Before + public void setup() throws Exception { + if (initialized) { + return; + } + + externalResource = new CatalogManagerExternalResource(); + externalResource.before(); + + URL resource = getClass().getResource("/datasets/opencga/v5.0.0-task-7756/"); + MongoBackupUtils.restore(externalResource.getCatalogManager(), externalResource.getOpencgaHome(), + Paths.get(resource.toURI()).resolve("mongodb")); + + // Setup catalog manager with test database + catalogManager = externalResource.getCatalogManager(); + + token = catalogManager.getUserManager().login("test", "test", "Test_P4ss").first().getToken(); + + // Run migration + runMigration(ClinicalMigrationTask7756.class); + + initialized = true; + } + + private void runMigration(Class migration) throws CatalogException { + Migration annotation = migration.getAnnotation(Migration.class); + catalogManager.getMigrationManager() + .runManualMigration(annotation.version(), annotation.id(), externalResource.getOpencgaHome(), new ObjectMap(), + externalResource.getAdminToken()); + } + + @Test + public void testIndividualQCMigration() throws CatalogException { + // Test Individual 1 (with QC data) + Individual individual = catalogManager.getIndividualManager().get("testStudy", "ind1", QueryOptions.empty(), token).first(); + + assertNotNull(individual); + assertNotNull(individual.getQualityControl()); + assertNotNull(individual.getQualityControl().getInferredSexReports()); + assertEquals(1, individual.getQualityControl().getInferredSexReports().size()); + assertEquals("CoverageRatio", individual.getQualityControl().getInferredSexReports().get(0).getMethod()); + + assertNotNull(individual.getQualityControl().getMendelianErrorReports()); + assertEquals(1, individual.getQualityControl().getMendelianErrorReports().size()); + assertEquals(0, individual.getQualityControl().getMendelianErrorReports().get(0).getNumErrors()); + + // Test Individual 2 (without QC data) + Individual individual2 = catalogManager.getIndividualManager().get("testStudy", "ind2", QueryOptions.empty(), token).first(); + + assertNotNull(individual2); + // After migration, should have an empty QC object instead of null + assertNotNull(individual2.getQualityControl()); + } + + @Test + public void testFamilyQCMigration() throws CatalogException { + // Test Family 1 (with QC data) + Family family = catalogManager.getFamilyManager().get("testStudy", "fam1", QueryOptions.empty(), token).first(); + + assertNotNull(family); + assertNotNull(family.getQualityControl()); + assertNotNull(family.getQualityControl().getRelatedness()); + assertEquals(1, family.getQualityControl().getRelatedness().size()); + assertEquals("PLINK/IBD", family.getQualityControl().getRelatedness().get(0).getMethod()); + + // Test Family 2 (without QC data) + Family family2 = catalogManager.getFamilyManager().get("testStudy", "fam2", QueryOptions.empty(), token).first(); + + assertNotNull(family2); + // After migration, should have an empty QC object instead of null + assertNotNull(family2.getQualityControl()); + assertTrue(family2.getQualityControl().getRelatedness().isEmpty()); + } + + @Test + public void testSampleQCMigration() throws CatalogException { + // Test Sample 1 (with variant QC data) + Sample sample1 = catalogManager.getSampleManager().get("testStudy", "sample1", QueryOptions.empty(), token).first(); + + assertNotNull(sample1); + assertNotNull(sample1.getQualityControl()); + assertNotNull(sample1.getQualityControl().getVariant()); + assertNotNull(sample1.getQualityControl().getVariant().getVariantStats()); + assertEquals(1, sample1.getQualityControl().getVariant().getVariantStats().size()); + assertEquals("stat1", sample1.getQualityControl().getVariant().getVariantStats().get(0).getId()); + + assertNotNull(sample1.getQualityControl().getVariant().getSignatures()); + assertEquals(1, sample1.getQualityControl().getVariant().getSignatures().size()); + assertEquals("sig1", sample1.getQualityControl().getVariant().getSignatures().get(0).getId()); + + assertNotNull(sample1.getQualityControl().getVariant().getGenomePlot()); + assertEquals("plot1", sample1.getQualityControl().getVariant().getGenomePlot().getId()); + + // Test Sample 2 (with files QC data) + Sample sample2 = catalogManager.getSampleManager().get("testStudy", "sample2", QueryOptions.empty(), token).first(); + + assertNotNull(sample2); + assertNotNull(sample2.getQualityControl()); + assertNotNull(sample2.getQualityControl().getFiles()); + assertEquals(1, sample2.getQualityControl().getFiles().size()); + assertEquals("file1", sample2.getQualityControl().getFiles().get(0)); + + // Test Sample 3 (without QC data) + Sample sample3 = catalogManager.getSampleManager().get("testStudy", "sample3", QueryOptions.empty(), token).first(); + + assertNotNull(sample3); + // After migration, should have an empty QC object instead of null + assertNotNull(sample3.getQualityControl()); + } + + @Test + public void testClinicalAnalysisCvdbIndexMigration() throws CatalogException { + // Test Clinical Analysis 1 (with cvdbIndex) + ClinicalAnalysis clinicalAnalysis = catalogManager.getClinicalAnalysisManager().get("testStudy", "case1", QueryOptions.empty(), token).first(); + + assertNotNull(clinicalAnalysis); + assertNotNull(clinicalAnalysis.getInternal().getCvdbIndex().getStatus()); + assertTrue(StringUtils.isEmpty(clinicalAnalysis.getInternal().getCvdbIndex().getJobId())); + } + + @Test + public void testStudyConfigurationMigration() throws CatalogException { + // Get a study after migration + org.opencb.opencga.core.models.study.Study study = catalogManager.getStudyManager().get("testStudy", new QueryOptions(QueryOptions.INCLUDE, "internal"), token).first(); + + assertNotNull("Study should not be null", study); + assertNotNull("Study internal should not be null", study.getInternal()); + assertNotNull("Study configuration should not be null", study.getInternal().getConfiguration()); + assertNotNull("Study catalog configuration should not be null", study.getInternal().getConfiguration().getCatalog()); + + // Verify the CVDB configuration exists + assertNotNull("CVDB configuration should not be null", + study.getInternal().getConfiguration().getCatalog().getCvdb()); + + // Verify the Variant Quality Control configuration exists + assertNotNull("Variant Quality Control configuration should not be null", + study.getInternal().getConfiguration().getCatalog().getVariantQualityControl()); + + // Check that default configuration was applied correctly by checking + // that defaults match CatalogStudyConfiguration.defaultConfiguration() + CatalogStudyConfiguration defaultConfig = CatalogStudyConfiguration.defaultConfiguration(); + assertEquals("CVDB configuration should match default", + defaultConfig.getCvdb().isActive(), + study.getInternal().getConfiguration().getCatalog().getCvdb().isActive()); + assertEquals("Variant Quality Control configuration should match default", + defaultConfig.getVariantQualityControl().isActive(), + study.getInternal().getConfiguration().getCatalog().getVariantQualityControl().isActive()); + } + +} diff --git a/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/README.txt b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/README.txt new file mode 100644 index 00000000000..bc2219d7acc --- /dev/null +++ b/opencga-app/src/test/resources/datasets/opencga/v5.0.0-task-7756/README.txt @@ -0,0 +1,59 @@ +# Commands executed to generate the dataset + +padmin="4dMiNiStR4t0R." +puser="Test_P4ss" +./opencga-admin.sh catalog install --force +echo "$padmin" | ./opencga-admin.sh catalog install --force +echo "$padmin" | ./opencga.sh users login -u opencga -p +vim ../conf/client-configuration.yml +echo "$padmin" | ./opencga.sh users login -u opencga -p +./opencga.sh organizations create --id test +./opencga.sh users create --email demo@opencga.com --id test --name Demo --password $puser --organization test +./opencga.sh organizations update --organization test --owner test +echo "$puser" | ./opencga.sh users login -u test -p + +# Create project and study +./opencga.sh projects create --id testProject --name "Test Project" --description "Migration test" --organism-assembly GRCh38 --organism-scientific-name "Homo sapiens" +./opencga.sh studies create --id testStudy --name "Test Study" --project testProject + +# Create individuals with different quality control configurations +# Individual 1: With QC data +./opencga.sh individuals create --id ind1 --name "Individual1" --sex-id MALE +echo '{"qualityControl": {"inferredSexReports":[{"method":"CoverageRatio"}], "mendelianErrorReports":[{"numErrors":0}]}}' > ind_qc_update.json +./opencga.sh individuals update --individuals ind1 --json-file ind_qc_update.json +rm ind_qc_update.json + +# Individual 2: Without QC data +./opencga.sh individuals create --id ind2 --name "Individual2" --sex-id FEMALE + +# Create families with different quality control configurations +# Family 1: With QC data +./opencga.sh individuals create --id father --name "Father" --sex-id MALE +./opencga.sh individuals create --id mother --name "Mother" --sex-id FEMALE +./opencga.sh individuals create --id child --name "Child" --sex-id MALE --father-id father --mother-id mother +./opencga.sh families create --id fam1 --name "Family1" --members father,mother,child +echo '{"qualityControl": {"relatedness":[{"method":"PLINK/IBD"}]}}' > fam_qc_update.json +./opencga.sh families update --families fam1 --json-file fam_qc_update.json +rm fam_qc_update.json + +# Family 2: Without QC data +./opencga.sh families create --id fam2 --name "Family2" + +# Create samples with different quality control configurations +# Sample 1: With variant QC data +./opencga.sh samples create --id sample1 --individual-id ind1 +echo '{"qualityControl": {"variant":{"variantStats":[{"id":"stat1"}],"signatures":[{"id":"sig1"}],"genomePlot":{"id": "plot1"}}}}' > sam_qc_update.json +./opencga.sh samples update --samples sample1 --json-file sam_qc_update.json + +# Sample 2: With files QC data +./opencga.sh samples create --id sample2 --individual-id ind1 +echo '{"qualityControl": {"files": ["file1"]}}' > sam_qc_update.json +./opencga.sh samples update --samples sample2 --json-file sam_qc_update.json +rm sam_qc_update.json + +# Sample 3: Without QC data +./opencga.sh samples create --id sample3 --individual-id ind2 + +# Create clinical analysis with different cvdbIndex configurations +# Clinical Analysis 1: With cvdbIndex +./opencga.sh clinical create --id case1 --type SINGLE --proband-id ind1 \ No newline at end of file From a65232dd6bc33e516002e9e04a364b1401c95e51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20Furi=C3=B3?= Date: Thu, 26 Jun 2025 14:45:42 +0200 Subject: [PATCH 07/13] core: Update opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Pedro Furió --- .../opencga/core/models/study/CatalogStudyConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java index 472c4a6461c..50078a1583b 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java @@ -22,7 +22,7 @@ public static CatalogStudyConfiguration defaultConfiguration() { @Override public String toString() { - final StringBuilder sb = new StringBuilder("StudyCatalogConfiguration{"); + final StringBuilder sb = new StringBuilder("CatalogStudyConfiguration{"); sb.append("cvdb=").append(cvdb); sb.append(", variantQualityControl=").append(variantQualityControl); sb.append('}'); From 5216fd3d1cef372fea3672c6f511e857c9e4b3c8 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 26 Jun 2025 15:12:30 +0200 Subject: [PATCH 08/13] core: apply copilot suggestions, #TASK-7556 --- .../models/common/QualityControlStatus.java | 5 ---- .../core/models/family/FamilyInternal.java | 1 - .../core/models/sample/SampleInternal.java | 5 ++-- .../monitor/daemons/CatalogService.java | 27 ------------------- 4 files changed, 3 insertions(+), 35 deletions(-) delete mode 100644 opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/CatalogService.java diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java index 15791e3f89b..9d31bc5b98f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java @@ -28,11 +28,6 @@ public QualityControlStatus() { this(PENDING, ""); } - @Override - public String getId() { - return super.getId(); - } - public static QualityControlStatus init() { return new QualityControlStatus(); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyInternal.java index 5c5606ee5c2..964b86a9144 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/family/FamilyInternal.java @@ -45,7 +45,6 @@ public String toString() { final StringBuilder sb = new StringBuilder("FamilyInternal{"); sb.append("status=").append(status); sb.append(", qualityControlStatus=").append(qualityControlStatus); - sb.append(", status=").append(status); sb.append(", registrationDate='").append(registrationDate).append('\''); sb.append(", lastModified='").append(lastModified).append('\''); sb.append('}'); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java index b209ee43e6a..0e49ab953e4 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/sample/SampleInternal.java @@ -130,11 +130,12 @@ public boolean equals(Object o) { return false; } SampleInternal that = (SampleInternal) o; - return Objects.equals(rga, that.rga) && Objects.equals(variant, that.variant); + return Objects.equals(rga, that.rga) && Objects.equals(variant, that.variant) + && Objects.equals(qualityControlStatus, that.qualityControlStatus); } @Override public int hashCode() { - return Objects.hash(rga, variant); + return Objects.hash(rga, variant, qualityControlStatus); } } diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/CatalogService.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/CatalogService.java deleted file mode 100644 index de2f4d75004..00000000000 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/CatalogService.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.opencb.opencga.master.monitor.daemons; - -import org.opencb.opencga.catalog.managers.CatalogManager; - -import java.io.Closeable; -import java.io.IOException; - -public class CatalogService extends MonitorParentDaemon implements Closeable { - - - public CatalogService(int interval, String token, CatalogManager catalogManager) { - super(interval, token, catalogManager); - } - - @Override - public void apply() throws Exception { - - } - - private void checkCVDB() { - } - - @Override - public void close() throws IOException { - - } -} From c0a24a3fa635633302407fdf46e7e54c28ee76c7 Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 26 Jun 2025 15:38:09 +0200 Subject: [PATCH 09/13] catalog: add new indexes, #TASK-7756 --- opencga-catalog/src/main/resources/catalog-indexes.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opencga-catalog/src/main/resources/catalog-indexes.txt b/opencga-catalog/src/main/resources/catalog-indexes.txt index eab521fc907..f51ee1de872 100644 --- a/opencga-catalog/src/main/resources/catalog-indexes.txt +++ b/opencga-catalog/src/main/resources/catalog-indexes.txt @@ -105,6 +105,7 @@ {"collections": ["sample", "sample_archive"], "fields": {"internal.variant.index.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["sample", "sample_archive"], "fields": {"internal.rga.status": 1, "studyUid": 1}, "options": {}} +{"collections": ["sample", "sample_archive"], "fields": {"internal.qualityControlStatus.id": 1, "studyUid": 1}, "options": {}} {"collections": ["individual", "individual_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["individual", "individual_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} @@ -138,6 +139,7 @@ {"collections": ["individual", "individual_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} {"collections": ["individual", "individual_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["individual", "individual_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["individual", "individual_archive"], "fields": {"internal.qualityControlStatus.id": 1, "studyUid": 1}, "options": {}} {"collections": ["cohort"], "fields": {"uuid": 1}, "options": {"unique": true}} {"collections": ["cohort"], "fields": {"uid": 1}, "options": {"unique": true}} @@ -182,6 +184,7 @@ {"collections": ["family", "family_archive"], "fields": {"_ias.id": 1, "_ias.value": 1, "studyUid": 1}, "options": {}} {"collections": ["family", "family_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["family", "family_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["family", "family_archive"], "fields": {"internal.qualityControlStatus.id": 1, "studyUid": 1}, "options": {}} {"collections": ["panel", "panel_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["panel", "panel_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} @@ -230,6 +233,7 @@ {"collections": ["clinical", "clinical_archive"], "fields": {"studyUid": 1, "_acl": 1}, "options": {}} {"collections": ["clinical", "clinical_archive"], "fields": {"status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["clinical", "clinical_archive"], "fields": {"internal.status.id": 1, "studyUid": 1}, "options": {}} +{"collections": ["clinical", "clinical_archive"], "fields": {"internal.cvdbIndex.status.id": 1, "studyUid": 1}, "options": {}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uuid": 1, "version": 1}, "options": {"unique": true}} {"collections": ["interpretation", "interpretation_archive"], "fields": {"uid": 1, "version": 1}, "options": {"unique": true}} From 8aed10123bbe8036d0ab14ae20f53a066fbc858d Mon Sep 17 00:00:00 2001 From: pfurio Date: Thu, 26 Jun 2025 15:55:06 +0200 Subject: [PATCH 10/13] app: check migration idempotent, #TASK-7756 --- .../opencga/app/migrations/MigrationTask7756Test.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationTask7756Test.java b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationTask7756Test.java index a7688d2ff2c..df6ddf2ac19 100644 --- a/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationTask7756Test.java +++ b/opencga-app/src/test/java/org/opencb/opencga/app/migrations/MigrationTask7756Test.java @@ -55,9 +55,13 @@ public void setup() throws Exception { } private void runMigration(Class migration) throws CatalogException { + runMigration(migration, false); + } + + private void runMigration(Class migration, boolean force) throws CatalogException { Migration annotation = migration.getAnnotation(Migration.class); catalogManager.getMigrationManager() - .runManualMigration(annotation.version(), annotation.id(), externalResource.getOpencgaHome(), new ObjectMap(), + .runManualMigration(annotation.version(), annotation.id(), externalResource.getOpencgaHome(), force, false, new ObjectMap(), externalResource.getAdminToken()); } @@ -179,4 +183,9 @@ public void testStudyConfigurationMigration() throws CatalogException { study.getInternal().getConfiguration().getCatalog().getVariantQualityControl().isActive()); } + @Test + public void idempotentMigration() throws CatalogException { + runMigration(ClinicalMigrationTask7756.class, true); + } + } From 1c88d406cfa22d8c00548945982bbdd30db99720 Mon Sep 17 00:00:00 2001 From: pfurio Date: Tue, 1 Jul 2025 16:44:40 +0200 Subject: [PATCH 11/13] catalog: add automatic QC statuses, #TASK-7756 --- .../db/api/ClinicalAnalysisDBAdaptor.java | 1 + .../catalog/db/api/FamilyDBAdaptor.java | 1 + .../catalog/db/api/IndividualDBAdaptor.java | 1 + .../catalog/db/api/SampleDBAdaptor.java | 1 + .../ClinicalAnalysisMongoDBAdaptor.java | 3 +- .../db/mongodb/FamilyMongoDBAdaptor.java | 2 +- .../db/mongodb/IndividualMongoDBAdaptor.java | 3 +- .../db/mongodb/SampleMongoDBAdaptor.java | 3 +- .../managers/ClinicalAnalysisManager.java | 19 ++- .../catalog/managers/FamilyManager.java | 47 +++++++ .../catalog/managers/IndividualManager.java | 88 ++++++++++++- .../catalog/managers/SampleManager.java | 124 +++++++++++++----- .../core/models/clinical/CvdbIndex.java | 16 ++- .../models/common/QualityControlStatus.java | 14 +- .../study/CatalogServiceConfiguration.java | 6 +- .../study/CatalogStudyConfiguration.java | 20 ++- .../CvdbCatalogServiceConfiguration.java | 47 +++++++ .../QualityControlServiceConfiguration.java | 53 ++++++++ .../monitor/daemons/MonitorParentDaemon.java | 21 +++ 19 files changed, 404 insertions(+), 66 deletions(-) create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/study/CvdbCatalogServiceConfiguration.java create mode 100644 opencga-core/src/main/java/org/opencb/opencga/core/models/study/QualityControlServiceConfiguration.java diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java index 6b6d1813a0b..0a08150c929 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/ClinicalAnalysisDBAdaptor.java @@ -63,6 +63,7 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), INTERNAL_CVDB_INDEX("internal.cvdbIndex", OBJECT, ""), + INTERNAL_CVDB_INDEX_PREVIOUS_STATUS("internal.cvdbIndex.previousStatus", OBJECT, ""), INTERNAL_CVDB_INDEX_STATUS("internal.cvdbIndex.status", OBJECT, ""), INTERNAL_CVDB_INDEX_JOB_ID("internal.cvdbIndex.jobId", STRING, ""), INTERNAL_CVDB_INDEX_STATUS_ID("internal.cvdbIndex.status.id", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java index 6fa44461c01..355e30d8421 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FamilyDBAdaptor.java @@ -64,6 +64,7 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_MSG("internal.status.msg", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), + INTERNAL_QUALITY_CONTROL_STATUS("internal.qualityControlStatus", OBJECT, ""), INTERNAL_QUALITY_CONTROL_STATUS_ID("internal.qualityControlStatus.id", TEXT, ""), RELEASE("release", INTEGER, ""), SNAPSHOT("snapshot", INTEGER, ""), // Last version of individual at release = snapshot diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java index 73b1a084835..f90953c34b1 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/IndividualDBAdaptor.java @@ -68,6 +68,7 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_ID("internal.status.id", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), INTERNAL_RGA("internal.rga", OBJECT, ""), + INTERNAL_QUALITY_CONTROL_STATUS("internal.qualityControlStatus", OBJECT, ""), INTERNAL_QUALITY_CONTROL_STATUS_ID("internal.qualityControlStatus.id", TEXT, ""), POPULATION_NAME("population.name", TEXT, ""), POPULATION_SUBPOPULATION("population.subpopulation", TEXT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java index e88a2fc7910..7d1749b80d8 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/SampleDBAdaptor.java @@ -133,6 +133,7 @@ enum QueryParams implements QueryParam { INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX("internal.variant.secondarySampleIndex", TEXT_ARRAY, ""), INTERNAL_VARIANT_ANNOTATION_INDEX("internal.variant.annotationIndex", TEXT_ARRAY, ""), INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX("internal.variant.secondaryAnnotationIndex", TEXT_ARRAY, ""), + INTERNAL_QUALITY_CONTROL_STATUS("internal.qualityControlStatus", OBJECT, ""), INTERNAL_QUALITY_CONTROL_STATUS_ID("internal.qualityControlStatus.id", TEXT, ""), RELEASE("release", INTEGER, ""), // Release where the sample was created SNAPSHOT("snapshot", INTEGER, ""), // Last version of sample at release = snapshot diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index db520d4fa14..7bd76190bda 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -464,7 +464,8 @@ UpdateDocument parseAndValidateUpdateParams(ObjectMap parameters, List create(String studyStr, ClinicalAnalysis CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING_INDEX, "User '" + userId + "' created case with status '" + clinicalAnalysis.getStatus().getId() + "', which is of type" + " CLOSED. Automatically setting CVDB index status to " + CvdbIndexStatus.PENDING_INDEX); - clinicalAnalysis.getInternal().setCvdbIndex(new CvdbIndex("", cvdbIndexStatus)); + clinicalAnalysis.getInternal().getCvdbIndex().setStatus(cvdbIndexStatus); events.add(new Event(Event.Type.INFO, clinicalAnalysis.getId(), msg)); } @@ -1473,6 +1473,10 @@ private OpenCGAResult update(String organizationId, Study stud CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING_REMOVE, "User '" + userId + "' requested to remove the status '" + clinicalAnalysis.getStatus().getId() + "' of type " + ClinicalStatusValue.ClinicalStatusType.CLOSED + " to set it to '" + updateParams.getStatus().getId() + "'"); + // Update previous status with the current status + parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_PREVIOUS_STATUS.key(), + clinicalAnalysis.getInternal().getCvdbIndex().getStatus()); + // And set the new status parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_STATUS.key(), cvdbIndexStatus); } } @@ -1769,6 +1773,10 @@ private OpenCGAResult update(String organizationId, Study stud CvdbIndexStatus cvdbIndexStatus = new CvdbIndexStatus(CvdbIndexStatus.PENDING_INDEX, "User '" + userId + "' changed case to status '" + updateParamsClone.getStatus().getId() + "', which is of type" + " CLOSED. Automatically changing CVDB index status to " + CvdbIndexStatus.PENDING_INDEX); + // Update previous status with the current status + parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_PREVIOUS_STATUS.key(), + clinicalAnalysis.getInternal().getCvdbIndex().getStatus()); + // And set the new status parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_STATUS.key(), cvdbIndexStatus); } else if (clinicalAnalysis.getInternal().getCvdbIndex().getStatus().getId() .equals(CvdbIndexStatus.PENDING_REMOVE)) { @@ -1776,6 +1784,10 @@ private OpenCGAResult update(String organizationId, Study stud + "' changed case to status '" + updateParamsClone.getStatus().getId() + "', which is of type" + " CLOSED. CVDB index was already in " + CvdbIndexStatus.PENDING_REMOVE + ", so automatically" + " changing CVDB index status to " + CvdbIndexStatus.PENDING_OVERWRITE); + // Update previous status with the current status + parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_PREVIOUS_STATUS.key(), + clinicalAnalysis.getInternal().getCvdbIndex().getStatus()); + // And set the new status parameters.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_STATUS.key(), cvdbIndexStatus); } else { logger.warn("CVDB index status is unexpectedly set to '{}'. Although the user is closing the case, OpenCGA" @@ -1917,6 +1929,9 @@ public OpenCGAResult updateCvdbIndex(String studyFqn, ClinicalAnalysis clinic if (index.getStatus() != null) { validateCvdbStatusTransition(clinical, index.getStatus().getId()); ObjectMap valueAsMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(index.getStatus())); + ObjectMap previousStatus = new ObjectMap(getUpdateObjectMapper() + .writeValueAsString(clinical.getInternal().getCvdbIndex().getStatus())); + params.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_PREVIOUS_STATUS.key(), previousStatus); params.put(ClinicalAnalysisDBAdaptor.QueryParams.INTERNAL_CVDB_INDEX_STATUS.key(), valueAsMap); } if (StringUtils.isNotEmpty(index.getJobId())) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java index 06f3eb718ba..fb047e20001 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java @@ -48,6 +48,7 @@ import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.family.*; import org.opencb.opencga.core.models.individual.Individual; import org.opencb.opencga.core.models.individual.IndividualReferenceParam; @@ -67,6 +68,7 @@ import static org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager.checkPermissions; import static org.opencb.opencga.core.common.JacksonUtils.getDefaultObjectMapper; +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; /** * Created by pfurio on 02/05/17. @@ -1079,6 +1081,19 @@ private OpenCGAResult update(String organizationId, Study study, Family checkUpdateAnnotations(organizationId, study, family, parameters, options, VariableSet.AnnotableDataModels.FAMILY, getFamilyDBAdaptor(organizationId), userId); + if (updateParams != null && updateParams.getQualityControl() != null) { + QualityControlStatus qualityControlStatus = new QualityControlStatus(QualityControlStatus.READY); + ObjectMap valueAsObjectMap; + try { + valueAsObjectMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(qualityControlStatus)); + } catch (JsonProcessingException e) { + throw new CatalogException("Internal error serializing quality control status.\n" + e.getMessage(), e); + } + // If user is updating the quality control object, we set its status to READY + logger.info("Setting internal quality control status automatically to READY for family '{}'", family.getId()); + parameters.put(FamilyDBAdaptor.QueryParams.INTERNAL_QUALITY_CONTROL_STATUS.key(), valueAsObjectMap); + } + OpenCGAResult update = getFamilyDBAdaptor(organizationId).update(family.getUid(), parameters, study.getVariableSets(), options); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { @@ -1090,6 +1105,38 @@ private OpenCGAResult update(String organizationId, Study study, Family return update; } + public OpenCGAResult updateInternalQualityControlStatus(String studyFqn, Family family, QualityControlStatus qualityControlStatus, + String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = getStudyDBAdaptor(organizationId).get(family.getStudyUid(), StudyManager.INCLUDE_STUDY_IDS).first(); + + ObjectMap auditParams = new ObjectMap() + .append("studyFqn", studyFqn) + .append("family", family.getId()) + .append("internal.qualityControlStatus", qualityControlStatus) + .append("token", token); + + long studyId = study.getUid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); + + ObjectMap params; + try { + ObjectMap valueAsObjectMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(qualityControlStatus)); + params = new ObjectMap(FamilyDBAdaptor.QueryParams.INTERNAL_QUALITY_CONTROL_STATUS.key(), valueAsObjectMap); + } catch (JsonProcessingException e) { + throw new CatalogException("Cannot parse FamilyInternalQualityControlStatus object: " + e.getMessage(), e); + } + OpenCGAResult update = getFamilyDBAdaptor(organizationId).update(family.getUid(), params, QueryOptions.empty()); + auditManager.audit(organizationId, userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.FAMILY, family.getId(), + family.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + return new OpenCGAResult<>(update.getTime(), update.getEvents(), 1, Collections.emptyList(), 1); + } + public Map> calculateFamilyGenotypes(String studyStr, String clinicalAnalysisId, String familyId, ClinicalProperty.ModeOfInheritance moi, String disorderId, Penetrance penetrance, String token) throws CatalogException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java index 9fb42f7da22..63aa3313b6a 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/IndividualManager.java @@ -41,6 +41,7 @@ import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.common.AnnotationSet; import org.opencb.opencga.core.models.common.Enums; +import org.opencb.opencga.core.models.common.QualityControlStatus; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.individual.*; import org.opencb.opencga.core.models.sample.Sample; @@ -58,6 +59,7 @@ import java.util.stream.Collectors; import static org.opencb.opencga.catalog.auth.authorization.CatalogAuthorizationManager.checkPermissions; +import static org.opencb.opencga.core.common.JacksonUtils.getUpdateObjectMapper; /** * Created by hpccoll1 on 19/06/15. @@ -896,7 +898,7 @@ public OpenCGAResult update(String studyStr, Query query, Individual while (iterator.hasNext()) { Individual individual = iterator.next(); try { - OpenCGAResult updateResult = update(organizationId, study, individual, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, individual, updateParams, options, userId, token); result.append(updateResult); auditManager.auditUpdate(organizationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), @@ -956,7 +958,7 @@ public OpenCGAResult update(String studyStr, String individualId, In individualId = individual.getId(); individualUuid = individual.getUuid(); - OpenCGAResult updateResult = update(organizationId, study, individual, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, individual, updateParams, options, userId, token); result.append(updateResult); auditManager.auditUpdate(organizationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), @@ -1035,7 +1037,7 @@ public OpenCGAResult update(String studyStr, List individual individualId = individual.getId(); individualUuid = individual.getUuid(); - OpenCGAResult updateResult = update(organizationId, study, individual, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, individual, updateParams, options, userId, token); result.append(updateResult); auditManager.auditUpdate(organizationId, userId, Enums.Resource.INDIVIDUAL, individual.getId(), individual.getUuid(), @@ -1056,7 +1058,7 @@ public OpenCGAResult update(String studyStr, List individual } private OpenCGAResult update(String organizationId, Study study, Individual individual, IndividualUpdateParams updateParams, - QueryOptions options, String userId) throws CatalogException { + QueryOptions options, String userId, String token) throws CatalogException { ObjectMap parameters = new ObjectMap(); if (updateParams != null) { try { @@ -1173,8 +1175,53 @@ private OpenCGAResult update(String organizationId, Study study, Individual indi checkUpdateAnnotations(organizationId, study, individual, parameters, options, VariableSet.AnnotableDataModels.INDIVIDUAL, getIndividualDBAdaptor(organizationId), userId); + if (updateParams != null && updateParams.getQualityControl() != null) { + QualityControlStatus qualityControlStatus = new QualityControlStatus(QualityControlStatus.READY); + ObjectMap valueAsObjectMap; + try { + valueAsObjectMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(qualityControlStatus)); + } catch (JsonProcessingException e) { + throw new CatalogException("Internal error serializing quality control status.\n" + e.getMessage(), e); + } + // If user is updating the quality control object, we set its status to READY + logger.info("Setting internal quality control status automatically to READY for individual '{}'", individual.getId()); + parameters.put(IndividualDBAdaptor.QueryParams.INTERNAL_QUALITY_CONTROL_STATUS.key(), valueAsObjectMap); + } + OpenCGAResult update = getIndividualDBAdaptor(organizationId).update(individual.getUid(), parameters, study.getVariableSets(), options); + + if (updateParams != null && updateParams.getQualityControl() != null) { + // Check if individual belongs to any family and the quality control status is not ready + Query query = new Query() + .append(FamilyDBAdaptor.QueryParams.MEMBER_UID.key(), individual.getUid()) + .append(FamilyDBAdaptor.QueryParams.INTERNAL_QUALITY_CONTROL_STATUS_ID.key(), + Arrays.asList(QualityControlStatus.NONE, QualityControlStatus.ERROR)); + OpenCGAResult search = catalogManager.getFamilyManager().search(study.getFqn(), query, + FamilyManager.INCLUDE_FAMILY_MEMBERS, token); + if (search.getNumResults() > 0) { + // Check if all individual QC status are set to READY for each family. If so, we can set the family QC status to PENDING + // so it is processed by the Quality Control Manager. + for (Family family : search.getResults()) { + boolean calculateQC = true; + for (Individual member : family.getMembers()) { + if (member.getInternal() == null || member.getInternal().getQualityControlStatus() == null + || !QualityControlStatus.READY.equals(member.getInternal().getQualityControlStatus().getId())) { + calculateQC = false; + break; + } + } + if (calculateQC) { + logger.info("Setting internal quality control status automatically to PENDING for family '{}'", family.getId()); + // All members have a READY QC status, so we can set the family QC status to PENDING so it is processed + QualityControlStatus qualityControlStatus = new QualityControlStatus(QualityControlStatus.PENDING); + catalogManager.getFamilyManager().updateInternalQualityControlStatus( + study.getFqn(), family, qualityControlStatus, token); + } + } + } + } + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated individual OpenCGAResult result = getIndividualDBAdaptor(organizationId).get(study.getUid(), @@ -1184,6 +1231,39 @@ private OpenCGAResult update(String organizationId, Study study, Individual indi return update; } + public OpenCGAResult updateInternalQualityControlStatus(String studyFqn, Individual individual, + QualityControlStatus qualityControlStatus, String token) + throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = getStudyDBAdaptor(organizationId).get(individual.getStudyUid(), StudyManager.INCLUDE_STUDY_IDS).first(); + + ObjectMap auditParams = new ObjectMap() + .append("studyFqn", studyFqn) + .append("individual", individual.getId()) + .append("internal.qualityControlStatus", qualityControlStatus) + .append("token", token); + + long studyId = study.getUid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); + + ObjectMap params; + try { + ObjectMap valueAsObjectMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(qualityControlStatus)); + params = new ObjectMap(IndividualDBAdaptor.QueryParams.INTERNAL_QUALITY_CONTROL_STATUS.key(), valueAsObjectMap); + } catch (JsonProcessingException e) { + throw new CatalogException("Cannot parse IndividualInternalQualityControlStatus object: " + e.getMessage(), e); + } + OpenCGAResult update = getIndividualDBAdaptor(organizationId).update(individual.getUid(), params, QueryOptions.empty()); + auditManager.audit(organizationId, userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.INDIVIDUAL, individual.getId(), + individual.getUuid(), study.getId(), study.getUuid(), auditParams, + new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + return new OpenCGAResult<>(update.getTime(), update.getEvents(), 1, Collections.emptyList(), 1); + } + @Override public OpenCGAResult rank(String studyStr, Query query, String field, int numResults, boolean asc, String token) throws CatalogException { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java index 7c2b6112b5c..402e286c538 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/SampleManager.java @@ -42,10 +42,7 @@ import org.opencb.opencga.core.models.audit.AuditRecord; import org.opencb.opencga.core.models.cohort.Cohort; import org.opencb.opencga.core.models.cohort.CohortStatus; -import org.opencb.opencga.core.models.common.AnnotationSet; -import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.common.ExternalSource; -import org.opencb.opencga.core.models.common.RgaIndex; +import org.opencb.opencga.core.models.common.*; import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.file.FileInternal; @@ -796,38 +793,67 @@ public OpenCGAResult updateRgaIndexes(String studyStr, List samp return result; } + public OpenCGAResult updateSampleInternalQualityControlStatus(String studyFqn, Sample sample, + QualityControlStatus qualityControlStatus, String token) + throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + String userId = tokenPayload.getUserId(organizationId); + Study study = getStudyDBAdaptor(organizationId).get(sample.getStudyUid(), StudyManager.INCLUDE_STUDY_IDS).first(); + + ObjectMap auditParams = new ObjectMap() + .append("studyFqn", studyFqn) + .append("sample", sample.getId()) + .append("internal.qualityControlStatus", qualityControlStatus) + .append("token", token); + + long studyId = study.getUid(); + authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); + + ObjectMap params; + try { + ObjectMap valueAsObjectMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(qualityControlStatus)); + params = new ObjectMap(SampleDBAdaptor.QueryParams.INTERNAL_QUALITY_CONTROL_STATUS.key(), valueAsObjectMap); + } catch (JsonProcessingException e) { + throw new CatalogException("Cannot parse SampleInternalQualityControlStatus object: " + e.getMessage(), e); + } + OpenCGAResult update = getSampleDBAdaptor(organizationId).update(sample.getUid(), params, QueryOptions.empty()); + auditManager.audit(organizationId, userId, Enums.Action.UPDATE_INTERNAL, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), + study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); + + return new OpenCGAResult<>(update.getTime(), update.getEvents(), 1, Collections.emptyList(), 1); + } + public OpenCGAResult updateSampleInternalVariantIndex(String studyFqn, Sample sample, SampleInternalVariantIndex index, String token) throws CatalogException { - return updateSampleInternalVariant(studyFqn, sample, index, SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_INDEX.key(), token); + SampleInternalVariant sampleInternalVariant = new SampleInternalVariant().setIndex(index); + return updateSampleInternalVariant(studyFqn, sample, sampleInternalVariant, token); } public OpenCGAResult updateSampleInternalVariantSecondarySampleIndex(String studyFqn, Sample sample, SampleInternalVariantSecondarySampleIndex index, String token) throws CatalogException { - return updateSampleInternalVariant(studyFqn, sample, index, Arrays.asList( - SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_SECONDARY_SAMPLE_INDEX.key(), - SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_GENOTYPE_INDEX.key()), token); + SampleInternalVariant sampleInternalVariant = new SampleInternalVariant() + .setSecondarySampleIndex(index) + .setSampleGenotypeIndex(index); + return updateSampleInternalVariant(studyFqn, sample, sampleInternalVariant, token); } public OpenCGAResult updateSampleInternalVariantAnnotationIndex( String studyFqn, Sample sample, SampleInternalVariantAnnotationIndex index, String token) throws CatalogException { - return updateSampleInternalVariant(studyFqn, sample, index, - SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_ANNOTATION_INDEX.key(), token); + SampleInternalVariant sampleInternalVariant = new SampleInternalVariant().setAnnotationIndex(index); + return updateSampleInternalVariant(studyFqn, sample, sampleInternalVariant, token); } public OpenCGAResult updateSampleInternalVariantSecondaryAnnotationIndex(String studyFqn, Sample sample, SampleInternalVariantSecondaryAnnotationIndex index, String token) throws CatalogException { - return updateSampleInternalVariant(studyFqn, sample, index, - SampleDBAdaptor.QueryParams.INTERNAL_VARIANT_SECONDARY_ANNOTATION_INDEX.key(), token); + SampleInternalVariant sampleInternalVariant = new SampleInternalVariant().setSecondaryAnnotationIndex(index); + return updateSampleInternalVariant(studyFqn, sample, sampleInternalVariant, token); } - private OpenCGAResult updateSampleInternalVariant(String studyFqn, Sample sample, Object value, String fieldKey, String token) - throws CatalogException { - return updateSampleInternalVariant(studyFqn, sample, value, Collections.singletonList(fieldKey), token); - } - - private OpenCGAResult updateSampleInternalVariant(String studyFqn, Sample sample, Object value, List fieldKeys, + private OpenCGAResult updateSampleInternalVariant(String studyFqn, Sample sample, SampleInternalVariant internalVariant, String token) throws CatalogException { JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyFqn, tokenPayload); @@ -838,10 +864,8 @@ private OpenCGAResult updateSampleInternalVariant(String studyFqn, Sample sam ObjectMap auditParams = new ObjectMap() .append("studyFqn", studyFqn) .append("sample", sample.getId()) + .append("internal.variant", internalVariant) .append("token", token); - for (String fieldKey : fieldKeys) { - auditParams.append(fieldKey, value); - } long studyId = study.getUid(); authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, studyId, userId); @@ -849,9 +873,16 @@ private OpenCGAResult updateSampleInternalVariant(String studyFqn, Sample sam ObjectMap params; try { params = new ObjectMap(); - ObjectMap valueAsObjectMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(value)); - for (String fieldKey : fieldKeys) { - params.append(fieldKey, valueAsObjectMap); + ObjectMap valueAsObjectMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(internalVariant)); + for (Map.Entry entry : valueAsObjectMap.entrySet()) { + params.put(SampleDBAdaptor.QueryParams.INTERNAL_VARIANT.key() + "." + entry.getKey(), entry.getValue()); + } + if (internalVariant.getIndex() != null && internalVariant.getIndex().getStatus() != null + && IndexStatus.READY.equals(internalVariant.getIndex().getStatus().getId())) { + // If the index is ready, update QualityControl status to PENDING so it can be calculated + QualityControlStatus qcStatus = new QualityControlStatus(QualityControlStatus.PENDING); + ObjectMap qcStatusAsMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(qcStatus)); + params.put(SampleDBAdaptor.QueryParams.INTERNAL_QUALITY_CONTROL_STATUS.key(), qcStatusAsMap); } } catch (JsonProcessingException e) { throw new CatalogException("Cannot parse SampleInternalVariant object: " + e.getMessage(), e); @@ -1052,7 +1083,7 @@ public OpenCGAResult update(String studyStr, Query query, SampleUpdatePa while (iterator.hasNext()) { Sample sample = iterator.next(); try { - OpenCGAResult updateResult = update(organizationId, study, sample, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, sample, updateParams, options, userId, token); result.append(updateResult); auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), @@ -1109,7 +1140,7 @@ public OpenCGAResult update(String studyStr, String sampleId, SampleUpda sampleId = sample.getId(); sampleUuid = sample.getUuid(); - OpenCGAResult updateResult = update(organizationId, study, sample, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, sample, updateParams, options, userId, token); result.append(updateResult); auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), @@ -1188,7 +1219,7 @@ public OpenCGAResult update(String studyStr, List sampleIds, Sam sampleId = sample.getId(); sampleUuid = sample.getUuid(); - OpenCGAResult updateResult = update(organizationId, study, sample, updateParams, options, userId); + OpenCGAResult updateResult = update(organizationId, study, sample, updateParams, options, userId, token); result.append(updateResult); auditManager.auditUpdate(organizationId, operationId, userId, Enums.Resource.SAMPLE, sample.getId(), sample.getUuid(), @@ -1209,7 +1240,7 @@ public OpenCGAResult update(String studyStr, List sampleIds, Sam } private OpenCGAResult update(String organizationId, Study study, Sample sample, SampleUpdateParams updateParams, QueryOptions options, - String userId) throws CatalogException { + String userId, String token) throws CatalogException { options = ParamUtils.defaultObject(options, QueryOptions::new); SampleUpdateParams updateParamsClone; @@ -1253,7 +1284,7 @@ private OpenCGAResult update(String organizationId, Study study, Sample sample, // Check permissions... // Only check write annotation permissions if the user wants to update the annotation sets - if (updateParamsClone != null && updateParamsClone.getAnnotationSets() != null) { + if (updateParamsClone.getAnnotationSets() != null) { authorizationManager.checkSamplePermission(organizationId, study.getUid(), sample.getUid(), userId, SamplePermissions.WRITE_ANNOTATIONS); } @@ -1264,11 +1295,11 @@ private OpenCGAResult update(String organizationId, Study study, Sample sample, SamplePermissions.WRITE); } - if (updateParamsClone != null && updateParamsClone.getId() != null) { + if (updateParamsClone.getId() != null) { ParamUtils.checkIdentifier(updateParamsClone.getId(), SampleDBAdaptor.QueryParams.ID.key()); } - if (updateParamsClone != null && StringUtils.isNotEmpty(updateParamsClone.getIndividualId())) { + if (StringUtils.isNotEmpty(updateParamsClone.getIndividualId())) { // Check individual id exists OpenCGAResult individualDataResult = catalogManager.getIndividualManager().internalGet(organizationId, study.getUid(), updateParamsClone.getIndividualId(), IndividualManager.INCLUDE_INDIVIDUAL_IDS, userId); @@ -1283,8 +1314,39 @@ private OpenCGAResult update(String organizationId, Study study, Sample sample, checkUpdateAnnotations(organizationId, study, sample, parameters, options, VariableSet.AnnotableDataModels.SAMPLE, getSampleDBAdaptor(organizationId), userId); + if (updateParamsClone.getQualityControl() != null) { + QualityControlStatus qualityControlStatus = new QualityControlStatus(QualityControlStatus.READY); + ObjectMap valueAsObjectMap; + try { + valueAsObjectMap = new ObjectMap(getUpdateObjectMapper().writeValueAsString(qualityControlStatus)); + } catch (JsonProcessingException e) { + throw new CatalogException("Internal error serializing quality control status.\n" + e.getMessage(), e); + } + // If user is updating the quality control object, we set its status to READY + logger.info("Setting internal quality control status automatically to READY for sample '{}'", sample.getId()); + parameters.put(SampleDBAdaptor.QueryParams.INTERNAL_QUALITY_CONTROL_STATUS.key(), valueAsObjectMap); + } + OpenCGAResult update = getSampleDBAdaptor(organizationId).update(sample.getUid(), parameters, study.getVariableSets(), options); + + if (updateParamsClone.getQualityControl() != null) { + // Check if corresponding individual exists and the quality control status is not ready + Query query = new Query() + .append(IndividualDBAdaptor.QueryParams.SAMPLE_UIDS.key(), sample.getUid()) + .append(IndividualDBAdaptor.QueryParams.INTERNAL_QUALITY_CONTROL_STATUS_ID.key(), + Arrays.asList(QualityControlStatus.NONE, QualityControlStatus.ERROR)); + OpenCGAResult search = catalogManager.getIndividualManager().search(study.getFqn(), query, + IndividualManager.INCLUDE_INDIVIDUAL_IDS, token); + if (search.getNumResults() == 1) { + // Update Individual quality control status to PENDING + logger.info("Setting internal quality control status automatically to PENDING for individual '{}'", search.first().getId()); + QualityControlStatus qualityControlStatus = new QualityControlStatus(QualityControlStatus.PENDING); + catalogManager.getIndividualManager().updateInternalQualityControlStatus(study.getFqn(), search.first(), + qualityControlStatus, token); + } + } + if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { // Fetch updated sample OpenCGAResult queryResult = getSampleDBAdaptor(organizationId).get(study.getUid(), diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndex.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndex.java index 4e2af0fc6d3..c092fd16517 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndex.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/clinical/CvdbIndex.java @@ -3,24 +3,27 @@ public class CvdbIndex { private String jobId; + private CvdbIndexStatus previousStatus; private CvdbIndexStatus status; public CvdbIndex() { } - public CvdbIndex(String jobId, CvdbIndexStatus status) { + public CvdbIndex(String jobId, CvdbIndexStatus previousStatus, CvdbIndexStatus status) { this.jobId = jobId; + this.previousStatus = previousStatus; this.status = status; } public static CvdbIndex init() { - return new CvdbIndex("", CvdbIndexStatus.init()); + return new CvdbIndex("", CvdbIndexStatus.init(), CvdbIndexStatus.init()); } @Override public String toString() { final StringBuilder sb = new StringBuilder("CvdbIndex{"); sb.append("jobId='").append(jobId).append('\''); + sb.append(", previousStatus=").append(previousStatus); sb.append(", status=").append(status); sb.append('}'); return sb.toString(); @@ -35,6 +38,15 @@ public CvdbIndex setJobId(String jobId) { return this; } + public CvdbIndexStatus getPreviousStatus() { + return previousStatus; + } + + public CvdbIndex setPreviousStatus(CvdbIndexStatus previousStatus) { + this.previousStatus = previousStatus; + return this; + } + public CvdbIndexStatus getStatus() { return status; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java index 9d31bc5b98f..408b46273cd 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/common/QualityControlStatus.java @@ -6,11 +6,11 @@ public class QualityControlStatus extends InternalStatus { public static final String ERROR = "ERROR"; + public static final String NONE = "NONE"; public static final String PENDING = "PENDING"; - public static final String PENDING_REMOVE = "PENDING_REMOVE"; - public static final String PENDING_OVERWRITE = "PENDING_OVERWRITE"; + public static final String QUEUED = "QUEUED"; - public static final List STATUS_LIST = Arrays.asList(PENDING, PENDING_REMOVE, PENDING_OVERWRITE, READY, ERROR); + public static final List STATUS_LIST = Arrays.asList(NONE, PENDING, QUEUED, READY, ERROR); public QualityControlStatus(String status, String message) { if (isValid(status)) { @@ -25,7 +25,7 @@ public QualityControlStatus(String status) { } public QualityControlStatus() { - this(PENDING, ""); + this(NONE, ""); } public static QualityControlStatus init() { @@ -34,9 +34,9 @@ public static QualityControlStatus init() { public static boolean isValid(String status) { return status != null - && (status.equals(PENDING) - || status.equals(PENDING_REMOVE) - || status.equals(PENDING_OVERWRITE) + && (status.equals(NONE) + || status.equals(PENDING) + || status.equals(QUEUED) || status.equals(READY) || status.equals(ERROR)); } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java index 87fbd03f522..4289bd70caa 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java @@ -1,6 +1,6 @@ package org.opencb.opencga.core.models.study; -public class CatalogServiceConfiguration { +public abstract class CatalogServiceConfiguration { private boolean active; private int concurrentJobs; @@ -13,10 +13,6 @@ public CatalogServiceConfiguration(boolean active, int concurrentJobs) { this.concurrentJobs = concurrentJobs; } - public static CatalogServiceConfiguration defaultConfiguration() { - return new CatalogServiceConfiguration(false, 5); - } - @Override public String toString() { final StringBuilder sb = new StringBuilder("CatalogServiceConfiguration{"); diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java index 50078a1583b..2e9c837a7e1 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java @@ -2,22 +2,20 @@ public class CatalogStudyConfiguration { - private CatalogServiceConfiguration cvdb; - private CatalogServiceConfiguration variantQualityControl; + private CvdbCatalogServiceConfiguration cvdb; + private QualityControlServiceConfiguration variantQualityControl; public CatalogStudyConfiguration() { } - public CatalogStudyConfiguration(CatalogServiceConfiguration cvdb, CatalogServiceConfiguration variantQualityControl) { + public CatalogStudyConfiguration(CvdbCatalogServiceConfiguration cvdb, QualityControlServiceConfiguration variantQualityControl) { this.cvdb = cvdb; this.variantQualityControl = variantQualityControl; } public static CatalogStudyConfiguration defaultConfiguration() { - return new CatalogStudyConfiguration( - CatalogServiceConfiguration.defaultConfiguration(), // cvdb - CatalogServiceConfiguration.defaultConfiguration() // variantQualityControl - ); + return new CatalogStudyConfiguration(CvdbCatalogServiceConfiguration.defaultConfiguration(), + QualityControlServiceConfiguration.defaultConfiguration()); } @Override @@ -29,20 +27,20 @@ public String toString() { return sb.toString(); } - public CatalogServiceConfiguration getCvdb() { + public CvdbCatalogServiceConfiguration getCvdb() { return cvdb; } - public CatalogStudyConfiguration setCvdb(CatalogServiceConfiguration cvdb) { + public CatalogStudyConfiguration setCvdb(CvdbCatalogServiceConfiguration cvdb) { this.cvdb = cvdb; return this; } - public CatalogServiceConfiguration getVariantQualityControl() { + public QualityControlServiceConfiguration getVariantQualityControl() { return variantQualityControl; } - public CatalogStudyConfiguration setVariantQualityControl(CatalogServiceConfiguration variantQualityControl) { + public CatalogStudyConfiguration setVariantQualityControl(QualityControlServiceConfiguration variantQualityControl) { this.variantQualityControl = variantQualityControl; return this; } diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CvdbCatalogServiceConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CvdbCatalogServiceConfiguration.java new file mode 100644 index 00000000000..a22d492709b --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CvdbCatalogServiceConfiguration.java @@ -0,0 +1,47 @@ +package org.opencb.opencga.core.models.study; + +public class CvdbCatalogServiceConfiguration extends CatalogServiceConfiguration { + + private int batchSize; + + public CvdbCatalogServiceConfiguration() { + } + + public CvdbCatalogServiceConfiguration(boolean active, int concurrentJobs, int batchSize) { + super(active, concurrentJobs); + this.batchSize = batchSize; + } + + public static CvdbCatalogServiceConfiguration defaultConfiguration() { + return new CvdbCatalogServiceConfiguration(false, 5, 10); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CvdbCatalogServiceConfiguration{"); + sb.append("batchSize=").append(batchSize); + sb.append('}'); + return sb.toString(); + } + + public int getBatchSize() { + return batchSize; + } + + public CvdbCatalogServiceConfiguration setBatchSize(int batchSize) { + this.batchSize = batchSize; + return this; + } + + @Override + public CvdbCatalogServiceConfiguration setActive(boolean active) { + super.setActive(active); + return this; + } + + @Override + public CvdbCatalogServiceConfiguration setConcurrentJobs(int concurrentJobs) { + super.setConcurrentJobs(concurrentJobs); + return this; + } +} diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/QualityControlServiceConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/QualityControlServiceConfiguration.java new file mode 100644 index 00000000000..a5bd2a11e26 --- /dev/null +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/QualityControlServiceConfiguration.java @@ -0,0 +1,53 @@ +package org.opencb.opencga.core.models.study; + +public class QualityControlServiceConfiguration extends CatalogServiceConfiguration { + + private Policy policy; + + public QualityControlServiceConfiguration() { + } + + public QualityControlServiceConfiguration(boolean active, int concurrentJobs, Policy policy) { + super(active, concurrentJobs); + this.policy = policy; + } + + public static QualityControlServiceConfiguration defaultConfiguration() { + return new QualityControlServiceConfiguration(false, 5, Policy.NIGHTLY); + } + + public enum Policy { + IMMEDIATELY, + NIGHTLY, + WEEKLY + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("QualityControlServiceConfiguration{"); + sb.append("policy=").append(policy); + sb.append('}'); + return sb.toString(); + } + + public Policy getPolicy() { + return policy; + } + + public QualityControlServiceConfiguration setPolicy(Policy policy) { + this.policy = policy; + return this; + } + + @Override + public QualityControlServiceConfiguration setActive(boolean active) { + super.setActive(active); + return this; + } + + @Override + public QualityControlServiceConfiguration setConcurrentJobs(int concurrentJobs) { + super.setConcurrentJobs(concurrentJobs); + return this; + } +} diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java index 393cb2afdd6..778ba561121 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java @@ -24,6 +24,9 @@ import java.io.Closeable; import java.io.IOException; +import java.time.DayOfWeek; +import java.time.LocalDate; +import java.time.LocalTime; /** * Created by imedina on 16/06/16. @@ -89,6 +92,24 @@ public void run() { } } + protected static boolean isNightTime() { + boolean isNightTime = false; + // Check date time is between 00:00 and 05:00 + //TODO: Define night time in configuration + LocalTime now = LocalTime.now(); + int hour = now.getHour(); + if (hour < 5) { + isNightTime = true; + } + return isNightTime; + } + + protected static boolean isWeekend() { + LocalDate today = LocalDate.now(); + DayOfWeek dayOfWeek = today.getDayOfWeek(); + return dayOfWeek == DayOfWeek.SATURDAY || dayOfWeek == DayOfWeek.SUNDAY; + } + public void init() throws Exception { } From 7f70e4b48fe08b5138936e92a0878decc4815ab0 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 2 Jul 2025 11:08:29 +0200 Subject: [PATCH 12/13] master: remove unused methods from ParentDaemon, #TASK-7756 --- .../monitor/daemons/MonitorParentDaemon.java | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java index 778ba561121..393cb2afdd6 100644 --- a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java +++ b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/MonitorParentDaemon.java @@ -24,9 +24,6 @@ import java.io.Closeable; import java.io.IOException; -import java.time.DayOfWeek; -import java.time.LocalDate; -import java.time.LocalTime; /** * Created by imedina on 16/06/16. @@ -92,24 +89,6 @@ public void run() { } } - protected static boolean isNightTime() { - boolean isNightTime = false; - // Check date time is between 00:00 and 05:00 - //TODO: Define night time in configuration - LocalTime now = LocalTime.now(); - int hour = now.getHour(); - if (hour < 5) { - isNightTime = true; - } - return isNightTime; - } - - protected static boolean isWeekend() { - LocalDate today = LocalDate.now(); - DayOfWeek dayOfWeek = today.getDayOfWeek(); - return dayOfWeek == DayOfWeek.SATURDAY || dayOfWeek == DayOfWeek.SUNDAY; - } - public void init() throws Exception { } From 34d095941478cd3ab676e7d6b3ae525045632f40 Mon Sep 17 00:00:00 2001 From: pfurio Date: Wed, 2 Jul 2025 14:11:19 +0200 Subject: [PATCH 13/13] core: add DataField documentation, #TASK-7756 --- .../core/models/study/CatalogServiceConfiguration.java | 5 +++++ .../opencga/core/models/study/CatalogStudyConfiguration.java | 5 +++++ .../core/models/study/CvdbCatalogServiceConfiguration.java | 3 +++ .../models/study/QualityControlServiceConfiguration.java | 4 ++++ .../core/models/study/configuration/StudyConfiguration.java | 3 +++ 5 files changed, 20 insertions(+) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java index 4289bd70caa..9e28e779cf5 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogServiceConfiguration.java @@ -1,8 +1,13 @@ package org.opencb.opencga.core.models.study; +import org.opencb.commons.annotations.DataField; + public abstract class CatalogServiceConfiguration { + @DataField(id = "active", description = "Indicates whether the service is active or not") private boolean active; + + @DataField(id = "concurrentJobs", description = "Number of concurrent jobs that can be run by this service") private int concurrentJobs; public CatalogServiceConfiguration() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java index 2e9c837a7e1..3edb1c1a2ca 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CatalogStudyConfiguration.java @@ -1,8 +1,13 @@ package org.opencb.opencga.core.models.study; +import org.opencb.commons.annotations.DataField; + public class CatalogStudyConfiguration { + @DataField(id = "cvdb", description = "Configuration for the CVDB catalog service") private CvdbCatalogServiceConfiguration cvdb; + + @DataField(id = "variantQualityControl", description = "Configuration for the variant quality control service") private QualityControlServiceConfiguration variantQualityControl; public CatalogStudyConfiguration() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CvdbCatalogServiceConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CvdbCatalogServiceConfiguration.java index a22d492709b..268e8e6ba8f 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CvdbCatalogServiceConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/CvdbCatalogServiceConfiguration.java @@ -1,7 +1,10 @@ package org.opencb.opencga.core.models.study; +import org.opencb.commons.annotations.DataField; + public class CvdbCatalogServiceConfiguration extends CatalogServiceConfiguration { + @DataField(id = "batchSize", description = "Maximum number of cases that can be launched per job") private int batchSize; public CvdbCatalogServiceConfiguration() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/QualityControlServiceConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/QualityControlServiceConfiguration.java index a5bd2a11e26..4af5ef301e7 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/QualityControlServiceConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/QualityControlServiceConfiguration.java @@ -1,7 +1,11 @@ package org.opencb.opencga.core.models.study; +import org.opencb.commons.annotations.DataField; + public class QualityControlServiceConfiguration extends CatalogServiceConfiguration { + @DataField(id = "policy", description = "Policy for running the quality control services. WEEKLY to process them only during weekends, " + + "NIGHTLY to process them only during night time (00-05AM) and IMMEDIATELY to process them as soon as possible.") private Policy policy; public QualityControlServiceConfiguration() { diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/StudyConfiguration.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/StudyConfiguration.java index 6e2c2b025b4..ee786811d3c 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/StudyConfiguration.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/study/configuration/StudyConfiguration.java @@ -1,5 +1,6 @@ package org.opencb.opencga.core.models.study.configuration; +import org.opencb.commons.annotations.DataField; import org.opencb.commons.datastore.core.ObjectMap; import org.opencb.opencga.core.config.storage.SampleIndexConfiguration; import org.opencb.opencga.core.models.study.CatalogStudyConfiguration; @@ -9,6 +10,8 @@ public class StudyConfiguration { private ClinicalAnalysisStudyConfiguration clinical; private StudyVariantEngineConfiguration variantEngine; + + @DataField(id = "catalog", description = "Configuration for the catalog services running in the background related to the study") private CatalogStudyConfiguration catalog;