TASK-5198 - Catalog changes to improve Clinical reports

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalInterpretationManager.java

@@ -824,7 +824,7 @@ public ClinicalAnalyst getAnalyst(String token) throws ToolException {
             OpenCGAResult<User> userQueryResult = catalogManager.getUserManager().get(userId, new QueryOptions(QueryOptions.INCLUDE,
                     Arrays.asList(UserDBAdaptor.QueryParams.EMAIL.key(), UserDBAdaptor.QueryParams.ORGANIZATION.key())), token);
             User user = userQueryResult.first();
-            return new ClinicalAnalyst(userId, user.getName(), user.getEmail(), "", "");
+            return new ClinicalAnalyst(userId, user.getName(), user.getEmail(), "", Collections.emptyMap());
         } catch (CatalogException e) {
             throw new ToolException(e);
         }

opencga-analysis/src/main/java/org/opencb/opencga/analysis/clinical/ClinicalTsvAnnotationLoader.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2015-2020 OpenCB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opencb.opencga.analysis.clinical;
+
+import org.opencb.commons.datastore.core.ObjectMap;
+import org.opencb.commons.datastore.core.Query;
+import org.opencb.commons.datastore.core.QueryOptions;
+import org.opencb.opencga.analysis.annotations.TsvAnnotationLoader;
+import org.opencb.opencga.catalog.exceptions.CatalogException;
+import org.opencb.opencga.catalog.managers.AnnotationSetManager;
+import org.opencb.opencga.catalog.utils.Constants;
+import org.opencb.opencga.catalog.utils.ParamUtils;
+import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams;
+import org.opencb.opencga.core.models.common.AnnotationSet;
+import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.tools.annotations.Tool;
+
+import java.util.Collections;
+
+@Tool(id = ClinicalTsvAnnotationLoader.ID, resource = Enums.Resource.CLINICAL_ANALYSIS, type = Tool.Type.OPERATION,
+        description = "Load annotations from TSV file.")
+public class ClinicalTsvAnnotationLoader extends TsvAnnotationLoader {
+    public final static String ID = "clinical-tsv-load";
+
+    @Override
+    public int count(Query query) throws CatalogException {
+        return catalogManager.getClinicalAnalysisManager().count(study, query, token).getNumResults();
+    }
+
+    @Override
+    public void addAnnotationSet(String entryId, AnnotationSet annotationSet, QueryOptions options) throws CatalogException {
+        ClinicalAnalysisUpdateParams updateParams = new ClinicalAnalysisUpdateParams()
+                .setAnnotationSets(Collections.singletonList(annotationSet));
+        QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions();
+        queryOptions.put(Constants.ACTIONS, new ObjectMap(AnnotationSetManager.ANNOTATION_SETS, ParamUtils.BasicUpdateAction.ADD));
+
+        catalogManager.getClinicalAnalysisManager().update(study, entryId, updateParams, queryOptions, token);
+    }
+}

opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/InternalCliOptionsParser.java

@@ -226,6 +226,7 @@ public InternalCliOptionsParser() {
         clinicalSubCommands.addCommand(RGA_INDEX_RUN_COMMAND, clinicalCommandOptions.rgaSecondaryIndexCommandOptions);
         clinicalSubCommands.addCommand(RGA_AUX_INDEX_RUN_COMMAND, clinicalCommandOptions.rgaAuxiliarSecondaryIndexCommandOptions);
         clinicalSubCommands.addCommand(EXOMISER_INTERPRETATION_RUN_COMMAND, clinicalCommandOptions.exomiserInterpretationCommandOptions);
+        clinicalSubCommands.addCommand("tsv-load", clinicalCommandOptions.tsvLoad);
 
         fileCommandOptions = new FileCommandOptions(commonCommandOptions, jCommander);
         jCommander.addCommand("files", fileCommandOptions);

opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/executors/ClinicalCommandExecutor.java

@@ -24,6 +24,7 @@
 import org.opencb.commons.datastore.core.ObjectMap;
 import org.opencb.commons.datastore.core.Query;
 import org.opencb.commons.datastore.core.QueryOptions;
+import org.opencb.opencga.analysis.clinical.ClinicalTsvAnnotationLoader;
 import org.opencb.opencga.analysis.clinical.exomiser.ExomiserInterpretationAnalysis;
 import org.opencb.opencga.analysis.clinical.rga.AuxiliarRgaAnalysis;
 import org.opencb.opencga.analysis.clinical.rga.RgaAnalysis;
@@ -96,6 +97,9 @@ public void execute() throws Exception {
             case EXOMISER_INTERPRETATION_RUN_COMMAND:
                 exomiserInterpretation();
                 break;
+            case "tsv-load":
+                tsvLoad();
+                break;
             default:
                 logger.error("Subcommand not valid");
                 break;
@@ -314,4 +318,19 @@ private void exomiserInterpretation() throws Exception {
 //        exomiserInterpretationAnalysis.setPrimary(cliOptions.primary);
         exomiserInterpretationAnalysis.start();
     }
+
+    private void tsvLoad() throws ToolException {
+        ClinicalCommandOptions.TsvLoad options = clinicalCommandOptions.tsvLoad;
+
+        Path outDir = Paths.get(options.outDir);
+
+        ClinicalTsvAnnotationLoader annotationLoader = new ClinicalTsvAnnotationLoader();
+        annotationLoader.setAnnotationSetId(options.annotationSetId);
+        annotationLoader.setVariableSetId(options.variableSetId);
+        annotationLoader.setPath(options.filePath);
+        annotationLoader.setStudy(options.study);
+
+        annotationLoader.setUp(opencgaHome.toString(), new ObjectMap(), outDir, options.commonOptions.token);
+        annotationLoader.start();
+    }
 }

opencga-app/src/main/java/org/opencb/opencga/app/cli/internal/options/ClinicalCommandOptions.java

@@ -5,6 +5,7 @@
 import com.beust.jcommander.Parameters;
 import com.beust.jcommander.ParametersDelegate;
 import org.opencb.biodata.models.clinical.ClinicalProperty;
+import org.opencb.opencga.analysis.clinical.ClinicalTsvAnnotationLoader;
 import org.opencb.opencga.analysis.clinical.exomiser.ExomiserInterpretationAnalysis;
 import org.opencb.opencga.analysis.clinical.rga.AuxiliarRgaAnalysis;
 import org.opencb.opencga.analysis.clinical.rga.RgaAnalysis;
@@ -14,6 +15,7 @@
 import org.opencb.opencga.analysis.clinical.zetta.ZettaInterpretationAnalysis;
 import org.opencb.opencga.app.cli.GeneralCliOptions;
 import org.opencb.opencga.app.cli.internal.InternalCliOptionsParser;
+import org.opencb.opencga.core.api.ParamConstants;
 import org.opencb.opencga.core.models.clinical.RgaAnalysisParams;
 import org.opencb.opencga.storage.app.cli.client.options.StorageVariantCommandOptions.BasicVariantQueryOptions;
 
@@ -33,6 +35,7 @@ public class ClinicalCommandOptions {
     public final RgaSecondaryIndexCommandOptions rgaSecondaryIndexCommandOptions;
     public final RgaAuxiliarSecondaryIndexCommandOptions rgaAuxiliarSecondaryIndexCommandOptions;
     public final ExomiserInterpretationCommandOptions exomiserInterpretationCommandOptions;
+    public final TsvLoad tsvLoad;
 
     public JCommander jCommander;
     public GeneralCliOptions.CommonCommandOptions commonCommandOptions;
@@ -53,6 +56,7 @@ public ClinicalCommandOptions(GeneralCliOptions.CommonCommandOptions commonComma
         this.rgaSecondaryIndexCommandOptions = new RgaSecondaryIndexCommandOptions();
         this.rgaAuxiliarSecondaryIndexCommandOptions = new RgaAuxiliarSecondaryIndexCommandOptions();
         this.exomiserInterpretationCommandOptions = new ExomiserInterpretationCommandOptions();
+        this.tsvLoad = new TsvLoad();
     }
 
     @Parameters(commandNames = {TieringCommandOptions.TIERING_INTERPRETATION_RUN_COMMAND}, commandDescription =
@@ -335,4 +339,29 @@ public class ExomiserInterpretationCommandOptions extends GeneralCliOptions.Stud
         @Parameter(names = {"-o", "--outdir"}, description = "Directory where output files will be saved", arity = 1)
         public String outdir;
     }
+
+    @Parameters(commandNames = {"tsv-load"}, commandDescription = "Load annotations from a TSV file")
+    public class TsvLoad extends GeneralCliOptions.StudyOption {
+
+        public static final String TSV_LOAD_COMMAND = ClinicalTsvAnnotationLoader.ID;
+
+        @ParametersDelegate
+        public GeneralCliOptions.CommonCommandOptions commonOptions = commonCommandOptions;
+
+        @ParametersDelegate
+        public InternalCliOptionsParser.JobOptions jobOptions = internalJobOptions;
+
+        @Parameter(names = {"--file"}, description = "Path to the TSV file.", required = true, arity = 1)
+        public String filePath;
+
+        @Parameter(names = {"--variable-set-id"}, description = ParamConstants.VARIABLE_SET_DESCRIPTION, required = true, arity = 1)
+        public String variableSetId;
+
+        @Parameter(names = {"--annotation-set-id"}, description = "AnnotationSet id that will be given to the new annotations.",
+                required = true, arity = 1)
+        public String annotationSetId;
+
+        @Parameter(names = {"-o", "--outdir"}, description = "Directory where output files will be saved", required = true, arity = 1)
+        public String outDir;
+    }
 }

opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_0/catalog/AddAnnotationSetsInClinicalAnalysisMigration.java

@@ -0,0 +1,43 @@
+package org.opencb.opencga.app.migrations.v2_12_0.catalog;
+
+
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.result.UpdateResult;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.opencb.opencga.catalog.db.mongodb.AnnotationMongoDBAdaptor;
+import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory;
+import org.opencb.opencga.catalog.migration.Migration;
+import org.opencb.opencga.catalog.migration.MigrationTool;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+@Migration(id = "add_annotation_sets_to_clinical_analysis" ,
+        description = "Add private annotation fields to ClinicalAnalysis documents #TASK-5198",
+        version = "2.12.0",
+        domain = Migration.MigrationDomain.CATALOG,
+        language = Migration.MigrationLanguage.JAVA,
+        date = 20231116
+)
+public class AddAnnotationSetsInClinicalAnalysisMigration extends MigrationTool {
+
+    @Override
+    protected void run() throws Exception {
+        Bson query = Filters.exists(AnnotationMongoDBAdaptor.AnnotationSetParams.ANNOTATION_SETS.key(), false);
+        Document update = new Document("$set", new Document()
+                .append(AnnotationMongoDBAdaptor.AnnotationSetParams.ANNOTATION_SETS.key(), Collections.emptyList())
+                .append(AnnotationMongoDBAdaptor.AnnotationSetParams.INTERNAL_ANNOTATION_SETS.key(), Collections.emptyList())
+                .append(AnnotationMongoDBAdaptor.AnnotationSetParams.PRIVATE_VARIABLE_SET_MAP.key(), Collections.emptyMap())
+                .append(AnnotationMongoDBAdaptor.AnnotationSetParams.PRIVATE_INTERNAL_VARIABLE_SET_MAP.key(), Collections.emptyMap())
+        );
+        // Initialise private fields in all Clinical Analysis documents
+        for (String collection : Arrays.asList(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION,
+                MongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION)) {
+            MongoCollection<Document> mongoCollection = getMongoCollection(collection);
+            UpdateResult updateResult = mongoCollection.updateMany(query, update);
+            logger.info("{} clinical analysis documents updated from the {} collection", updateResult.getModifiedCount(), collection);
+        }
+    }
+}

opencga-app/src/main/java/org/opencb/opencga/app/migrations/v2_12_0/catalog/CompleteClinicalReportDataModelMigration.java

@@ -0,0 +1,62 @@
+package org.opencb.opencga.app.migrations.v2_12_0.catalog;
+
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Projections;
+import com.mongodb.client.model.UpdateOneModel;
+import org.bson.Document;
+import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptor;
+import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory;
+import org.opencb.opencga.catalog.migration.Migration;
+import org.opencb.opencga.catalog.migration.MigrationTool;
+
+import java.util.Arrays;
+import java.util.Collections;
+
+import static com.mongodb.client.model.Filters.eq;
+
+@Migration(id = "complete_clinical_report_data_model" ,
+        description = "Complete Clinical Report data model #TASK-5198",
+        version = "2.12.0",
+        domain = Migration.MigrationDomain.CATALOG,
+        language = Migration.MigrationLanguage.JAVA,
+        date = 20231128
+)
+public class CompleteClinicalReportDataModelMigration extends MigrationTool {
+
+    @Override
+    protected void run() throws Exception {
+        migrateCollection(
+                Arrays.asList(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION, MongoDBAdaptorFactory.DELETED_CLINICAL_ANALYSIS_COLLECTION),
+                Filters.exists("analyst"),
+                Projections.include(Arrays.asList("analyst", "report")),
+                (document, bulk) -> {
+                    Document analyst = document.get("analyst", Document.class);
+                    analyst.remove("assignedBy");
+                    analyst.remove("date");
+
+                    Document report = document.get("report", Document.class);
+                    if (report != null) {
+                        report.put("comments", Collections.emptyList());
+                        report.put("supportingEvidences", Collections.emptyList());
+                        report.put("files", Collections.emptyList());
+                    }
+
+                    MongoDBAdaptor.UpdateDocument updateDocument = new MongoDBAdaptor.UpdateDocument();
+                    updateDocument.getSet().put("report", report);
+                    updateDocument.getSet().put("request", new Document());
+                    updateDocument.getSet().put("responsible", new Document()
+                            .append("id", analyst.get("id"))
+                            .append("name", analyst.get("name"))
+                            .append("email", analyst.get("email"))
+                    );
+                    updateDocument.getSet().put("analysts", Collections.singletonList(analyst));
+                    updateDocument.getUnset().add("analyst");
+
+                    bulk.add(new UpdateOneModel<>(
+                                    eq("_id", document.get("_id")),
+                                    updateDocument.toFinalUpdateDocument()
+                            )
+                    );
+                });
+    }
+}