diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java index 86e62f1ebc1..2690fb210ab 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java @@ -68,6 +68,7 @@ enum QueryParams implements QueryParam { INTERNAL_STATUS_DESCRIPTION("internal.status.description", TEXT, ""), INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""), RELATED_FILES("relatedFiles", TEXT_ARRAY, ""), + RELATED_FILES_FILE_UID("relatedFiles.file.uid", LONG, ""), RELATED_FILES_RELATION("relatedFiles.relation", TEXT, ""), SIZE("size", INTEGER, ""), EXPERIMENT("experiment", OBJECT, ""), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java index dd5b90a22d8..078554b4737 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java @@ -31,9 +31,7 @@ import org.opencb.commons.datastore.core.*; import org.opencb.commons.datastore.mongodb.MongoDBCollection; import org.opencb.commons.datastore.mongodb.MongoDBIterator; -import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor; -import org.opencb.opencga.catalog.db.api.DBIterator; -import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor; +import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.db.mongodb.converters.ClinicalAnalysisConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.ClinicalAnalysisCatalogMongoDBIterator; import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; @@ -151,6 +149,10 @@ static void fixFilesForRemoval(ObjectMap parameters, String key) { for (Object file : parameters.getAsList(key)) { if (file instanceof File) { fileParamList.add(new Document("uid", ((File) file).getUid())); + } else if (file instanceof Document) { + fileParamList.add(new Document("uid", ((Document) file).get("uid"))); + } else { + throw new IllegalArgumentException("Expected a File or Document object"); } } parameters.put(key, fileParamList); @@ -814,6 +816,25 @@ OpenCGAResult privateDelete(ClientSession clientSession, ClinicalAnalysis cli return endWrite(tmpStartTime, 1, 0, 0, 1, Collections.emptyList()); } + void removeFileReferences(ClientSession clientSession, long studyUid, long fileUid, Document file) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + ObjectMap parameters = new ObjectMap(FILES.key(), Collections.singletonList(file)); + ObjectMap actionMap = new ObjectMap(FILES.key(), ParamUtils.BasicUpdateAction.REMOVE); + QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); + + Query query = new Query() + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.FILES_UID.key(), fileUid); + OpenCGAResult result = get(query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS); + for (ClinicalAnalysis clinicalAnalysis : result.getResults()) { + logger.debug("Removing file references from Clinical Analysis {}", clinicalAnalysis.getId()); + ClinicalAudit clinicalAudit = new ClinicalAudit("OPENCGA", ClinicalAudit.Action.UPDATE_CLINICAL_ANALYSIS, "File " + + file.getString(FileDBAdaptor.QueryParams.PATH.key()) + " was deleted. Remove file references from case.", + TimeUtils.getTime()); + transactionalUpdate(clientSession, clinicalAnalysis, parameters, null, Collections.singletonList(clinicalAudit), options); + } + } + @Override public OpenCGAResult restore(long id, QueryOptions queryOptions) throws CatalogDBException { return null; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java index 38d24ed6674..f256a7e9cec 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java @@ -40,6 +40,7 @@ import org.opencb.opencga.catalog.managers.FileUtils; import org.opencb.opencga.catalog.managers.SampleManager; import org.opencb.opencga.catalog.utils.Constants; +import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction; import org.opencb.opencga.catalog.utils.UuidUtils; import org.opencb.opencga.core.api.ParamConstants; @@ -785,7 +786,8 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, lon document.getSet().put(QueryParams.RELATED_FILES.key(), relatedFileDocument); break; case REMOVE: - document.getPullAll().put(QueryParams.RELATED_FILES.key(), relatedFileDocument); + List documentList = fixRelatedFilesForRemoval(relatedFileDocument); + document.getPull().put(QueryParams.RELATED_FILES.key(), documentList); break; case ADD: document.getAddToSet().put(QueryParams.RELATED_FILES.key(), relatedFileDocument); @@ -886,6 +888,18 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, lon return document; } + private List fixRelatedFilesForRemoval(List relatedFiles) { + if (CollectionUtils.isEmpty(relatedFiles)) { + return Collections.emptyList(); + } + + List relatedFilesCopy = new ArrayList<>(relatedFiles.size()); + for (Document relatedFile : relatedFiles) { + relatedFilesCopy.add(new Document("file", new Document("uid", relatedFile.get("file", Document.class).get("uid")))); + } + return relatedFilesCopy; + } + @Override public OpenCGAResult delete(File file) throws CatalogDBException { throw new UnsupportedOperationException("Use delete passing status field."); @@ -1007,7 +1021,9 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document fileDo Document tmpFile = iterator.next(); long tmpFileUid = tmpFile.getLong(PRIVATE_UID); + removeFileReferences(clientSession, studyUid, tmpFileUid, tmpFile); dbAdaptorFactory.getCatalogJobDBAdaptor().removeFileReferences(clientSession, studyUid, tmpFileUid, tmpFile); + dbAdaptorFactory.getClinicalAnalysisDBAdaptor().removeFileReferences(clientSession, studyUid, tmpFileUid, tmpFile); // Set status nestedPut(QueryParams.INTERNAL_STATUS.key(), getMongoDBDocument(new FileStatus(status), "status"), tmpFile); @@ -1035,6 +1051,28 @@ OpenCGAResult privateDelete(ClientSession clientSession, Document fileDo } } + void removeFileReferences(ClientSession clientSession, long studyUid, long fileUid, Document fileDoc) + throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException { + File file = fileConverter.convertToDataModelType(fileDoc); + FileRelatedFile relatedFile = new FileRelatedFile(file, null); + ObjectMap parameters = new ObjectMap(QueryParams.RELATED_FILES.key(), Collections.singletonList(relatedFile)); + ObjectMap actionMap = new ObjectMap(QueryParams.RELATED_FILES.key(), ParamUtils.BasicUpdateAction.REMOVE); + QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap); + + Query query = new Query() + .append(QueryParams.STUDY_UID.key(), studyUid) + .append(QueryParams.RELATED_FILES_FILE_UID.key(), fileUid); + UpdateDocument updateDocument = getValidatedUpdateParams(clientSession, studyUid, parameters, query, options); + Document updateDoc = updateDocument.toFinalUpdateDocument(); + if (!updateDoc.isEmpty()) { + Bson bsonQuery = parseQuery(query); + OpenCGAResult result = transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument); + if (result.getNumUpdated() > 0) { + logger.debug("File '{}' removed from related files array from {} files.", file.getPath(), result.getNumUpdated()); + } + } + } + // OpenCGAResult privateDelete(ClientSession clientSession, File file, String status) throws CatalogDBException { // long tmpStartTime = startQuery(); // logger.debug("Deleting file {} ({})", file.getPath(), file.getUid()); @@ -1491,6 +1529,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user) case TAGS: case SIZE: case SOFTWARE_NAME: + case RELATED_FILES_FILE_UID: case JOB_ID: addAutoOrQuery(queryParam.key(), queryParam.key(), myQuery, queryParam.type(), andBsonList); break; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FileConverter.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FileConverter.java index 267666f77f0..c4db86b7408 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FileConverter.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FileConverter.java @@ -107,7 +107,7 @@ public List convertRelatedFiles(List relatedFileList) if (ListUtils.isNotEmpty(relatedFileList)) { for (FileRelatedFile relatedFile : relatedFileList) { relatedFiles.add(new Document() - .append("relation", relatedFile.getRelation().name()) + .append("relation", relatedFile.getRelation() != null ? relatedFile.getRelation().name() : null) .append("file", new Document("uid", relatedFile.getFile().getUid())) ); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java index 0832d689440..2d411253440 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java @@ -30,10 +30,7 @@ import org.opencb.commons.utils.ListUtils; import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager; import org.opencb.opencga.catalog.db.DBAdaptorFactory; -import org.opencb.opencga.catalog.db.api.DBIterator; -import org.opencb.opencga.catalog.db.api.FileDBAdaptor; -import org.opencb.opencga.catalog.db.api.SampleDBAdaptor; -import org.opencb.opencga.catalog.db.api.StudyDBAdaptor; +import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.*; import org.opencb.opencga.catalog.io.IOManager; @@ -1538,7 +1535,7 @@ public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions return delete(studyStr, fileIds, options, false, token); } - public OpenCGAResult delete(String studyStr, List fileIds, ObjectMap params, boolean ignoreException, String token) + public OpenCGAResult delete(String studyStr, List fileIds, QueryOptions options, boolean ignoreException, String token) throws CatalogException { String userId = catalogManager.getUserManager().getUserId(token); Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE, @@ -1549,13 +1546,15 @@ public OpenCGAResult delete(String studyStr, List fileIds, ObjectMap par ObjectMap auditParams = new ObjectMap() .append("study", studyStr) .append("fileIds", fileIds) - .append("params", params) + .append("options", options) .append("ignoreException", ignoreException) .append("token", token); + QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions(); + // We need to avoid processing subfolders or subfiles of an already processed folder independently Set processedPaths = new HashSet<>(); - boolean physicalDelete = params.getBoolean(Constants.SKIP_TRASH, false); + boolean physicalDelete = queryOptions.getBoolean(Constants.SKIP_TRASH, false); auditManager.initAuditBatch(operationUuid); OpenCGAResult result = OpenCGAResult.empty(File.class); @@ -1836,7 +1835,7 @@ public OpenCGAResult unlink(@Nullable String studyId, String fileId, Strin } catch (CatalogException e) { auditManager.audit(userId, Enums.Action.UNLINK, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError())); - throw e; + throw new CatalogException("Could not unlink file '" + fileId + "'", e); } } @@ -3100,6 +3099,16 @@ private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List // TODO: Validate no file/folder within any registered directory is not registered in OpenCGA } + Query clinicalQuery = new Query() + .append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) + .append(ClinicalAnalysisDBAdaptor.QueryParams.FILES_UID.key(), file.getUid()) + .append(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), true); + OpenCGAResult count = clinicalDBAdaptor.count(clinicalQuery); + if (count.getNumMatches() > 0) { + throw new CatalogException("The file " + file.getName() + " is part of " + count.getNumMatches() + " clinical analyses"); + } + + // Check the original files are not being indexed at the moment if (!indexFiles.isEmpty()) { Query query = new Query(FileDBAdaptor.QueryParams.UID.key(), new ArrayList<>(indexFiles)); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java index c01d915498c..6cf76e3ff12 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FileManagerTest.java @@ -38,11 +38,17 @@ import org.opencb.opencga.catalog.io.IOManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.ParamUtils; +import org.opencb.opencga.core.common.JacksonUtils; import org.opencb.opencga.core.common.TimeUtils; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.models.AclEntryList; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysis; +import org.opencb.opencga.core.models.clinical.ClinicalAnalysisUpdateParams; import org.opencb.opencga.core.models.common.AnnotationSet; +import org.opencb.opencga.core.models.family.Family; import org.opencb.opencga.core.models.file.*; +import org.opencb.opencga.core.models.individual.Individual; +import org.opencb.opencga.core.models.panel.Panel; import org.opencb.opencga.core.models.sample.Sample; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.models.user.Account; @@ -1052,7 +1058,7 @@ public void testUnlinkFile() throws CatalogException, IOException { // We send the unlink command again thrown.expect(CatalogException.class); - thrown.expectMessage("not found"); + thrown.expectMessage("not unlink"); fileManager.unlink(studyFqn, "myDirectory/data/test/folder/test_0.5K.txt", token); } @@ -2222,6 +2228,97 @@ public void deleteFolderTest4() throws CatalogException, IOException { } } + @Test + public void deleteFileInClinicalAnalysis() throws CatalogException, IOException { + // START DATA PREPARATION FOR TEST !!! + String bamFile = getClass().getResource("/biofiles/NA19600.chrom20.small.bam").getFile(); + File file = fileManager.link(studyFqn, new FileLinkParams(bamFile, "", "", "", null, null, null, null, null), false, token).first(); + + Family family1 = DummyModelUtils.getDummyFamily("familyId1"); + catalogManager.getFamilyManager().create(studyFqn, family1, QueryOptions.empty(), token); + + // Associate BAM file to sample + String sampleId = family1.getMembers().get(0).getSamples().get(0).getId(); + catalogManager.getFileManager().update(studyFqn, file.getId(), new FileUpdateParams().setSampleIds(Collections.singletonList(sampleId)), + QueryOptions.empty(), token); + + Panel myPanel = DummyModelUtils.getDummyPanel("myPanel"); + catalogManager.getPanelManager().create(studyFqn, myPanel, QueryOptions.empty(), token); + + Family copy = JacksonUtils.copy(family1, Family.class); + for (Individual member : copy.getMembers()) { + // Only use the first sample + member.setSamples(Collections.singletonList(member.getSamples().get(0))); + } + + ClinicalAnalysis clinicalAnalysis1 = DummyModelUtils.getDummyClinicalAnalysis(copy.getMembers().get(0), copy, Collections.singletonList(myPanel)); + clinicalAnalysis1 = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis1, INCLUDE_RESULT, token).first(); + assertEquals(1, clinicalAnalysis1.getFiles().size()); + assertEquals(file.getPath(), clinicalAnalysis1.getFiles().get(0).getPath()); + assertFalse(clinicalAnalysis1.isLocked()); + + ClinicalAnalysis clinicalAnalysis2 = DummyModelUtils.getDummyClinicalAnalysis(copy.getMembers().get(0), copy, Collections.singletonList(myPanel)); + clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().create(studyFqn, clinicalAnalysis2, INCLUDE_RESULT, token).first(); + assertEquals(1, clinicalAnalysis2.getFiles().size()); + assertEquals(file.getPath(), clinicalAnalysis2.getFiles().get(0).getPath()); + assertFalse(clinicalAnalysis2.isLocked()); + + // Lock clinicalAnalysis2 + clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis2.getId(), + new ClinicalAnalysisUpdateParams().setLocked(true), INCLUDE_RESULT, token).first(); + assertTrue(clinicalAnalysis2.isLocked()); + // END DATA PREPARATION FOR TEST !!! + + // Mark as pending delete + catalogManager.getFileManager().fileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.PENDING_DELETE), QueryOptions.empty()); + CatalogException catalogException = assertThrows(CatalogException.class, () -> catalogManager.getFileManager().unlink(studyFqn, file.getId(), token)); + assertTrue(catalogException.getMessage().contains("Could not unlink")); + assertTrue(catalogException.getCause().getMessage().contains("clinical analyses")); + + // Unlock clinicalAnalysis2 + clinicalAnalysis2 = catalogManager.getClinicalAnalysisManager().update(studyFqn, clinicalAnalysis2.getId(), + new ClinicalAnalysisUpdateParams().setLocked(false), INCLUDE_RESULT, token).first(); + assertFalse(clinicalAnalysis2.isLocked()); + + // Unlink file + catalogManager.getFileManager().unlink(studyFqn, file.getId(), token); + + Sample sample = catalogManager.getSampleManager().get(studyFqn, sampleId, QueryOptions.empty(), token).first(); + assertEquals(0, sample.getFileIds().size()); + + OpenCGAResult search = catalogManager.getClinicalAnalysisManager().search(studyFqn, new Query(), QueryOptions.empty(), token); + assertEquals(2, search.getNumResults()); + for (ClinicalAnalysis clinicalAnalysis : search.getResults()) { + assertEquals(0, clinicalAnalysis.getFiles().size()); + assertEquals("OPENCGA", clinicalAnalysis.getAudit().get(clinicalAnalysis.getAudit().size() - 1).getAuthor()); + assertTrue(clinicalAnalysis.getAudit().get(clinicalAnalysis.getAudit().size() - 1).getMessage().contains("was deleted. Remove file references from case")); + } + } + + @Test + public void deleteFileUserInRelatedFilesTest() throws CatalogException { + fileManager.update(studyFqn, "data/test/folder/test_1K.txt.gz", + new FileUpdateParams().setRelatedFiles(Collections.singletonList( + new SmallRelatedFileParams("data/test/folder/test_0.5K.txt", FileRelatedFile.Relation.PART_OF_PAIR))), + null, token); + File file = fileManager.get(studyFqn, "data/test/folder/test_1K.txt.gz", QueryOptions.empty(), token).first(); + assertFalse(file.getRelatedFiles().isEmpty()); + assertEquals(1, file.getRelatedFiles().size()); + assertEquals("data/test/folder/test_0.5K.txt", file.getRelatedFiles().get(0).getFile().getPath()); + + file = fileManager.get(studyFqn, "data/test/folder/test_0.5K.txt", FileManager.INCLUDE_FILE_IDS, token).first(); + + // Mark as pending delete + catalogManager.getFileManager().fileDBAdaptor.update(file.getUid(), new ObjectMap(FileDBAdaptor.QueryParams.INTERNAL_STATUS_ID.key(), FileStatus.PENDING_DELETE), QueryOptions.empty()); + // Delete test_0.5K file + QueryOptions options = new QueryOptions(Constants.SKIP_TRASH, true); + fileManager.delete(studyFqn, Collections.singletonList("data/test/folder/test_0.5K.txt"), options, token); + + // Ensure there are no more references to test_0.5K file + file = fileManager.get(studyFqn, "data/test/folder/test_1K.txt.gz", QueryOptions.empty(), token).first(); + assertTrue(file.getRelatedFiles().isEmpty()); + } + private File createBasicDirectoryFileTestEnvironment(List folderFiles) throws CatalogException { File folder = fileManager.createFolder(studyFqn, Paths.get("folder").toString(), false, null, QueryOptions.empty(), token).first();