Skip to content

Commit

Permalink
Merge branch 'release-2.12.x' into TASK-6231
Browse files Browse the repository at this point in the history
  • Loading branch information
@juanfeSanahuja
juanfeSanahuja authored May 14, 2024
2 parents 2edb506 + f295315 commit 28e916f
Show file tree
Hide file tree
Showing 6 changed files with 181 additions and 14 deletions.
1 change: 1 addition & 0 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ enum QueryParams implements QueryParam {
INTERNAL_STATUS_DESCRIPTION("internal.status.description", TEXT, ""),
INTERNAL_STATUS_DATE("internal.status.date", TEXT, ""),
RELATED_FILES("relatedFiles", TEXT_ARRAY, ""),
RELATED_FILES_FILE_UID("relatedFiles.file.uid", LONG, ""),
RELATED_FILES_RELATION("relatedFiles.relation", TEXT, ""),
SIZE("size", INTEGER, ""),
EXPERIMENT("experiment", OBJECT, ""),
Expand Down
27 changes: 24 additions & 3 deletions ...g/src/main/java/org/opencb/opencga/catalog/db/mongodb/ClinicalAnalysisMongoDBAdaptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,7 @@
import org.opencb.commons.datastore.core.*;
import org.opencb.commons.datastore.mongodb.MongoDBCollection;
import org.opencb.commons.datastore.mongodb.MongoDBIterator;
import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor;
import org.opencb.opencga.catalog.db.api.DBIterator;
import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor;
import org.opencb.opencga.catalog.db.api.*;
import org.opencb.opencga.catalog.db.mongodb.converters.ClinicalAnalysisConverter;
import org.opencb.opencga.catalog.db.mongodb.iterators.ClinicalAnalysisCatalogMongoDBIterator;
import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException;
Expand Down Expand Up @@ -151,6 +149,10 @@ static void fixFilesForRemoval(ObjectMap parameters, String key) {
for (Object file : parameters.getAsList(key)) {
if (file instanceof File) {
fileParamList.add(new Document("uid", ((File) file).getUid()));
} else if (file instanceof Document) {
fileParamList.add(new Document("uid", ((Document) file).get("uid")));
} else {
throw new IllegalArgumentException("Expected a File or Document object");
}
}
parameters.put(key, fileParamList);
Expand Down Expand Up @@ -814,6 +816,25 @@ OpenCGAResult<?> privateDelete(ClientSession clientSession, ClinicalAnalysis cli
return endWrite(tmpStartTime, 1, 0, 0, 1, Collections.emptyList());
}

void removeFileReferences(ClientSession clientSession, long studyUid, long fileUid, Document file)
throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException {
ObjectMap parameters = new ObjectMap(FILES.key(), Collections.singletonList(file));
ObjectMap actionMap = new ObjectMap(FILES.key(), ParamUtils.BasicUpdateAction.REMOVE);
QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap);

Query query = new Query()
.append(QueryParams.STUDY_UID.key(), studyUid)
.append(QueryParams.FILES_UID.key(), fileUid);
OpenCGAResult<ClinicalAnalysis> result = get(query, ClinicalAnalysisManager.INCLUDE_CLINICAL_IDS);
for (ClinicalAnalysis clinicalAnalysis : result.getResults()) {
logger.debug("Removing file references from Clinical Analysis {}", clinicalAnalysis.getId());
ClinicalAudit clinicalAudit = new ClinicalAudit("OPENCGA", ClinicalAudit.Action.UPDATE_CLINICAL_ANALYSIS, "File "
+ file.getString(FileDBAdaptor.QueryParams.PATH.key()) + " was deleted. Remove file references from case.",
TimeUtils.getTime());
transactionalUpdate(clientSession, clinicalAnalysis, parameters, null, Collections.singletonList(clinicalAudit), options);
}
}

@Override
public OpenCGAResult restore(long id, QueryOptions queryOptions) throws CatalogDBException {
return null;
Expand Down
41 changes: 40 additions & 1 deletion opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.opencb.opencga.catalog.managers.FileUtils;
import org.opencb.opencga.catalog.managers.SampleManager;
import org.opencb.opencga.catalog.utils.Constants;
import org.opencb.opencga.catalog.utils.ParamUtils;
import org.opencb.opencga.catalog.utils.ParamUtils.BasicUpdateAction;
import org.opencb.opencga.catalog.utils.UuidUtils;
import org.opencb.opencga.core.api.ParamConstants;
Expand Down Expand Up @@ -785,7 +786,8 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, lon
document.getSet().put(QueryParams.RELATED_FILES.key(), relatedFileDocument);
break;
case REMOVE:
document.getPullAll().put(QueryParams.RELATED_FILES.key(), relatedFileDocument);
List<Document> documentList = fixRelatedFilesForRemoval(relatedFileDocument);
document.getPull().put(QueryParams.RELATED_FILES.key(), documentList);
break;
case ADD:
document.getAddToSet().put(QueryParams.RELATED_FILES.key(), relatedFileDocument);
Expand Down Expand Up @@ -886,6 +888,18 @@ private UpdateDocument getValidatedUpdateParams(ClientSession clientSession, lon
return document;
}

private List<Document> fixRelatedFilesForRemoval(List<Document> relatedFiles) {
if (CollectionUtils.isEmpty(relatedFiles)) {
return Collections.emptyList();
}

List<Document> relatedFilesCopy = new ArrayList<>(relatedFiles.size());
for (Document relatedFile : relatedFiles) {
relatedFilesCopy.add(new Document("file", new Document("uid", relatedFile.get("file", Document.class).get("uid"))));
}
return relatedFilesCopy;
}

@Override
public OpenCGAResult delete(File file) throws CatalogDBException {
throw new UnsupportedOperationException("Use delete passing status field.");
Expand Down Expand Up @@ -1007,7 +1021,9 @@ OpenCGAResult<Object> privateDelete(ClientSession clientSession, Document fileDo
Document tmpFile = iterator.next();
long tmpFileUid = tmpFile.getLong(PRIVATE_UID);

removeFileReferences(clientSession, studyUid, tmpFileUid, tmpFile);
dbAdaptorFactory.getCatalogJobDBAdaptor().removeFileReferences(clientSession, studyUid, tmpFileUid, tmpFile);
dbAdaptorFactory.getClinicalAnalysisDBAdaptor().removeFileReferences(clientSession, studyUid, tmpFileUid, tmpFile);

// Set status
nestedPut(QueryParams.INTERNAL_STATUS.key(), getMongoDBDocument(new FileStatus(status), "status"), tmpFile);
Expand Down Expand Up @@ -1035,6 +1051,28 @@ OpenCGAResult<Object> privateDelete(ClientSession clientSession, Document fileDo
}
}

void removeFileReferences(ClientSession clientSession, long studyUid, long fileUid, Document fileDoc)
throws CatalogParameterException, CatalogDBException, CatalogAuthorizationException {
File file = fileConverter.convertToDataModelType(fileDoc);
FileRelatedFile relatedFile = new FileRelatedFile(file, null);
ObjectMap parameters = new ObjectMap(QueryParams.RELATED_FILES.key(), Collections.singletonList(relatedFile));
ObjectMap actionMap = new ObjectMap(QueryParams.RELATED_FILES.key(), ParamUtils.BasicUpdateAction.REMOVE);
QueryOptions options = new QueryOptions(Constants.ACTIONS, actionMap);

Query query = new Query()
.append(QueryParams.STUDY_UID.key(), studyUid)
.append(QueryParams.RELATED_FILES_FILE_UID.key(), fileUid);
UpdateDocument updateDocument = getValidatedUpdateParams(clientSession, studyUid, parameters, query, options);
Document updateDoc = updateDocument.toFinalUpdateDocument();
if (!updateDoc.isEmpty()) {
Bson bsonQuery = parseQuery(query);
OpenCGAResult<File> result = transactionalUpdate(clientSession, studyUid, bsonQuery, updateDocument);
if (result.getNumUpdated() > 0) {
logger.debug("File '{}' removed from related files array from {} files.", file.getPath(), result.getNumUpdated());
}
}
}

// OpenCGAResult<Object> privateDelete(ClientSession clientSession, File file, String status) throws CatalogDBException {
// long tmpStartTime = startQuery();
// logger.debug("Deleting file {} ({})", file.getPath(), file.getUid());
Expand Down Expand Up @@ -1491,6 +1529,7 @@ private Bson parseQuery(Query query, Document extraQuery, String user)
case TAGS:
case SIZE:
case SOFTWARE_NAME:
case RELATED_FILES_FILE_UID:
case JOB_ID:
addAutoOrQuery(queryParam.key(), queryParam.key(), myQuery, queryParam.type(), andBsonList);
break;
Expand Down
2 changes: 1 addition & 1 deletion ...catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/converters/FileConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public List<Document> convertRelatedFiles(List<FileRelatedFile> relatedFileList)
if (ListUtils.isNotEmpty(relatedFileList)) {
for (FileRelatedFile relatedFile : relatedFileList) {
relatedFiles.add(new Document()
.append("relation", relatedFile.getRelation().name())
.append("relation", relatedFile.getRelation() != null ? relatedFile.getRelation().name() : null)
.append("file", new Document("uid", relatedFile.getFile().getUid()))
);
}
Expand Down
25 changes: 17 additions & 8 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,7 @@
import org.opencb.commons.utils.ListUtils;
import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager;
import org.opencb.opencga.catalog.db.DBAdaptorFactory;
import org.opencb.opencga.catalog.db.api.DBIterator;
import org.opencb.opencga.catalog.db.api.FileDBAdaptor;
import org.opencb.opencga.catalog.db.api.SampleDBAdaptor;
import org.opencb.opencga.catalog.db.api.StudyDBAdaptor;
import org.opencb.opencga.catalog.db.api.*;
import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory;
import org.opencb.opencga.catalog.exceptions.*;
import org.opencb.opencga.catalog.io.IOManager;
Expand Down Expand Up @@ -1538,7 +1535,7 @@ public OpenCGAResult delete(String studyStr, List<String> fileIds, QueryOptions
return delete(studyStr, fileIds, options, false, token);
}

public OpenCGAResult delete(String studyStr, List<String> fileIds, ObjectMap params, boolean ignoreException, String token)
public OpenCGAResult delete(String studyStr, List<String> fileIds, QueryOptions options, boolean ignoreException, String token)
throws CatalogException {
String userId = catalogManager.getUserManager().getUserId(token);
Study study = studyManager.resolveId(studyStr, userId, new QueryOptions(QueryOptions.INCLUDE,
Expand All @@ -1549,13 +1546,15 @@ public OpenCGAResult delete(String studyStr, List<String> fileIds, ObjectMap par
ObjectMap auditParams = new ObjectMap()
.append("study", studyStr)
.append("fileIds", fileIds)
.append("params", params)
.append("options", options)
.append("ignoreException", ignoreException)
.append("token", token);

QueryOptions queryOptions = options != null ? new QueryOptions(options) : new QueryOptions();

// We need to avoid processing subfolders or subfiles of an already processed folder independently
Set<String> processedPaths = new HashSet<>();
boolean physicalDelete = params.getBoolean(Constants.SKIP_TRASH, false);
boolean physicalDelete = queryOptions.getBoolean(Constants.SKIP_TRASH, false);

auditManager.initAuditBatch(operationUuid);
OpenCGAResult<File> result = OpenCGAResult.empty(File.class);
Expand Down Expand Up @@ -1836,7 +1835,7 @@ public OpenCGAResult<File> unlink(@Nullable String studyId, String fileId, Strin
} catch (CatalogException e) {
auditManager.audit(userId, Enums.Action.UNLINK, Enums.Resource.FILE, fileId, "", study.getId(), study.getUuid(),
auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()));
throw e;
throw new CatalogException("Could not unlink file '" + fileId + "'", e);
}
}

Expand Down Expand Up @@ -3100,6 +3099,16 @@ private void checkCanDeleteFile(Study study, String fileId, boolean unlink, List
// TODO: Validate no file/folder within any registered directory is not registered in OpenCGA
}

Query clinicalQuery = new Query()
.append(ClinicalAnalysisDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid())
.append(ClinicalAnalysisDBAdaptor.QueryParams.FILES_UID.key(), file.getUid())
.append(ClinicalAnalysisDBAdaptor.QueryParams.LOCKED.key(), true);
OpenCGAResult<Long> count = clinicalDBAdaptor.count(clinicalQuery);
if (count.getNumMatches() > 0) {
throw new CatalogException("The file " + file.getName() + " is part of " + count.getNumMatches() + " clinical analyses");
}


// Check the original files are not being indexed at the moment
if (!indexFiles.isEmpty()) {
Query query = new Query(FileDBAdaptor.QueryParams.UID.key(), new ArrayList<>(indexFiles));
Expand Down
Loading

0 comments on commit 28e916f

Please sign in to comment.