Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

TASK-5716 - Renaming a file is not reflected on the list of files contained by samples #2410

Merged
merged 4 commits into from
Apr 4, 2024
Merged

TASK-5716 - Renaming a file is not reflected on the list of files contained by samples #2410

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d095e97
catalog: implement file move in MongoDBAdaptor, #TASK-5716
pfurio Mar 7, 2024
85767cf
Merge branch 'release-2.12.x' into TASK-5716
pfurio Mar 21, 2024
d0f46fe
Merge branch 'release-2.12.x' into TASK-5716
pfurio Mar 25, 2024
fe319af
Merge branch 'release-2.12.x' into TASK-5716
pfurio Apr 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ private void status() throws CatalogException, JsonProcessingException {
if (catalogManager.existsCatalogDB()) {
result.put("installed", true);

MongoDBAdaptorFactory factory = new MongoDBAdaptorFactory(configuration);
MongoDBAdaptorFactory factory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory());
MongoDBCollection metaCollection = factory.getMongoDBCollectionMap().get(MongoDBAdaptorFactory.METADATA_COLLECTION);
Document metaDocument = metaCollection.find(new Document(), QueryOptions.empty()).first();

Expand Down
26 changes: 0 additions & 26 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/FileDBAdaptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,32 +287,6 @@ OpenCGAResult<File> get(long fileId, QueryOptions options)
OpenCGAResult<File> getAllInStudy(long studyId, QueryOptions options)
throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException;

/***
* Retrieves all the files present in the folder.
*
* @param studyId Study id where the files will be extracted from.
* @param path Directory where the files will be extracted from.
* @param options Options to filter the file output.
* @return A OpenCGAResult object containing the files present in the folder of the given study.
* @throws CatalogDBException when the study or the path does not exist.
*/
OpenCGAResult<File> getAllFilesInFolder(long studyId, String path, QueryOptions options) throws CatalogDBException;

/***
* Renames the file.
*
* @param fileId Id of the file to be renamed.
* @param filePath New file or directory name (containing the full path).
* @param fileUri New file uri (containing the full path).
* @param options Options to filter the file output.
* @return A OpenCGAResult object.
* @throws CatalogDBException when the filePath already exists.
* @throws CatalogParameterException if there is any formatting error.
* @throws CatalogAuthorizationException if the user is not authorised to perform the query.
*/
OpenCGAResult rename(long fileId, String filePath, String fileUri, QueryOptions options)
throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException;

/**
* Removes the mark of the permission rule (if existed) from all the entries from the study to notify that permission rule would need to
* be applied.
Expand Down
289 changes: 177 additions & 112 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/FileMongoDBAdaptor.java
View file
Open in desktop

Large diffs are not rendered by default.

7 changes: 5 additions & 2 deletions ...ga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.opencb.opencga.catalog.db.api.MigrationDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogDBException;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.io.IOManagerFactory;
import org.opencb.opencga.core.config.Admin;
import org.opencb.opencga.core.config.Configuration;
import org.slf4j.Logger;
Expand Down Expand Up @@ -140,6 +141,7 @@ public class MongoDBAdaptorFactory implements DBAdaptorFactory {
);

static final String METADATA_OBJECT_ID = "METADATA";
private final IOManagerFactory ioManagerFactory;
private final MongoDataStoreManager mongoManager;
private final MongoDBConfiguration configuration;
private final String database;
Expand All @@ -165,7 +167,7 @@ public class MongoDBAdaptorFactory implements DBAdaptorFactory {

private Logger logger;

public MongoDBAdaptorFactory(Configuration catalogConfiguration) throws CatalogDBException {
public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactory ioManagerFactory) throws CatalogDBException {
List<DataStoreServerAddress> dataStoreServerAddresses = new LinkedList<>();
for (String host : catalogConfiguration.getCatalog().getDatabase().getHosts()) {
if (host.contains(":")) {
Expand All @@ -188,6 +190,7 @@ public MongoDBAdaptorFactory(Configuration catalogConfiguration) throws CatalogD
this.mongoManager = new MongoDataStoreManager(dataStoreServerAddresses);
this.configuration = mongoDBConfiguration;
this.database = getCatalogDatabase(catalogConfiguration.getDatabasePrefix());
this.ioManagerFactory = ioManagerFactory;

logger = LoggerFactory.getLogger(this.getClass());
connect(catalogConfiguration);
Expand Down Expand Up @@ -422,7 +425,7 @@ private void connect(Configuration catalogConfiguration) throws CatalogDBExcepti

collections.put(AUDIT_COLLECTION, auditCollection);

fileDBAdaptor = new FileMongoDBAdaptor(fileCollection, deletedFileCollection, catalogConfiguration, this);
fileDBAdaptor = new FileMongoDBAdaptor(fileCollection, deletedFileCollection, catalogConfiguration, this, ioManagerFactory);
familyDBAdaptor = new FamilyMongoDBAdaptor(familyCollection, familyArchivedCollection, deletedFamilyCollection,
catalogConfiguration, this);
individualDBAdaptor = new IndividualMongoDBAdaptor(individualCollection, individualArchivedCollection, deletedIndividualCollection,
Expand Down
4 changes: 2 additions & 2 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,10 @@ public class CatalogManager implements AutoCloseable {

public CatalogManager(Configuration configuration) throws CatalogException {
this.configuration = configuration;
logger.debug("CatalogManager configureDBAdaptorFactory");
catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration);
logger.debug("CatalogManager configureIOManager");
configureIOManager(configuration);
logger.debug("CatalogManager configureDBAdaptorFactory");
catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory);
logger.debug("CatalogManager configureManager");
configureManagers(configuration);
}
Expand Down
142 changes: 48 additions & 94 deletions opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FileManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2226,13 +2226,6 @@ private OpenCGAResult<File> update(Study study, File file, FileUpdateParams upda
userId, false);
}

//Name must be changed with "rename".
if (updateParams != null && StringUtils.isNotEmpty(updateParams.getName())) {
logger.info("Rename file using update method!");
rename(study.getFqn(), file.getPath(), updateParams.getName(), token);
parameters.remove(FileDBAdaptor.QueryParams.NAME.key());
}

checkUpdateAnnotations(study, file, parameters, options, VariableSet.AnnotableDataModels.FILE, fileDBAdaptor, userId);

OpenCGAResult update = fileDBAdaptor.update(file.getUid(), parameters, study.getVariableSets(), options);
Expand Down Expand Up @@ -2288,12 +2281,6 @@ public OpenCGAResult<File> update(String studyStr, String entryStr, ObjectMap pa
catalogManager.getSampleManager().internalGet(study.getUid(), sampleIds, SampleManager.INCLUDE_SAMPLE_IDS, userId, false);
}

//Name must be changed with "rename".
if (parameters.containsKey(FileDBAdaptor.QueryParams.NAME.key())) {
logger.info("Rename file using update method!");
rename(studyStr, file.getPath(), parameters.getString(FileDBAdaptor.QueryParams.NAME.key()), token);
}

OpenCGAResult<File> queryResult = unsafeUpdate(study, file, parameters, options, userId);
auditManager.auditUpdate(userId, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(),
auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS));
Expand Down Expand Up @@ -2323,6 +2310,52 @@ OpenCGAResult<File> unsafeUpdate(Study study, File file, ObjectMap parameters, Q
return fileDBAdaptor.get(file.getUid(), options);
}

public OpenCGAResult<File> move(String studyStr, String entryStr, String targetPathStr, QueryOptions options, String token)
throws CatalogException {
String userId = userManager.getUserId(token);
Study study = studyManager.resolveId(studyStr, userId);

ObjectMap auditParams = new ObjectMap()
.append("study", studyStr)
.append("file", entryStr)
.append("target", targetPathStr)
.append("options", options)
.append("token", token);

String fileId = entryStr;
String fileUuid = "";
try {
File file = internalGet(study.getUid(), entryStr, QueryOptions.empty(), userId).first();
fileId = file.getId();
fileUuid = file.getUuid();
// Check user has write permissions on file/folder
authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.WRITE);

OpenCGAResult<File> parents = getParents(study.getUid(), targetPathStr, false, INCLUDE_FILE_IDS);
// Check user can write in target path
File parentFolder = parents.first();
authorizationManager.checkFilePermission(study.getUid(), parentFolder.getUid(), userId, FilePermissions.WRITE);

ObjectMap parameters = new ObjectMap(FileDBAdaptor.QueryParams.PATH.key(), targetPathStr);
OpenCGAResult<File> update = fileDBAdaptor.update(file.getUid(), parameters, Collections.emptyList(), QueryOptions.empty());

auditManager.audit(userId, Enums.Action.MOVE, Enums.Resource.FILE, file.getId(), file.getUuid(), study.getId(), study.getUuid(),
auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS));

if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) {
// Fetch updated file
OpenCGAResult<File> result = fileDBAdaptor.get(study.getUid(),
new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), options, userId);
update.setResults(result.getResults());
}
return update;
} catch (Exception e) {
auditManager.audit(userId, Enums.Action.MOVE, Enums.Resource.FILE, fileId, fileUuid, study.getId(), study.getUuid(),
auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, new Error(0, fileId, e.getMessage())));
throw e;
}
}

public OpenCGAResult<File> link(String studyStr, FileLinkParams params, boolean parents, String token) throws CatalogException {
// We make two attempts to link to ensure the behaviour remains even if it is being called at the same time link from different
// threads
Expand Down Expand Up @@ -2416,68 +2449,6 @@ public OpenCGAResult groupBy(@Nullable String studyStr, Query query, List<String
return ParamUtils.defaultObject(queryResult, OpenCGAResult::new);
}

OpenCGAResult<File> rename(String studyStr, String fileStr, String newName, String sessionId) throws CatalogException {
ParamUtils.checkFileName(newName, "name");

String userId = userManager.getUserId(sessionId);
Study study = studyManager.resolveId(studyStr, userId);

File file = internalGet(study.getUid(), fileStr, EXCLUDE_FILE_ATTRIBUTES, userId).first();
authorizationManager.checkFilePermission(study.getUid(), file.getUid(), userId, FilePermissions.WRITE);

if (file.getName().equals(newName)) {
OpenCGAResult result = OpenCGAResult.empty();
result.setEvents(Collections.singletonList(new Event(Event.Type.WARNING, newName, "File already had that name.")));
return result;
}

if (isRootFolder(file)) {
throw new CatalogException("Can not rename root folder");
}

String oldPath = file.getPath();
Path parent = Paths.get(oldPath).getParent();
String newPath;
if (parent == null) {
newPath = newName;
} else {
newPath = parent.resolve(newName).toString();
}

IOManager ioManager = null;
try {
ioManager = ioManagerFactory.get(file.getUri());
} catch (IOException e) {
throw CatalogIOException.ioManagerException(file.getUri(), e);
}
URI oldUri = file.getUri();
URI newUri = Paths.get(oldUri).getParent().resolve(newName).toUri();
// URI studyUri = file.getUri();
boolean isExternal = file.isExternal(); //If the file URI is not null, the file is external located.

switch (file.getType()) {
case DIRECTORY:
if (!isExternal) { //Only rename non external files
// TODO? check if something in the subtree is not READY?
if (ioManager.exists(oldUri)) {
ioManager.rename(oldUri, newUri); // io.move() 1
}
}
fileDBAdaptor.rename(file.getUid(), newPath, newUri.toString(), null);
break;
case FILE:
if (!isExternal) { //Only rename non external files
ioManager.rename(oldUri, newUri);
}
fileDBAdaptor.rename(file.getUid(), newPath, newUri.toString(), null);
break;
default:
throw new CatalogException("Unknown file type " + file.getType());
}

return fileDBAdaptor.get(file.getUid(), QueryOptions.empty());
}

public OpenCGAResult<FileContent> grep(String studyId, String fileId, String pattern, boolean ignoreCase, int numLines, String token)
throws CatalogException {
long startTime = System.currentTimeMillis();
Expand Down Expand Up @@ -2865,7 +2836,7 @@ public OpenCGAResult<File> getParents(String studyStr, String path, boolean root
String userId = userManager.getUserId(token);
Study study = studyManager.resolveId(studyStr, userId);

List<String> paths = calculateAllPossiblePaths(path);
List<String> paths = org.opencb.opencga.catalog.managers.FileUtils.calculateAllPossiblePaths(path);

Query query = new Query(FileDBAdaptor.QueryParams.PATH.key(), paths);
query.put(FileDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid());
Expand Down Expand Up @@ -2918,23 +2889,6 @@ private List<File> getRecursiveFilesAndFolders(long studyUid, List<File> fileLis
return fileListCopy;
}

private List<String> calculateAllPossiblePaths(String filePath) {
String path = "";
String[] split = filePath.split("/");
List<String> paths = new ArrayList<>(split.length + 1);
paths.add(""); //Add study root folder
//Add intermediate folders
//Do not add the last split, could be a file or a folder..
//Depending on this, it could end with '/' or not.
for (int i = 0; i < split.length - 1; i++) {
String f = split[i];
path = path + f + "/";
paths.add(path);
}
paths.add(filePath); //Add the file path
return paths;
}

//FIXME: This should use org.opencb.opencga.storage.core.variant.io.VariantReaderUtils
private String getMainVariantFile(String name) {
if (name.endsWith(".variants.avro.gz") || name.endsWith(".variants.proto.gz") || name.endsWith(".variants.json.gz")) {
Expand All @@ -2960,7 +2914,7 @@ private String getVariantMetadataFile(String path) {

private OpenCGAResult<File> getParents(long studyUid, String filePath, boolean rootFirst, QueryOptions options)
throws CatalogException {
List<String> paths = calculateAllPossiblePaths(filePath);
List<String> paths = org.opencb.opencga.catalog.managers.FileUtils.calculateAllPossiblePaths(filePath);

Query query = new Query(FileDBAdaptor.QueryParams.PATH.key(), paths);
query.put(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid);
Expand Down
Loading
Loading