Skip to content

TASK-6346 - Load multiple VCFs with the same name #2612

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: develop
Choose a base branch
from
Open

TASK-6346 - Load multiple VCFs with the same name #2612

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
059a2e2
storage: Allow loading duplicated file names in variant-storage. #TAS…
j-coll Jul 14, 2025
4861d8f
storage: Add method rename and move files in storage. #TASK-6346
j-coll Jul 14, 2025
f2c0241
storage: Add method move-bulk files in storage. #TASK-6346
j-coll Jul 21, 2025
34cdae4
catalog: allow updating the workspace, #TASK-6346
pfurio Jul 28, 2025
7c9d67d
catalog: allow updating the URI of moved files, #TASK-6346
pfurio Jul 29, 2025
fdef721
storage: Fix delete file with duplicated file name. #TASK-6346
j-coll Jul 30, 2025
2a91bd1
storage: Use file URIs at CatalogStorageMetadataSynchronizer. #TASK-6346
j-coll Jul 30, 2025
6bc3f0d
catalog: comment check exists in old path, #TASK-6346
pfurio Sep 3, 2025
018ef1b
Merge branch 'develop' into TASK-6346
pfurio Sep 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions ...s/src/main/java/org/opencb/opencga/analysis/variant/manager/VariantCatalogQueryUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ public Query parseQuery(Query query, QueryOptions queryOptions, CellBaseUtils ce
sampleFilterValidator.processFilter(query, VariantQueryParam.GENOTYPE, release, token, defaultStudyStr);
fileFilterValidator.processFilter(query, VariantQueryParam.FILE, release, token, defaultStudyStr);
fileFilterValidator.processFilter(query, VariantQueryParam.INCLUDE_FILE, release, token, defaultStudyStr);
fileFilterValidator.processFilter(query, VariantQueryParam.FILE_DATA, release, token, defaultStudyStr);
cohortFilterValidator.processFilter(query, VariantQueryParam.COHORT, release, token, defaultStudyStr);
cohortFilterValidator.processFilter(query, VariantQueryParam.STATS_ALT, release, token, defaultStudyStr);
cohortFilterValidator.processFilter(query, VariantQueryParam.STATS_REF, release, token, defaultStudyStr);
Expand Down Expand Up @@ -1386,6 +1387,14 @@ public List<Trio> getTrios(
return trios;
}

public static String toStorageFileName(File file) {
return toStorageFilePath(file);
}

public static String toStorageFilePath(File file) {
return file.getUri().getPath();
}

public abstract class FilterValidator {
protected final QueryOptions RELEASE_OPTIONS = new QueryOptions(INCLUDE, Arrays.asList(
FileDBAdaptor.QueryParams.ID.key(),
Expand Down Expand Up @@ -1532,8 +1541,8 @@ protected List<String> validate(String defaultStudyStr, List<String> values, Int
throws CatalogException {
if (release == null) {
DataResult<File> files = catalogManager.getFileManager().get(defaultStudyStr, values,
FileManager.INCLUDE_FILE_IDS, sessionId);
return files.getResults().stream().map(File::getName).collect(Collectors.toList());
FileManager.INCLUDE_FILE_URI, sessionId);
return files.getResults().stream().map(VariantCatalogQueryUtils::toStorageFileName).collect(Collectors.toList());
} else {
return validate(defaultStudyStr, values, release, param, catalogManager.getFileManager(), File::getName,
file -> file.getInternal().getVariant().getIndex().getRelease(), file -> {
Expand All @@ -1545,6 +1554,7 @@ protected List<String> validate(String defaultStudyStr, List<String> values, Int

}
}

}

public class SampleFilterValidator extends FilterValidator {
Expand Down
4 changes: 2 additions & 2 deletions ...org/opencb/opencga/analysis/variant/manager/operations/VariantDeleteOperationManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package org.opencb.opencga.analysis.variant.manager.operations;

import org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils;
import org.opencb.opencga.analysis.variant.manager.VariantStorageManager;
import org.opencb.opencga.analysis.variant.metadata.CatalogStorageMetadataSynchronizer;
import org.opencb.opencga.catalog.exceptions.CatalogException;
Expand Down Expand Up @@ -89,8 +90,7 @@ public void removeFile(String study, List<String> inputFiles, URI outdir, String
+ (fileMetadata == null ? " File not found in storage." : ""));
}
}
fileNames.add(file.getName());
// filePaths.add(file.getPath());
fileNames.add(VariantCatalogQueryUtils.toStorageFileName(file));
}

}
Expand Down
11 changes: 6 additions & 5 deletions ...pencb/opencga/analysis/variant/manager/operations/VariantFileIndexerOperationManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,8 @@ private List<URI> findFilesToIndex(ObjectMap params, String token) throws Catalo
String virtualFile = null;
for (File fileToIndex : filesToIndex) {
if (!fileNamesToIndexSet.add(fileToIndex.getName())) {
throw new CatalogException("Unable to " + step + " multiple files with the same name");
throw new CatalogException("Unable to " + step + " multiple files with the same name in the same job. "
+ "Please split into multiple jobs.");
}
if (FileUtils.isPartial(fileToIndex)) {
String thisVirtualFile = FileUtils.getVirtualFileFromPartial(fileToIndex).getName();
Expand Down Expand Up @@ -369,15 +370,15 @@ private void updateFileInfo(String study, List<File> filesToIndex, VariantReader
Integer release, boolean saveIntermediateFiles, ObjectMap options, String sessionId)
throws CatalogException, IOException {

Map<String, StoragePipelineResult> map;
Map<URI, StoragePipelineResult> map;
try {
map = storagePipelineResults
.stream()
.collect(Collectors.toMap(s -> {
String input = s.getInput().getPath();
String inputFileName = Paths.get(input).getFileName().toString();
String inputFileName = Paths.get(input).toString();
// Input file may be the transformed one. Convert into original file.
return VariantReaderUtils.getOriginalFromTransformedFile(inputFileName);
return UriUtils.createUriSafe(VariantReaderUtils.getOriginalFromTransformedFile(inputFileName));
}, i -> i));
} catch (IllegalStateException e) {
throw e;
Expand All @@ -387,7 +388,7 @@ private void updateFileInfo(String study, List<File> filesToIndex, VariantReader
// Fetch from catalog. {@link #copyResult} may modify the content
indexedFile = catalogManager.getFileManager().get(study, indexedFile.getId(), null, sessionId).first();
// Suppose that the missing results are due to errors, and those files were not indexed.
StoragePipelineResult storagePipelineResult = map.get(indexedFile.getName());
StoragePipelineResult storagePipelineResult = map.get(indexedFile.getUri());

boolean jobFailed = storagePipelineResult == null || storagePipelineResult.getLoadError() != null
|| storagePipelineResult.getTransformError() != null;
Expand Down
54 changes: 28 additions & 26 deletions ...java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.analysis.variant.manager.VariantCatalogQueryUtils;
import org.opencb.opencga.analysis.variant.operations.VariantIndexOperationTool;
import org.opencb.opencga.catalog.db.api.*;
import org.opencb.opencga.catalog.exceptions.CatalogException;
Expand Down Expand Up @@ -58,6 +59,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.URI;
import java.nio.file.Paths;
import java.util.*;
import java.util.function.Function;
Expand Down Expand Up @@ -443,17 +445,17 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
// FIXME: This method call should be relocated
modified |= synchronizeSampleIndexConfiguration(study, token);

Map<String, Integer> fileNameMap = new HashMap<>();
Map<URI, Integer> fileURIMap = new HashMap<>();
Map<Integer, String> filePathMap = new HashMap<>();
Set<Integer> virtualFiles = new HashSet<>();
Map<String, Set<String>> fileSamplesMap = new HashMap<>();
Map<URI, Set<String>> fileSamplesMap = new HashMap<>();
LinkedHashSet<Integer> indexedFilesFromStorage = new LinkedHashSet<>();
Set<String> annotationReadyFilesFromStorage = new HashSet<>();
Set<String> secondaryIndexReadyFilesFromStorage = new HashSet<>();
Set<URI> annotationReadyFilesFromStorage = new HashSet<>();
Set<URI> secondaryIndexReadyFilesFromStorage = new HashSet<>();
Set<Integer> allSamples = new HashSet<>();

// -------------------------------------------------------------------
logger.info("Read file metadata from Storage");
logger.info("Read file metadata from Storage" + (CollectionUtils.isEmpty(files) ? " for all files" : " for " + files.size() + " files"));
// -------------------------------------------------------------------
Iterable<FileMetadata> filesIterable;
boolean fullSynchronize;
Expand All @@ -465,7 +467,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
filesIterable = () -> {
Iterator<FileMetadata> iteratorMain = files.stream()
.map(f -> {
FileMetadata fm = metadataManager.getFileMetadata(study.getId(), f.getName());
FileMetadata fm = metadataManager.getFileMetadata(study.getId(), VariantCatalogQueryUtils.toStorageFileName(f));
if (fm != null) {
if (fm.getType() == FileMetadata.Type.PARTIAL) {
virtualFiles.add(fm.getAttributes().getInt(FileMetadata.VIRTUAL_PARENT));
Expand All @@ -483,7 +485,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
};
}
for (FileMetadata fileMetadata : filesIterable) {
fileNameMap.put(fileMetadata.getName(), fileMetadata.getId());
fileURIMap.put(fileMetadata.getURI(), fileMetadata.getId());
filePathMap.put(fileMetadata.getId(), fileMetadata.getPath());
Set<String> samples;
if (fullSynchronize && !fileMetadata.isIndexed()) {
Expand All @@ -494,10 +496,10 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
indexedFilesFromStorage.add(fileMetadata.getId());
}
if (fileMetadata.getAnnotationStatus() == TaskMetadata.Status.READY) {
annotationReadyFilesFromStorage.add(fileMetadata.getName());
annotationReadyFilesFromStorage.add(fileMetadata.getURI());
}
if (fileMetadata.getSecondaryAnnotationIndexStatus() == TaskMetadata.Status.READY) {
secondaryIndexReadyFilesFromStorage.add(fileMetadata.getName());
secondaryIndexReadyFilesFromStorage.add(fileMetadata.getURI());
}
if (fileMetadata.getSamples() == null) {
logger.warn("File '{}' with null samples", fileMetadata.getName());
Expand Down Expand Up @@ -526,14 +528,14 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
}
});
}
fileSamplesMap.put(fileMetadata.getName(), samples);
fileSamplesMap.put(fileMetadata.getURI(), samples);
allSamples.addAll(fileMetadata.getSamples());
if (samples.size() > 100) {
// Try to reuse value.
// If the file holds more than 100 samples, it's most likely this same set of samples is already present
for (Set<String> value : fileSamplesMap.values()) {
if (value.equals(samples)) {
fileSamplesMap.put(fileMetadata.getName(), value);
fileSamplesMap.put(fileMetadata.getURI(), value);
break;
}
}
Expand All @@ -548,8 +550,8 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
for (Integer virtualFile : virtualFiles) {
File file = catalogManager.getFileManager()
.get(study.getName(), filePathMap.get(virtualFile), INDEXED_FILES_QUERY_OPTIONS, token).first();
boolean annotationIndexReady = annotationReadyFilesFromStorage.contains(file.getName());
boolean secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(file.getName());
boolean annotationIndexReady = annotationReadyFilesFromStorage.contains(file.getUri());
boolean secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(file.getUri());
if (synchronizeIndexedFile(study, file, fileSamplesMap, annotationIndexReady, secondaryIndexReady, token)) {
modified = true;
}
Expand All @@ -573,8 +575,8 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
.iterator(study.getName(), query, INDEXED_FILES_QUERY_OPTIONS, token)) {
while (iterator.hasNext()) {
File file = iterator.next();
boolean annotationIndexReady = annotationReadyFilesFromStorage.contains(file.getName());
boolean secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(file.getName());
boolean annotationIndexReady = annotationReadyFilesFromStorage.contains(file.getUri());
boolean secondaryIndexReady = secondaryIndexReadyFilesFromStorage.contains(file.getUri());
if (synchronizeIndexedFile(study, file, fileSamplesMap, annotationIndexReady, secondaryIndexReady, token)) {
modifiedFiles++;
modified = true;
Expand Down Expand Up @@ -631,7 +633,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
.iterator(study.getName(), indexedFilesQuery, INDEXED_FILES_QUERY_OPTIONS, token)) {
while (iterator.hasNext()) {
File file = iterator.next();
Integer fileId = fileNameMap.get(file.getName());
Integer fileId = fileURIMap.get(file.getUri());
if (fileId == null || !indexedFilesFromStorage.contains(fileId)) {
String newStatus;
FileInternalVariantIndex index = file.getInternal().getVariant().getIndex();
Expand All @@ -640,7 +642,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
} else {
newStatus = VariantIndexStatus.NONE;
}
logger.info("File \"{}\" change status from {} to {}", file.getName(),
logger.info("File \"{}\" change status from {} to {}", file.getId(),
VariantIndexStatus.READY, newStatus);
index.setStatus(new VariantIndexStatus(newStatus, "Not indexed, regarding Storage Metadata"));
catalogManager.getFileManager().updateFileInternalVariantIndex(study.getName(), file, index, token);
Expand All @@ -653,7 +655,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
// -------------------------------------------------------------------
logger.info("Synchronize indexStatus=INDEXING files up to Catalog");
// -------------------------------------------------------------------
Set<String> loadingFilesRegardingCatalog = new HashSet<>();
Set<URI> loadingFilesRegardingCatalog = new HashSet<>();
Query runningIndexFilesQuery;
if (CollectionUtils.isEmpty(files)) {
runningIndexFilesQuery = RUNNING_INDEX_FILES_QUERY;
Expand All @@ -667,7 +669,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
.iterator(study.getName(), runningIndexFilesQuery, INDEXED_FILES_QUERY_OPTIONS, token)) {
while (iterator.hasNext()) {
File file = iterator.next();
Integer fileId = fileNameMap.get(file.getName());
Integer fileId = fileURIMap.get(file.getUri());
FileMetadata fileMetadata;
if (fileId == null) {
fileMetadata = null;
Expand Down Expand Up @@ -698,18 +700,18 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
} else {
newStatus = VariantIndexStatus.NONE;
}
logger.info("File \"{}\" change status from {} to {}", file.getName(),
logger.info("File \"{}\" change status from {} to {}", file.getId(),
prevStatus, newStatus);
index.setStatus(new VariantIndexStatus(newStatus, "Error loading. Reset status to " + newStatus));

catalogManager.getFileManager().updateFileInternalVariantIndex(study.getName(), file, index, token);
modified = true;
} else {
// Running job. Might be transforming, or have just started. Do not modify the status!
loadingFilesRegardingCatalog.add(file.getName());
loadingFilesRegardingCatalog.add(file.getUri());
}
} else {
loadingFilesRegardingCatalog.add(file.getName());
loadingFilesRegardingCatalog.add(file.getUri());
}
}
}
Expand Down Expand Up @@ -754,7 +756,7 @@ protected boolean synchronizeFiles(StudyMetadata study, List<File> files, String
return modified;
}

private boolean synchronizeIndexedFile(StudyMetadata study, File file, Map<String, Set<String>> fileSamplesMap,
private boolean synchronizeIndexedFile(StudyMetadata study, File file, Map<URI, Set<String>> fileSamplesMap,
boolean annotationIndexReady, boolean secondaryIndexReady, String token)
throws CatalogException {
boolean modified = false;
Expand All @@ -766,7 +768,7 @@ private boolean synchronizeIndexedFile(StudyMetadata study, File file, Map<Strin
if (index.getStatus() == null) {
index.setStatus(new VariantIndexStatus());
}
logger.debug("File \"{}\" change status from {} to {}", file.getName(), status, VariantIndexStatus.READY);
logger.debug("File \"{}\" change status from {} to {}", file.getId(), status, VariantIndexStatus.READY);
index.setStatus(new VariantIndexStatus(VariantIndexStatus.READY, "Indexed, regarding Storage Metadata"));

catalogManager.getFileManager().updateFileInternalVariantIndex(study.getName(), file, index, token);
Expand Down Expand Up @@ -797,11 +799,11 @@ private boolean synchronizeIndexedFile(StudyMetadata study, File file, Map<Strin
modified = true;
}

Set<String> storageSamples = fileSamplesMap.get(file.getName());
Set<String> storageSamples = fileSamplesMap.get(file.getUri());
Set<String> catalogSamples = new HashSet<>(file.getSampleIds());
if (storageSamples == null) {
storageSamples = new HashSet<>();
Integer fileId = metadataManager.getFileId(study.getId(), file.getName());
Integer fileId = metadataManager.getFileId(study.getId(), VariantCatalogQueryUtils.toStorageFileName(file));
for (Integer sampleId : metadataManager.getSampleIdsFromFileId(study.getId(), fileId)) {
storageSamples.add(metadataManager.getSampleName(study.getId(), sampleId));
}
Expand Down
Loading
Loading