From c6f84efeb2804b6a44e3f58c3b4ffbd6a5eff4cf Mon Sep 17 00:00:00 2001 From: pfurio Date: Mon, 7 Oct 2024 12:07:01 +0200 Subject: [PATCH] catalog: create default RESOURCE folder for studies, #TASK-6442 --- .../executors/CatalogCommandExecutor.java | 3 +- .../catalog/db/api/StudyDBAdaptor.java | 3 +- .../db/mongodb/MongoDBAdaptorFactory.java | 13 ++- .../OrganizationMongoDBAdaptorFactory.java | 6 +- .../db/mongodb/StudyMongoDBAdaptor.java | 65 ++++++++--- .../opencga/catalog/io/CatalogIOManager.java | 5 +- .../catalog/managers/CatalogManager.java | 6 +- .../catalog/managers/StudyManager.java | 107 +++++++++--------- .../catalog/migration/MigrationManager.java | 5 +- .../mongodb/AbstractMongoDBAdaptorTest.java | 3 +- .../catalog/db/mongodb/MongoBackupUtils.java | 6 +- .../db/mongodb/ProjectMongoDBAdaptorTest.java | 2 +- .../db/mongodb/StudyMongoDBAdaptorTest.java | 4 +- .../CatalogManagerExternalResource.java | 4 +- .../migration/MigrationManagerTest.java | 2 +- .../opencga/core/api/FieldConstants.java | 3 +- .../opencb/opencga/core/models/file/File.java | 16 ++- 17 files changed, 158 insertions(+), 95 deletions(-) diff --git a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java index 090802f3741..154d81e6696 100644 --- a/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java +++ b/opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java @@ -138,7 +138,8 @@ private void status() throws CatalogException, JsonProcessingException { result.put("installed", true); } else { String oldDatabase = configuration.getDatabasePrefix() + "_catalog"; - MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory()); + MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory(), + catalogManager.getCatalogIOManager()); MongoDataStore oldDatastore = mongoDBAdaptorFactory.getMongoManager().get(oldDatabase, mongoDBAdaptorFactory.getMongoDbConfiguration()); try { if (oldDatastore.getCollectionNames().contains("metadata")) { diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java index 0afd3f255d4..25dd369d8c9 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java @@ -29,7 +29,6 @@ import org.opencb.opencga.catalog.exceptions.CatalogParameterException; import org.opencb.opencga.catalog.utils.ParamUtils; import org.opencb.opencga.core.models.common.Enums; -import org.opencb.opencga.core.models.file.File; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.response.OpenCGAResult; @@ -191,7 +190,7 @@ default void checkId(long studyId) throws CatalogDBException { OpenCGAResult nativeInsert(Map study) throws CatalogDBException; - OpenCGAResult insert(Project project, Study study, List files, QueryOptions options) throws CatalogDBException; + OpenCGAResult insert(Project project, Study study, QueryOptions options) throws CatalogDBException; boolean hasStudyPermission(long studyId, String user, StudyPermissions.Permissions permission) throws CatalogDBException; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java index e262c3d3595..75ffb1106ee 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java @@ -29,6 +29,7 @@ import org.opencb.opencga.catalog.db.api.*; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.catalog.managers.NoteManager; import org.opencb.opencga.core.api.ParamConstants; @@ -53,6 +54,7 @@ public class MongoDBAdaptorFactory implements DBAdaptorFactory { private final IOManagerFactory ioManagerFactory; private final MongoDataStoreManager mongoManager; private final MongoDBConfiguration mongoDbConfiguration; + private final CatalogIOManager catalogIOManager; private final Configuration configuration; private static final String ORGANIZATION_PREFIX = "ORG_"; @@ -67,7 +69,8 @@ private enum OrganizationTag { private final Logger logger; - public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactory ioManagerFactory) throws CatalogDBException { + public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager) + throws CatalogDBException { List dataStoreServerAddresses = new LinkedList<>(); for (String host : catalogConfiguration.getCatalog().getDatabase().getHosts()) { if (host.contains(":")) { @@ -91,6 +94,7 @@ public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactor this.mongoDbConfiguration = mongoDBConfiguration; this.configuration = catalogConfiguration; this.ioManagerFactory = ioManagerFactory; + this.catalogIOManager = catalogIOManager; logger = LoggerFactory.getLogger(this.getClass()); connect(catalogConfiguration); @@ -207,7 +211,8 @@ private OrganizationSummary getOrganizationSummary(Note note) { private OrganizationMongoDBAdaptorFactory configureOrganizationMongoDBAdaptorFactory(String organizationId, Configuration configuration) throws CatalogDBException { - return new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, ioManagerFactory); + return new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, catalogIOManager, + ioManagerFactory); } public OrganizationMongoDBAdaptorFactory getOrganizationMongoDBAdaptorFactory(String organization) throws CatalogDBException { @@ -231,7 +236,7 @@ private OrganizationMongoDBAdaptorFactory getOrganizationMongoDBAdaptorFactory(S // Organization is present, so create new OrganizationMongoDBAdaptorFactory for the organization OrganizationMongoDBAdaptorFactory organizationMongoDBAdaptorFactory = new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, - ioManagerFactory); + catalogIOManager, ioManagerFactory); organizationDBAdaptorMap.put(organizationId, organizationMongoDBAdaptorFactory); return organizationMongoDBAdaptorFactory; } @@ -268,7 +273,7 @@ public OpenCGAResult createOrganization(Organization organization, try { // Create organization OrganizationMongoDBAdaptorFactory organizationDBAdaptorFactory = new OrganizationMongoDBAdaptorFactory(organization.getId(), - mongoManager, mongoDbConfiguration, configuration, ioManagerFactory); + mongoManager, mongoDbConfiguration, configuration, catalogIOManager, ioManagerFactory); organizationDBAdaptorMap.put(organization.getId(), organizationDBAdaptorFactory); OrganizationSummary organizationSummary = new OrganizationSummary(organization.getId(), diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java index 9884bbccc6c..95e60382fa3 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java @@ -13,6 +13,7 @@ import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.core.config.Admin; import org.opencb.opencga.core.config.Configuration; @@ -145,7 +146,8 @@ public class OrganizationMongoDBAdaptorFactory { public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreManager mongoDataStoreManager, MongoDBConfiguration mongoDBConfiguration, Configuration configuration, - IOManagerFactory ioManagerFactory) throws CatalogDBException { + CatalogIOManager catalogIOManager, IOManagerFactory ioManagerFactory) + throws CatalogDBException { logger = LoggerFactory.getLogger(OrganizationMongoDBAdaptorFactory.class); this.mongoManager = mongoDataStoreManager; this.organizationId = organizationId; @@ -210,7 +212,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa sampleDBAdaptor = new SampleMongoDBAdaptor(sampleCollection, sampleArchivedCollection, deletedSampleCollection, configuration, this); - studyDBAdaptor = new StudyMongoDBAdaptor(studyCollection, deletedStudyCollection, configuration, this); + studyDBAdaptor = new StudyMongoDBAdaptor(studyCollection, deletedStudyCollection, catalogIOManager, configuration, this); userDBAdaptor = new UserMongoDBAdaptor(userCollection, deletedUserCollection, configuration, this); cohortDBAdaptor = new CohortMongoDBAdaptor(cohortCollection, deletedCohortCollection, configuration, this); panelDBAdaptor = new PanelMongoDBAdaptor(panelCollection, panelArchivedCollection, deletedPanelCollection, configuration, this); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java index c91fd4c3665..c68fdc99654 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java @@ -36,10 +36,8 @@ import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter; import org.opencb.opencga.catalog.db.mongodb.converters.VariableSetConverter; import org.opencb.opencga.catalog.db.mongodb.iterators.StudyCatalogMongoDBIterator; -import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException; -import org.opencb.opencga.catalog.exceptions.CatalogDBException; -import org.opencb.opencga.catalog.exceptions.CatalogException; -import org.opencb.opencga.catalog.exceptions.CatalogParameterException; +import org.opencb.opencga.catalog.exceptions.*; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.utils.Constants; import org.opencb.opencga.catalog.utils.FqnUtils; import org.opencb.opencga.catalog.utils.ParamUtils; @@ -51,6 +49,7 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.common.InternalStatus; import org.opencb.opencga.core.models.file.File; +import org.opencb.opencga.core.models.file.FileInternal; import org.opencb.opencga.core.models.project.Project; import org.opencb.opencga.core.models.study.*; import org.opencb.opencga.core.response.OpenCGAResult; @@ -58,6 +57,7 @@ import javax.annotation.Nullable; import java.net.URI; +import java.nio.file.Paths; import java.util.*; import java.util.function.Consumer; import java.util.function.Function; @@ -80,11 +80,15 @@ public class StudyMongoDBAdaptor extends CatalogMongoDBAdaptor implements StudyD private StudyConverter studyConverter; private VariableSetConverter variableSetConverter; - public StudyMongoDBAdaptor(MongoDBCollection studyCollection, MongoDBCollection deletedStudyCollection, Configuration configuration, + private final CatalogIOManager ioManager; + + public StudyMongoDBAdaptor(MongoDBCollection studyCollection, MongoDBCollection deletedStudyCollection, + CatalogIOManager catalogIOManager, Configuration configuration, OrganizationMongoDBAdaptorFactory dbAdaptorFactory) { super(configuration, LoggerFactory.getLogger(StudyMongoDBAdaptor.class)); this.dbAdaptorFactory = dbAdaptorFactory; this.studyCollection = studyCollection; + this.ioManager = catalogIOManager; this.deletedStudyCollection = deletedStudyCollection; this.studyConverter = new StudyConverter(); this.variableSetConverter = new VariableSetConverter(); @@ -199,23 +203,22 @@ public OpenCGAResult nativeInsert(Map study) throws Catal // } @Override - public OpenCGAResult insert(Project project, Study study, List files, QueryOptions options) throws CatalogDBException { + public OpenCGAResult insert(Project project, Study study, QueryOptions options) throws CatalogDBException { try { return runTransaction(clientSession -> { long tmpStartTime = startQuery(); logger.debug("Starting study insert transaction for study id '{}'", study.getId()); - insert(clientSession, project, study, files); + insert(clientSession, project, study); return endWrite(tmpStartTime, 1, 1, 0, 0, null); }); } catch (Exception e) { - logger.error("Could not create study {}: {}", study.getId(), e.getMessage()); - throw new CatalogDBException(e); + throw new CatalogDBException("Could not create study '" + study.getFqn() + "'", e); } } - Study insert(ClientSession clientSession, Project project, Study study, List files) - throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException { + Study insert(ClientSession clientSession, Project project, Study study) + throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException, CatalogIOException { if (project.getUid() < 0) { throw CatalogDBException.uidNotFound("Project", project.getUid()); } @@ -243,12 +246,23 @@ Study insert(ClientSession clientSession, Project project, Study study, List(variableSets.size()); for (VariableSet variableSet : variableSets) { + long vsetUid = getNewUid(clientSession); + variableSet.setUid(vsetUid); variableSetDocuments.add(variableSetConverter.convertToStorageType(variableSet)); } } study.setVariableSets(null); - //Create DBObject + FqnUtils.FQN fqn = FqnUtils.parse(study.getFqn()); + String organization = fqn.getOrganization(); + try { + URI studyUri = ioManager.getStudyUri(organization, Long.toString(project.getUid()), Long.toString(study.getUid())); + study.setUri(studyUri); + } catch (CatalogIOException e) { + throw new CatalogIOException("Could not obtain study uri.", e); + } + + //Create Document and store in the database Document studyObject = studyConverter.convertToStorageType(study); studyObject.put(PRIVATE_UID, studyUid); studyObject.put(QueryParams.VARIABLE_SET.key(), variableSetDocuments); @@ -264,14 +278,29 @@ Study insert(ClientSession clientSession, Project project, Study study, List files = Arrays.asList( + new File(".", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "", study.getUri(), + "study root folder", FileInternal.init(), 0, project.getCurrentRelease()), + new File("JOBS", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "JOBS/", + ioManager.getJobsUri(), "Default jobs folder", FileInternal.init(), 0, project.getCurrentRelease()), + new File("RESOURCES", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "RESOURCES/", + Paths.get(study.getUri()).resolve("RESOURCES").toUri(), "Default resources folder", FileInternal.init(), 0, + project.getCurrentRelease()).setResource(true) + ); + + // Create default folders + for (File file : files) { + dbAdaptorFactory.getCatalogFileDBAdaptor().insert(clientSession, study.getUid(), file, Collections.emptyList(), + Collections.emptyList(), Collections.emptyList()); + } + + try { + ioManager.createStudy(organization, Long.toString(project.getUid()), Long.toString(study.getUid())); + } catch (CatalogIOException e) { + throw new CatalogIOException("Could not create study folder '" + study.getUri() + "' in file system.", e); } return study; diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java index 699bc9c99da..1d51ce61351 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java @@ -16,6 +16,7 @@ package org.opencb.opencga.catalog.io; +import org.apache.commons.lang3.StringUtils; import org.opencb.opencga.catalog.exceptions.CatalogIOException; import org.opencb.opencga.core.common.UriUtils; import org.opencb.opencga.core.config.Configuration; @@ -85,7 +86,7 @@ public void createDefaultOpenCGAFolders() throws CatalogIOException { } protected void checkParam(String param) throws CatalogIOException { - if (param == null || param.equals("")) { + if (StringUtils.isEmpty(param)) { throw new CatalogIOException("Parameter '" + param + "' not valid"); } } @@ -107,7 +108,7 @@ public URI getProjectUri(String organizationId, String projectId) throws Catalog return Paths.get(getProjectsUri(organizationId)).resolve(projectId.endsWith("/") ? projectId : (projectId + "/")).toUri(); } - private URI getStudyUri(String organizationId, String projectId, String studyId) throws CatalogIOException { + public URI getStudyUri(String organizationId, String projectId, String studyId) throws CatalogIOException { checkParam(studyId); return Paths.get(getProjectUri(organizationId, projectId)).resolve(studyId.endsWith("/") ? studyId : (studyId + "/")).toUri(); } diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java index 030505395d0..eabad72ea28 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java @@ -105,7 +105,7 @@ private void init() throws CatalogException { logger.debug("CatalogManager configureIOManager"); configureIOManager(configuration); logger.debug("CatalogManager configureDBAdaptorFactory"); - catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory); + catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory, catalogIOManager); authorizationDBAdaptorFactory = new AuthorizationMongoDBAdaptorFactory((MongoDBAdaptorFactory) catalogDBAdaptorFactory, configuration); authenticationFactory = new AuthenticationFactory(catalogDBAdaptorFactory, configuration); @@ -365,6 +365,10 @@ public IOManagerFactory getIoManagerFactory() { return ioManagerFactory; } + public CatalogIOManager getCatalogIOManager() { + return catalogIOManager; + } + private void configureIOManager(Configuration configuration) throws CatalogIOException { ioManagerFactory = new IOManagerFactory(); catalogIOManager = new CatalogIOManager(configuration); diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java index 70f933f6430..6c397914599 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/StudyManager.java @@ -54,7 +54,6 @@ import org.opencb.opencga.core.models.common.Enums; import org.opencb.opencga.core.models.family.FamilyPermissions; import org.opencb.opencga.core.models.file.File; -import org.opencb.opencga.core.models.file.FileInternal; import org.opencb.opencga.core.models.file.FilePermissions; import org.opencb.opencga.core.models.file.FileStatus; import org.opencb.opencga.core.models.individual.IndividualPermissions; @@ -439,15 +438,9 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions study.setAdditionalInfo(ParamUtils.defaultObject(study.getAdditionalInfo(), Collections::emptyList)); study.setAttributes(ParamUtils.defaultObject(study.getAttributes(), HashMap::new)); - study.setVariableSets(ParamUtils.defaultObject(study.getVariableSets(), ArrayList::new)); - - LinkedList files = new LinkedList<>(); - File rootFile = new File(".", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "", null, "study root folder", - FileInternal.init(), 0, project.getCurrentRelease()); - File jobsFile = new File("JOBS", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "JOBS/", - catalogIOManager.getJobsUri(), "Default jobs folder", FileInternal.init(), 0, project.getCurrentRelease()); - files.add(rootFile); - files.add(jobsFile); + // Scan and add default VariableSets + List variableSetList = scanDefaultVariableSets(study.getRelease()); + study.setVariableSets(variableSetList); // Get organization owner and admins Organization organization = catalogManager.getOrganizationManager().get(organizationId, @@ -463,37 +456,14 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions study.setGroups(groups); /* CreateStudy */ - getStudyDBAdaptor(organizationId).insert(project, study, files, options); - OpenCGAResult result = getStudy(organizationId, projectUid, study.getUuid(), options); - study = result.getResults().get(0); - - URI uri; - try { - uri = catalogIOManager.createStudy(organizationId, Long.toString(project.getUid()), Long.toString(study.getUid())); - } catch (CatalogIOException e) { - try { - getStudyDBAdaptor(organizationId).delete(study); - } catch (Exception e1) { - logger.error("Can't delete study after failure creating study", e1); - } - throw e; - } - - // Update uri of study - getStudyDBAdaptor(organizationId).update(study.getUid(), new ObjectMap("uri", uri), QueryOptions.empty()); - study.setUri(uri); - - long rootFileId = getFileDBAdaptor(organizationId).getId(study.getUid(), ""); //Set studyUri to the root folder too - getFileDBAdaptor(organizationId).update(rootFileId, new ObjectMap("uri", uri), QueryOptions.empty()); - - // Read and process installation variable sets - createDefaultVariableSets(organizationId, study, token); + OpenCGAResult result = getStudyDBAdaptor(organizationId).insert(project, study, options); auditManager.auditCreate(organizationId, userId, Enums.Resource.STUDY, study.getId(), study.getUuid(), study.getId(), study.getUuid(), auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS)); if (options.getBoolean(ParamConstants.INCLUDE_RESULT_PARAM)) { - result.setResults(Arrays.asList(study)); + OpenCGAResult tmpResult = getStudy(organizationId, projectUid, study.getUuid(), options); + result.setResults(tmpResult.getResults()); } return result; } catch (CatalogException e) { @@ -503,36 +473,47 @@ public OpenCGAResult create(String projectStr, Study study, QueryOptions } } - public void createDefaultVariableSets(String studyStr, String token) throws CatalogException { - JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); - CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); - String organizationId = catalogFqn.getOrganizationId(); - Study study = resolveId(catalogFqn, null, tokenPayload); - createDefaultVariableSets(organizationId, study, token); - } - - private void createDefaultVariableSets(String organizationId, Study study, String token) throws CatalogException { + public List scanDefaultVariableSets(int release) throws CatalogException { Set variablesets = new Reflections(new ResourcesScanner(), "variablesets/").getResources(Pattern.compile(".*\\.json")); + List variableSetList = new ArrayList<>(variablesets.size()); for (String variableSetFile : variablesets) { VariableSet vs; try { vs = JacksonUtils.getDefaultNonNullObjectMapper().readValue( getClass().getClassLoader().getResourceAsStream(variableSetFile), VariableSet.class); } catch (IOException e) { - logger.error("Could not parse variable set '{}'", variableSetFile, e); - continue; + throw new CatalogException("Could not parse default variable set '" + variableSetFile + "'.", e); } if (vs != null) { if (vs.getAttributes() == null) { vs.setAttributes(new HashMap<>()); } vs.getAttributes().put("resource", variableSetFile); + validateNewVariableSet(vs, release); + variableSetList.add(vs); + } else { + throw new CatalogException("Default VariableSet object is null after parsing file '" + variableSetFile + "'"); + } + } + return variableSetList; + } - if (study.getVariableSets().stream().anyMatch(tvs -> tvs.getId().equals(vs.getId()))) { - logger.debug("Skip already existing variable set " + vs.getId()); - } else { - createVariableSet(organizationId, study, vs, token); - } + public void createDefaultVariableSets(String studyStr, String token) throws CatalogException { + JwtPayload tokenPayload = catalogManager.getUserManager().validateToken(token); + CatalogFqn catalogFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload); + String organizationId = catalogFqn.getOrganizationId(); + Study study = resolveId(catalogFqn, null, tokenPayload); + createDefaultVariableSets(organizationId, study, token); + } + + private void createDefaultVariableSets(String organizationId, Study study, String token) throws CatalogException { + List variableSetList = scanDefaultVariableSets(study.getRelease()); + + for (VariableSet vs : variableSetList) { + if (study.getVariableSets().stream().anyMatch(tvs -> tvs.getId().equals(vs.getId()))) { + logger.debug("Skip already existing variable set " + vs.getId()); + } else { + createVariableSet(organizationId, study, vs, token); } } } @@ -1408,6 +1389,28 @@ public OpenCGAResult getVariableSetSummary(String studyStr, return new OpenCGAResult<>(dbTime, Collections.emptyList(), 1, Collections.singletonList(variableSetSummary), 1); } + private void validateNewVariableSet(VariableSet variableSet, int release) throws CatalogException { + ParamUtils.checkParameter(variableSet.getId(), "id"); + ParamUtils.checkObj(variableSet.getVariables(), "Variables from VariableSet"); + + variableSet.setDescription(ParamUtils.defaultString(variableSet.getDescription(), "")); + variableSet.setAttributes(ParamUtils.defaultObject(variableSet.getAttributes(), new HashMap<>())); + variableSet.setEntities(ParamUtils.defaultObject(variableSet.getEntities(), Collections.emptyList())); + variableSet.setName(ParamUtils.defaultString(variableSet.getName(), variableSet.getId())); + variableSet.setRelease(release); + + for (Variable variable : variableSet.getVariables()) { + ParamUtils.checkParameter(variable.getId(), "variable ID"); + ParamUtils.checkObj(variable.getType(), "variable Type"); + variable.setAllowedValues(ParamUtils.defaultObject(variable.getAllowedValues(), Collections.emptyList())); + variable.setAttributes(ParamUtils.defaultObject(variable.getAttributes(), Collections.emptyMap())); + variable.setCategory(ParamUtils.defaultString(variable.getCategory(), "")); + variable.setDependsOn(ParamUtils.defaultString(variable.getDependsOn(), "")); + variable.setDescription(ParamUtils.defaultString(variable.getDescription(), "")); + variable.setName(ParamUtils.defaultString(variable.getName(), variable.getId())); +// variable.setRank(defaultString(variable.getDescription(), "")); + } + } /* * Variables Methods diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java index aacf4949226..7b1f979c863 100644 --- a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java +++ b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/migration/MigrationManager.java @@ -19,6 +19,7 @@ import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.CatalogDBException; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.managers.CatalogManager; import org.opencb.opencga.core.api.ParamConstants; import org.opencb.opencga.core.common.TimeUtils; @@ -127,8 +128,10 @@ public void runMigration(String version, Collection d String token) throws CatalogException, IOException { runMigration(ParamConstants.ADMIN_ORGANIZATION, version, domains, languages, offline, appHome, params, token); + CatalogIOManager catalogIOManager = new CatalogIOManager(configuration); // ***** Starts code to remove in future versions. Reload MongoDBAdaptorFactory to avoid Notes migration issue. *****/ - try (MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory())) { + try (MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, + catalogManager.getIoManagerFactory(), catalogIOManager)) { for (String organizationId : mongoDBAdaptorFactory.getOrganizationIds()) { // ***** Finish code to remove in future versions. Reload MongoDBAdaptorFactory to avoid Notes migration issue. *****/ diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java index 632b13694cc..8fb999325bb 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/AbstractMongoDBAdaptorTest.java @@ -31,7 +31,8 @@ public class AbstractMongoDBAdaptorTest extends AbstractManagerTest { @Before public void setUp() throws Exception { super.setUp(); - dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory()); + dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), catalogManager.getIoManagerFactory(), + catalogManager.getCatalogIOManager()); catalogUserDBAdaptor = (UserMongoDBAdaptor) dbAdaptorFactory.getCatalogUserDBAdaptor(organizationId); catalogStudyDBAdaptor = (StudyMongoDBAdaptor) dbAdaptorFactory.getCatalogStudyDBAdaptor(organizationId); catalogProjectDBAdaptor = (ProjectMongoDBAdaptor) dbAdaptorFactory.getCatalogProjectDbAdaptor(organizationId); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java index 252085c353b..d6c11dd6568 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/MongoBackupUtils.java @@ -43,7 +43,7 @@ public class MongoBackupUtils { public static void dump(CatalogManager catalogManager, Path opencgaHome) throws CatalogDBException { StopWatch stopWatch = StopWatch.createStarted(); try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), - catalogManager.getIoManagerFactory())) { + catalogManager.getIoManagerFactory(), catalogManager.getCatalogIOManager())) { MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(dbAdaptorFactory.getOrganizationIds().get(0)) .getMongoDataStore().getMongoClient(); MongoDatabase dumpDatabase = mongoClient.getDatabase("test_dump"); @@ -91,7 +91,7 @@ public static void dump(CatalogManager catalogManager, Path opencgaHome) throws public static void restore(CatalogManager catalogManager, Path opencgaHome) throws Exception { try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), - catalogManager.getIoManagerFactory())) { + catalogManager.getIoManagerFactory(), catalogManager.getCatalogIOManager())) { MongoClient mongoClient = dbAdaptorFactory.getOrganizationMongoDBAdaptorFactory(ParamConstants.ADMIN_ORGANIZATION) .getMongoDataStore().getMongoClient(); @@ -147,7 +147,7 @@ private static void restore(CatalogManager catalogManager, Path opencgaHome, Obj logger.info("Restore opencga from source " + source + " for organizations " + organizationIds); StopWatch stopWatch = StopWatch.createStarted(); try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), - catalogManager.getIoManagerFactory())) { + catalogManager.getIoManagerFactory(), catalogManager.getCatalogIOManager())) { for (String existingOrganizationId : dbAdaptorFactory.getOrganizationIds()) { // We need to completely remove databases that were not backed up so tests that attempt to create them again don't fail diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java index 50f710ae3eb..fc00605e5fa 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/ProjectMongoDBAdaptorTest.java @@ -138,7 +138,7 @@ public void renameProjectTest() throws CatalogException { Project p2 = getProject("myp2"); // Add study - catalogStudyDBAdaptor.insert(p1, new Study().setId("study").setFqn(FqnUtils.buildFqn(organizationId, "myp1", "study")), null, null); + catalogStudyDBAdaptor.insert(p1, new Study().setId("study").setFqn(FqnUtils.buildFqn(organizationId, "myp1", "study")), null); catalogProjectDBAdaptor.update(p1.getUid(), new ObjectMap(ProjectDBAdaptor.QueryParams.ID.key(), "newpmp"), QueryOptions.empty()); Project project = getProject("newpmp"); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java index f81f28b37a6..2e8fd510215 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptorTest.java @@ -69,13 +69,13 @@ private Study getMinimalStudyInstance(String projectId, String id) { @Test public void createStudyWithSameIdInDifferentProject() throws CatalogException { Project project = getProject(project1); - OpenCGAResult create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project1, "studyId"), null, null); + OpenCGAResult create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project1, "studyId"), null); assertEquals(1, create.getNumInserted()); Study ph1 = getStudy(project.getUid(), "studyId"); assertNotNull(ph1); project = getProject(project2); - create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project2, "studyId"), null, null); + create = catalogStudyDBAdaptor.insert(project, getMinimalStudyInstance(project2, "studyId"), null); assertEquals(1, create.getNumInserted()); ph1 = getStudy(project.getUid(), "studyId"); assertNotNull(ph1); diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java index 09c94ee33d3..15d5a21f095 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/CatalogManagerExternalResource.java @@ -24,6 +24,7 @@ import org.opencb.opencga.catalog.auth.authentication.JwtManager; import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory; import org.opencb.opencga.catalog.exceptions.CatalogException; +import org.opencb.opencga.catalog.io.CatalogIOManager; import org.opencb.opencga.catalog.io.IOManagerFactory; import org.opencb.opencga.core.common.PasswordUtils; import org.opencb.opencga.core.common.TimeUtils; @@ -137,7 +138,8 @@ public Path getOpencgaHome() { } public static void clearCatalog(Configuration configuration) throws CatalogException, URISyntaxException { - try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(configuration, new IOManagerFactory())) { + CatalogIOManager catalogIOManager = new CatalogIOManager(configuration); + try (MongoDBAdaptorFactory dbAdaptorFactory = new MongoDBAdaptorFactory(configuration, new IOManagerFactory(), catalogIOManager)) { dbAdaptorFactory.deleteCatalogDB(); } diff --git a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java index b7d6b72f62f..12508962113 100644 --- a/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java +++ b/opencga-catalog/src/test/java/org/opencb/opencga/catalog/migration/MigrationManagerTest.java @@ -135,7 +135,7 @@ protected void run() throws Exception { public void setUp() throws Exception { super.setUp(); try (MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(catalogManager.getConfiguration(), - catalogManager.getIoManagerFactory())) { + catalogManager.getIoManagerFactory(), catalogManager.getCatalogIOManager())) { for (String organizationId : mongoDBAdaptorFactory.getOrganizationIds()) { mongoDBAdaptorFactory.getMongoDataStore(organizationId) .getCollection(OrganizationMongoDBAdaptorFactory.MIGRATION_COLLECTION) diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java index dbde894f78f..1de3a3cf976 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/api/FieldConstants.java @@ -289,7 +289,8 @@ public class FieldConstants { public static final String FILE_CHECKSUM = "The checksum of the file."; public static final String FILE_PATH = "The path of the file."; public static final String FILE_URI = "The uri of the file."; - public static final String FILE_EXTERNAL = "Indicates the file is external or not."; + public static final String FILE_EXTERNAL = "Indicates whether the file comes from an external path or not."; + public static final String FILE_RESOURCE = "Indicates the file is treated as a resource."; public static final String FILE_SIZE = "The size of the file."; public static final String FILE_SOFTWARE = "Software related with file."; public static final String FILE_EXPERIMENT = "File experiment."; diff --git a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/File.java b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/File.java index 581cc6d6dd1..e5a2c8b90be 100644 --- a/opencga-core/src/main/java/org/opencb/opencga/core/models/file/File.java +++ b/opencga-core/src/main/java/org/opencb/opencga/core/models/file/File.java @@ -121,10 +121,12 @@ public class File extends Annotable { description = FieldConstants.GENERIC_DESCRIPTION_DESCRIPTION) private String description; - @DataField(id = "external", - description = FieldConstants.FILE_EXTERNAL) + @DataField(id = "external", description = FieldConstants.FILE_EXTERNAL) private boolean external; + @DataField(id = "resource", description = FieldConstants.FILE_RESOURCE) + private boolean resource; + @DataField(id = "size", description = FieldConstants.FILE_SIZE) private long size; @@ -262,6 +264,7 @@ public String toString() { sb.append(", modificationDate='").append(modificationDate).append('\''); sb.append(", description='").append(description).append('\''); sb.append(", external=").append(external); + sb.append(", resource=").append(resource); sb.append(", size=").append(size); sb.append(", software=").append(software); sb.append(", experiment=").append(experiment); @@ -419,6 +422,15 @@ public File setExternal(boolean external) { return this; } + public boolean isResource() { + return resource; + } + + public File setResource(boolean resource) { + this.resource = resource; + return this; + } + public long getSize() { return size; }