catalog: create default RESOURCE folder for studies, #TASK-6442

opencga-app/src/main/java/org/opencb/opencga/app/cli/admin/executors/CatalogCommandExecutor.java

@@ -138,7 +138,8 @@ private void status() throws CatalogException, JsonProcessingException {
                 result.put("installed", true);
             } else {
                 String oldDatabase = configuration.getDatabasePrefix() + "_catalog";
-                MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory());
+                MongoDBAdaptorFactory mongoDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, catalogManager.getIoManagerFactory(),
+                        catalogManager.getCatalogIOManager());
                 MongoDataStore oldDatastore = mongoDBAdaptorFactory.getMongoManager().get(oldDatabase, mongoDBAdaptorFactory.getMongoDbConfiguration());
                 try {
                     if (oldDatastore.getCollectionNames().contains("metadata")) {

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/api/StudyDBAdaptor.java

@@ -29,7 +29,6 @@
 import org.opencb.opencga.catalog.exceptions.CatalogParameterException;
 import org.opencb.opencga.catalog.utils.ParamUtils;
 import org.opencb.opencga.core.models.common.Enums;
-import org.opencb.opencga.core.models.file.File;
 import org.opencb.opencga.core.models.project.Project;
 import org.opencb.opencga.core.models.study.*;
 import org.opencb.opencga.core.response.OpenCGAResult;
@@ -191,7 +190,7 @@ default void checkId(long studyId) throws CatalogDBException {
 
     OpenCGAResult<Study> nativeInsert(Map<String, Object> study) throws CatalogDBException;
 
-    OpenCGAResult<Study> insert(Project project, Study study, List<File> files, QueryOptions options) throws CatalogDBException;
+    OpenCGAResult<Study> insert(Project project, Study study, QueryOptions options) throws CatalogDBException;
 
     boolean hasStudyPermission(long studyId, String user, StudyPermissions.Permissions permission) throws CatalogDBException;
 

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/MongoDBAdaptorFactory.java

@@ -29,6 +29,7 @@
 import org.opencb.opencga.catalog.db.api.*;
 import org.opencb.opencga.catalog.exceptions.CatalogDBException;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
+import org.opencb.opencga.catalog.io.CatalogIOManager;
 import org.opencb.opencga.catalog.io.IOManagerFactory;
 import org.opencb.opencga.catalog.managers.NoteManager;
 import org.opencb.opencga.core.api.ParamConstants;
@@ -53,6 +54,7 @@ public class MongoDBAdaptorFactory implements DBAdaptorFactory {
     private final IOManagerFactory ioManagerFactory;
     private final MongoDataStoreManager mongoManager;
     private final MongoDBConfiguration mongoDbConfiguration;
+    private final CatalogIOManager catalogIOManager;
     private final Configuration configuration;
 
     private static final String ORGANIZATION_PREFIX = "ORG_";
@@ -67,7 +69,8 @@ private enum OrganizationTag {
 
     private final Logger logger;
 
-    public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactory ioManagerFactory) throws CatalogDBException {
+    public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactory ioManagerFactory, CatalogIOManager catalogIOManager)
+            throws CatalogDBException {
         List<DataStoreServerAddress> dataStoreServerAddresses = new LinkedList<>();
         for (String host : catalogConfiguration.getCatalog().getDatabase().getHosts()) {
             if (host.contains(":")) {
@@ -91,6 +94,7 @@ public MongoDBAdaptorFactory(Configuration catalogConfiguration, IOManagerFactor
         this.mongoDbConfiguration = mongoDBConfiguration;
         this.configuration = catalogConfiguration;
         this.ioManagerFactory = ioManagerFactory;
+        this.catalogIOManager = catalogIOManager;
 
         logger = LoggerFactory.getLogger(this.getClass());
         connect(catalogConfiguration);
@@ -207,7 +211,8 @@ private OrganizationSummary getOrganizationSummary(Note note) {
 
     private OrganizationMongoDBAdaptorFactory configureOrganizationMongoDBAdaptorFactory(String organizationId, Configuration configuration)
             throws CatalogDBException {
-        return new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, ioManagerFactory);
+        return new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration, catalogIOManager,
+                ioManagerFactory);
     }
 
     public OrganizationMongoDBAdaptorFactory getOrganizationMongoDBAdaptorFactory(String organization) throws CatalogDBException {
@@ -231,7 +236,7 @@ private OrganizationMongoDBAdaptorFactory getOrganizationMongoDBAdaptorFactory(S
                         // Organization is present, so create new OrganizationMongoDBAdaptorFactory for the organization
                         OrganizationMongoDBAdaptorFactory organizationMongoDBAdaptorFactory =
                                 new OrganizationMongoDBAdaptorFactory(organizationId, mongoManager, mongoDbConfiguration, configuration,
-                                        ioManagerFactory);
+                                        catalogIOManager, ioManagerFactory);
                         organizationDBAdaptorMap.put(organizationId, organizationMongoDBAdaptorFactory);
                         return organizationMongoDBAdaptorFactory;
                     }
@@ -268,7 +273,7 @@ public OpenCGAResult<Organization> createOrganization(Organization organization,
         try {
             // Create organization
             OrganizationMongoDBAdaptorFactory organizationDBAdaptorFactory = new OrganizationMongoDBAdaptorFactory(organization.getId(),
-                    mongoManager, mongoDbConfiguration, configuration, ioManagerFactory);
+                    mongoManager, mongoDbConfiguration, configuration, catalogIOManager, ioManagerFactory);
             organizationDBAdaptorMap.put(organization.getId(), organizationDBAdaptorFactory);
 
             OrganizationSummary organizationSummary = new OrganizationSummary(organization.getId(),

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/OrganizationMongoDBAdaptorFactory.java

@@ -13,6 +13,7 @@
 import org.opencb.commons.datastore.mongodb.MongoDataStoreManager;
 import org.opencb.opencga.catalog.exceptions.CatalogDBException;
 import org.opencb.opencga.catalog.exceptions.CatalogException;
+import org.opencb.opencga.catalog.io.CatalogIOManager;
 import org.opencb.opencga.catalog.io.IOManagerFactory;
 import org.opencb.opencga.core.config.Admin;
 import org.opencb.opencga.core.config.Configuration;
@@ -145,7 +146,8 @@ public class OrganizationMongoDBAdaptorFactory {
 
     public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreManager mongoDataStoreManager,
                                              MongoDBConfiguration mongoDBConfiguration, Configuration configuration,
-                                             IOManagerFactory ioManagerFactory) throws CatalogDBException {
+                                             CatalogIOManager catalogIOManager, IOManagerFactory ioManagerFactory)
+            throws CatalogDBException {
         logger = LoggerFactory.getLogger(OrganizationMongoDBAdaptorFactory.class);
         this.mongoManager = mongoDataStoreManager;
         this.organizationId = organizationId;
@@ -210,7 +212,7 @@ public OrganizationMongoDBAdaptorFactory(String organizationId, MongoDataStoreMa
 
         sampleDBAdaptor = new SampleMongoDBAdaptor(sampleCollection, sampleArchivedCollection, deletedSampleCollection, configuration,
                 this);
-        studyDBAdaptor = new StudyMongoDBAdaptor(studyCollection, deletedStudyCollection, configuration, this);
+        studyDBAdaptor = new StudyMongoDBAdaptor(studyCollection, deletedStudyCollection, catalogIOManager, configuration, this);
         userDBAdaptor = new UserMongoDBAdaptor(userCollection, deletedUserCollection, configuration, this);
         cohortDBAdaptor = new CohortMongoDBAdaptor(cohortCollection, deletedCohortCollection, configuration, this);
         panelDBAdaptor = new PanelMongoDBAdaptor(panelCollection, panelArchivedCollection, deletedPanelCollection, configuration, this);

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/StudyMongoDBAdaptor.java

@@ -36,10 +36,8 @@
 import org.opencb.opencga.catalog.db.mongodb.converters.StudyConverter;
 import org.opencb.opencga.catalog.db.mongodb.converters.VariableSetConverter;
 import org.opencb.opencga.catalog.db.mongodb.iterators.StudyCatalogMongoDBIterator;
-import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException;
-import org.opencb.opencga.catalog.exceptions.CatalogDBException;
-import org.opencb.opencga.catalog.exceptions.CatalogException;
-import org.opencb.opencga.catalog.exceptions.CatalogParameterException;
+import org.opencb.opencga.catalog.exceptions.*;
+import org.opencb.opencga.catalog.io.CatalogIOManager;
 import org.opencb.opencga.catalog.utils.Constants;
 import org.opencb.opencga.catalog.utils.FqnUtils;
 import org.opencb.opencga.catalog.utils.ParamUtils;
@@ -51,13 +49,15 @@
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.common.InternalStatus;
 import org.opencb.opencga.core.models.file.File;
+import org.opencb.opencga.core.models.file.FileInternal;
 import org.opencb.opencga.core.models.project.Project;
 import org.opencb.opencga.core.models.study.*;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nullable;
 import java.net.URI;
+import java.nio.file.Paths;
 import java.util.*;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -80,11 +80,15 @@ public class StudyMongoDBAdaptor extends CatalogMongoDBAdaptor implements StudyD
     private StudyConverter studyConverter;
     private VariableSetConverter variableSetConverter;
 
-    public StudyMongoDBAdaptor(MongoDBCollection studyCollection, MongoDBCollection deletedStudyCollection, Configuration configuration,
+    private final CatalogIOManager ioManager;
+
+    public StudyMongoDBAdaptor(MongoDBCollection studyCollection, MongoDBCollection deletedStudyCollection,
+                               CatalogIOManager catalogIOManager, Configuration configuration,
                                OrganizationMongoDBAdaptorFactory dbAdaptorFactory) {
         super(configuration, LoggerFactory.getLogger(StudyMongoDBAdaptor.class));
         this.dbAdaptorFactory = dbAdaptorFactory;
         this.studyCollection = studyCollection;
+        this.ioManager = catalogIOManager;
         this.deletedStudyCollection = deletedStudyCollection;
         this.studyConverter = new StudyConverter();
         this.variableSetConverter = new VariableSetConverter();
@@ -199,23 +203,22 @@ public OpenCGAResult<Study> nativeInsert(Map<String, Object> study) throws Catal
 //    }
 
     @Override
-    public OpenCGAResult<Study> insert(Project project, Study study, List<File> files, QueryOptions options) throws CatalogDBException {
+    public OpenCGAResult<Study> insert(Project project, Study study, QueryOptions options) throws CatalogDBException {
         try {
             return runTransaction(clientSession -> {
                 long tmpStartTime = startQuery();
                 logger.debug("Starting study insert transaction for study id '{}'", study.getId());
 
-                insert(clientSession, project, study, files);
+                insert(clientSession, project, study);
                 return endWrite(tmpStartTime, 1, 1, 0, 0, null);
             });
         } catch (Exception e) {
-            logger.error("Could not create study {}: {}", study.getId(), e.getMessage());
-            throw new CatalogDBException(e);
+            throw new CatalogDBException("Could not create study '" + study.getFqn() + "'", e);
         }
     }
 
-    Study insert(ClientSession clientSession, Project project, Study study, List<File> files)
-            throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException {
+    Study insert(ClientSession clientSession, Project project, Study study)
+            throws CatalogDBException, CatalogParameterException, CatalogAuthorizationException, CatalogIOException {
         if (project.getUid() < 0) {
             throw CatalogDBException.uidNotFound("Project", project.getUid());
         }
@@ -243,12 +246,23 @@ Study insert(ClientSession clientSession, Project project, Study study, List<Fil
         if (variableSets != null) {
             variableSetDocuments = new ArrayList<>(variableSets.size());
             for (VariableSet variableSet : variableSets) {
+                long vsetUid = getNewUid(clientSession);
+                variableSet.setUid(vsetUid);
                 variableSetDocuments.add(variableSetConverter.convertToStorageType(variableSet));
             }
         }
         study.setVariableSets(null);
 
-        //Create DBObject
+        FqnUtils.FQN fqn = FqnUtils.parse(study.getFqn());
+        String organization = fqn.getOrganization();
+        try {
+            URI studyUri = ioManager.getStudyUri(organization, Long.toString(project.getUid()), Long.toString(study.getUid()));
+            study.setUri(studyUri);
+        } catch (CatalogIOException e) {
+            throw new CatalogIOException("Could not obtain study uri.", e);
+        }
+
+        //Create Document and store in the database
         Document studyObject = studyConverter.convertToStorageType(study);
         studyObject.put(PRIVATE_UID, studyUid);
         studyObject.put(QueryParams.VARIABLE_SET.key(), variableSetDocuments);
@@ -264,14 +278,29 @@ Study insert(ClientSession clientSession, Project project, Study study, List<Fil
                 StringUtils.isNotEmpty(study.getCreationDate()) ? TimeUtils.toDate(study.getCreationDate()) : TimeUtils.getDate());
         studyObject.put(PRIVATE_MODIFICATION_DATE,
                 StringUtils.isNotEmpty(study.getModificationDate()) ? TimeUtils.toDate(study.getModificationDate()) : TimeUtils.getDate());
-
         studyCollection.insert(clientSession, studyObject, null);
 
-        if (files != null) {
-            for (File file : files) {
-                dbAdaptorFactory.getCatalogFileDBAdaptor().insert(clientSession, study.getUid(), file, Collections.emptyList(),
-                        Collections.emptyList(), Collections.emptyList());
-            }
+        // Create default folders for study
+        List<File> files = Arrays.asList(
+                new File(".", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "", study.getUri(),
+                        "study root folder", FileInternal.init(), 0, project.getCurrentRelease()),
+                new File("JOBS", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "JOBS/",
+                        ioManager.getJobsUri(), "Default jobs folder", FileInternal.init(), 0, project.getCurrentRelease()),
+                new File("RESOURCES", File.Type.DIRECTORY, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "RESOURCES/",
+                        Paths.get(study.getUri()).resolve("RESOURCES").toUri(), "Default resources folder", FileInternal.init(), 0,
+                        project.getCurrentRelease()).setResource(true)
+        );
+
+        // Create default folders
+        for (File file : files) {
+            dbAdaptorFactory.getCatalogFileDBAdaptor().insert(clientSession, study.getUid(), file, Collections.emptyList(),
+                    Collections.emptyList(), Collections.emptyList());
+        }
+
+        try {
+            ioManager.createStudy(organization, Long.toString(project.getUid()), Long.toString(study.getUid()));
+        } catch (CatalogIOException e) {
+            throw new CatalogIOException("Could not create study folder '" + study.getUri() + "' in file system.", e);
         }
 
         return study;

opencga-catalog/src/main/java/org/opencb/opencga/catalog/io/CatalogIOManager.java

@@ -16,6 +16,7 @@
 
 package org.opencb.opencga.catalog.io;
 
+import org.apache.commons.lang3.StringUtils;
 import org.opencb.opencga.catalog.exceptions.CatalogIOException;
 import org.opencb.opencga.core.common.UriUtils;
 import org.opencb.opencga.core.config.Configuration;
@@ -85,7 +86,7 @@ public void createDefaultOpenCGAFolders() throws CatalogIOException {
     }
 
     protected void checkParam(String param) throws CatalogIOException {
-        if (param == null || param.equals("")) {
+        if (StringUtils.isEmpty(param)) {
             throw new CatalogIOException("Parameter '" + param + "' not valid");
         }
     }
@@ -107,7 +108,7 @@ public URI getProjectUri(String organizationId, String projectId) throws Catalog
         return Paths.get(getProjectsUri(organizationId)).resolve(projectId.endsWith("/") ? projectId : (projectId + "/")).toUri();
     }
 
-    private URI getStudyUri(String organizationId, String projectId, String studyId) throws CatalogIOException {
+    public URI getStudyUri(String organizationId, String projectId, String studyId) throws CatalogIOException {
         checkParam(studyId);
         return Paths.get(getProjectUri(organizationId, projectId)).resolve(studyId.endsWith("/") ? studyId : (studyId + "/")).toUri();
     }

opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/CatalogManager.java

@@ -105,7 +105,7 @@ private void init() throws CatalogException {
         logger.debug("CatalogManager configureIOManager");
         configureIOManager(configuration);
         logger.debug("CatalogManager configureDBAdaptorFactory");
-        catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory);
+        catalogDBAdaptorFactory = new MongoDBAdaptorFactory(configuration, ioManagerFactory, catalogIOManager);
         authorizationDBAdaptorFactory = new AuthorizationMongoDBAdaptorFactory((MongoDBAdaptorFactory) catalogDBAdaptorFactory,
                 configuration);
         authenticationFactory = new AuthenticationFactory(catalogDBAdaptorFactory, configuration);
@@ -365,6 +365,10 @@ public IOManagerFactory getIoManagerFactory() {
         return ioManagerFactory;
     }
 
+    public CatalogIOManager getCatalogIOManager() {
+        return catalogIOManager;
+    }
+
     private void configureIOManager(Configuration configuration) throws CatalogIOException {
         ioManagerFactory = new IOManagerFactory();
         catalogIOManager = new CatalogIOManager(configuration);