feat: allow up/download table CSV by name or id (#3544)

* feat: allow download table by name or id * refactored so it becomes more central available * add test; fix bug * refactor * sonar * sonar * sonare warning * fix: removed null check in getSchema * fix: changed mergeMetadata to check for name and id table names * Added sanitizing of names in zip entries * fix get entry in TableStoreForCsvInZipFile --------- Co-authored-by: Harm Brugge <[email protected]>
molgenis · Sep 16, 2024 · 325041c · 325041c
1 parent 282b6e5
commit 325041c
Show file tree

Hide file tree

Showing 10 changed files with 83 additions and 30 deletions.
diff --git a/backend/molgenis-emx2-io/src/main/java/org/molgenis/emx2/io/ImportDataTask.java b/backend/molgenis-emx2-io/src/main/java/org/molgenis/emx2/io/ImportDataTask.java
@@ -56,14 +56,14 @@ public void run() {
     }
 
     // check what files we skipped
-    Collection<String> tableNames = schema.getTableNames();
     try {
-      for (String sheet : tableStore.getTableNames()) {
+      for (String tableName : tableStore.getTableNames()) {
+        String sheet = tableName.toLowerCase().replace(" ", "");
         if (!sheet.startsWith("_files/")
             && !"molgenis".equals(sheet)
             && !"molgenis_settings".equals(sheet)
             && !"molgenis_members".equals(sheet)
-            && !tableNames.contains(sheet)) {
+            && !schema.hasTableWithNameOrIdCaseInsensitive(sheet)) {
           this.addSubTask(
                   "Sheet with name '" + sheet + "' was skipped: no table with that name found")
               .setSkipped();

diff --git a/...enis-emx2-io/src/main/java/org/molgenis/emx2/io/tablestore/TableStoreForCsvInZipFile.java b/...enis-emx2-io/src/main/java/org/molgenis/emx2/io/tablestore/TableStoreForCsvInZipFile.java
@@ -176,11 +176,19 @@ public BinaryFileWrapper getBinaryFileWrapper(String name) {
     }
   }
 
-  // magic function to allow file in subfolder
   private ZipEntry getEntry(ZipFile zf, String name) {
+    String nameWithoutSpaces = name.replace(" ", "").toLowerCase();
     List<? extends ZipEntry> result =
         // find all files that have name as prefix
-        zf.stream().filter(e -> new File(e.getName()).getName().startsWith(name + ".")).toList();
+        zf.stream()
+            .filter(
+                entry ->
+                    new File(entry.getName())
+                        .getName()
+                        .replace(" ", "")
+                        .toLowerCase()
+                        .startsWith(nameWithoutSpaces + "."))
+            .toList();
     if (result.size() > 1) {
       throw new MolgenisException(
           "Import failed, contains multiple files of name " + name + " in different subfolders");

diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlSchema.java
@@ -430,6 +430,40 @@ public SqlTable getTableById(String id) {
     else return null;
   }
 
+  @Override
+  public Table getTableByNameOrIdCaseInsensitive(String tableName) {
+    Table table = getTable(tableName);
+    if (table == null) {
+      table = getTableById(tableName);
+    }
+    if (table == null) {
+      Optional<String> name =
+          getTableNames().stream()
+              .filter(
+                  value ->
+                      value
+                          .toLowerCase()
+                          .replace(" ", "")
+                          .equals(tableName.replace(" ", "").toLowerCase()))
+              .findFirst();
+      if (name.isPresent()) {
+        table = getTable(name.get());
+      }
+    }
+    return table;
+  }
+
+  @Override
+  public boolean hasTableWithNameOrIdCaseInsensitive(String tableName) {
+    return getTableNames().stream()
+        .anyMatch(
+            value ->
+                value
+                    .toLowerCase()
+                    .replace(" ", "")
+                    .equals(tableName.replace(" ", "").toLowerCase()));
+  }
+
   public DSLContext getJooq() {
     return ((SqlDatabase) getDatabase()).getJooq();
   }

diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/CsvApi.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/CsvApi.java
@@ -55,13 +55,13 @@ private static String discardMetadata(Request request, Response response) {
 
   static String mergeMetadata(Request request, Response response) {
     String fileName = request.headers("fileName");
-    boolean fileNameMatchesTable = getSchema(request).getTableNames().contains(fileName);
+    boolean fileNameMatchesTable = getSchema(request).hasTableWithNameOrIdCaseInsensitive(fileName);
 
     if (fileNameMatchesTable) { // so we assume it isn't meta data
-      Table table = MolgenisWebservice.getTableById(request, fileName);
+      Table table = MolgenisWebservice.getTableByIdOrName(request, fileName);
       if (request.queryParams("async") != null) {
         TableStoreForCsvInMemory tableStore = new TableStoreForCsvInMemory();
-        tableStore.setCsvString(fileName, request.body());
+        tableStore.setCsvString(table.getName(), request.body());
         String id = TaskApi.submit(new ImportTableTask(tableStore, table, false));
         return new TaskReference(id, table.getSchema()).toString();
       } else {
@@ -96,7 +96,7 @@ static String getMetadata(Request request, Response response) throws IOException
   }
 
   private static String tableRetrieve(Request request, Response response) throws IOException {
-    Table table = MolgenisWebservice.getTableById(request);
+    Table table = MolgenisWebservice.getTableByIdOrName(request);
     TableStoreForCsvInMemory store = new TableStoreForCsvInMemory(getSeperator(request));
     store.writeTable(
         table.getName(), getDownloadColumns(request, table), getDownloadRows(request, table));
@@ -132,7 +132,7 @@ public static List<Row> getDownloadRows(Request request, Table table)
   }
 
   private static String tableUpdate(Request request, Response response) {
-    int count = MolgenisWebservice.getTableById(request).save(getRowList(request));
+    int count = MolgenisWebservice.getTableByIdOrName(request).save(getRowList(request));
     response.status(200);
     response.type(ACCEPT_CSV);
     return "" + count;
@@ -143,7 +143,7 @@ private static Iterable<Row> getRowList(Request request) {
   }
 
   private static String tableDelete(Request request, Response response) {
-    int count = MolgenisWebservice.getTableById(request).delete(getRowList(request));
+    int count = MolgenisWebservice.getTableByIdOrName(request).delete(getRowList(request));
     response.type(ACCEPT_CSV);
     response.status(200);
     return "" + count;

diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/ExcelApi.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/ExcelApi.java
@@ -95,7 +95,7 @@ static String getExcel(Request request, Response response) throws IOException {
   }
 
   static String getExcelTable(Request request, Response response) throws IOException {
-    Table table = MolgenisWebservice.getTableById(request);
+    Table table = MolgenisWebservice.getTableByIdOrName(request);
     Path tempDir = Files.createTempDirectory(MolgenisWebservice.TEMPFILES_DELETE_ON_EXIT);
     tempDir.toFile().deleteOnExit();
     try (OutputStream outputStream = response.raw().getOutputStream()) {

diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/MolgenisWebservice.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/MolgenisWebservice.java
@@ -226,29 +226,27 @@ private static String openApiYaml(Request request, Response response) throws IOE
    * @return the table object corresponding to the table id. Never null.
    * @throws MolgenisException if the table or the schema is not found or accessible.
    */
-  public static Table getTableById(Request request) {
-    Table table = getTableById(request, request.params(TABLE));
-    if (table == null) {
-      throw new MolgenisException("Table " + request.params(TABLE) + " unknown");
-    }
-    return table;
+  public static Table getTableByIdOrName(Request request) {
+    return getTableByIdOrName(request, request.params(TABLE));
   }
 
   /**
    * Get the table by its id.
    *
    * @param request the request
-   * @return the table object corresponding to the table id. Never null.
+   * @return the table object corresponding to the table id or name. Never null.
    * @throws MolgenisException if the schema is not found or accessible.
    */
-  public static Table getTableById(Request request, String tableName) {
-    String schemaName = request.params(SCHEMA);
-    Schema schema =
-        sessionManager.getSession(request).getDatabase().getSchema(sanitize(schemaName));
+  public static Table getTableByIdOrName(Request request, String tableName) {
+    Schema schema = getSchema(request);
     if (schema == null) {
-      throw new MolgenisException("Schema " + schemaName + " unknown or access denied");
+      throw new MolgenisException("Schema " + request.params(SCHEMA) + " unknown");
+    }
+    Table table = schema.getTableByNameOrIdCaseInsensitive(tableName);
+    if (table == null) {
+      throw new MolgenisException("Table " + tableName + " unknown");
     }
-    return schema.getTableById(sanitize(tableName));
+    return table;
   }
 
   public static String sanitize(String string) {

diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/RDFApi.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/RDFApi.java
@@ -141,7 +141,7 @@ private static int rdfForTable(Request request, Response response, RDFFormat for
       throws IOException {
     format = selectFormat(request, format); // defines format if null
 
-    Table table = getTableById(request);
+    Table table = getTableByIdOrName(request);
     String rowId = null;
     if (request.queryString() != null && !request.queryString().isBlank()) {
       rowId = request.queryString();
@@ -161,8 +161,7 @@ private static int rdfForTable(Request request, Response response, RDFFormat for
   private static int rdfForRow(Request request, Response response, RDFFormat format)
       throws IOException {
     format = selectFormat(request, format); // defines format if null
-
-    Table table = getTableById(request);
+    Table table = getTableByIdOrName(request);
     String rowId = sanitize(request.params("row"));
 
     final String baseURL = extractBaseURL(request);
@@ -180,7 +179,7 @@ private static int rdfForColumn(Request request, Response response, RDFFormat fo
       throws IOException {
     format = selectFormat(request, format); // defines format if null
 
-    Table table = getTableById(request);
+    Table table = getTableByIdOrName(request);
     String columnName = sanitize(request.params("column"));
 
     final String baseURL = extractBaseURL(request);

diff --git a/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/ZipApi.java b/backend/molgenis-emx2-webapi/src/main/java/org/molgenis/emx2/web/ZipApi.java
@@ -121,7 +121,7 @@ static String postZip(Request request, Response response)
   }
 
   static String getZipTable(Request request, Response response) throws IOException {
-    Table table = MolgenisWebservice.getTableById(request);
+    Table table = MolgenisWebservice.getTableByIdOrName(request);
     boolean includeSystemColumns = includeSystemColumns(request);
     if (table == null) throw new MolgenisException("Table " + request.params(TABLE) + " unknown");
     Path tempDir = Files.createTempDirectory(MolgenisWebservice.TEMPFILES_DELETE_ON_EXIT);

diff --git a/backend/molgenis-emx2-webapi/src/test/java/org.molgenis.emx2.web/WebApiSmokeTests.java b/backend/molgenis-emx2-webapi/src/test/java/org.molgenis.emx2.web/WebApiSmokeTests.java
@@ -269,6 +269,8 @@ public void testCsvApi_csvUploadDownload() throws IOException {
     byte[] contentsPetData = getContentAsByteArray(ACCEPT_CSV, "/pet store/api/csv/Pet");
     byte[] contentsUserData = getContentAsByteArray(ACCEPT_CSV, "/pet store/api/csv/User");
     byte[] contentsTagData = getContentAsByteArray(ACCEPT_CSV, "/pet store/api/csv/Tag");
+    byte[] contentsTableWithSpacesData =
+        getContentAsByteArray(ACCEPT_CSV, "/pet store/api/csv/" + TABLE_WITH_SPACES);
 
     // create tmp files for csv metadata and data
     File contentsMetaFile = createTempFile(contentsMeta, ".csv");
@@ -277,6 +279,7 @@ public void testCsvApi_csvUploadDownload() throws IOException {
     File contentsPetDataFile = createTempFile(contentsPetData, ".csv");
     File contentsUserDataFile = createTempFile(contentsUserData, ".csv");
     File contentsTagDataFile = createTempFile(contentsTagData, ".csv");
+    File contentsTableWithSpacesDataFile = createTempFile(contentsTableWithSpacesData, ".csv");
 
     // upload csv metadata and data into the new schema
     // here we use 'body' (instead of 'multiPart' in e.g. testCsvApi_zipUploadDownload) because csv,
@@ -286,13 +289,17 @@ public void testCsvApi_csvUploadDownload() throws IOException {
     acceptFileUpload(contentsTagDataFile, "Tag", false);
     acceptFileUpload(contentsPetDataFile, "Pet", false);
     acceptFileUpload(contentsUserDataFile, "User", false);
+    acceptFileUpload(contentsTableWithSpacesDataFile, TABLE_WITH_SPACES, false);
 
     // download csv from the new schema
     String contentsMetaNew = getContentAsString("/api/csv");
     String contentsCategoryDataNew = getContentAsString("/api/csv/Category");
     String contentsPetDataNew = getContentAsString("/api/csv/Pet");
     String contentsUserDataNew = getContentAsString("/api/csv/User");
     String contentsTagDataNew = getContentAsString("/api/csv/Tag");
+    String contentsTableWithSpacesDataNew =
+        getContentAsString(
+            "/api/csv/" + TABLE_WITH_SPACES.toUpperCase()); // to test for case insensitive match
 
     // test if existing and new schema are equal
     assertArrayEquals(toSortedArray(new String(contentsMeta)), toSortedArray(contentsMetaNew));
@@ -304,6 +311,9 @@ public void testCsvApi_csvUploadDownload() throws IOException {
         toSortedArray(new String(contentsUserData)), toSortedArray(contentsUserDataNew));
     assertArrayEquals(
         toSortedArray(new String(contentsTagData)), toSortedArray(contentsTagDataNew));
+    assertArrayEquals(
+        toSortedArray(new String(contentsTableWithSpacesData)),
+        toSortedArray(contentsTableWithSpacesDataNew));
 
     // Test async
     String response = acceptFileUpload(contentsOrderDataFile, "Order", true);

diff --git a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Schema.java
@@ -73,4 +73,8 @@ public interface Schema {
   boolean hasSetting(String emailHost);
 
   Table getTableById(String id);
+
+  Table getTableByNameOrIdCaseInsensitive(String name);
+
+  boolean hasTableWithNameOrIdCaseInsensitive(String fileName);
 }