Merge branch 'master' into RubenOPS-patch-1

molgenis · Sep 19, 2024 · 96a28ba · 96a28ba
2 parents e7ae0ba + 6816ba8
commit 96a28ba
Show file tree

Hide file tree

Showing 15 changed files with 221 additions and 168 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -9,7 +9,7 @@ version: 2.1
 jobs:
   build:
     docker:
-      - image: eclipse-temurin:17.0.10_7-jdk-jammy
+      - image: eclipse-temurin:17.0.12_7-jdk-jammy
 
     working_directory: ~/repo
     resource_class: large

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 eclipse-temurin:17.0.10_7-jdk-focal
+FROM --platform=linux/amd64 eclipse-temurin:17.0.12_7-jdk-focal
 VOLUME /data
 VOLUME /config
 VOLUME /logs

diff --git a/armadillo/build.gradle b/armadillo/build.gradle
@@ -46,7 +46,7 @@ dependencies {
     //other
     implementation 'org.rosuda.REngine:REngine:2.1.0'
     implementation 'org.rosuda.REngine:Rserve:1.8.1'
-    implementation 'io.swagger.core.v3:swagger-annotations:2.2.20'
+    implementation 'io.swagger.core.v3:swagger-annotations:2.2.23'
     implementation 'com.google.auto.value:auto-value-annotations:1.10.4'
     implementation 'org.obiba.datashield:ds4j-core:2.1.0'
     implementation 'org.obiba.datashield:ds4j-r:2.1.0'

diff --git a/armadillo/src/main/java/org/molgenis/armadillo/info/RProcessEndpoint.java b/armadillo/src/main/java/org/molgenis/armadillo/info/RProcessEndpoint.java
@@ -15,6 +15,8 @@
 import org.springframework.boot.actuate.endpoint.annotation.ReadOperation;
 import org.springframework.stereotype.Component;
 
+import static org.molgenis.armadillo.security.RunAs.runAsSystem;
+
 @Component
 @Endpoint(id = "rserveProcesses")
 public class RProcessEndpoint {
@@ -41,7 +43,7 @@ public List<REnvironment> getRServeEnvironments() {
 
   <T> T doWithConnection(String environmentName, Function<RServerConnection, T> action) {
     var environment =
-        profileService.getAll().stream()
+            runAsSystem(profileService::getAll).stream()
             .filter(it -> environmentName.equals(it.getName()))
             .map(ProfileConfig::toEnvironmentConfigProps)
             .findFirst()

diff --git a/armadillo/src/main/java/org/molgenis/armadillo/security/AuthConfig.java b/armadillo/src/main/java/org/molgenis/armadillo/security/AuthConfig.java
@@ -75,6 +75,7 @@ protected SecurityFilterChain oauthAndBasic(HttpSecurity http) throws Exception
                     "/v3/**",
                     "/swagger-ui/**",
                     "/ui/**",
+                    "/actuator/prometheus",
                     "/swagger-ui.html")
                 .permitAll()
                 .requestMatchers(EndpointRequest.to(InfoEndpoint.class, HealthEndpoint.class))

diff --git a/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloStorageService.java b/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloStorageService.java
@@ -223,28 +223,34 @@ private static Workspace toWorkspace(ObjectMetadata item) {
         .build();
   }
 
+  private void trySaveWorkspace (ArmadilloWorkspace workspace, Principal principal, String id) {
+    try {
+      storageService.save(
+              workspace.createInputStream(),
+              getUserBucketName(principal),
+              getWorkspaceObjectName(id),
+              APPLICATION_OCTET_STREAM);
+    } catch (StorageException e) {
+      throw new StorageException(e);
+    }
+  }
   public void saveWorkspace(InputStream is, Principal principal, String id) {
     // Load root dir
     File drive = new File("/");
     long usableSpace = drive.getUsableSpace();
-
-    ArmadilloWorkspace workspace = storageService.getWorkSpace(is);
-    long fileSize = workspace.getSize();
-    if (usableSpace > fileSize * 2L) {
-      try {
-        storageService.save(
-            workspace.createInputStream(),
-            getUserBucketName(principal),
-            getWorkspaceObjectName(id),
-            APPLICATION_OCTET_STREAM);
-      } catch (StorageException e) {
-        throw new StorageException(e);
+    try {
+      ArmadilloWorkspace workspace = storageService.getWorkSpace(is);
+      long fileSize = workspace.getSize();
+      if (usableSpace > fileSize * 2L) {
+        trySaveWorkspace(workspace, principal, id);
+      } else {
+        throw new StorageException(
+                format(
+                        "Can't save workspace: workspace too big (%s), not enough space left on device. Try to make your workspace smaller and/or contact the administrator to increase diskspace.",
+                        getHumanReadableByteCount(fileSize)));
       }
-    } else {
-      throw new StorageException(
-          format(
-              "Can't save workspace: workspace too big (%s), not enough space left on device. Try to make your workspace smaller and/or contact the administrator to increase diskspace.",
-              getHumanReadableByteCount(fileSize)));
+    } catch (StorageException e) {
+      throw new StorageException(e.getMessage().replace("load", "save"));
     }
   }
 

diff --git a/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloWorkspace.java b/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloWorkspace.java
@@ -1,13 +1,13 @@
 package org.molgenis.armadillo.storage;
 
 import java.io.ByteArrayInputStream;
-import java.io.IOException;
 import java.io.InputStream;
 import org.apache.commons.io.IOUtils;
 import org.molgenis.armadillo.exceptions.StorageException;
 
 public class ArmadilloWorkspace {
   byte[] content;
+  public static final String WORKSPACE_TOO_BIG_ERROR = "Unable to load workspace. Maximum supported workspace size is 2GB";
 
   public ArmadilloWorkspace(InputStream is) {
     content = toByteArray(is);
@@ -16,8 +16,10 @@ public ArmadilloWorkspace(InputStream is) {
   private byte[] toByteArray(InputStream is) {
     try {
       return IOUtils.toByteArray(is);
-    } catch (IOException e) {
-      throw new StorageException("Unable to read workspace");
+    } catch (OutOfMemoryError e) {
+      throw new StorageException(WORKSPACE_TOO_BIG_ERROR);
+    } catch (Exception e) {
+      throw new StorageException("Unable to load workspace, because: " + e);
     }
   }
 

diff --git a/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloStorageServiceTest.java b/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloStorageServiceTest.java
@@ -518,6 +518,16 @@ void testSaveWorkspaceReturnsErrorWhenTooBig() {
         StorageException.class, () -> armadilloStorage.saveWorkspace(is, principal, "test"));
   }
 
+  @Test
+  void testSaveWorkspaceReturnsErrorWhenBiggerThan2Gbs() {
+    when(storageService.getWorkSpace(is)).thenThrow(new StorageException(ArmadilloWorkspace.WORKSPACE_TOO_BIG_ERROR));
+    try {
+      armadilloStorage.saveWorkspace(is, principal, "test");
+    } catch (StorageException e) {
+      assertEquals("Unable to save workspace. Maximum supported workspace size is 2GB", e.getMessage());
+    }
+  }
+
   @Test
   void testSaveWorkspaceChecksBucketName() {
     ArmadilloWorkspace workspaceMock = mock(ArmadilloWorkspace.class);

diff --git a/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloWorkspaceTest.java b/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloWorkspaceTest.java
@@ -28,4 +28,15 @@ void testGetByteOfInputStreamThrowsError() throws IOException {
     when(isMock.read(any(byte[].class))).thenThrow(IOException.class);
     assertThrows(StorageException.class, () -> new ArmadilloWorkspace(isMock));
   }
+
+  @Test
+  void testGetByteOfInputStreamThrowsMemoryError() throws IOException {
+    InputStream isMock = mock(InputStream.class);
+    when(isMock.read(any(byte[].class))).thenThrow(OutOfMemoryError.class);
+    try {
+      new ArmadilloWorkspace(isMock);
+    } catch(StorageException e) {
+      assertEquals(ArmadilloWorkspace.WORKSPACE_TOO_BIG_ERROR, e.getMessage());
+    }
+  }
 }
diff --git a/docs/ops/installing.md b/docs/ops/installing.md
@@ -47,6 +47,12 @@ The values needed are:
 
 You need a SSL certificate to configuring the front-end proxy and make the browser use **https** before putting data on the server.
 
+If SELinux is enabled, run the following command to ensure that the application can connect to required services:
+
+```bash
+setsebool -P httpd_can_network_connect on
+```
+
 ## Installing Armadillo as service
 
 We run Armadillo in production as a Linux service on Ubuntu, ensuring it gets restarted when the server is rebooted. You might be able to reproduce also on CentOS (using yum instead of apt).
@@ -184,4 +190,4 @@ For questions on other linux release you can email [email protected]
 ## What's next?
 
 * [For the server owner or data manager who need to put data on to the server](https://molgenis.github.io/molgenis-r-armadillo/)
-* [For the researcher who want to start analyzing the data on the server](https://molgenis.github.io/molgenis-r-datashield/)
+* [For the researcher who want to start analyzing the data on the server](https://molgenis.github.io/molgenis-r-datashield/)
diff --git a/r/build.gradle b/r/build.gradle
@@ -35,6 +35,7 @@ dependencies {
     implementation 'com.google.auto.value:auto-value-annotations:1.10.4'
     implementation 'org.apache.commons:commons-text:1.11.0'
     implementation 'org.json:json:20240303'
+    implementation 'org.apache.httpcomponents.client5:httpclient5:5.3.1'
 
     //test
     testImplementation('org.springframework.boot:spring-boot-starter-test') {