Implemented an endpoint to handle file uploads

.github/workflows/main.yml

@@ -138,7 +138,7 @@ jobs:
           sleep 15
           ./gradlew integrationTest -PdockerComposeBuild=false
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: integration-test-logs

buildSrc/src/main/kotlin/Versions.kt

@@ -10,13 +10,17 @@ object Versions {
     const val radarJersey = "0.11.1"
     const val radarCommons = "1.1.3"
     const val radarSchemas = "0.8.9"
+    const val assertJ = "3.27.3"
+    const val mockk = "1.13.16"
     const val jackson = "2.15.3"
     const val log4j2 = "2.23.1"
     const val lzfse = "0.1.1"
     const val radarAuth = "2.1.1"
     const val avro = "1.11.4"
     const val confluent = "7.6.0"
     const val kafka = "$confluent-ce"
+    const val minio = "8.5.10"
+    const val multipart = "3.1.10"
 
     const val mockitoKotlin = "5.3.1"
     const val grizzly = "4.0.2"

gateway.yml

@@ -55,3 +55,29 @@ auth:
         #rsa: []
     # jwks URLs to fetch public keys from
     #publicKeyUrls: []
+# Storage settings for the application
+storageCondition:
+  # Enables or disables file upload functionality in the application
+  fileUploadEnabled: true
+  # Specifies the type of storage used for storing uploaded files.
+  radarStorageType: s3
+# Amazon S3 storage configurations
+s3:
+  # The endpoint URL for the S3-compatible storage service.
+  # This can be a local MinIO server (e.g., http://localhost:9000) or AWS S3.
+  url: http://localhost:9000
+  # The access key used for authentication with the S3 storage.
+  accessKey: access-key
+  # The secret key used for authentication with the S3 storage.
+  secretKey: secret-key
+  # The name of the S3 bucket where files will be uploaded.
+  bucketName: radar
+  # specifies the region for the bucket
+  region: eu-west-2
+  # Path-related settings for organizing stored files.
+  path:
+    # A prefix to be added to the stored file paths (optional).
+    prefix:
+    # Determines whether files should be organized into daily subdirectories.
+    # If true, files will be stored in a structure like "bucketName/yyyy-MM-dd/".
+    collectPerDay: true

radar-gateway/build.gradle.kts

@@ -61,12 +61,15 @@ dependencies {
     implementation(kotlin("stdlib-jdk8"))
     implementation(kotlin("reflect"))
 
+    implementation("org.glassfish.jersey.media:jersey-media-multipart:${Versions.multipart}")
+
     implementation("org.radarbase:radar-commons:${Versions.radarCommons}")
     implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}")
     implementation("org.radarbase:radar-jersey:${Versions.radarJersey}")
     implementation("org.radarbase:managementportal-client:${Versions.radarAuth}")
     implementation("org.radarbase:lzfse-decode:${Versions.lzfse}")
     implementation("org.radarbase:radar-auth:${Versions.radarAuth}")
+    implementation("io.minio:minio:${Versions.minio}")
 
     implementation("org.apache.kafka:kafka-clients:${Versions.kafka}")
     implementation("io.confluent:kafka-avro-serializer:${Versions.confluent}")
@@ -93,6 +96,8 @@ dependencies {
     integrationTestImplementation("io.ktor:ktor-serialization-kotlinx-json")
 
     testImplementation("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}")
+    testImplementation("org.assertj:assertj-core:${Versions.assertJ}")
+    testImplementation("io.mockk:mockk:${Versions.mockk}")
     integrationTestImplementation("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}")
     integrationTestImplementation("org.radarbase:radar-commons-testing:${Versions.radarCommons}")
 }

radar-gateway/src/integrationTest/docker/etc/gateway.yml

@@ -9,3 +9,29 @@ kafka:
 
 auth:
   managementPortalUrl: http://managementportal-app:8080/managementportal
+# Storage settings for the application
+storageCondition:
+  # Enables or disables file upload functionality in the application
+  fileUploadEnabled: false
+  # Specifies the type of storage used for storing uploaded files.
+  radarStorageType: s3
+# Amazon S3 storage configurations
+s3:
+  # The endpoint URL for the S3-compatible storage service.
+  # This can be a local MinIO server (e.g., http://localhost:9000) or AWS S3.
+  url: http://localhost:9000
+  # The access key used for authentication with the S3 storage.
+  #  accessKey: access-key
+  # The secret key used for authentication with the S3 storage.
+  #  secretKey: secret-key
+  # The name of the S3 bucket where files will be uploaded.
+  #  bucketName: radar
+  # Path-related settings for organizing stored files.
+  # specifies the region for the bucket
+  # region: eu-west-2
+  path:
+    # A prefix to be added to the stored file paths (optional).
+    #    prefix:
+    # Determines whether files should be organized into daily subdirectories.
+    # If true, files will be stored in a structure like "bucketName/yyyy-MM-dd/".
+    collectPerDay: true

radar-gateway/src/main/kotlin/org/radarbase/gateway/Main.kt

@@ -27,6 +27,7 @@ fun main(args: Array<String>) {
 
     try {
         config.validate()
+        config.checkEnvironmentVars()
     } catch (ex: IllegalStateException) {
         logger.error("Configuration incomplete: {}", ex.message)
         exitProcess(1)

radar-gateway/src/main/kotlin/org/radarbase/gateway/config/GatewayConfig.kt

@@ -12,6 +12,10 @@ data class GatewayConfig(
     val kafka: KafkaConfig = KafkaConfig(),
     /** Server configurations. */
     val server: GatewayServerConfig = GatewayServerConfig(),
+    /** AWS s3 storage configuration */
+    val s3: S3StorageConfig = S3StorageConfig(),
+    /** Whether to enable or disable the configurations based on the storage conditions */
+    val storageCondition: StorageConditionConfig = StorageConditionConfig(),
 ) {
     /** Fill in some default values for the configuration. */
     fun withDefaults(): GatewayConfig = copy(kafka = kafka.withDefaults())
@@ -24,4 +28,8 @@ data class GatewayConfig(
         kafka.validate()
         auth.validate()
     }
+
+    fun checkEnvironmentVars() {
+        s3.checkEnvironmentVars()
+    }
 }

radar-gateway/src/main/kotlin/org/radarbase/gateway/config/S3StorageConfig.kt

@@ -0,0 +1,35 @@
+package org.radarbase.gateway.config
+
+import org.radarbase.gateway.utils.Env.AWS_ACCESS_KEY_ID
+import org.radarbase.gateway.utils.Env.AWS_DEFAULT_REGION
+import org.radarbase.gateway.utils.Env.AWS_ENDPOINT_URL_S3
+import org.radarbase.gateway.utils.Env.AWS_S3_BUCKET_NAME
+import org.radarbase.gateway.utils.Env.AWS_SECRET_ACCESS_KEY
+
+data class S3StorageConfig(
+    var url: String? = null,
+    var accessKey: String? = null,
+    var secretKey: String? = null,
+    var bucketName: String? = null,
+    var region: String? = null,
+    var path: S3StoragePathConfig = S3StoragePathConfig(),
+) {
+    fun checkEnvironmentVars() {
+        url ?: run {
+            url = System.getenv(AWS_ENDPOINT_URL_S3)
+        }
+        accessKey ?: run {
+            accessKey = System.getenv(AWS_ACCESS_KEY_ID)
+        }
+        secretKey ?: run {
+            secretKey = System.getenv(AWS_SECRET_ACCESS_KEY)
+        }
+        bucketName ?: run {
+            bucketName = System.getenv(AWS_S3_BUCKET_NAME)
+        }
+        region ?: run {
+            region = System.getenv(AWS_DEFAULT_REGION)
+        }
+        path.checkEnvironmentVars()
+    }
+}

radar-gateway/src/main/kotlin/org/radarbase/gateway/config/S3StoragePathConfig.kt

@@ -0,0 +1,14 @@
+package org.radarbase.gateway.config
+
+import org.radarbase.gateway.utils.Env.AWS_S3_PATH_PREFIX
+
+data class S3StoragePathConfig(
+    var prefix: String? = null,
+    var collectPerDay: Boolean = true,
+) {
+    fun checkEnvironmentVars() {
+        prefix ?: run {
+            prefix = System.getenv(AWS_S3_PATH_PREFIX)
+        }
+    }
+}

radar-gateway/src/main/kotlin/org/radarbase/gateway/config/StorageConditionConfig.kt

@@ -0,0 +1,6 @@
+package org.radarbase.gateway.config
+
+data class StorageConditionConfig(
+    val fileUploadEnabled: Boolean = true,
+    val radarStorageType: String = "s3",
+)

radar-gateway/src/main/kotlin/org/radarbase/gateway/exception/FileStorageException.kt

@@ -0,0 +1,11 @@
+package org.radarbase.gateway.exception
+
+import jakarta.ws.rs.core.Response
+import org.radarbase.jersey.exception.HttpApplicationException
+
+class FileStorageException(message: String) :
+    HttpApplicationException(
+        Response.Status.INTERNAL_SERVER_ERROR,
+        Response.Status.INTERNAL_SERVER_ERROR.name.lowercase(),
+        message,
+    )

radar-gateway/src/main/kotlin/org/radarbase/gateway/exception/InvalidFileDetailsException.kt

@@ -0,0 +1,11 @@
+package org.radarbase.gateway.exception
+
+import jakarta.ws.rs.core.Response
+import org.radarbase.jersey.exception.HttpApplicationException
+
+class InvalidFileDetailsException(message: String) :
+    HttpApplicationException(
+        Response.Status.EXPECTATION_FAILED,
+        Response.Status.EXPECTATION_FAILED.name.lowercase(),
+        message,
+    )

radar-gateway/src/main/kotlin/org/radarbase/gateway/exception/InvalidPathDetailsException.kt

@@ -0,0 +1,11 @@
+package org.radarbase.gateway.exception
+
+import jakarta.ws.rs.core.Response
+import org.radarbase.jersey.exception.HttpApplicationException
+
+class InvalidPathDetailsException(message: String) :
+    HttpApplicationException(
+        Response.Status.EXPECTATION_FAILED,
+        Response.Status.EXPECTATION_FAILED.name.lowercase(),
+        message,
+    )

radar-gateway/src/main/kotlin/org/radarbase/gateway/filter/FileUploadFilter.kt

@@ -0,0 +1,31 @@
+package org.radarbase.gateway.filter
+
+import jakarta.annotation.Priority
+import jakarta.inject.Singleton
+import jakarta.ws.rs.Priorities
+import jakarta.ws.rs.container.ContainerRequestContext
+import jakarta.ws.rs.container.ContainerRequestFilter
+import jakarta.ws.rs.core.Context
+import jakarta.ws.rs.core.Response
+import jakarta.ws.rs.ext.Provider
+import org.radarbase.gateway.config.GatewayConfig
+import org.radarbase.gateway.inject.ProcessFileUpload
+
+@Provider
+@Singleton
+@Priority(Priorities.USER)
+@ProcessFileUpload
+class FileUploadFilter(
+    @Context private val config: GatewayConfig,
+) : ContainerRequestFilter {
+
+    override fun filter(requestContext: ContainerRequestContext?) {
+        if (!config.storageCondition.fileUploadEnabled) {
+            requestContext?.abortWith(
+                Response.status(Response.Status.FORBIDDEN)
+                    .entity("File uploading is not configured")
+                    .build(),
+            )
+        }
+    }
+}

radar-gateway/src/main/kotlin/org/radarbase/gateway/inject/FileStorageEnhancer.kt

@@ -0,0 +1,37 @@
+package org.radarbase.gateway.inject
+
+import jakarta.inject.Singleton
+import org.glassfish.jersey.internal.inject.AbstractBinder
+import org.glassfish.jersey.media.multipart.MultiPartFeature
+import org.radarbase.gateway.config.GatewayConfig
+import org.radarbase.gateway.config.S3StorageConfig
+import org.radarbase.gateway.service.storage.RadarMinioClient
+import org.radarbase.gateway.service.storage.RadarMinioClientFactory
+import org.radarbase.gateway.service.storage.S3StorageService
+import org.radarbase.gateway.service.storage.StorageService
+import org.radarbase.jersey.enhancer.JerseyResourceEnhancer
+
+class FileStorageEnhancer(
+    private val config: GatewayConfig,
+) : JerseyResourceEnhancer {
+
+    override val classes: Array<Class<*>> = buildList(1) {
+        add(MultiPartFeature::class.java)
+    }.toTypedArray()
+
+    override fun AbstractBinder.enhance() {
+        bind(config.s3)
+            .to(S3StorageConfig::class.java)
+            .`in`(Singleton::class.java)
+
+        if (config.storageCondition.radarStorageType == "s3") {
+            bindFactory(RadarMinioClientFactory::class.java)
+                .to(RadarMinioClient::class.java)
+                .`in`(Singleton::class.java)
+
+            bind(S3StorageService::class.java)
+                .to(StorageService::class.java)
+                .`in`(Singleton::class.java)
+        }
+    }
+}

radar-gateway/src/main/kotlin/org/radarbase/gateway/inject/ManagementPortalEnhancerFactory.kt

@@ -18,12 +18,15 @@ class ManagementPortalEnhancerFactory(private val config: GatewayConfig) : Enhan
             jwtIssuer = config.auth.issuer,
             jwksUrls = config.auth.publicKeyUrls ?: emptyList(),
         )
-        return listOf(
-            GatewayResourceEnhancer(config),
-            Enhancers.radar(authConfig),
-            Enhancers.managementPortal(authConfig),
-            Enhancers.health,
-            Enhancers.exception,
-        )
+        return buildList {
+            add(GatewayResourceEnhancer(config))
+            add(Enhancers.radar(authConfig))
+            add(Enhancers.managementPortal(authConfig))
+            add(Enhancers.health)
+            add(Enhancers.exception)
+            if (config.storageCondition.fileUploadEnabled) {
+                add(FileStorageEnhancer(config))
+            }
+        }
     }
 }

radar-gateway/src/main/kotlin/org/radarbase/gateway/inject/ProcessFileUpload.kt

@@ -0,0 +1,13 @@
+package org.radarbase.gateway.inject
+
+import jakarta.ws.rs.NameBinding
+
+@NameBinding
+@Target(
+    AnnotationTarget.CLASS,
+    AnnotationTarget.FUNCTION,
+    AnnotationTarget.PROPERTY_GETTER,
+    AnnotationTarget.PROPERTY_SETTER,
+)
+@Retention(AnnotationRetention.RUNTIME)
+annotation class ProcessFileUpload