diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 566af60d..3e85e3bc 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -7,16 +7,19 @@ plugins { } project.extra.apply { - set("okhttpVersion", "4.0.1") + set("okhttpVersion", "4.2.0") set("kafkaVersion", "2.3.0") - set("jacksonVersion", "2.9.9.1") - set("jacksonDataVersion", "2.9.9") + set("jacksonVersion", "2.9.10") + set("jacksonDataVersion", "2.9.10") } repositories { jcenter() + mavenLocal() maven(url = "http://packages.confluent.io/maven/") maven(url = "https://dl.bintray.com/radar-cns/org.radarcns") + maven(url = "https://dl.bintray.com/radar-base/org.radarbase") + maven(url = "https://repo.thehyve.nl/content/repositories/snapshots") maven(url = "http://oss.jfrog.org/artifactory/oss-snapshot-local/") } diff --git a/integration-test/src/integrationTest/java/org/radarbase/connect/upload/UploadSourceTaskTest.kt b/integration-test/src/integrationTest/java/org/radarbase/connect/upload/UploadSourceTaskTest.kt index 7f3a9e2a..cc83172e 100644 --- a/integration-test/src/integrationTest/java/org/radarbase/connect/upload/UploadSourceTaskTest.kt +++ b/integration-test/src/integrationTest/java/org/radarbase/connect/upload/UploadSourceTaskTest.kt @@ -60,9 +60,10 @@ class UploadSourceTaskTest { "upload.source.client.tokenUrl" to tokenUrl, "upload.source.backend.baseUrl" to baseUri, "upload.source.poll.interval.ms" to "10000", - "upload.source.record.converter.classes" to - "org.radarbase.connect.upload.converter.AccelerometerCsvRecordConverter,org.radarbase.connect.upload.converter.altoida.AltoidaZipFileRecordConverter" - + "upload.source.record.converter.classes" to listOf( + "org.radarbase.connect.upload.converter.AccelerometerCsvRecordConverter", + "org.radarbase.connect.upload.converter.altoida.AltoidaZipFileRecordConverter" + ).joinToString(separator=",") ) sourceTask.start(settings) @@ -102,7 +103,6 @@ class UploadSourceTaskTest { @Test @DisplayName("Records of no registered converters should not be polled") fun noConverterFound() { - val sourceType = "acceleration-zip" val fileName = "TEST_ACC.zip" val createdRecord = createRecordAndUploadContent(accessToken, sourceType, fileName) @@ -116,17 +116,14 @@ class UploadSourceTaskTest { val metadata = retrieveRecordMetadata(accessToken, createdRecord.id!!) assertNotNull(metadata) assertEquals("READY", metadata.status) - } @Test @DisplayName("Should mark FAILED if the record data does not match the source-type") fun incorrectSourceTypeForRecord() { - val sourceType = "phone-acceleration" val fileName = "TEST_ACC.zip" val createdRecord = createRecordAndUploadContent(accessToken, sourceType, fileName) - assertNotNull(createdRecord) assertNotNull(createdRecord.id) val sourceRecords = sourceTask.poll() @@ -136,6 +133,5 @@ class UploadSourceTaskTest { val metadata = retrieveRecordMetadata(accessToken, createdRecord.id!!) assertNotNull(metadata) assertEquals("FAILED", metadata.status) - } } diff --git a/integration-test/src/integrationTest/java/org/radarbase/connect/upload/api/UploadBackendClientIntegrationTest.kt b/integration-test/src/integrationTest/java/org/radarbase/connect/upload/api/UploadBackendClientIntegrationTest.kt index 189da966..73e16110 100644 --- a/integration-test/src/integrationTest/java/org/radarbase/connect/upload/api/UploadBackendClientIntegrationTest.kt +++ b/integration-test/src/integrationTest/java/org/radarbase/connect/upload/api/UploadBackendClientIntegrationTest.kt @@ -45,8 +45,6 @@ import java.io.File @TestInstance(TestInstance.Lifecycle.PER_CLASS) class UploadBackendClientIntegrationTest { - - private lateinit var uploadBackendClient: UploadBackendClient private lateinit var logRepository: LogRepository @@ -113,7 +111,7 @@ class UploadBackendClientIntegrationTest { val converter = AccelerometerCsvRecordConverter() converter.initialize(sourceType, uploadBackendClient, logRepository, emptyMap()) - val recordToProcess = records.records.filter { recordDTO -> recordDTO.sourceType == sourceTypeName }.first() + val recordToProcess = records.records.first { recordDTO -> recordDTO.sourceType == sourceTypeName } createdRecord.metadata = uploadBackendClient.updateStatus(recordToProcess.id!!, recordToProcess.metadata!!.copy(status = "PROCESSING", message = "The record is being processed")) val convertedRecords = converter.convert(records.records.first()) assertNotNull(convertedRecords) @@ -137,9 +135,9 @@ class UploadBackendClientIntegrationTest { } private fun retrieveFile(recordId: RecordDTO) { - uploadBackendClient.retrieveFile(recordId, fileName).use { response -> + uploadBackendClient.retrieveFile(recordId, fileName) { response -> assertNotNull(response) - val responseData = response!!.bytes() + val responseData = response.bytes() assertThat(responseData.size.toLong(), equalTo(File(fileName).length())) assertThat(responseData, equalTo(File(fileName).readBytes())) } diff --git a/integration-test/src/integrationTest/java/org/radarbase/connect/upload/util/TestBase.kt b/integration-test/src/integrationTest/java/org/radarbase/connect/upload/util/TestBase.kt index e6ab3f72..55635fdd 100644 --- a/integration-test/src/integrationTest/java/org/radarbase/connect/upload/util/TestBase.kt +++ b/integration-test/src/integrationTest/java/org/radarbase/connect/upload/util/TestBase.kt @@ -36,7 +36,9 @@ import okhttp3.RequestBody.Companion.toRequestBody import org.hamcrest.CoreMatchers import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers +import org.hamcrest.Matchers.* import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertTrue import org.radarbase.connect.upload.api.* import org.radarbase.connect.upload.auth.ClientCredentialsAuthorizer import org.radarbase.upload.Config @@ -64,22 +66,22 @@ class TestBase { const val uploadConnectSecret = "upload_secret" - const val BEARER = "Bearer " + private const val BEARER = "Bearer " - const val USER = "sub-1" + private const val USER = "sub-1" - const val PROJECT = "radar" + private const val PROJECT = "radar" - const val SOURCE = "03d28e5c-e005-46d4-a9b3-279c27fbbc83" + private const val SOURCE = "03d28e5c-e005-46d4-a9b3-279c27fbbc83" - val APPLICATION_JSON = "application/json; charset=utf-8".toMediaType() + private val APPLICATION_JSON = "application/json; charset=utf-8".toMediaType() - val TEXT_CSV = "text/csv; charset=utf-8".toMediaType() + private val TEXT_CSV = "text/csv; charset=utf-8".toMediaType() val httpClient = OkHttpClient() - val sourceType = SourceTypeDTO( + private val sourceType = SourceTypeDTO( name = sourceTypeName, topics = mutableSetOf("test_topic"), contentTypes = mutableSetOf("application/text"), @@ -88,26 +90,28 @@ class TestBase { configuration = mutableMapOf("setting1" to "value1", "setting2" to "value2") ) - val altoidaZip = SourceTypeDTO( + private val altoidaZip = SourceTypeDTO( name = "altoida-zip", topics = mutableSetOf("test_topic"), contentTypes = mutableSetOf("application/zip"), timeRequired = false, sourceIdRequired = false, - configuration = emptyMap() + configuration = mutableMapOf() ) - val accelerationZip = SourceTypeDTO( + private val accelerationZip = SourceTypeDTO( name = "acceleration-zip", topics = mutableSetOf("test_topic_Acc"), contentTypes = mutableSetOf("application/zip"), timeRequired = false, sourceIdRequired = false, - configuration = emptyMap() + configuration = mutableMapOf() ) val uploadBackendConfig = Config( managementPortalUrl = "http://localhost:8090/managementportal", + clientId = "radar_upload_backend", + clientSecret = "secret", baseUri = URI.create(baseUri), jdbcDriver = "org.postgresql.Driver", jdbcUrl = "jdbc:postgresql://localhost:5434/uploadconnector", @@ -116,7 +120,7 @@ class TestBase { sourceTypes = listOf(sourceType, altoidaZip, accelerationZip) ) - val mapper = ObjectMapper(JsonFactory()) + private val mapper: ObjectMapper = ObjectMapper(JsonFactory()) .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) .registerModule(KotlinModule()) .registerModule(JavaTimeModule()) @@ -126,10 +130,9 @@ class TestBase { httpClient, uploadConnectClient, uploadConnectSecret, - tokenUrl - ) + tokenUrl) - fun call( + private fun call( httpClient: OkHttpClient, expectedStatus: Int, requestSupplier: (Request.Builder) -> Request.Builder @@ -142,56 +145,62 @@ class TestBase { println(tree) tree } - assertThat(response.code, CoreMatchers.`is`(expectedStatus)) + assertThat(response.code, `is`(expectedStatus)) body } } - fun Any.toJsonString(): String = mapper.writeValueAsString(this) + private fun call( + httpClient: OkHttpClient, + expectedStatus: Int, + parseClass: Class, + requestSupplier: Request.Builder.() -> Request.Builder + ): T { + val request = requestSupplier(Request.Builder()).build() + println(request.url) + return httpClient.newCall(request).execute().use { response -> + assertThat(response.code, `is`(expectedStatus)) + assertThat(response.body, not(nullValue())) + mapper.readValue(response.body?.byteStream(), parseClass) + .also { assertThat(it, not(nullValue())) } + .also { println(it!!.toJsonString()) } + } + } + + private fun Any.toJsonString(): String = mapper.writeValueAsString(this) - fun call( + private fun call( httpClient: OkHttpClient, expectedStatus: Response.Status, requestSupplier: (Request.Builder) -> Request.Builder - ): JsonNode? { - return call(httpClient, expectedStatus.statusCode, requestSupplier) - } + ): JsonNode? = call(httpClient, expectedStatus.statusCode, requestSupplier) - fun call( + private fun call( httpClient: OkHttpClient, expectedStatus: Response.Status, stringProperty: String, - requestSupplier: (Request.Builder) -> Request.Builder - ): String { - return call(httpClient, expectedStatus, requestSupplier)?.get(stringProperty)?.asText() - ?: throw AssertionError("String property $stringProperty not found") - } + requestSupplier: Request.Builder.() -> Request.Builder + ): String = call(httpClient, expectedStatus, requestSupplier) + ?.get(stringProperty) + ?.asText() + ?: throw AssertionError("String property $stringProperty not found") fun retrieveRecordMetadata(accessToken: String, recordId: Long): RecordMetadataDTO { - - val requestToUploadFile = Request.Builder() - .url("$baseUri/records/$recordId/metadata") - .get() - .addHeader("Authorization", BEARER + accessToken) - .build() - val response = httpClient.newCall(requestToUploadFile).execute() - Assertions.assertTrue(response.isSuccessful) - - return mapper.readValue(response.body?.string(), RecordMetadataDTO::class.java) + return call(httpClient,200, RecordMetadataDTO::class.java) { + url("$baseUri/records/$recordId/metadata") + addHeader("Authorization", BEARER + accessToken) + } } - fun getAccessToken() : String { - return call(httpClient, Response.Status.OK, "access_token") { - it.url(tokenUrl) - .addHeader("Authorization", Credentials.basic("radar_upload_test_client", "test")) - .post(FormBody.Builder() - .add("grant_type", "client_credentials") - .build()) - } + fun getAccessToken() : String = call(httpClient, Response.Status.OK, "access_token") { + url(tokenUrl) + addHeader("Authorization", Credentials.basic("radar_upload_test_client", "test")) + post(FormBody.Builder() + .add("grant_type", "client_credentials") + .build()) } fun createRecordAndUploadContent(accessToken: String, sourceType: String, fileName: String): RecordDTO { - val record = RecordDTO( id = null, data = RecordDataDTO( @@ -204,20 +213,14 @@ class TestBase { metadata = null ) - val request = Request.Builder() - .url("$baseUri/records") - .post(record.toJsonString().toRequestBody(APPLICATION_JSON)) - .addHeader("Authorization", BEARER + accessToken) - .addHeader("Content-type", "application/json") - .build() - - val response = httpClient.newCall(request).execute() - Assertions.assertTrue(response.isSuccessful) - - val recordCreated = mapper.readValue(response.body?.string(), RecordDTO::class.java) - Assertions.assertNotNull(recordCreated) - Assertions.assertNotNull(recordCreated.id) - assertThat(recordCreated?.id!!, Matchers.greaterThan(0L)) + val recordCreated = call(httpClient, 201, RecordDTO::class.java) { + url("$baseUri/records") + post(record.toJsonString().toRequestBody(APPLICATION_JSON)) + addHeader("Authorization", BEARER + accessToken) + addHeader("Content-type", "application/json") + } + assertThat(recordCreated.id, not(nullValue())) + assertThat(recordCreated.id!!, greaterThan(0L)) //Test uploading request contentFile for created record uploadContent(recordCreated.id!!, fileName, accessToken) @@ -229,36 +232,21 @@ class TestBase { //Test uploading request contentFile val file = File(fileName) - val requestToUploadFile = Request.Builder() - .url("$baseUri/records/$recordId/contents/$fileName") - .put(file.asRequestBody(TEXT_CSV)) - .addHeader("Authorization", BEARER + clientUserToken) - .build() - - val uploadResponse = httpClient.newCall(requestToUploadFile).execute() - Assertions.assertTrue(uploadResponse.isSuccessful) - - val content = mapper.readValue(uploadResponse.body?.string(), ContentsDTO::class.java) - Assertions.assertNotNull(content) - Assertions.assertEquals(fileName, content.fileName) + val content = call(httpClient, 201, ContentsDTO::class.java) { + url("$baseUri/records/$recordId/contents/$fileName") + put(file.asRequestBody(TEXT_CSV)) + addHeader("Authorization", BEARER + clientUserToken) + } + assertThat(content.fileName, equalTo(fileName)) } - private fun markReady(recordId: Long, clientUserToken: String) { - //Test marking record READY - val requestToUploadFile = Request.Builder() - .url("$baseUri/records/$recordId/metadata") - .post("{\"status\":\"READY\",\"revision\":1}".toRequestBody("application/json".toMediaType())) - .addHeader("Authorization", BEARER + clientUserToken) - .build() - - val uploadResponse = httpClient.newCall(requestToUploadFile).execute() - Assertions.assertTrue(uploadResponse.isSuccessful) - - val metadata = mapper.readValue(uploadResponse.body?.string(), RecordMetadataDTO::class.java) - Assertions.assertNotNull(metadata) - Assertions.assertEquals("READY", metadata.status) + val metadata = call(httpClient, 200, RecordMetadataDTO::class.java) { + url("$baseUri/records/$recordId/metadata") + post("{\"status\":\"READY\",\"revision\":1}".toRequestBody("application/json".toMediaType())) + addHeader("Authorization", BEARER + clientUserToken) + } + assertThat(metadata.status, equalTo("READY")) } - } } diff --git a/kafka-connect-upload-source/build.gradle.kts b/kafka-connect-upload-source/build.gradle.kts index ce9bb143..51a244e6 100644 --- a/kafka-connect-upload-source/build.gradle.kts +++ b/kafka-connect-upload-source/build.gradle.kts @@ -8,9 +8,9 @@ plugins { project.extra.apply { set("kafkaVersion", "2.3.0") - set("okhttpVersion", "3.14.2") - set("jacksonVersion", "2.9.9.1") - set("jacksonDataVersion", "2.9.9") + set("okhttpVersion", "4.2.0") + set("jacksonVersion", "2.9.10") + set("jacksonDataVersion", "2.9.10") set("openCsvVersion", "4.6") set("confluentVersion", "5.3.0") set("radarSchemaVersion", "0.5.2-SNAPSHOT") @@ -44,12 +44,10 @@ dependencies { implementation("com.fasterxml.jackson.module:jackson-module-kotlin:${project.extra["jacksonDataVersion"]}") implementation("com.opencsv:opencsv:${project.extra["openCsvVersion"]}") - // Included in connector runtime compileOnly("org.apache.kafka:connect-api:${project.extra["kafkaVersion"]}") implementation(kotlin("stdlib-jdk8")) - testImplementation("org.junit.jupiter:junit-jupiter:5.4.2") testImplementation("org.hamcrest:hamcrest-all:1.3") testImplementation("org.apache.kafka:connect-api:${project.extra["kafkaVersion"]}") diff --git a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/UploadSourceTask.kt b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/UploadSourceTask.kt index dc9b843d..5a973c91 100644 --- a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/UploadSourceTask.kt +++ b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/UploadSourceTask.kt @@ -35,6 +35,7 @@ import org.radarbase.connect.upload.converter.ConverterLogRepository import org.radarbase.connect.upload.converter.LogRepository import org.radarbase.connect.upload.exception.ConflictException import org.radarbase.connect.upload.exception.ConversionFailedException +import org.radarbase.connect.upload.exception.ConversionTemporarilyFailedException import org.radarbase.connect.upload.util.VersionUtil import org.slf4j.LoggerFactory import java.time.Duration @@ -88,6 +89,7 @@ class UploadSourceTask : SourceTask() { override fun stop() { logger.debug("Stopping source task") + uploadAllLogs() uploadClient.close() converters.forEach(Converter::close) } @@ -95,78 +97,71 @@ class UploadSourceTask : SourceTask() { override fun version(): String = VersionUtil.getVersion() override fun poll(): List { - while (true) { - val timeout = ChronoUnit.MILLIS.between(Instant.now(), getNextPollingTime()) - if (timeout > 0) { - logger.info("Waiting {} milliseconds for next polling time", timeout) - Thread.sleep(timeout) - } - - return pollRecords() + val timeout = ChronoUnit.MILLIS.between(Instant.now(), getNextPollingTime()) + if (timeout > 0) { + logger.info("Waiting {} milliseconds for next polling time", timeout) + Thread.sleep(timeout) + } + logger.info("Polling new records...") + val records: List = try { + uploadClient.pollRecords(PollDTO(1, converters.map { it.sourceType })).records + } catch (exe: Exception) { + logger.info("Could not successfully poll records. Waiting for next polling...") + return emptyList() } + lastPooledAt = Instant.now() + logger.info("Received ${records.size} records at $lastPooledAt") + + return records.flatMap { record -> processRecord(record) ?: emptyList() } } private fun getNextPollingTime(): Instant { return lastPooledAt.plus(Duration.of(pollInterval, ChronoUnit.MILLIS)) } - private fun pollRecords(): List { - logger.info("Polling new records...") - val sourceRecords = mutableListOf() - var records: List? = null - try { - records = uploadClient.pollRecords(PollDTO(1, converters.map { it.sourceType })).records - - lastPooledAt = Instant.now() - logger.info("Received ${records.size} records at $lastPooledAt") - } catch (exe: Exception) { - logger.info("Could not successfully poll records. Waiting for next polling...") + private fun processRecord(record: RecordDTO): List? { + return try { + val converter = converters.find { it.sourceType == record.sourceType } + ?: throw ConversionTemporarilyFailedException("Could not find converter ${record.sourceType} for record ${record.id}") + + markProcessing(record) ?: return null + + converter.convert(record).result + } catch (exe: ConversionFailedException) { + logger.error("Could not convert record ${record.id}", exe) + updateRecordFailure(record, exe) + null + } catch (exe: ConversionTemporarilyFailedException) { + logger.error("Could not convert record ${record.id} due to temporary failure", exe) + updateRecordTemporaryFailure(record, exe) + null } - if (records != null) { - records@ for (record in records) { - val converter = converters.find { it.sourceType == record.sourceType } - try { - if (converter == null) { - // technically this should not happen, since we query only for supported converters. - // I am just leaving this check, just to be sure, that we don't have any corner case. - logger.error("Could not find converter ${record.sourceType} for record ${record.id}") - continue@records - } else { - record.metadata = uploadClient.updateStatus( - record.id!!, - record.metadata!!.copy(status = "PROCESSING") - ) - logger.debug("Updated metadata ${record.id} to PROCESSING") - } - } catch (exe: Exception) { - when (exe) { - is ConflictException -> { - logger.warn("Conflicting request was made. Skipping this record") - continue@records - } - else -> throw exe - } - } + } - try { - val result = converter.convert(record) - result.result?.takeIf(List<*>::isNotEmpty)?.let { - sourceRecords.addAll(it) - } - } catch (exe: ConversionFailedException) { - logger.error("Could not convert record ${record.id}", exe) - updateRecordFailure(record) + private fun markProcessing(record: RecordDTO): RecordDTO? { + return try { + record.apply { + metadata = uploadClient.updateStatus( + record.id!!, + record.metadata!!.copy(status = "PROCESSING") + ) + logger.debug("Updated metadata $id to PROCESSING") + } + } catch (exe: Exception) { + when (exe) { + is ConflictException -> { + logger.warn("Conflicting request was made. Skipping this record") + null } - + else -> throw ConversionTemporarilyFailedException("Cannot update record metadata", exe) } } - return sourceRecords } - private fun updateRecordFailure(record: RecordDTO, reason: String? = "Could not convert this record. Please refer to the conversion logs for more details") { - logger.info("Update record conversion failure") + private fun updateRecordFailure(record: RecordDTO, exe: Exception, reason: String = "Could not convert this record. Please refer to the conversion logs for more details") { + logRepository.error(logger, record.id!!, reason, exe) val metadata = uploadClient.retrieveRecordMetadata(record.id!!) val updatedMetadata = uploadClient.updateStatus(record.id!!, metadata.copy( status = "FAILED", @@ -175,7 +170,22 @@ class UploadSourceTask : SourceTask() { if (updatedMetadata.status == "FAILED") { logger.info("Uploading logs to backend") - logRepository.uploadLogs(record.id!!) + uploadLogs(record.id!!) + } + } + + private fun updateRecordTemporaryFailure(record: RecordDTO, exe: Exception, reason: String = "Temporarily could not convert this record. Please refer to the conversion logs for more details") { + logger.info("Update record conversion failure") + logRepository.error(logger, record.id!!, reason, exe) + val metadata = uploadClient.retrieveRecordMetadata(record.id!!) + val updatedMetadata = uploadClient.updateStatus(record.id!!, metadata.copy( + status = "READY", + message = reason + )) + + if (updatedMetadata.status == "READY") { + logger.info("Uploading logs to backend") + uploadLogs(record.id!!) } } @@ -194,13 +204,27 @@ class UploadSourceTask : SourceTask() { if (updatedMetadata.status == "SUCCEEDED") { logger.info("Uploading logs to backend") - logRepository.uploadLogs(recordId.toLong()) + uploadLogs(recordId.toLong()) } } } + private fun uploadAllLogs(reset: Boolean = true) { + logger.info("Uploading all remaining logs") + logRepository.recordIds.forEach { r -> + logRepository.extract(r, reset) + ?.let { uploadClient.addLogs(it) } + } + logger.info("All record logs are uploaded") + } + + private fun uploadLogs(recordId: Long, reset: Boolean = true) { + logger.info("Sending record $recordId logs...") + logRepository.extract(recordId, reset) + ?.let { uploadClient.addLogs(it) } + } + companion object { private val logger = LoggerFactory.getLogger(UploadSourceTask::class.java) } - } diff --git a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/api/UploadBackendClient.kt b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/api/UploadBackendClient.kt index bd8cd8ee..f7716f5f 100644 --- a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/api/UploadBackendClient.kt +++ b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/api/UploadBackendClient.kt @@ -25,11 +25,16 @@ import com.fasterxml.jackson.databind.SerializationFeature import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import com.fasterxml.jackson.module.kotlin.KotlinModule import okhttp3.* +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import okio.BufferedSink +import org.radarbase.connect.upload.converter.Log import org.radarbase.connect.upload.exception.BadGatewayException import org.radarbase.connect.upload.exception.ConflictException import org.radarbase.connect.upload.exception.NotAuthorizedException import org.slf4j.LoggerFactory import java.io.Closeable +import java.io.IOException class UploadBackendClient( auth: Authenticator, @@ -45,99 +50,84 @@ class UploadBackendClient( uploadBackendBaseUrl = uploadBackendBaseUrl.trimEnd('/') } - fun pollRecords(configuration: PollDTO): RecordContainerDTO { - val request = Request.Builder() - .url("$uploadBackendBaseUrl/records/poll") - .post(RequestBody.create(APPLICATION_JSON, configuration.toJsonString())) - .build() - val response = httpClient.executeRequest(request) - return mapper.readValue(response.body()?.string(), RecordContainerDTO::class.java) + fun pollRecords(configuration: PollDTO): RecordContainerDTO = httpClient.executeRequest { + url("$uploadBackendBaseUrl/records/poll") + post(configuration.toJsonBody()) } - fun requestConnectorConfig(name: String): SourceTypeDTO { - val request = Request.Builder() - .url("$uploadBackendBaseUrl/source-types/${name}/") - .get() - .build() - val response = httpClient.executeRequest(request) - return mapper.readValue(response.body()?.string(), SourceTypeDTO::class.java) + fun requestConnectorConfig(name: String): SourceTypeDTO = httpClient.executeRequest { + url("$uploadBackendBaseUrl/source-types/${name}/") } - fun requestAllConnectors(): SourceTypeContainerDTO { - val request = Request.Builder() - .url("$uploadBackendBaseUrl/source-types") - .get() - .build() - val response = httpClient.executeRequest(request) - return mapper.readValue(response.body()?.string(), SourceTypeContainerDTO::class.java) + fun requestAllConnectors(): SourceTypeContainerDTO = httpClient.executeRequest { + url("$uploadBackendBaseUrl/source-types") } - fun retrieveFile(record: RecordDTO, fileName: String): ResponseBody? { - val request = Request.Builder() - .url("$uploadBackendBaseUrl/records/${record.id}/contents/$fileName") - .get() - .build() - return httpClient.executeRequest(request).body() + fun retrieveFile(record: RecordDTO, fileName: String, handling: (ResponseBody) -> T): T { + return httpClient.executeRequest({ + url("$uploadBackendBaseUrl/records/${record.id}/contents/$fileName") + }) { + handling(it.body ?: throw IOException("No file content response body")) + } } - fun retrieveRecordMetadata(recordId: Long): RecordMetadataDTO { - val request = Request.Builder() - .url("$uploadBackendBaseUrl/records/$recordId/metadata") - .get() - .build() - val response = httpClient.executeRequest(request) - return mapper.readValue(response.body()?.string(), RecordMetadataDTO::class.java) + fun retrieveRecordMetadata(recordId: Long): RecordMetadataDTO = httpClient.executeRequest { + url("$uploadBackendBaseUrl/records/$recordId/metadata") } - fun updateStatus(recordId: Long, newStatus: RecordMetadataDTO): RecordMetadataDTO { - val request = Request.Builder() - .url("$uploadBackendBaseUrl/records/$recordId/metadata") - .post(RequestBody.create(APPLICATION_JSON, newStatus.toJsonString())) - .build() - val response = httpClient.executeRequest(request) - return mapper.readValue(response.body()?.string(), RecordMetadataDTO::class.java) + fun updateStatus(recordId: Long, newStatus: RecordMetadataDTO): RecordMetadataDTO = httpClient.executeRequest { + url("$uploadBackendBaseUrl/records/$recordId/metadata") + post(newStatus.toJsonBody()) } - fun addLogs(recordId: Long, status: LogsDto): RecordMetadataDTO { - val request = Request.Builder() - .url("$uploadBackendBaseUrl/records/$recordId/logs") - .put(RequestBody.create(TEXT_PLAIN, status.toJsonString())) - .build() - val response = httpClient.executeRequest(request) - return mapper.readValue(response.body()?.charStream(), RecordMetadataDTO::class.java) + fun addLogs(log: Log): RecordMetadataDTO = httpClient.executeRequest { + url("$uploadBackendBaseUrl/records/${log.recordId}/logs") + put(object : RequestBody() { + override fun contentType() = TEXT_PLAIN + + override fun writeTo(sink: BufferedSink) = log.asString(sink) + }) } override fun close() { } - private fun OkHttpClient.executeRequest(request: Request): Response { - val response = this.newCall(request).execute() - if (response.isSuccessful) { - logger.info("Request to ${request.url()} is SUCCESSFUL") - return response - } else { - logger.info("Request to ${request.url()} has FAILED with response-code ${response.code()}") - when (response.code()) { - 401 -> throw NotAuthorizedException("access token is not provided or is invalid : ${response.message()}") - 403 -> throw NotAuthorizedException("access token is not authorized to perform this request") - 409 -> throw ConflictException("Conflicting request exception: ${response.message()}") + private inline fun OkHttpClient.executeRequest( + noinline requestBuilder: Request.Builder.() -> Request.Builder): T = executeRequest(requestBuilder) { response -> + mapper.readValue(response.body?.byteStream(), T::class.java) + ?: throw IOException("Received invalid response") + } + + private fun OkHttpClient.executeRequest(requestBuilder: Request.Builder.() -> Request.Builder, handling: (Response) -> T): T { + val request = Request.Builder().requestBuilder().build() + return this.newCall(request).execute().use { response -> + if (response.isSuccessful) { + logger.info("Request to ${request.url} is SUCCESSFUL") + return handling(response) + } else { + logger.info("Request to ${request.url} has FAILED with response-code ${response.code}") + when (response.code) { + 401 -> throw NotAuthorizedException("access token is not provided or is invalid : ${response.message}") + 403 -> throw NotAuthorizedException("access token is not authorized to perform this request") + 409 -> throw ConflictException("Conflicting request exception: ${response.message}") + } + throw BadGatewayException("Failed to make request to ${request.url}: Error code ${response.code}: ${response.body?.string()}") } - throw BadGatewayException("Failed to make request to ${request.url()}: Error code ${response.code()}: ${response.body()?.string()}") } - } - private fun Any.toJsonString(): String = mapper.writeValueAsString(this) + private fun Any.toJsonBody(mediaType: MediaType = APPLICATION_JSON): RequestBody = mapper + .writeValueAsString(this) + .toRequestBody(mediaType) companion object { private val logger = LoggerFactory.getLogger(UploadBackendClient::class.java) - private val APPLICATION_JSON = MediaType.parse("application/json; charset=utf-8") - private val TEXT_PLAIN = MediaType.parse("text/plain; charset=utf-8") + private val APPLICATION_JSON = "application/json; charset=utf-8".toMediaType() + private val TEXT_PLAIN = "text/plain; charset=utf-8".toMediaType() private var mapper: ObjectMapper = ObjectMapper() .registerModule(KotlinModule()) .registerModule(JavaTimeModule()) .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) } - } diff --git a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/auth/Authorizer.kt b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/auth/Authorizer.kt index 03ea62e3..7918a171 100644 --- a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/auth/Authorizer.kt +++ b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/auth/Authorizer.kt @@ -37,22 +37,21 @@ class ClientCredentialsAuthorizer( lateinit var token: OauthToken override fun authenticate(route: Route?, response: Response): Request? { - if (response.code() != 401) { - logger.debug("Received ${response.code()} at the authenticator. Skipping this request...") + if (response.code != 401) { + logger.debug("Received ${response.code} at the authenticator. Skipping this request...") return null } var accessToken = accessToken() - if (response.code() == 401 - && "Bearer $accessToken" == response.request().header("Authorization")) { + if ("Bearer $accessToken" == response.request.header("Authorization")) { logger.debug("Request failed with token existing token. Requesting new token") Thread.sleep(60000L) accessToken = accessToken(true) } - logger.debug("Response request ${response.request()} and code ${response.code()}") + logger.debug("Response request ${response.request} and code ${response.code}") try { - return response.request().newBuilder() + return response.request.newBuilder() .header("Authorization", "Bearer $accessToken") .build() } catch (exe: Exception) { @@ -81,16 +80,17 @@ class ClientCredentialsAuthorizer( .post(form) .build() - val response = httpClient.newCall(request).execute() - if (response.isSuccessful) { - logger.info("Request to get access token was SUCCESSFUL") - try { - return UploadSourceConnectorConfig.mapper.readValue(response.body()?.charStream(), OauthToken::class.java) - } catch (exe: IOException) { - throw NotAuthorizedException("Could not convert response into a valid access token ${exe.message}") + return httpClient.newCall(request).execute().use { response -> + if (response.isSuccessful) { + logger.info("Request to get access token was SUCCESSFUL") + try { + UploadSourceConnectorConfig.mapper.readValue(response.body?.charStream(), OauthToken::class.java) + } catch (exe: IOException) { + throw NotAuthorizedException("Could not convert response into a valid access token ${exe.message}") + } + } else { + throw NotAuthorizedException("Request to get access token failed with response code ${response.code} and ${response.body?.string()}") } - } else { - throw NotAuthorizedException("Request to get access token failed with response code ${response.code()} and ${response.body()?.string()}") } } diff --git a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/Converter.kt b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/Converter.kt index 6978609e..33a1427b 100644 --- a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/Converter.kt +++ b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/Converter.kt @@ -52,7 +52,6 @@ interface Converter : Closeable { data class ConversionResult(val record: RecordDTO, val result: List?) - data class TopicData( var endOfFileOffSet: Boolean, val topic: String, diff --git a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/ConverterLogRepository.kt b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/ConverterLogRepository.kt index dd5cf80f..5bf8ff92 100644 --- a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/ConverterLogRepository.kt +++ b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/ConverterLogRepository.kt @@ -19,11 +19,18 @@ package org.radarbase.connect.upload.converter +import okio.BufferedSink +import okio.Sink import org.radarbase.connect.upload.UploadSourceConnectorConfig import org.radarbase.connect.upload.api.LogsDto import org.radarbase.connect.upload.api.UploadBackendClient import org.slf4j.Logger import org.slf4j.LoggerFactory +import java.io.ByteArrayOutputStream +import java.io.PrintStream +import java.io.Writer +import java.nio.charset.StandardCharsets.UTF_8 +import java.time.Instant import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentLinkedQueue @@ -31,74 +38,79 @@ enum class LogLevel { INFO, DEBUG, WARN, ERROR } -data class Log( +data class LogRecord( var logLevel: LogLevel, - var message: String -) + var message: String) { + val time = Instant.now() +} + +data class Log(val recordId: Long, val records: Collection) { + fun asString(writer: BufferedSink) { + records.forEach { log -> + writer.writeUtf8("${log.time} - [${log.logLevel}] ${log.message}\n") + } + } +} interface LogRepository { fun info(logger: Logger, recordId: Long, logMessage: String) fun debug(logger: Logger, recordId: Long, logMessage: String) fun warn(logger: Logger, recordId: Long, logMessage: String) fun error(logger: Logger, recordId: Long, logMessage: String, exe: Exception? = null) - fun uploadLogs(recordId: Long) - fun uploadAllLogs() + val recordIds: Set + fun extract(recordId: Long, reset: Boolean = false): Log? } class ConverterLogRepository( - val uploadClient: UploadBackendClient): LogRepository { - private val logContainer = ConcurrentHashMap>() + private val uploadClient: UploadBackendClient): LogRepository { + private val logContainer = ConcurrentHashMap>() - private fun get(recordId: Long): ConcurrentLinkedQueue = + private fun get(recordId: Long): ConcurrentLinkedQueue = logContainer.getOrPut(recordId, { ConcurrentLinkedQueue() }) - override fun info(logger: Logger, recordId: Long, logMessage: String) { - get(recordId).add(Log(LogLevel.INFO, logMessage)) + get(recordId).add(LogRecord(LogLevel.INFO, logMessage)) logger.info(logMessage) } override fun debug(logger: Logger, recordId: Long, logMessage: String) { - get(recordId).add(Log(LogLevel.DEBUG, logMessage)) + get(recordId).add(LogRecord(LogLevel.DEBUG, logMessage)) logger.debug(logMessage) } override fun warn(logger: Logger, recordId: Long, logMessage: String) { - get(recordId).add(Log(LogLevel.WARN, logMessage)) + get(recordId).add(LogRecord(LogLevel.WARN, logMessage)) logger.warn(logMessage) } override fun error(logger: Logger, recordId: Long, logMessage: String, exe: Exception?) { - get(recordId).add(Log(LogLevel.ERROR, "$logMessage: ${exe?.stackTrace?.toString()}")) - logger.error(logMessage, exe) - } - - override fun uploadLogs(recordId: Long) { - val listOfLogs = logContainer.getValue(recordId) - - if (listOfLogs.isNotEmpty()) { - logger.info("Sending record $recordId logs...") - val logs = LogsDto().apply { - contents = UploadSourceConnectorConfig.mapper.writeValueAsString(listOfLogs) + val message = if (exe != null) { + val trace = ByteArrayOutputStream().use { byteOut -> + PrintStream(byteOut).use { printOut -> + exe.printStackTrace(printOut) + } + byteOut.toString(UTF_8) } - logger.info(UploadSourceConnectorConfig.mapper.writeValueAsString(logs.contents)) - uploadClient.addLogs(recordId, logs) - logContainer.remove(recordId) + "$logMessage: $exe$trace" + } else { + logMessage } + get(recordId).add(LogRecord(LogLevel.ERROR, message)) + logger.error(logMessage, exe) } - override fun uploadAllLogs() { - logger.info("Uploading all remaining logs") - if (logContainer.isNotEmpty()) { - logContainer.map { entry -> uploadLogs(entry.key) } + override val recordIds: Set + get() = logContainer.keys + + override fun extract(recordId: Long, reset: Boolean): Log? { + val recordQueue = if (reset) { + logContainer.remove(recordId) } else { - logger.info("All record logs are uploaded") + logContainer[recordId] } + return recordQueue + ?.takeIf { it.isNotEmpty() } + ?.let { Log(recordId, it) } } - - companion object { - private val logger = LoggerFactory.getLogger(ConverterLogRepository::class.java) - } - } diff --git a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/RecordConverter.kt b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/RecordConverter.kt index 5cc30966..e2bb7d20 100644 --- a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/RecordConverter.kt +++ b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/converter/RecordConverter.kt @@ -28,15 +28,16 @@ import org.radarbase.connect.upload.converter.Converter.Companion.END_OF_RECORD_ import org.radarbase.connect.upload.converter.Converter.Companion.RECORD_ID_KEY import org.radarbase.connect.upload.converter.Converter.Companion.REVISION_KEY import org.radarbase.connect.upload.exception.ConversionFailedException +import org.radarbase.connect.upload.exception.ConversionTemporarilyFailedException import org.radarcns.kafka.ObservationKey import org.slf4j.LoggerFactory +import java.io.IOException import java.io.InputStream import java.lang.Exception import java.time.Instant -abstract class RecordConverter(override val sourceType: String, val avroData: AvroData = AvroData(20)) : Converter { - +abstract class RecordConverter(override val sourceType: String, private val avroData: AvroData = AvroData(20)) : Converter { private lateinit var connectorConfig: SourceTypeDTO private lateinit var client: UploadBackendClient private lateinit var settings: Map @@ -59,9 +60,6 @@ abstract class RecordConverter(override val sourceType: String, val avroData: Av } override fun close() { - if (this::logsRepository.isInitialized) { - this.logsRepository.uploadAllLogs() - } if (this::client.isInitialized) { this.client.close() } @@ -70,82 +68,57 @@ abstract class RecordConverter(override val sourceType: String, val avroData: Av val logRepository get() = this.logsRepository override fun convert(record: RecordDTO): ConversionResult { - val recordId = record.id!! - logsRepository.info(logger, recordId,"Converting record : record-id $recordId") + val recordId = checkNotNull(record.id) + logsRepository.info(logger, recordId,"Converting record: record-id $recordId") try { - record.validateRecord() - - val key = record.computeObservationKey(avroData) - - val recordContents = record.data!!.contents!! - - val sourceRecords = recordContents.map contentMap@{ content -> - - val response = client.retrieveFile(record, content.fileName) - // if receiving a content fails, mark that record as READY and stop converting - if(response == null) { - client.updateStatus(recordId, record.metadata!!.copy( - status = "READY", - message = "Could not retrieve file ${content.fileName} from record with id $recordId" - )) - - logsRepository.error(logger, recordId,"Could not retrieve file ${content.fileName} from record with id $recordId") - return@convert ConversionResult(record, emptyList()) - } - - val timeReceived = Instant.now().epochSecond - - response.use responseResource@{ res -> - return@contentMap processData(content, res.byteStream(), record, timeReceived.toDouble()) - .map topicDataMap@{ topicData -> - val valRecord = avroData.toConnectData(topicData.value.schema, topicData.value) - val offset = mutableMapOf( - END_OF_RECORD_KEY to topicData.endOfFileOffSet, - RECORD_ID_KEY to recordId, - REVISION_KEY to record.metadata?.revision - ) - return@topicDataMap SourceRecord(getPartition(), offset, topicData.topic, key.schema(), key.value(), valRecord.schema(), valRecord.value()) + val recordData = checkNotNull(record.data) { "Record data cannot be null" } + val recordContents = checkNotNull(recordData.contents) { "Record data has empty content" } + val recordMetadata = checkNotNull(record.metadata) { "Record meta-data cannot be null" } + + val key = recordData.computeObservationKey(avroData) + + val sourceRecords: List = recordContents + .flatMap { content -> + try { + client.retrieveFile(record, content.fileName) { body -> + val timeReceived = System.currentTimeMillis() / 1000.0 + + processData(content, body.byteStream(), record, timeReceived) } - } - } - return ConversionResult(record, sourceRecords.flatMap { it.toList() }) + } catch (ex: IOException) { + throw ConversionTemporarilyFailedException("Could not retrieve file ${content.fileName} from record with id $recordId", ex) + } + } + .map { topicData -> + val valRecord = avroData.toConnectData(topicData.value.schema, topicData.value) + val offset = mutableMapOf( + END_OF_RECORD_KEY to topicData.endOfFileOffSet, + RECORD_ID_KEY to recordId, + REVISION_KEY to recordMetadata.revision + ) + SourceRecord(getPartition(), offset, topicData.topic, key.schema(), key.value(), valRecord.schema(), valRecord.value()) + } + return ConversionResult(record, sourceRecords) } catch (exe: Exception){ logsRepository.error(logger, recordId, "Could not convert record $recordId", exe) throw ConversionFailedException("Could not convert record $recordId",exe) } } - private fun commitLogs(record: RecordDTO, client: UploadBackendClient): RecordMetadataDTO { - logger.debug("Sending record logs..") - val logs = LogsDto().apply { - contents = UploadSourceConnectorConfig.mapper.writeValueAsString(logsRepository) - } - logger.info(UploadSourceConnectorConfig.mapper.writeValueAsString(logsRepository)) - return client.addLogs(record.id!!, logs) - } - /** process file content with the record data. The implementing method should close response-body. */ abstract fun processData(contents: ContentsDTO, inputStream: InputStream, record: RecordDTO, timeReceived: Double) : List override fun getPartition(): MutableMap = mutableMapOf("source-type" to sourceType) - private fun RecordDTO.validateRecord() { - this.id ?: throw IllegalStateException("Record id cannot be null") - this.metadata ?: throw IllegalStateException("Record meta-data cannot be null") - this.data ?: throw IllegalStateException("Record data cannot be null") - this.data?.contents ?: throw IllegalStateException("Record data has empty content") - } - - private fun RecordDTO.computeObservationKey(avroData: AvroData): SchemaAndValue { - val data = this.data ?: throw IllegalStateException("Cannot process record without data") + private fun RecordDataDTO.computeObservationKey(avroData: AvroData): SchemaAndValue { return avroData.toConnectData( ObservationKey.getClassSchema(), ObservationKey( - data.projectId, - data.userId, - data.sourceId + projectId, + userId, + sourceId ) ) } diff --git a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/exception/Exceptions.kt b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/exception/Exceptions.kt index 47c1d803..0a8b8524 100644 --- a/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/exception/Exceptions.kt +++ b/kafka-connect-upload-source/src/main/java/org/radarbase/connect/upload/exception/Exceptions.kt @@ -30,3 +30,5 @@ class InvalidFormatException(message: String) : RuntimeException(message) class DataProcessorNotFoundException(message: String) : RuntimeException(message) class ConversionFailedException(message: String, cause: Throwable? = null) : RuntimeException(message, cause) + +class ConversionTemporarilyFailedException(message: String, cause: Throwable? = null) : RuntimeException(message, cause) diff --git a/radar-upload-backend/build.gradle.kts b/radar-upload-backend/build.gradle.kts index 8c01527c..cd9c9d03 100644 --- a/radar-upload-backend/build.gradle.kts +++ b/radar-upload-backend/build.gradle.kts @@ -16,20 +16,24 @@ application { project.extra.apply { set("okhttpVersion", "4.2.0") set("radarMpVersion", "0.5.7") + set("radarAuthVersion", "0.1.0") set("radarCommonsVersion", "0.12.2") set("radarSchemasVersion", "0.5.2") set("jacksonVersion", "2.9.9.2") set("jacksonDataVersion", "2.9.9") - set("slf4jVersion", "1.7.26") + set("slf4jVersion", "1.7.27") set("logbackVersion", "1.2.3") set("grizzlyVersion", "2.4.4") set("jerseyVersion", "2.29.1") + // skip 5.4.5: https://hibernate.atlassian.net/browse/HHH-13625 set("hibernateVersion", "5.4.4.Final") } repositories { jcenter() maven(url = "https://dl.bintray.com/radar-cns/org.radarcns") + maven(url = "https://dl.bintray.com/radar-base/org.radarbase") + maven(url = "https://repo.thehyve.nl/content/repositories/snapshots") } dependencies { @@ -48,7 +52,7 @@ dependencies { implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${project.extra["jacksonDataVersion"]}") implementation("com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${project.extra["jacksonDataVersion"]}") - implementation("org.radarcns:radar-auth:${project.extra["radarMpVersion"]}") + implementation("org.radarbase:radar-auth-jersey:${project.extra["radarAuthVersion"]}") implementation("org.slf4j:slf4j-api:${project.extra["slf4jVersion"]}") diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/Main.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/Main.kt index 9d6478ff..9d74664f 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/Main.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/Main.kt @@ -31,7 +31,6 @@ import java.nio.file.Files import java.nio.file.Paths import kotlin.system.exitProcess - val logger: Logger = LoggerFactory.getLogger("org.radarbase.upload.Main") fun loadConfig(args: Array): Config { diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/api/SourceTypeMapperImpl.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/api/SourceTypeMapperImpl.kt index 6c8efdd8..d7f18d22 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/api/SourceTypeMapperImpl.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/api/SourceTypeMapperImpl.kt @@ -22,29 +22,23 @@ package org.radarbase.upload.api import org.radarbase.upload.doa.entity.SourceType class SourceTypeMapperImpl : SourceTypeMapper { - override fun fromSourceType(sourceType: SourceType) = SourceTypeDTO( name = sourceType.name, topics = sourceType.topics, contentTypes = sourceType.contentTypes, timeRequired = sourceType.timeRequired, sourceIdRequired = sourceType.sourceIdRequired, - configuration = sourceType.configuration - ) + configuration = sourceType.configuration) override fun fromSourceTypes(sourceTypes: List) = SourceTypeContainerDTO( - sourceTypes = sourceTypes.map(::fromSourceType) - ) + sourceTypes = sourceTypes.map(::fromSourceType)) - override fun toSourceType(sourceType: SourceTypeDTO): SourceType { - val entity = SourceType() - entity.name = sourceType.name - entity.topics = sourceType.topics ?: mutableSetOf() - entity.contentTypes = sourceType.contentTypes ?: mutableSetOf() - entity.sourceIdRequired = sourceType.sourceIdRequired ?: false - entity.timeRequired = sourceType.timeRequired ?: false - entity.configuration = sourceType.configuration ?: mutableMapOf() - return entity + override fun toSourceType(sourceType: SourceTypeDTO) = SourceType().apply { + name = sourceType.name + topics = sourceType.topics?.toMutableSet() ?: mutableSetOf() + contentTypes = sourceType.contentTypes?.toMutableSet() ?: mutableSetOf() + sourceIdRequired = sourceType.sourceIdRequired ?: false + timeRequired = sourceType.timeRequired ?: false + configuration = sourceType.configuration?.toMutableMap() ?: mutableMapOf() } - } diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/Auth.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/auth/Auth.kt deleted file mode 100644 index 101072c3..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/Auth.kt +++ /dev/null @@ -1,45 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.auth - -import org.radarcns.auth.authorization.Permission -import java.time.Instant - -interface Auth { - val defaultProject: String? - val userId: String? - val bearerToken: String? - val expiresAt: Instant? - - fun checkSourcePermission(permission: Permission, projectId: String?, userId: String?, sourceId: String?) - fun checkUserPermission(permission: Permission, projectId: String?, userId: String?) - fun checkProjectPermission(permission: Permission, projectId: String?) - fun hasRole(projectId: String, role: String): Boolean - fun hasPermission(permission: Permission): Boolean - fun hasPermissionOnProject(permission: Permission, projectId: String): Boolean - fun hasPermissionOnSubject(permission: Permission, projectId: String, userId: String): Boolean - fun authorizedProjects(permission: Permission): AccessRestriction - val isClientCredentials: Boolean -} - -sealed class AccessRestriction - -object AllAccess : AccessRestriction() -data class RestrictedAccess(var access: Set) : AccessRestriction() diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/AuthValidator.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/auth/AuthValidator.kt deleted file mode 100644 index d8c443b8..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/AuthValidator.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.auth - -import org.radarbase.upload.filter.AuthenticationFilter -import org.radarcns.auth.exception.TokenValidationException -import javax.ws.rs.NotAuthorizedException -import javax.ws.rs.container.ContainerRequestContext - -interface AuthValidator { - @Throws(TokenValidationException::class, NotAuthorizedException::class) - fun verify(request: ContainerRequestContext): Auth? - - fun getToken(request: ContainerRequestContext): String? { - val authorizationHeader = request.getHeaderString("Authorization") - - // Check if the HTTP Authorization header is present and formatted correctly - if (authorizationHeader != null - && authorizationHeader.startsWith(AuthenticationFilter.BEARER, ignoreCase = true)) { - // Extract the token from the HTTP Authorization header - return authorizationHeader.substring(AuthenticationFilter.BEARER.length).trim { it <= ' ' } - } - - // Extract the token from the Authorization cookie - val authorizationCookie = request.cookies["authorizationBearer"] - if (authorizationCookie != null) { - return authorizationCookie.value - } - - return null - } -} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/Authenticated.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/auth/Authenticated.kt deleted file mode 100644 index d2e12b56..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/Authenticated.kt +++ /dev/null @@ -1,31 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.auth - -import javax.ws.rs.NameBinding - -/** - * Annotation for requests that should be authenticated. - */ -@NameBinding -@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION, - AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) -@Retention(AnnotationRetention.RUNTIME) -annotation class Authenticated diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/ManagementPortalAuth.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/auth/ManagementPortalAuth.kt deleted file mode 100644 index 69c00691..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/ManagementPortalAuth.kt +++ /dev/null @@ -1,101 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.auth - -import org.radarcns.auth.authorization.AuthoritiesConstants.SYS_ADMIN -import org.radarcns.auth.authorization.Permission -import org.radarcns.auth.authorization.Permission.MEASUREMENT_CREATE -import org.radarcns.auth.token.RadarToken -import java.time.Instant -import javax.ws.rs.BadRequestException -import javax.ws.rs.ForbiddenException - -/** - * Parsed JWT for validating authorization of data contents. - */ -class ManagementPortalAuth(private val token: RadarToken) : Auth { - override val defaultProject = token.roles.keys - .firstOrNull { token.hasPermissionOnProject(MEASUREMENT_CREATE, it) } - override val userId: String? = token.subject.takeUnless { it.isEmpty() } - - override val bearerToken: String? = token.token - - override val expiresAt: Instant? = token.expiresAt.toInstant() - - override fun checkSourcePermission(permission: Permission, projectId: String?, userId: String?, sourceId: String?) { - if (!token.hasPermissionOnSource(permission, - projectId ?: throw BadRequestException("Missing project ID in request"), - userId ?: throw BadRequestException("Missing user ID in request"), - sourceId ?: throw BadRequestException("Missing source ID in request"))) { - throw ForbiddenException("No $permission permission for " + - "project $projectId with user $userId and source $sourceId " + - "using token ${token.token}") - } - } - - - override fun checkProjectPermission(permission: Permission, projectId: String?) { - if (!token.hasPermissionOnProject(permission, - projectId ?: throw BadRequestException("Missing project ID in request"))) { - throw ForbiddenException("No $permission permission for " + - "project $projectId " + - "using token ${token.token}") - } - } - - - override fun checkUserPermission(permission: Permission, projectId: String?, userId: String?) { - if (!token.hasPermissionOnSubject(permission, - projectId ?: throw BadRequestException("Missing project ID in request"), - userId ?: throw BadRequestException("Missing user ID in request"))) { - throw ForbiddenException("No permission to create measurement for " + - "project $projectId with user $userId " + - "using token ${token.token}") - } - } - - override fun hasRole(projectId: String, role: String) = token.roles - .getOrDefault(projectId, emptyList()) - .contains(role) - - override fun hasPermission(permission: Permission) = token.hasPermission(permission) - - override fun hasPermissionOnProject(permission: Permission, projectId: String): Boolean { - return token.hasPermissionOnProject(permission, projectId) - } - - override fun hasPermissionOnSubject(permission: Permission, projectId: String, userId: String): Boolean { - return token.hasPermissionOnSubject(permission, projectId, userId) - } - - override fun authorizedProjects(permission: Permission): AccessRestriction { - if (((token.authorities.contains(SYS_ADMIN) && permission.isAuthorityAllowed(SYS_ADMIN)) - || isClientCredentials) && permission.scopeName() in token.scopes) { - return AllAccess - } - - return RestrictedAccess(token.roles.filter { project -> - project.value.any { permission.isAuthorityAllowed(it) } - }.keys) - } - - override val isClientCredentials - get() = "client_credentials" == token.grantType -} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/NeedsPermission.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/auth/NeedsPermission.kt deleted file mode 100644 index 34ba129b..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/NeedsPermission.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.auth - -import org.radarcns.auth.authorization.Permission - -/** - * Indicates that a method needs an authenticated user that has a certain permission. - */ -@Target(AnnotationTarget.FUNCTION, - AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) -@Retention(AnnotationRetention.RUNTIME) -annotation class NeedsPermission( - /** - * Entity that the permission is needed on. - */ - val entity: Permission.Entity, - /** - * Operation on given entity that the permission is needed for. - */ - val operation: Permission.Operation) diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/NeedsPermissionOnProject.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/auth/NeedsPermissionOnProject.kt deleted file mode 100644 index dfa67dff..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/NeedsPermissionOnProject.kt +++ /dev/null @@ -1,40 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.auth - -import org.radarcns.auth.authorization.Permission - -/** - * Indicates that a method needs an authenticated user that has a certain permission. - */ -@Target(AnnotationTarget.FUNCTION, - AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) -@Retention(AnnotationRetention.RUNTIME) -annotation class NeedsPermissionOnProject( - /** - * Entity that the permission is needed on. - */ - val entity: Permission.Entity, - /** - * Operation on given entity that the permission is needed for. - */ - val operation: Permission.Operation, - /** Project path parameter */ - val projectPathParam: String) diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/NeedsPermissionOnUser.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/auth/NeedsPermissionOnUser.kt deleted file mode 100644 index 27ade0dc..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/NeedsPermissionOnUser.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.auth - -import org.radarcns.auth.authorization.Permission - -/** - * Indicates that a method needs an authenticated user that has a certain permission. - */ -@Target(AnnotationTarget.FUNCTION, - AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER) -@Retention(AnnotationRetention.RUNTIME) -annotation class NeedsPermissionOnUser( - /** - * Entity that the permission is needed on. - */ - val entity: Permission.Entity, - /** - * Operation on given entity that the permission is needed for. - */ - val operation: Permission.Operation, - /** Project path parameter. */ - val projectPathParam: String, - /** User path parameter. */ - val userPathParam: String) diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/RadarSecurityContext.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/auth/RadarSecurityContext.kt deleted file mode 100644 index 544658f5..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/RadarSecurityContext.kt +++ /dev/null @@ -1,50 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.auth - -import java.security.Principal -import javax.ws.rs.core.SecurityContext - -/** - * Security context from currently parsed authentication. - */ -class RadarSecurityContext( - /** Get the parsed authentication. */ - val auth: Auth) : SecurityContext { - - override fun getUserPrincipal() = Principal { auth.userId } - - /** - * Maps roles in the shape `"project:role"` to a Management Portal role. Global roles - * take the shape of `":global_role"`. This allows for example a - * `@RolesAllowed(":SYS_ADMIN")` annotation to resolve correctly. - * @param role role to be mapped - * @return `true` if the authentication contains given project/role, - * `false` otherwise - */ - override fun isUserInRole(role: String): Boolean { - val projectRole = role.split(":") - return projectRole.size == 2 && auth.hasRole(projectRole[0], projectRole[1]) - } - - override fun isSecure() = true - - override fun getAuthenticationScheme() = "JWT" -} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/exception/HttpApplicationExceptionMapper.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/exception/HttpApplicationExceptionMapper.kt deleted file mode 100644 index 29c0f08d..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/exception/HttpApplicationExceptionMapper.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.exception - -import com.fasterxml.jackson.core.util.BufferRecyclers -import org.glassfish.jersey.message.internal.ReaderWriter -import org.slf4j.LoggerFactory -import javax.ws.rs.core.Context -import javax.ws.rs.core.Response -import javax.ws.rs.core.UriInfo -import javax.ws.rs.ext.ExceptionMapper -import javax.ws.rs.ext.Provider - -@Provider -class HttpApplicationExceptionMapper : ExceptionMapper { - @Context - private lateinit var uriInfo: UriInfo - - override fun toResponse(exception: HttpApplicationException): Response { - logger.error("[{}] {} - {}: {}", exception.status, uriInfo.absolutePath, exception.code, exception.detailedMessage) - - val stringEncoder = BufferRecyclers.getJsonStringEncoder() - val quotedError = stringEncoder.quoteAsUTF8(exception.code).toString(ReaderWriter.UTF8) - val quotedDescription = stringEncoder.quoteAsUTF8(exception.detailedMessage).toString(ReaderWriter.UTF8) - return Response.status(exception.status) - .header("Content-Type", "application/json; charset=utf-8") - .entity("{\"error\":\"$quotedError\"," - + "\"error_description\":\"$quotedDescription\"}") - .build() - - } - - companion object { - private val logger = LoggerFactory.getLogger(HttpApplicationExceptionMapper::class.java) - } -} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/filter/AuthenticationFilter.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/filter/AuthenticationFilter.kt deleted file mode 100644 index 20b97ef2..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/filter/AuthenticationFilter.kt +++ /dev/null @@ -1,78 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.filter - -import org.radarbase.upload.auth.AuthValidator -import org.radarbase.upload.auth.Authenticated -import org.radarbase.upload.auth.RadarSecurityContext -import org.radarcns.auth.exception.TokenValidationException -import org.slf4j.LoggerFactory -import javax.annotation.Priority -import javax.ws.rs.Priorities -import javax.ws.rs.container.ContainerRequestContext -import javax.ws.rs.container.ContainerRequestFilter -import javax.ws.rs.core.Context -import javax.ws.rs.core.Response -import javax.ws.rs.ext.Provider - -/** - * Authenticates user by a JWT in the bearer signed by the Management Portal. - */ -@Provider -@Authenticated -@Priority(Priorities.AUTHENTICATION) -class AuthenticationFilter : ContainerRequestFilter { - - @Context - private lateinit var validator: AuthValidator - - override fun filter(requestContext: ContainerRequestContext) { - val radarToken = try { - validator.verify(requestContext) - } catch (ex: TokenValidationException) { - logger.warn("[401] {}: {}", requestContext.uriInfo.path, ex.message, ex) - requestContext.abortWith( - Response.status(Response.Status.UNAUTHORIZED) - .header("WWW-Authenticate", - BEARER_REALM - + " error=\"invalid_token\"" - + " error_description=\"${ex.message}\"") - .build()) - null - } - logger.debug("Verified token: $radarToken for request ${requestContext.uriInfo.path}" ) - if (radarToken == null) { - logger.debug("[401] {}: Could not find a valid token in the header", - requestContext.uriInfo.path) - requestContext.abortWith(Response.status(Response.Status.UNAUTHORIZED) - .header("WWW-Authenticate", BEARER_REALM) - .build()) - } else { - requestContext.securityContext = RadarSecurityContext(radarToken) - } - } - - companion object { - private val logger = LoggerFactory.getLogger(AuthenticationFilter::class.java) - - const val BEARER_REALM: String = "Bearer realm=\"Upload server\"" - const val BEARER: String = "Bearer " - } -} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/filter/PermissionFilter.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/filter/PermissionFilter.kt deleted file mode 100644 index 7bc80318..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/filter/PermissionFilter.kt +++ /dev/null @@ -1,116 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.filter - -import org.radarbase.upload.auth.Auth -import org.radarbase.upload.auth.NeedsPermission -import org.radarbase.upload.auth.NeedsPermissionOnProject -import org.radarbase.upload.auth.NeedsPermissionOnUser -import org.radarbase.upload.service.MPService -import org.radarcns.auth.authorization.Permission -import org.slf4j.LoggerFactory -import javax.ws.rs.container.ContainerRequestContext -import javax.ws.rs.container.ContainerRequestFilter -import javax.ws.rs.container.ResourceInfo -import javax.ws.rs.core.Context -import javax.ws.rs.core.Response -import javax.ws.rs.core.UriInfo - -/** - * Check that the token has given permissions. - */ -class PermissionFilter : ContainerRequestFilter { - - @Context - private lateinit var resourceInfo: ResourceInfo - - @Context - private lateinit var auth: Auth - - @Context - private lateinit var mpService: MPService - - @Context - private lateinit var uriInfo: UriInfo - - override fun filter(requestContext: ContainerRequestContext) { - val resourceMethod = resourceInfo.resourceMethod - - val userAnnotation = resourceMethod.getAnnotation(NeedsPermissionOnUser::class.java) - val projectAnnotation = resourceMethod.getAnnotation(NeedsPermissionOnProject::class.java) - val annotation = resourceMethod.getAnnotation(NeedsPermission::class.java) - - val (permission, project, isAuthenticated) = when { - userAnnotation != null -> { - val permission = Permission(userAnnotation.entity, userAnnotation.operation) - val projectId = uriInfo.pathParameters[userAnnotation.projectPathParam]?.firstOrNull() - val userId = uriInfo.pathParameters[userAnnotation.userPathParam]?.firstOrNull() - - Triple(permission, projectId, projectId != null - && userId != null - && auth.hasPermissionOnSubject(permission, projectId, userId)) - } - projectAnnotation != null -> { - val permission = Permission(projectAnnotation.entity, projectAnnotation.operation) - - val projectId = uriInfo.pathParameters[projectAnnotation.projectPathParam]?.firstOrNull() - - Triple(permission, projectId, projectId != null - && auth.hasPermissionOnProject(permission, projectId)) - } - annotation != null -> { - val permission = Permission(annotation.entity, annotation.operation) - - Triple(permission, null, auth.hasPermission(permission)) - } - else -> return - } - - if (!isAuthenticated) { - abortWithForbidden(requestContext, permission) - return - } - project?.let { mpService.ensureProject(it) } - } - - companion object { - private val logger = LoggerFactory.getLogger(PermissionFilter::class.java) - - /** - * Abort the request with a forbidden status. The caller must ensure that no other changes are - * made to the context (i.e., make a quick return). - * @param requestContext context to abort - * @param scope the permission that is needed. - */ - fun abortWithForbidden(requestContext: ContainerRequestContext, scope: Permission) { - val message = "$scope permission not given." - logger.warn("[403] {}: {}", - requestContext.uriInfo.path, message) - - requestContext.abortWith( - Response.status(Response.Status.FORBIDDEN) - .header("WWW-Authenticate", AuthenticationFilter.BEARER_REALM - + " error=\"insufficient_scope\"" - + " error_description=\"$message\"" - + " scope=\"$scope\"") - .build()) - } - } -} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/AuthFactory.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/AuthFactory.kt deleted file mode 100644 index 0a9a5be0..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/AuthFactory.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.inject - -import org.radarbase.upload.auth.Auth -import org.radarbase.upload.auth.RadarSecurityContext -import java.util.function.Supplier -import javax.ws.rs.container.ContainerRequestContext -import javax.ws.rs.core.Context - -/** Generates radar tokens from the security context. */ -class AuthFactory : Supplier { - @Context - private lateinit var context: ContainerRequestContext - - override fun get() = (context.securityContext as? RadarSecurityContext)?.auth - ?: throw IllegalStateException("Created null wrapper") -} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/AuthorizationFeature.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/AuthorizationFeature.kt deleted file mode 100644 index daf85c52..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/AuthorizationFeature.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.inject - -import org.radarbase.upload.auth.NeedsPermission -import org.radarbase.upload.auth.NeedsPermissionOnProject -import org.radarbase.upload.auth.NeedsPermissionOnUser -import org.radarbase.upload.filter.PermissionFilter -import javax.ws.rs.Priorities -import javax.ws.rs.container.DynamicFeature -import javax.ws.rs.container.ResourceInfo -import javax.ws.rs.core.FeatureContext -import javax.ws.rs.ext.Provider - -/** Authorization for different auth tags. */ -@Provider -class AuthorizationFeature : DynamicFeature { - override fun configure(resourceInfo: ResourceInfo, context: FeatureContext) { - val resourceMethod = resourceInfo.resourceMethod - if (resourceMethod.isAnnotationPresent(NeedsPermission::class.java) - || resourceMethod.isAnnotationPresent(NeedsPermissionOnProject::class.java) - || resourceMethod.isAnnotationPresent(NeedsPermissionOnUser::class.java)) { - context.register(PermissionFilter::class.java, Priorities.AUTHORIZATION) - } - } -} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/DoaEntityManagerFactory.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/DoaEntityManagerFactory.kt index 1798b19f..b78f49e3 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/DoaEntityManagerFactory.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/DoaEntityManagerFactory.kt @@ -37,8 +37,10 @@ class DoaEntityManagerFactory(@Context private val emf: EntityManagerFactory) : } override fun dispose(instance: EntityManager?) { - logger.debug("Disposing EntityManager") - instance?.close() + instance?.let { + logger.debug("Disposing EntityManager") + it.close() + } } companion object { diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/DoaEntityManagerFactoryFactory.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/DoaEntityManagerFactoryFactory.kt index 0b25c7c1..2b3f4128 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/DoaEntityManagerFactoryFactory.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/DoaEntityManagerFactoryFactory.kt @@ -76,5 +76,4 @@ class DoaEntityManagerFactoryFactory(@Context config: Config) : DisposableSuppli companion object { private val logger = LoggerFactory.getLogger(DoaEntityManagerFactoryFactory::class.java) } - } diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/ManagementPortalResourceConfig.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/ManagementPortalResourceConfig.kt index ca8612a3..63024677 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/ManagementPortalResourceConfig.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/ManagementPortalResourceConfig.kt @@ -21,18 +21,36 @@ package org.radarbase.upload.inject import org.glassfish.jersey.internal.inject.AbstractBinder import org.glassfish.jersey.server.ResourceConfig -import org.radarbase.upload.auth.AuthValidator +import org.radarbase.auth.jersey.* +import org.radarbase.upload.Config +import org.radarbase.upload.service.managementportal.MPClient +import org.radarbase.upload.service.managementportal.MPProjectService +import org.radarbase.upload.service.UploadProjectService import javax.inject.Singleton /** This binder needs to register all non-Jersey classes, otherwise initialization fails. */ class ManagementPortalResourceConfig : UploadResourceConfig() { - override fun registerAuthentication(resources: ResourceConfig) { - // none needed - } + override fun createEnhancers(config: Config): List = listOf( + RadarJerseyResourceEnhancer(AuthConfig( + managementPortalUrl = config.managementPortalUrl, + jwtResourceName = "res_upload")), + ManagementPortalResourceEnhancer()) + + override fun registerAuthentication(binder: AbstractBinder) { + binder.apply { + bind(MPClient::class.java) + .to(MPClient::class.java) + .`in`(Singleton::class.java) + + bind(MPProjectService::class.java) + .to(UploadProjectService::class.java) + .`in`(Singleton::class.java) + + bind(MPProjectService::class.java) + .to(ProjectService::class.java) + .`in`(Singleton::class.java) + + } - override fun registerAuthenticationUtilities(binder: AbstractBinder) { - binder.bind(RadarTokenValidator::class.java) - .to(AuthValidator::class.java) - .`in`(Singleton::class.java) } } diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/RadarTokenValidator.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/RadarTokenValidator.kt deleted file mode 100644 index 843242ae..00000000 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/RadarTokenValidator.kt +++ /dev/null @@ -1,65 +0,0 @@ -/* - * - * * Copyright 2019 The Hyve - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * * - * - */ - -package org.radarbase.upload.inject - -import org.radarbase.upload.Config -import org.radarbase.upload.auth.Auth -import org.radarbase.upload.auth.AuthValidator -import org.radarbase.upload.auth.ManagementPortalAuth -import org.radarcns.auth.authentication.TokenValidator -import org.radarcns.auth.config.TokenVerifierPublicKeyConfig -import org.slf4j.LoggerFactory -import java.io.IOException -import java.lang.Exception -import java.net.URI -import javax.ws.rs.container.ContainerRequestContext -import javax.ws.rs.core.Context - -/** Creates a TokenValidator based on the current management portal configuration. */ -class RadarTokenValidator constructor(@Context config: Config) : AuthValidator { - private val tokenValidator = try { - TokenValidator() - } catch (e: RuntimeException) { - TokenValidator(TokenVerifierPublicKeyConfig().apply { - publicKeyEndpoints = listOf(URI("${config.managementPortalUrl}/oauth/token_key")) - resourceName = config.jwtResourceName - }) - } - - init { - try { - this.tokenValidator.refresh() - logger.info("Refreshed Token Validator") - } catch (ex: Exception) { - logger.error("Failed to immediately initialize token validator, will try again later: {}", - ex.toString()) - } - } - - override fun verify(request: ContainerRequestContext): Auth? { - return getToken(request)?.let { - ManagementPortalAuth(tokenValidator.validateAccessToken(it)) - } - } - - companion object { - private val logger = LoggerFactory.getLogger(RadarTokenValidator::class.java) - } -} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/UploadResourceConfig.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/UploadResourceConfig.kt index 48f4271b..1cc7da78 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/inject/UploadResourceConfig.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/inject/UploadResourceConfig.kt @@ -28,20 +28,18 @@ import okhttp3.OkHttpClient import org.glassfish.jersey.internal.inject.AbstractBinder import org.glassfish.jersey.process.internal.RequestScoped import org.glassfish.jersey.server.ResourceConfig +import org.radarbase.auth.jersey.JerseyResourceEnhancer import org.radarbase.upload.Config import org.radarbase.upload.api.RecordMapper import org.radarbase.upload.api.RecordMapperImpl import org.radarbase.upload.api.SourceTypeMapper import org.radarbase.upload.api.SourceTypeMapperImpl -import org.radarbase.upload.auth.Auth -import org.radarbase.upload.auth.MPClient import org.radarbase.upload.doa.RecordRepository import org.radarbase.upload.doa.RecordRepositoryImpl import org.radarbase.upload.doa.SourceTypeRepository import org.radarbase.upload.doa.SourceTypeRepositoryImpl import org.radarbase.upload.dto.CallbackManager import org.radarbase.upload.dto.QueuedCallbackManager -import org.radarbase.upload.service.MPService import java.util.concurrent.TimeUnit import javax.inject.Singleton import javax.persistence.EntityManager @@ -55,26 +53,23 @@ abstract class UploadResourceConfig { .readTimeout(30, TimeUnit.SECONDS) .build() - fun resources(config: Config): ResourceConfig { - val resources = ResourceConfig().apply { - packages( - "org.radarbase.upload.auth", - "org.radarbase.upload.exception", - "org.radarbase.upload.filter", - "org.radarbase.upload.resource") - register(binder(config)) - register(ContextResolver { OBJECT_MAPPER }) - property("jersey.config.server.wadl.disableWadl", true) - } - registerAuthentication(resources) - return resources + fun resources(config: Config) = ResourceConfig().apply { + val enhancers = createEnhancers(config) + packages( + "org.radarbase.upload.exception", + "org.radarbase.upload.filter", + "org.radarbase.upload.resource") + enhancers.forEach { packages(*it.packages) } + register(binder(config, enhancers)) + register(ContextResolver { OBJECT_MAPPER }) + property("jersey.config.server.wadl.disableWadl", true) } - abstract fun registerAuthentication(resources: ResourceConfig) + abstract fun createEnhancers(config: Config): List - abstract fun registerAuthenticationUtilities(binder: AbstractBinder) + abstract fun registerAuthentication(binder: AbstractBinder) - private fun binder(config: Config) = object : AbstractBinder() { + private fun binder(config: Config, enhancers: List) = object : AbstractBinder() { override fun configure() { // Bind instances. These cannot use any injects themselves bind(config) @@ -90,21 +85,7 @@ abstract class UploadResourceConfig { .to(CallbackManager::class.java) .`in`(Singleton::class.java) - bind(MPClient::class.java) - .to(MPClient::class.java) - .`in`(Singleton::class.java) - - bind(MPService::class.java) - .to(MPService::class.java) - .`in`(Singleton::class.java) - // Bind factories. - bindFactory(AuthFactory::class.java) - .proxy(true) - .proxyForSameScope(true) - .to(Auth::class.java) - .`in`(RequestScoped::class.java) - bindFactory(DoaEntityManagerFactoryFactory::class.java) .to(EntityManagerFactory::class.java) .`in`(Singleton::class.java) @@ -129,12 +110,14 @@ abstract class UploadResourceConfig { .to(SourceTypeRepository::class.java) .`in`(Singleton::class.java) - registerAuthenticationUtilities(this) + enhancers.forEach { it.enhance(this) } + + registerAuthentication(this) } } companion object { - private val OBJECT_MAPPER = ObjectMapper() + private val OBJECT_MAPPER: ObjectMapper = ObjectMapper() .setSerializationInclusion(JsonInclude.Include.NON_NULL) .registerModule(JavaTimeModule()) .registerModule(KotlinModule()) diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/resource/ProjectResource.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/resource/ProjectResource.kt index 862c1338..12b8ae71 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/resource/ProjectResource.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/resource/ProjectResource.kt @@ -19,16 +19,16 @@ package org.radarbase.upload.resource -import org.radarbase.upload.auth.Auth -import org.radarbase.upload.auth.Authenticated -import org.radarbase.upload.auth.NeedsPermission -import org.radarbase.upload.auth.NeedsPermissionOnProject +import org.radarbase.auth.jersey.Auth +import org.radarbase.auth.jersey.Authenticated +import org.radarbase.auth.jersey.NeedsPermission import org.radarbase.upload.dto.Project import org.radarbase.upload.dto.ProjectList import org.radarbase.upload.dto.UserList -import org.radarbase.upload.service.MPService +import org.radarbase.upload.service.UploadProjectService import org.radarcns.auth.authorization.Permission import javax.annotation.Resource +import javax.inject.Singleton import javax.ws.rs.* import javax.ws.rs.core.Context import javax.ws.rs.core.MediaType @@ -38,25 +38,26 @@ import javax.ws.rs.core.MediaType @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @Resource +@Singleton class ProjectResource( - @Context private val mpService: MPService, + @Context private val projectService: UploadProjectService, @Context private val auth: Auth) { @GET @NeedsPermission(Permission.Entity.PROJECT, Permission.Operation.READ) - fun projects() = ProjectList(mpService.userProjects(auth)) + fun projects() = ProjectList(projectService.userProjects(auth)) @GET @Path("{projectId}/users") - @NeedsPermissionOnProject(Permission.Entity.PROJECT, Permission.Operation.READ, "projectId") + @NeedsPermission(Permission.Entity.PROJECT, Permission.Operation.READ, "projectId") fun users(@PathParam("projectId") projectId: String): UserList { - return UserList(mpService.projectUsers(projectId)) + return UserList(projectService.projectUsers(projectId)) } @GET @Path("{projectId}") - @NeedsPermissionOnProject(Permission.Entity.PROJECT, Permission.Operation.READ, "projectId") + @NeedsPermission(Permission.Entity.PROJECT, Permission.Operation.READ, "projectId") fun project(@PathParam("projectId") projectId: String): Project { - return mpService.project(projectId) + return projectService.project(projectId) } } diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/resource/RecordResource.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/resource/RecordResource.kt index 2857a1ce..8aed2c49 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/resource/RecordResource.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/resource/RecordResource.kt @@ -20,10 +20,10 @@ package org.radarbase.upload.resource import com.fasterxml.jackson.databind.ObjectMapper +import org.radarbase.auth.jersey.Auth +import org.radarbase.auth.jersey.Authenticated +import org.radarbase.auth.jersey.NeedsPermission import org.radarbase.upload.api.* -import org.radarbase.upload.auth.Auth -import org.radarbase.upload.auth.Authenticated -import org.radarbase.upload.auth.NeedsPermission import org.radarbase.upload.doa.RecordRepository import org.radarbase.upload.doa.SourceTypeRepository import org.radarbase.upload.doa.entity.Record @@ -37,6 +37,8 @@ import java.io.IOException import java.io.InputStream import java.net.URI import javax.annotation.Resource +import javax.inject.Provider +import javax.inject.Singleton import javax.ws.rs.* import javax.ws.rs.core.* import kotlin.math.max @@ -49,6 +51,7 @@ import org.radarbase.upload.exception.NotFoundException as RbNotFoundException @Consumes(MediaType.APPLICATION_JSON) @Resource @Authenticated +@Singleton class RecordResource { @Context @@ -64,7 +67,7 @@ class RecordResource { lateinit var uri: UriInfo @Context - lateinit var auth: Auth + lateinit var auth: Provider @Context lateinit var sourceTypeRepository: SourceTypeRepository @@ -80,9 +83,9 @@ class RecordResource { projectId ?: throw RbBadRequestException("missing_project", "Required project ID not provided.") if (userId != null) { - auth.checkUserPermission(SUBJECT_READ, projectId, userId) + auth.get().checkPermissionOnSubject(SUBJECT_READ, projectId, userId) } else { - auth.checkProjectPermission(PROJECT_READ, projectId) + auth.get().checkPermissionOnProject(PROJECT_READ, projectId) } val imposedLimit = min(max(size, 1), 100) @@ -94,7 +97,7 @@ class RecordResource { @POST @NeedsPermission(Entity.MEASUREMENT, Operation.CREATE) fun create(record: RecordDTO): Response { - validateNewRecord(record, auth) + validateNewRecord(record, auth.get()) val (doaRecord, metadata) = recordMapper.toRecord(record) val result = recordRepository.create(doaRecord, metadata, record.data?.contents) @@ -149,7 +152,7 @@ class RecordResource { data.projectId ?: throw RbBadRequestException("project_missing", "Record needs a project ID") data.userId ?: throw RbBadRequestException("user_missing", "Record needs a user ID") - auth.checkUserPermission(MEASUREMENT_CREATE, data.projectId, data.userId) + auth.checkPermissionOnSubject(MEASUREMENT_CREATE, data.projectId, data.userId) data.contents?.forEach { it.text ?: throw RbBadRequestException("field_missing", "Contents need explicit text value set in UTF-8 encoding.") @@ -202,8 +205,8 @@ class RecordResource { val record = recordRepository.read(recordId) ?: throw RbNotFoundException("record_not_found", "Record with ID $recordId does not exist") - permission?.let { - auth.checkUserPermission(it, record.projectId, record.userId) + if (permission != null) { + auth.get().checkPermissionOnSubject(permission, record.projectId, record.userId) } return record @@ -246,14 +249,14 @@ class RecordResource { @POST @Path("poll") fun poll(pollDTO: PollDTO): RecordContainerDTO { - if (auth.isClientCredentials) { + if (auth.get().token.grantType.equals("client_credentials", ignoreCase = true)) { val imposedLimit = pollDTO.limit .coerceAtLeast(1) .coerceAtMost(100) val records = recordRepository.poll(imposedLimit, pollDTO.supportedConverters) return recordMapper.fromRecords(records, page = Page(pageSize = imposedLimit)) } else { - throw NotAuthorizedException("Client is not authorized to poll records") + throw NotAuthorizedException("Only for internal use") } } diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/resource/SourceTypeResource.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/resource/SourceTypeResource.kt index ee834cbd..76be3c0f 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/resource/SourceTypeResource.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/resource/SourceTypeResource.kt @@ -19,12 +19,13 @@ package org.radarbase.upload.resource +import org.radarbase.auth.jersey.Authenticated import org.radarbase.upload.api.SourceTypeContainerDTO import org.radarbase.upload.api.SourceTypeDTO import org.radarbase.upload.api.SourceTypeMapper -import org.radarbase.upload.auth.Authenticated import org.radarbase.upload.doa.SourceTypeRepository import javax.annotation.Resource +import javax.inject.Singleton import javax.ws.rs.* import javax.ws.rs.core.Context import javax.ws.rs.core.MediaType @@ -34,6 +35,7 @@ import javax.ws.rs.core.MediaType @Consumes(MediaType.APPLICATION_JSON) @Resource @Authenticated +@Singleton class SourceTypeResource { @Context lateinit var sourceTypeRepository: SourceTypeRepository @@ -44,8 +46,10 @@ class SourceTypeResource { @GET fun query(@DefaultValue("20") @QueryParam("limit") limit: Int, @QueryParam("lastId") lastId: Long?): SourceTypeContainerDTO { + val imposedLimit = limit + .coerceAtLeast(1) + .coerceAtMost(100) - val imposedLimit = Math.min(Math.max(limit, 1), 100) val records = sourceTypeRepository.readAll(imposedLimit, lastId) return sourceTypeMapper.fromSourceTypes(records) diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/service/UploadProjectService.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/service/UploadProjectService.kt new file mode 100644 index 00000000..9dacd89d --- /dev/null +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/service/UploadProjectService.kt @@ -0,0 +1,12 @@ +package org.radarbase.upload.service + +import org.radarbase.auth.jersey.Auth +import org.radarbase.auth.jersey.ProjectService +import org.radarbase.upload.dto.Project +import org.radarbase.upload.dto.User + +interface UploadProjectService : ProjectService { + fun project(projectId: String): Project + fun userProjects(auth: Auth): List + fun projectUsers(projectId: String): List +} diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/MPClient.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/service/managementportal/MPClient.kt similarity index 98% rename from radar-upload-backend/src/main/java/org/radarbase/upload/auth/MPClient.kt rename to radar-upload-backend/src/main/java/org/radarbase/upload/service/managementportal/MPClient.kt index 96279e85..8f88760a 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/auth/MPClient.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/service/managementportal/MPClient.kt @@ -17,7 +17,7 @@ * */ -package org.radarbase.upload.auth +package org.radarbase.upload.service.managementportal import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.core.type.TypeReference @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import okhttp3.* import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.radarbase.auth.jersey.Auth import org.radarbase.upload.Config import org.radarbase.upload.dto.Project import org.radarbase.upload.dto.User diff --git a/radar-upload-backend/src/main/java/org/radarbase/upload/service/MPService.kt b/radar-upload-backend/src/main/java/org/radarbase/upload/service/managementportal/MPProjectService.kt similarity index 68% rename from radar-upload-backend/src/main/java/org/radarbase/upload/service/MPService.kt rename to radar-upload-backend/src/main/java/org/radarbase/upload/service/managementportal/MPProjectService.kt index c0263034..48d848d7 100644 --- a/radar-upload-backend/src/main/java/org/radarbase/upload/service/MPService.kt +++ b/radar-upload-backend/src/main/java/org/radarbase/upload/service/managementportal/MPProjectService.kt @@ -17,21 +17,21 @@ * */ -package org.radarbase.upload.service +package org.radarbase.upload.service.managementportal +import org.radarbase.auth.jersey.Auth import org.radarbase.upload.util.CachedSet -import org.radarbase.upload.auth.Auth -import org.radarbase.upload.auth.MPClient import org.radarbase.upload.dto.Project import org.radarbase.upload.dto.User import org.radarbase.upload.exception.NotFoundException +import org.radarbase.upload.service.UploadProjectService import org.radarcns.auth.authorization.Permission import java.time.Duration import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentMap import javax.ws.rs.core.Context -class MPService(@Context private val mpClient: MPClient) { +class MPProjectService(@Context private val mpClient: MPClient): UploadProjectService { private val projects = CachedSet( Duration.ofMinutes(30), Duration.ofMinutes(1)) { @@ -40,21 +40,21 @@ class MPService(@Context private val mpClient: MPClient) { private val participants: ConcurrentMap> = ConcurrentHashMap() - fun ensureProject(name: String) { - if (projects.find { it.id == name } == null) { - throw NotFoundException("project_not_found", "Project $name not found.") + override fun ensureProject(projectId: String) { + if (projects.find { it.id == projectId } == null) { + throw NotFoundException("project_not_found", "Project $projectId not found.") } } - fun userProjects(auth: Auth): List { + override fun userProjects(auth: Auth): List { return projects.get() - .filter { auth.hasPermissionOnProject(Permission.PROJECT_READ, it.id) } + .filter { auth.token.hasPermissionOnProject(Permission.PROJECT_READ, it.id) } } - fun project(name: String) : Project = projects.find { it.id == name } ?: - throw NotFoundException("project_not_found", "Project $name not found.") + override fun project(projectId: String) : Project = projects.find { it.id == projectId } ?: + throw NotFoundException("project_not_found", "Project $projectId not found.") - fun projectUsers(projectId: String): List { + override fun projectUsers(projectId: String): List { val projectParticipants = participants.computeIfAbsent(projectId) { CachedSet(Duration.ofMinutes(30), Duration.ofMinutes(1)) { mpClient.readParticipants(projectId) diff --git a/radar-upload-backend/src/test/java/org/radarbase/upload/doa/RecordRepositoryImplTest.kt b/radar-upload-backend/src/test/java/org/radarbase/upload/doa/RecordRepositoryImplTest.kt index d57a4940..697b62ea 100644 --- a/radar-upload-backend/src/test/java/org/radarbase/upload/doa/RecordRepositoryImplTest.kt +++ b/radar-upload-backend/src/test/java/org/radarbase/upload/doa/RecordRepositoryImplTest.kt @@ -273,5 +273,5 @@ internal class RecordRepositoryImplTest { fun close() { } - private fun RecordRepository.BlobReader.asString(): String = use { it.stream.readAllBytes().toString(UTF_8) } + private fun RecordRepository.BlobReader.asString(): String = use { it.stream.readBytes().toString(UTF_8) } }