Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use radar-auth-jersey #5

Merged
merged 12 commits into from
Oct 2, 2019
Merged

Use radar-auth-jersey #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
f16de5a
Use radar-auth-jersey
blootsvoets Aug 7, 2019
388a068
Update to latest radar-auth-jersey
blootsvoets Aug 7, 2019
4f44432
Merge branch 'dev' into radarAuthJersey
blootsvoets Sep 24, 2019
429dae6
Use UploadProjectService interface
blootsvoets Sep 24, 2019
4b60c3f
Merge branch 'dev' into radarAuthJersey
blootsvoets Sep 25, 2019
2bd1553
Merge branch 'dev' into radarAuthJersey
blootsvoets Sep 26, 2019
12d56fa
Update dependencies
blootsvoets Sep 26, 2019
a1f4043
Merge branch 'dev' into radarAuthJersey
blootsvoets Oct 1, 2019
24331ac
Use published radar-auth-jersey
blootsvoets Oct 1, 2019
ecca3a3
Merge remote-tracking branch 'origin/dev' into radarAuthJersey
blootsvoets Oct 1, 2019
b16fea5
Fix integration test repos
blootsvoets Oct 1, 2019
51d3214
Working with radar-auth-jersey library
blootsvoets Oct 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions integration-test/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,19 @@ plugins {
}

project.extra.apply {
set("okhttpVersion", "4.0.1")
set("okhttpVersion", "4.2.0")
set("kafkaVersion", "2.3.0")
set("jacksonVersion", "2.9.9.1")
set("jacksonDataVersion", "2.9.9")
set("jacksonVersion", "2.9.10")
set("jacksonDataVersion", "2.9.10")
}

repositories {
jcenter()
mavenLocal()
maven(url = "http://packages.confluent.io/maven/")
maven(url = "https://dl.bintray.com/radar-cns/org.radarcns")
maven(url = "https://dl.bintray.com/radar-base/org.radarbase")
maven(url = "https://repo.thehyve.nl/content/repositories/snapshots")
maven(url = "http://oss.jfrog.org/artifactory/oss-snapshot-local/")
}

Expand Down
12 changes: 4 additions & 8 deletions ...ration-test/src/integrationTest/java/org/radarbase/connect/upload/UploadSourceTaskTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,10 @@ class UploadSourceTaskTest {
"upload.source.client.tokenUrl" to tokenUrl,
"upload.source.backend.baseUrl" to baseUri,
"upload.source.poll.interval.ms" to "10000",
"upload.source.record.converter.classes" to
"org.radarbase.connect.upload.converter.AccelerometerCsvRecordConverter,org.radarbase.connect.upload.converter.altoida.AltoidaZipFileRecordConverter"

"upload.source.record.converter.classes" to listOf(
"org.radarbase.connect.upload.converter.AccelerometerCsvRecordConverter",
"org.radarbase.connect.upload.converter.altoida.AltoidaZipFileRecordConverter"
).joinToString(separator=",")
)

sourceTask.start(settings)
Expand Down Expand Up @@ -102,7 +103,6 @@ class UploadSourceTaskTest {
@Test
@DisplayName("Records of no registered converters should not be polled")
fun noConverterFound() {

val sourceType = "acceleration-zip"
val fileName = "TEST_ACC.zip"
val createdRecord = createRecordAndUploadContent(accessToken, sourceType, fileName)
Expand All @@ -116,17 +116,14 @@ class UploadSourceTaskTest {
val metadata = retrieveRecordMetadata(accessToken, createdRecord.id!!)
assertNotNull(metadata)
assertEquals("READY", metadata.status)

}

@Test
@DisplayName("Should mark FAILED if the record data does not match the source-type")
fun incorrectSourceTypeForRecord() {

val sourceType = "phone-acceleration"
val fileName = "TEST_ACC.zip"
val createdRecord = createRecordAndUploadContent(accessToken, sourceType, fileName)
assertNotNull(createdRecord)
assertNotNull(createdRecord.id)

val sourceRecords = sourceTask.poll()
Expand All @@ -136,6 +133,5 @@ class UploadSourceTaskTest {
val metadata = retrieveRecordMetadata(accessToken, createdRecord.id!!)
assertNotNull(metadata)
assertEquals("FAILED", metadata.status)

}
}
8 changes: 3 additions & 5 deletions ...tegrationTest/java/org/radarbase/connect/upload/api/UploadBackendClientIntegrationTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,6 @@ import java.io.File

@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class UploadBackendClientIntegrationTest {


private lateinit var uploadBackendClient: UploadBackendClient

private lateinit var logRepository: LogRepository
Expand Down Expand Up @@ -113,7 +111,7 @@ class UploadBackendClientIntegrationTest {
val converter = AccelerometerCsvRecordConverter()
converter.initialize(sourceType, uploadBackendClient, logRepository, emptyMap())

val recordToProcess = records.records.filter { recordDTO -> recordDTO.sourceType == sourceTypeName }.first()
val recordToProcess = records.records.first { recordDTO -> recordDTO.sourceType == sourceTypeName }
createdRecord.metadata = uploadBackendClient.updateStatus(recordToProcess.id!!, recordToProcess.metadata!!.copy(status = "PROCESSING", message = "The record is being processed"))
val convertedRecords = converter.convert(records.records.first())
assertNotNull(convertedRecords)
Expand All @@ -137,9 +135,9 @@ class UploadBackendClientIntegrationTest {
}

private fun retrieveFile(recordId: RecordDTO) {
uploadBackendClient.retrieveFile(recordId, fileName).use { response ->
uploadBackendClient.retrieveFile(recordId, fileName) { response ->
assertNotNull(response)
val responseData = response!!.bytes()
val responseData = response.bytes()
assertThat(responseData.size.toLong(), equalTo(File(fileName).length()))
assertThat(responseData, equalTo(File(fileName).readBytes()))
}
Expand Down
162 changes: 75 additions & 87 deletions integration-test/src/integrationTest/java/org/radarbase/connect/upload/util/TestBase.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ import okhttp3.RequestBody.Companion.toRequestBody
import org.hamcrest.CoreMatchers
import org.hamcrest.MatcherAssert.assertThat
import org.hamcrest.Matchers
import org.hamcrest.Matchers.*
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Assertions.assertTrue
import org.radarbase.connect.upload.api.*
import org.radarbase.connect.upload.auth.ClientCredentialsAuthorizer
import org.radarbase.upload.Config
Expand Down Expand Up @@ -64,22 +66,22 @@ class TestBase {

const val uploadConnectSecret = "upload_secret"

const val BEARER = "Bearer "
private const val BEARER = "Bearer "

const val USER = "sub-1"
private const val USER = "sub-1"

const val PROJECT = "radar"
private const val PROJECT = "radar"

const val SOURCE = "03d28e5c-e005-46d4-a9b3-279c27fbbc83"
private const val SOURCE = "03d28e5c-e005-46d4-a9b3-279c27fbbc83"

val APPLICATION_JSON = "application/json; charset=utf-8".toMediaType()
private val APPLICATION_JSON = "application/json; charset=utf-8".toMediaType()

val TEXT_CSV = "text/csv; charset=utf-8".toMediaType()
private val TEXT_CSV = "text/csv; charset=utf-8".toMediaType()

val httpClient = OkHttpClient()


val sourceType = SourceTypeDTO(
private val sourceType = SourceTypeDTO(
name = sourceTypeName,
topics = mutableSetOf("test_topic"),
contentTypes = mutableSetOf("application/text"),
Expand All @@ -88,26 +90,28 @@ class TestBase {
configuration = mutableMapOf("setting1" to "value1", "setting2" to "value2")
)

val altoidaZip = SourceTypeDTO(
private val altoidaZip = SourceTypeDTO(
name = "altoida-zip",
topics = mutableSetOf("test_topic"),
contentTypes = mutableSetOf("application/zip"),
timeRequired = false,
sourceIdRequired = false,
configuration = emptyMap()
configuration = mutableMapOf()
)

val accelerationZip = SourceTypeDTO(
private val accelerationZip = SourceTypeDTO(
name = "acceleration-zip",
topics = mutableSetOf("test_topic_Acc"),
contentTypes = mutableSetOf("application/zip"),
timeRequired = false,
sourceIdRequired = false,
configuration = emptyMap()
configuration = mutableMapOf()
)

val uploadBackendConfig = Config(
managementPortalUrl = "http://localhost:8090/managementportal",
clientId = "radar_upload_backend",
clientSecret = "secret",
baseUri = URI.create(baseUri),
jdbcDriver = "org.postgresql.Driver",
jdbcUrl = "jdbc:postgresql://localhost:5434/uploadconnector",
Expand All @@ -116,7 +120,7 @@ class TestBase {
sourceTypes = listOf(sourceType, altoidaZip, accelerationZip)
)

val mapper = ObjectMapper(JsonFactory())
private val mapper: ObjectMapper = ObjectMapper(JsonFactory())
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.registerModule(KotlinModule())
.registerModule(JavaTimeModule())
Expand All @@ -126,10 +130,9 @@ class TestBase {
httpClient,
uploadConnectClient,
uploadConnectSecret,
tokenUrl
)
tokenUrl)

fun call(
private fun call(
httpClient: OkHttpClient,
expectedStatus: Int,
requestSupplier: (Request.Builder) -> Request.Builder
Expand All @@ -142,56 +145,62 @@ class TestBase {
println(tree)
tree
}
assertThat(response.code, CoreMatchers.`is`(expectedStatus))
assertThat(response.code, `is`(expectedStatus))
body
}
}

fun Any.toJsonString(): String = mapper.writeValueAsString(this)
private fun<T> call(
httpClient: OkHttpClient,
expectedStatus: Int,
parseClass: Class<T>,
requestSupplier: Request.Builder.() -> Request.Builder
): T {
val request = requestSupplier(Request.Builder()).build()
println(request.url)
return httpClient.newCall(request).execute().use { response ->
assertThat(response.code, `is`(expectedStatus))
assertThat(response.body, not(nullValue()))
mapper.readValue(response.body?.byteStream(), parseClass)
.also { assertThat(it, not(nullValue())) }
.also { println(it!!.toJsonString()) }
}
}

private fun Any.toJsonString(): String = mapper.writeValueAsString(this)

fun call(
private fun call(
httpClient: OkHttpClient,
expectedStatus: Response.Status,
requestSupplier: (Request.Builder) -> Request.Builder
): JsonNode? {
return call(httpClient, expectedStatus.statusCode, requestSupplier)
}
): JsonNode? = call(httpClient, expectedStatus.statusCode, requestSupplier)

fun call(
private fun call(
httpClient: OkHttpClient,
expectedStatus: Response.Status,
stringProperty: String,
requestSupplier: (Request.Builder) -> Request.Builder
): String {
return call(httpClient, expectedStatus, requestSupplier)?.get(stringProperty)?.asText()
?: throw AssertionError("String property $stringProperty not found")
}
requestSupplier: Request.Builder.() -> Request.Builder
): String = call(httpClient, expectedStatus, requestSupplier)
?.get(stringProperty)
?.asText()
?: throw AssertionError("String property $stringProperty not found")

fun retrieveRecordMetadata(accessToken: String, recordId: Long): RecordMetadataDTO {

val requestToUploadFile = Request.Builder()
.url("$baseUri/records/$recordId/metadata")
.get()
.addHeader("Authorization", BEARER + accessToken)
.build()
val response = httpClient.newCall(requestToUploadFile).execute()
Assertions.assertTrue(response.isSuccessful)

return mapper.readValue(response.body?.string(), RecordMetadataDTO::class.java)
return call(httpClient,200, RecordMetadataDTO::class.java) {
url("$baseUri/records/$recordId/metadata")
addHeader("Authorization", BEARER + accessToken)
}
}

fun getAccessToken() : String {
return call(httpClient, Response.Status.OK, "access_token") {
it.url(tokenUrl)
.addHeader("Authorization", Credentials.basic("radar_upload_test_client", "test"))
.post(FormBody.Builder()
.add("grant_type", "client_credentials")
.build())
}
fun getAccessToken() : String = call(httpClient, Response.Status.OK, "access_token") {
url(tokenUrl)
addHeader("Authorization", Credentials.basic("radar_upload_test_client", "test"))
post(FormBody.Builder()
.add("grant_type", "client_credentials")
.build())
}

fun createRecordAndUploadContent(accessToken: String, sourceType: String, fileName: String): RecordDTO {

val record = RecordDTO(
id = null,
data = RecordDataDTO(
Expand All @@ -204,20 +213,14 @@ class TestBase {
metadata = null
)

val request = Request.Builder()
.url("$baseUri/records")
.post(record.toJsonString().toRequestBody(APPLICATION_JSON))
.addHeader("Authorization", BEARER + accessToken)
.addHeader("Content-type", "application/json")
.build()

val response = httpClient.newCall(request).execute()
Assertions.assertTrue(response.isSuccessful)

val recordCreated = mapper.readValue(response.body?.string(), RecordDTO::class.java)
Assertions.assertNotNull(recordCreated)
Assertions.assertNotNull(recordCreated.id)
assertThat(recordCreated?.id!!, Matchers.greaterThan(0L))
val recordCreated = call(httpClient, 201, RecordDTO::class.java) {
url("$baseUri/records")
post(record.toJsonString().toRequestBody(APPLICATION_JSON))
addHeader("Authorization", BEARER + accessToken)
addHeader("Content-type", "application/json")
}
assertThat(recordCreated.id, not(nullValue()))
assertThat(recordCreated.id!!, greaterThan(0L))

//Test uploading request contentFile for created record
uploadContent(recordCreated.id!!, fileName, accessToken)
Expand All @@ -229,36 +232,21 @@ class TestBase {
//Test uploading request contentFile
val file = File(fileName)

val requestToUploadFile = Request.Builder()
.url("$baseUri/records/$recordId/contents/$fileName")
.put(file.asRequestBody(TEXT_CSV))
.addHeader("Authorization", BEARER + clientUserToken)
.build()

val uploadResponse = httpClient.newCall(requestToUploadFile).execute()
Assertions.assertTrue(uploadResponse.isSuccessful)

val content = mapper.readValue(uploadResponse.body?.string(), ContentsDTO::class.java)
Assertions.assertNotNull(content)
Assertions.assertEquals(fileName, content.fileName)
val content = call(httpClient, 201, ContentsDTO::class.java) {
url("$baseUri/records/$recordId/contents/$fileName")
put(file.asRequestBody(TEXT_CSV))
addHeader("Authorization", BEARER + clientUserToken)
}
assertThat(content.fileName, equalTo(fileName))
}


private fun markReady(recordId: Long, clientUserToken: String) {
//Test marking record READY
val requestToUploadFile = Request.Builder()
.url("$baseUri/records/$recordId/metadata")
.post("{\"status\":\"READY\",\"revision\":1}".toRequestBody("application/json".toMediaType()))
.addHeader("Authorization", BEARER + clientUserToken)
.build()

val uploadResponse = httpClient.newCall(requestToUploadFile).execute()
Assertions.assertTrue(uploadResponse.isSuccessful)

val metadata = mapper.readValue(uploadResponse.body?.string(), RecordMetadataDTO::class.java)
Assertions.assertNotNull(metadata)
Assertions.assertEquals("READY", metadata.status)
val metadata = call(httpClient, 200, RecordMetadataDTO::class.java) {
url("$baseUri/records/$recordId/metadata")
post("{\"status\":\"READY\",\"revision\":1}".toRequestBody("application/json".toMediaType()))
addHeader("Authorization", BEARER + clientUserToken)
}
assertThat(metadata.status, equalTo("READY"))
}

}
}
8 changes: 3 additions & 5 deletions kafka-connect-upload-source/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ plugins {

project.extra.apply {
set("kafkaVersion", "2.3.0")
set("okhttpVersion", "3.14.2")
set("jacksonVersion", "2.9.9.1")
set("jacksonDataVersion", "2.9.9")
set("okhttpVersion", "4.2.0")
set("jacksonVersion", "2.9.10")
set("jacksonDataVersion", "2.9.10")
set("openCsvVersion", "4.6")
set("confluentVersion", "5.3.0")
set("radarSchemaVersion", "0.5.2-SNAPSHOT")
Expand Down Expand Up @@ -44,12 +44,10 @@ dependencies {
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:${project.extra["jacksonDataVersion"]}")
implementation("com.opencsv:opencsv:${project.extra["openCsvVersion"]}")


// Included in connector runtime
compileOnly("org.apache.kafka:connect-api:${project.extra["kafkaVersion"]}")
implementation(kotlin("stdlib-jdk8"))


testImplementation("org.junit.jupiter:junit-jupiter:5.4.2")
testImplementation("org.hamcrest:hamcrest-all:1.3")
testImplementation("org.apache.kafka:connect-api:${project.extra["kafkaVersion"]}")
Expand Down
Loading