diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 2f2ca641b..48595e2f8 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -10,6 +10,10 @@ defaults: run: working-directory: ./backend +concurrency: + group: ci-${{ github.ref }}-backend + cancel-in-progress: true + jobs: Tests: runs-on: ubuntu-latest diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0b361b4ab..3ea11dfeb 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -3,6 +3,10 @@ name: e2e on: push: +concurrency: + group: ci-${{ github.ref }}-e2e + cancel-in-progress: true + jobs: E2ETests: runs-on: ubuntu-latest diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 6fac8c9d5..aea84717b 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -10,6 +10,10 @@ defaults: run: working-directory: ./website +concurrency: + group: ci-${{ github.ref }}-website + cancel-in-progress: true + jobs: checks: name: Check format diff --git a/backend/src/main/kotlin/org/pathoplexus/backend/controller/ExceptionHandler.kt b/backend/src/main/kotlin/org/pathoplexus/backend/controller/ExceptionHandler.kt index 366e54dc1..a53b8b684 100644 --- a/backend/src/main/kotlin/org/pathoplexus/backend/controller/ExceptionHandler.kt +++ b/backend/src/main/kotlin/org/pathoplexus/backend/controller/ExceptionHandler.kt @@ -28,10 +28,10 @@ class ExceptionHandler : ResponseEntityExceptionHandler() { ) } - @ExceptionHandler(ConstraintViolationException::class) + @ExceptionHandler(ConstraintViolationException::class, BadRequestException::class) @ResponseStatus(HttpStatus.BAD_REQUEST) - fun handleBadRequestException(e: ConstraintViolationException): ResponseEntity { - log.warn(e) { "Caught ConstraintViolationException: ${e.message}" } + fun handleBadRequestException(e: Exception): ResponseEntity { + log.warn(e) { "Caught ${e.javaClass}: ${e.message}" } return responseEntity( HttpStatus.BAD_REQUEST, @@ -80,5 +80,6 @@ class ExceptionHandler : ResponseEntityExceptionHandler() { } } -class UnprocessableEntityException(message: String) : RuntimeException(message) +class BadRequestException(message: String, override val cause: Throwable? = null) : RuntimeException(message) class ForbiddenException(message: String) : RuntimeException(message) +class UnprocessableEntityException(message: String) : RuntimeException(message) diff --git a/backend/src/main/kotlin/org/pathoplexus/backend/controller/SubmissionController.kt b/backend/src/main/kotlin/org/pathoplexus/backend/controller/SubmissionController.kt index 1aa54ab8c..d91a567db 100644 --- a/backend/src/main/kotlin/org/pathoplexus/backend/controller/SubmissionController.kt +++ b/backend/src/main/kotlin/org/pathoplexus/backend/controller/SubmissionController.kt @@ -3,7 +3,6 @@ package org.pathoplexus.backend.controller import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.media.Content -import io.swagger.v3.oas.annotations.media.ExampleObject import io.swagger.v3.oas.annotations.media.Schema import io.swagger.v3.oas.annotations.responses.ApiResponse import jakarta.servlet.http.HttpServletRequest @@ -15,10 +14,10 @@ import org.pathoplexus.backend.model.SubmitModel import org.pathoplexus.backend.service.DatabaseService import org.pathoplexus.backend.service.FileData import org.pathoplexus.backend.service.OriginalData -import org.pathoplexus.backend.service.SequenceVersion +import org.pathoplexus.backend.service.SequenceValidation import org.pathoplexus.backend.service.SequenceVersionStatus +import org.pathoplexus.backend.service.SubmittedProcessedData import org.pathoplexus.backend.service.UnprocessedData -import org.pathoplexus.backend.service.ValidationResult import org.pathoplexus.backend.utils.FastaReader import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus @@ -65,6 +64,18 @@ private const val SUBMIT_REVIEWED_SEQUENCE_DESCRIPTION = private const val MAX_EXTRACTED_SEQUENCES = 100_000L +private const val SUBMIT_PROCESSED_DATA_DESCRIPTION = """ +Submit processed data as a stream of NDJSON. The schema is to be understood per line of the NDJSON stream. +This endpoint performs some server side validation and returns the validation result for every submitted sequence. +Any server side validation errors will be appended to the 'errors' field of the sequence. +On a technical error, this endpoint will roll back all previously inserted data. +""" +private const val SUBMIT_PROCESSED_DATA_RESPONSE_DESCRIPTION = "Contains an entry for every submitted sequence." +private const val SUBMIT_PROCESSED_DATA_ERROR_RESPONSE_DESCRIPTION = """ +On sequence version that cannot be written to the database, e.g. if the sequence id does not exist. +Rolls back the whole transaction. +""" + @RestController @Validated class SubmissionController( @@ -115,34 +126,23 @@ class SubmissionController( } @Operation( - description = "Submit processed data as a stream of NDJSON", + description = SUBMIT_PROCESSED_DATA_DESCRIPTION, requestBody = SwaggerRequestBody( content = [ Content( mediaType = MediaType.APPLICATION_NDJSON_VALUE, - schema = Schema(implementation = SequenceVersion::class), - examples = [ - ExampleObject( - name = "Example for submitting processed sequences. \n" + - " NOTE: Due to formatting issues with swagger, remove all newlines from the example.", - value = """{"sequenceId":"4","version":"1",data":{"date":"2020-12-25","host":"Homo sapiens","region":"Europe","country":"Switzerland","division":"Schaffhausen", "nucleotideSequences":{"main":"NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNAGATC..."}}}""", // ktlint-disable max-line-length - summary = "Processed data (remove all newlines from the example)", - ), - ExampleObject( - name = "Example for submitting processed sequences with errors. \n" + - " NOTE: Due to formatting issues with swagger, remove all newlines from the example.", - value = """{"sequenceId":"4","version":"1","data":{"errors":[{"field":"host",message:"Not that kind of host"}],"date":"2020-12-25","host":"google.com","region":"Europe","country":"Switzerland","division":"Schaffhausen", "nucleotideSequences":{"main":"NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNAGATC..."}}}""", // ktlint-disable max-line-length - summary = "Processed data with errors (remove all newlines from the example)", - ), - ], + schema = Schema(implementation = SubmittedProcessedData::class), ), ], ), ) + @ApiResponse(responseCode = "200", description = SUBMIT_PROCESSED_DATA_RESPONSE_DESCRIPTION) + @ApiResponse(responseCode = "400", description = "On invalid NDJSON line. Rolls back the whole transaction.") + @ApiResponse(responseCode = "422", description = SUBMIT_PROCESSED_DATA_ERROR_RESPONSE_DESCRIPTION) @PostMapping("/submit-processed-data", consumes = [MediaType.APPLICATION_NDJSON_VALUE]) fun submitProcessedData( request: HttpServletRequest, - ): List { + ): List { return databaseService.updateProcessedData(request.inputStream) } diff --git a/backend/src/main/kotlin/org/pathoplexus/backend/service/DatabaseService.kt b/backend/src/main/kotlin/org/pathoplexus/backend/service/DatabaseService.kt index 9078e6931..91c440a61 100644 --- a/backend/src/main/kotlin/org/pathoplexus/backend/service/DatabaseService.kt +++ b/backend/src/main/kotlin/org/pathoplexus/backend/service/DatabaseService.kt @@ -1,6 +1,7 @@ package org.pathoplexus.backend.service import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.core.JacksonException import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue @@ -27,6 +28,7 @@ import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.stringParam import org.jetbrains.exposed.sql.update import org.jetbrains.exposed.sql.wrapAsExpression +import org.pathoplexus.backend.controller.BadRequestException import org.pathoplexus.backend.controller.ForbiddenException import org.pathoplexus.backend.controller.UnprocessableEntityException import org.pathoplexus.backend.model.HeaderId @@ -130,64 +132,70 @@ class DatabaseService( } } - fun updateProcessedData(inputStream: InputStream): List { + fun updateProcessedData(inputStream: InputStream): List { log.info { "updating processed data" } val reader = BufferedReader(InputStreamReader(inputStream)) - val validationResults = mutableListOf() - - reader.lineSequence().forEach { line -> - val sequenceVersion = objectMapper.readValue(line) - val validationResult = sequenceValidatorService.validateSequence(sequenceVersion) - - if (sequenceValidatorService.isValidResult(validationResult)) { - val numInserted = insertProcessedData(sequenceVersion) - if (numInserted != 1) { - validationResults.add( - ValidationResult( - sequenceVersion.sequenceId, - emptyList(), - emptyList(), - emptyList(), - listOf(insertProcessedDataError(sequenceVersion)), - ), - ) - } - } else { - validationResults.add(validationResult) + return reader.lineSequence().map { line -> + val submittedProcessedData = try { + objectMapper.readValue(line) + } catch (e: JacksonException) { + throw BadRequestException("Failed to deserialize NDJSON line: ${e.message}", e) + } + val validationResult = sequenceValidatorService.validateSequence(submittedProcessedData) + + val numInserted = insertProcessedDataWithStatus(submittedProcessedData, validationResult) + if (numInserted != 1) { + throwInsertFailedException(submittedProcessedData) } - } - return validationResults + SequenceValidation(submittedProcessedData.sequenceId, submittedProcessedData.version, validationResult) + }.toList() } - private fun insertProcessedData(sequenceVersion: SequenceVersion): Int { - val newStatus = if (sequenceVersion.errors != null && - sequenceVersion.errors.isArray && - sequenceVersion.errors.size() > 0 - ) { - Status.NEEDS_REVIEW.name - } else { - Status.PROCESSED.name - } + private fun insertProcessedDataWithStatus( + submittedProcessedData: SubmittedProcessedData, + validationResult: ValidationResult, + ): Int { val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) + val validationErrors = when (validationResult) { + is ValidationResult.Error -> validationResult.validationErrors + is ValidationResult.Ok -> emptyList() + }.map { + PreprocessingAnnotation( + listOf( + PreprocessingAnnotationSource( + PreprocessingAnnotationSourceType.Metadata, + it.fieldName, + ), + ), + "${it.type}: ${it.message}", + ) + } + val computedErrors = validationErrors + submittedProcessedData.errors.orEmpty() + + val newStatus = when { + computedErrors.isEmpty() -> Status.PROCESSED + else -> Status.NEEDS_REVIEW + } + return SequencesTable.update( where = { - (SequencesTable.sequenceId eq sequenceVersion.sequenceId) and - (SequencesTable.version eq sequenceVersion.version) and + (SequencesTable.sequenceId eq submittedProcessedData.sequenceId) and + (SequencesTable.version eq submittedProcessedData.version) and (SequencesTable.status eq Status.PROCESSING.name) }, ) { - it[status] = newStatus - it[processedData] = sequenceVersion.data - it[errors] = sequenceVersion.errors - it[warnings] = sequenceVersion.warnings + it[status] = newStatus.name + it[processedData] = submittedProcessedData.data + it[errors] = computedErrors + it[warnings] = submittedProcessedData.warnings it[finishedProcessingAt] = now } } - private fun insertProcessedDataError(sequenceVersion: SequenceVersion): String { + private fun throwInsertFailedException(submittedProcessedData: SubmittedProcessedData): String { val selectedSequences = SequencesTable .slice( SequencesTable.sequenceId, @@ -196,17 +204,24 @@ class DatabaseService( ) .select( where = { - (SequencesTable.sequenceId eq sequenceVersion.sequenceId) and - (SequencesTable.version eq sequenceVersion.version) + (SequencesTable.sequenceId eq submittedProcessedData.sequenceId) and + (SequencesTable.version eq submittedProcessedData.version) }, ) - if (selectedSequences.count().toInt() == 0) { - return "SequenceId does not exist" + + val sequenceVersion = "${submittedProcessedData.sequenceId}.${submittedProcessedData.version}" + if (selectedSequences.count() == 0L) { + throw UnprocessableEntityException("Sequence version $sequenceVersion does not exist") } - if (selectedSequences.any { it[SequencesTable.status] != Status.PROCESSING.name }) { - return "SequenceId is not in processing state" + + val selectedSequence = selectedSequences.first() + if (selectedSequence[SequencesTable.status] != Status.PROCESSING.name) { + throw UnprocessableEntityException( + "Sequence version $sequenceVersion is in not in state ${Status.PROCESSING} " + + "(was ${selectedSequence[SequencesTable.status]})", + ) } - return "Unknown error" + throw RuntimeException("Update processed data: Unexpected error for sequence version $sequenceVersion") } fun approveProcessedData(submitter: String, sequenceIds: List) { @@ -266,7 +281,7 @@ class DatabaseService( (SequencesTable.version eq maxVersionQuery) }, ).limit(numberOfSequences).map { row -> - SequenceVersion( + SubmittedProcessedData( row[SequencesTable.sequenceId], row[SequencesTable.version], row[SequencesTable.processedData]!!, @@ -297,7 +312,7 @@ class DatabaseService( (SequencesTable.submitter eq submitter) }, ).limit(numberOfSequences).map { row -> - SequenceVersion( + SubmittedProcessedData( row[SequencesTable.sequenceId], row[SequencesTable.version], row[SequencesTable.processedData]!!, @@ -558,14 +573,47 @@ class DatabaseService( } } -data class SequenceVersion( +data class SubmittedProcessedData( val sequenceId: Long, val version: Long, - val data: JsonNode, - val errors: JsonNode? = null, - val warnings: JsonNode? = null, + val data: ProcessedData, + @Schema(description = "The preprocessing will be considered failed if this is not empty") + val errors: List? = null, + @Schema( + description = + "Issues where data is not necessarily wrong, but the submitter might want to look into those warnings.", + ) + val warnings: List? = null, ) +data class ProcessedData( + @Schema( + example = """{"date": "2020-01-01", "country": "Germany", "age": 42, "qc": 0.95}""", + description = "Key value pairs of metadata, correctly typed", + ) + val metadata: Map, + @Schema( + example = """{"segment1": "ACTG", "segment2": "GTCA"}""", + description = "The key is the segment name, the value is the nucleotide sequence", + ) + val unalignedNucleotideSequences: Map, +) + +data class PreprocessingAnnotation( + val source: List, + @Schema(description = "A descriptive message that helps the submitter to fix the issue") val message: String, +) + +data class PreprocessingAnnotationSource( + val type: PreprocessingAnnotationSourceType, + @Schema(description = "Field or sequence segment name") val name: String, +) + +enum class PreprocessingAnnotationSourceType { + Metadata, + NucleotideSequence, +} + data class SequenceVersionStatus( val sequenceId: Long, val version: Long, @@ -603,6 +651,12 @@ data class OriginalData( val unalignedNucleotideSequences: Map, ) +data class SequenceValidation( + val sequenceId: Long, + val version: Long, + val validation: ValidationResult, +) + enum class Status { @JsonProperty("RECEIVED") RECEIVED, diff --git a/backend/src/main/kotlin/org/pathoplexus/backend/service/SequenceValidatorService.kt b/backend/src/main/kotlin/org/pathoplexus/backend/service/SequenceValidatorService.kt index ce8aab64a..f400f6c6f 100644 --- a/backend/src/main/kotlin/org/pathoplexus/backend/service/SequenceValidatorService.kt +++ b/backend/src/main/kotlin/org/pathoplexus/backend/service/SequenceValidatorService.kt @@ -1,107 +1,189 @@ package org.pathoplexus.backend.service import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.NullNode +import io.swagger.v3.oas.annotations.media.Schema import org.pathoplexus.backend.model.Metadata import org.pathoplexus.backend.model.SchemaConfig -import org.springframework.beans.factory.annotation.Autowired +import org.pathoplexus.backend.service.ValidationErrorType.MissingRequiredField +import org.pathoplexus.backend.service.ValidationErrorType.TypeMismatch +import org.pathoplexus.backend.service.ValidationErrorType.UnknownField import org.springframework.stereotype.Component import java.time.LocalDate import java.time.format.DateTimeFormatter import java.time.format.DateTimeParseException +private const val DATE_FORMAT = "yyyy-MM-dd" +private const val PANGO_LINEAGE_REGEX_PATTERN = "[a-zA-Z]{1,3}(\\.\\d{1,3}){0,3}" +private val pangoLineageRegex = Regex(PANGO_LINEAGE_REGEX_PATTERN) + @Component -class SequenceValidatorService -@Autowired constructor(private val schemaConfig: SchemaConfig) { +class SequenceValidatorService(private val schemaConfig: SchemaConfig) { - fun isValidDate(dateStringCandidate: String): Boolean { - val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd") - return try { - LocalDate.parse(dateStringCandidate, formatter) - true - } catch (e: DateTimeParseException) { - false + fun validateSequence(submittedProcessedData: SubmittedProcessedData): ValidationResult { + var validationResult: ValidationResult = ValidationResult.Ok() + val metadataFields = schemaConfig.schema.metadata + + for (metadata in metadataFields) { + validationResult = validateKnownMetadataField(validationResult, metadata, submittedProcessedData) } - } - fun isValidPangoLineage(pangoLineageCandidate: String): Boolean { - return pangoLineageCandidate.matches(Regex("[a-zA-Z]{1,3}(\\.\\d{1,3}){0,3}")) + val knownFieldNames = metadataFields.map { it.name } + val unknownFields = submittedProcessedData.data.metadata.keys.subtract(knownFieldNames.toSet()) + for (unknownField in unknownFields) { + validationResult = validationResult.withErrorAppended(ValidationError.unknownField(unknownField)) + } + + return validationResult } - fun validateFieldType(fieldValue: JsonNode, metadata: Metadata): Boolean { - if (fieldValue.isNull) { - return true + private fun validateKnownMetadataField( + validationResult: ValidationResult, + metadata: Metadata, + submittedProcessedData: SubmittedProcessedData, + ): ValidationResult { + val fieldName = metadata.name + val fieldValue = submittedProcessedData.data.metadata[fieldName] + + if (metadata.required) { + if (fieldValue == null) { + return validationResult.withErrorAppended(ValidationError.missingRequiredField(fieldName)) + } + + if (fieldValue is NullNode) { + return validationResult.withErrorAppended(ValidationError.requiredFieldIsNull(fieldName)) + } } - return when (metadata.type) { - "string" -> fieldValue.isTextual - "int" -> fieldValue.isInt - "float" -> fieldValue.isFloat - "double" -> fieldValue.isDouble - "number" -> fieldValue.isNumber - "date" -> isValidDate(fieldValue.asText()) - "pango_lineage" -> isValidPangoLineage(fieldValue.asText()) - else -> false + + if (fieldValue != null) { + when (val validationError = validateType(fieldValue, metadata)) { + null -> {} + else -> return validationResult.withErrorAppended(validationError) + } } + return validationResult } - fun validateSequence(sequenceVersion: SequenceVersion): ValidationResult { - val missingFields = mutableListOf() - val typeMismatchFields = mutableListOf() - val unknownFields = mutableListOf() - - if (sequenceVersion.data["metadata"] == null) { - return ValidationResult( - sequenceVersion.sequenceId, - emptyList(), - emptyList(), - emptyList(), - listOf("Missing field: metadata"), - ) + fun validateType(fieldValue: JsonNode, metadata: Metadata): ValidationError? { + if (fieldValue.isNull) { + return null } - for (metadata in schemaConfig.schema.metadata) { - val fieldName = metadata.name - val fieldValue = sequenceVersion.data["metadata"][fieldName] - - if (fieldValue == null && metadata.required) { - missingFields.add(fieldName) + when (metadata.type) { + "date" -> { + if (!isValidDate(fieldValue.asText())) { + return ValidationError.invalidDate(metadata.name, fieldValue) + } + return null } - if (fieldValue != null) { - if (!validateFieldType(fieldValue, metadata)) { - typeMismatchFields.add(FieldError(fieldName, metadata.type, fieldValue.toString())) + "pango_lineage" -> { + if (!isValidPangoLineage(fieldValue.asText())) { + return ValidationError.invalidPangoLineage(metadata.name, fieldValue) } + return null } } - val knownFieldNames = schemaConfig.schema.metadata.map { it.name } + val isOfCorrectPrimitiveType = when (metadata.type) { + "string" -> fieldValue.isTextual + "integer" -> fieldValue.isInt + "float" -> fieldValue.isFloat + "double" -> fieldValue.isDouble + "number" -> fieldValue.isNumber + else -> throw RuntimeException( + "Found unknown metadata type in config: ${metadata.type}. Refactor this to an enum", + ) + } - sequenceVersion.data["metadata"].fieldNames().forEachRemaining { fieldName -> - if (!knownFieldNames.contains(fieldName)) { - unknownFields.add(fieldName) - } + return when (isOfCorrectPrimitiveType) { + true -> null + false -> ValidationError.typeMismatch(metadata.name, metadata.type, fieldValue) } + } - return ValidationResult(sequenceVersion.sequenceId, missingFields, typeMismatchFields, unknownFields) + fun isValidDate(dateStringCandidate: String): Boolean { + val formatter = DateTimeFormatter.ofPattern(DATE_FORMAT) + return try { + LocalDate.parse(dateStringCandidate, formatter) + true + } catch (e: DateTimeParseException) { + false + } } - fun isValidResult(validationResult: ValidationResult): Boolean { - return validationResult.missingRequiredFields.isEmpty() && - validationResult.fieldsWithTypeMismatch.isEmpty() && - validationResult.unknownFields.isEmpty() && - validationResult.genericErrors.isEmpty() + fun isValidPangoLineage(pangoLineageCandidate: String): Boolean { + return pangoLineageCandidate.matches(pangoLineageRegex) } } -data class FieldError( - val field: String, - val shouldBeType: String, - val fieldValue: String, +@Schema( + oneOf = [ValidationResult.Ok::class, ValidationResult.Error::class], + discriminatorProperty = "type", ) +sealed interface ValidationResult { + val type: String -data class ValidationResult( - val sequenceId: Long, - val missingRequiredFields: List, - val fieldsWithTypeMismatch: List, - val unknownFields: List, - val genericErrors: List = emptyList(), -) + fun withErrorAppended(validationError: ValidationError): ValidationResult + + class Ok : ValidationResult { + @Schema(allowableValues = ["Ok"]) + override val type = "Ok" + + override fun withErrorAppended(validationError: ValidationError) = Error(listOf(validationError)) + } + + class Error(val validationErrors: List) : ValidationResult { + @Schema(allowableValues = ["Error"]) + override val type = "Error" + + override fun withErrorAppended(validationError: ValidationError) = Error(validationErrors + validationError) + } +} + +class ValidationError(val type: ValidationErrorType, val fieldName: String, val message: String) { + companion object { + fun missingRequiredField(field: String) = ValidationError( + MissingRequiredField, + field, + "Missing the required field '$field'.", + ) + + fun requiredFieldIsNull(field: String) = ValidationError( + MissingRequiredField, + field, + "Field '$field' is null, but a value is required.", + ) + + fun typeMismatch(fieldName: String, expectedType: String, fieldValue: JsonNode) = ValidationError( + TypeMismatch, + fieldName, + "Expected type '$expectedType' for field '$fieldName', found value '$fieldValue'.", + ) + + fun invalidDate(fieldName: String, fieldValue: JsonNode) = ValidationError( + TypeMismatch, + fieldName, + "Expected type 'date' in format '$DATE_FORMAT' for field '$fieldName', found value '$fieldValue'.", + ) + + fun invalidPangoLineage(fieldName: String, fieldValue: JsonNode) = ValidationError( + TypeMismatch, + fieldName, + "Expected type 'pango_lineage' for field '$fieldName', found value '$fieldValue'. " + + "A pango lineage must be of the form $PANGO_LINEAGE_REGEX_PATTERN, e.g. 'XBB' or 'BA.1.5'.", + ) + + fun unknownField(fieldName: String) = ValidationError( + UnknownField, + fieldName, + "Found unknown field '$fieldName' in processed data.", + ) + } +} + +enum class ValidationErrorType { + MissingRequiredField, + TypeMismatch, + UnknownField, +} diff --git a/backend/src/main/kotlin/org/pathoplexus/backend/service/SequencesTable.kt b/backend/src/main/kotlin/org/pathoplexus/backend/service/SequencesTable.kt index 1bb02e380..07c37c92c 100644 --- a/backend/src/main/kotlin/org/pathoplexus/backend/service/SequencesTable.kt +++ b/backend/src/main/kotlin/org/pathoplexus/backend/service/SequencesTable.kt @@ -1,6 +1,5 @@ package org.pathoplexus.backend.service -import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import org.jetbrains.exposed.sql.Table @@ -27,9 +26,9 @@ object SequencesTable : Table("sequences") { val revoked = bool("revoked").default(false) val originalData = jacksonSerializableJsonb("original_data").nullable() - val processedData = jacksonSerializableJsonb("processed_data").nullable() - val errors = jacksonSerializableJsonb("errors").nullable() - val warnings = jacksonSerializableJsonb("warnings").nullable() + val processedData = jacksonSerializableJsonb("processed_data").nullable() + val errors = jacksonSerializableJsonb>("errors").nullable() + val warnings = jacksonSerializableJsonb>("warnings").nullable() override val primaryKey = PrimaryKey(sequenceId, version) } diff --git a/backend/src/main/resources/config.json b/backend/src/main/resources/config.json index 7fbbdf760..5a2ae41d4 100644 --- a/backend/src/main/resources/config.json +++ b/backend/src/main/resources/config.json @@ -47,8 +47,7 @@ { "name": "division", "type": "string", - "autocomplete": true, - "required": true + "autocomplete": true }, { "name": "location", @@ -110,6 +109,10 @@ { "name": "authors", "type": "string" + }, + { + "name": "qc", + "type": "double" } ], "tableColumns": ["strain", "country", "date", "pangoLineage"], diff --git a/backend/src/test/kotlin/org/pathoplexus/backend/controller/EndpointTestExtension.kt b/backend/src/test/kotlin/org/pathoplexus/backend/controller/EndpointTestExtension.kt index 6ce0d7a89..fa69552b9 100644 --- a/backend/src/test/kotlin/org/pathoplexus/backend/controller/EndpointTestExtension.kt +++ b/backend/src/test/kotlin/org/pathoplexus/backend/controller/EndpointTestExtension.kt @@ -19,7 +19,10 @@ import org.testcontainers.containers.PostgreSQLContainer @ActiveProfiles("with-database") @ExtendWith(EndpointTestExtension::class) @DirtiesContext -@Import(SubmissionControllerClient::class) +@Import( + SubmissionControllerClient::class, + SubmissionConvenienceClient::class, +) annotation class EndpointTest private const val SPRING_DATASOURCE_URL = "spring.datasource.url" diff --git a/backend/src/test/kotlin/org/pathoplexus/backend/controller/ExtractUnprocessedDataEndpointTest.kt b/backend/src/test/kotlin/org/pathoplexus/backend/controller/ExtractUnprocessedDataEndpointTest.kt index 5a7842255..1dec6277b 100644 --- a/backend/src/test/kotlin/org/pathoplexus/backend/controller/ExtractUnprocessedDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/pathoplexus/backend/controller/ExtractUnprocessedDataEndpointTest.kt @@ -13,7 +13,10 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPat import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status @EndpointTest -class ExtractUnprocessedDataEndpointTest(@Autowired val submissionControllerClient: SubmissionControllerClient) { +class ExtractUnprocessedDataEndpointTest( + @Autowired val convenienceClient: SubmissionConvenienceClient, + @Autowired val submissionControllerClient: SubmissionControllerClient, +) { @Test fun `GIVEN no sequences in database THEN returns empty response`() { @@ -25,7 +28,7 @@ class ExtractUnprocessedDataEndpointTest(@Autowired val submissionControllerClie @Test fun `WHEN extracting unprocessed data THEN only previously not extracted sequences are returned`() { - submissionControllerClient.submitDefaultFiles() + convenienceClient.submitDefaultFiles() val result7 = submissionControllerClient.extractUnprocessedData(7) val responseBody7 = result7.expectNdjsonAndGetContent() diff --git a/backend/src/test/kotlin/org/pathoplexus/backend/controller/PreparedProcessedData.kt b/backend/src/test/kotlin/org/pathoplexus/backend/controller/PreparedProcessedData.kt new file mode 100644 index 000000000..d01e28ee3 --- /dev/null +++ b/backend/src/test/kotlin/org/pathoplexus/backend/controller/PreparedProcessedData.kt @@ -0,0 +1,170 @@ +package org.pathoplexus.backend.controller + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.DoubleNode +import com.fasterxml.jackson.databind.node.IntNode +import com.fasterxml.jackson.databind.node.NullNode +import com.fasterxml.jackson.databind.node.TextNode +import org.pathoplexus.backend.controller.SubmitFiles.DefaultFiles +import org.pathoplexus.backend.service.PreprocessingAnnotation +import org.pathoplexus.backend.service.PreprocessingAnnotationSource +import org.pathoplexus.backend.service.PreprocessingAnnotationSourceType +import org.pathoplexus.backend.service.ProcessedData +import org.pathoplexus.backend.service.SubmittedProcessedData + +private val defaultProcessedData = ProcessedData( + metadata = mapOf( + "date" to TextNode("2002-12-15"), + "host" to TextNode("google.com"), + "region" to TextNode("Europe"), + "country" to TextNode("Spain"), + "age" to IntNode(42), + "qc" to DoubleNode(0.9), + "pangoLineage" to TextNode("XBB.1.5"), + ), + unalignedNucleotideSequences = mapOf( + "main" to "NNNNNN", + ), +) + +private val defaultSuccessfulSubmittedData = SubmittedProcessedData( + sequenceId = 1, + version = 1, + data = defaultProcessedData, + errors = null, + warnings = null, +) + +object PreparedProcessedData { + fun successfullyProcessed(sequenceId: Long = DefaultFiles.firstSequence) = + defaultSuccessfulSubmittedData.withValues( + sequenceId = sequenceId, + ) + + fun withNullForFields(sequenceId: Long = DefaultFiles.firstSequence, fields: List) = + defaultSuccessfulSubmittedData.withValues( + sequenceId = sequenceId, + data = defaultProcessedData.withValues( + metadata = defaultProcessedData.metadata + fields.map { it to NullNode.instance }, + ), + ) + + fun withUnknownMetadataField(sequenceId: Long = DefaultFiles.firstSequence, fields: List) = + defaultSuccessfulSubmittedData.withValues( + sequenceId = sequenceId, + data = defaultProcessedData.withValues( + metadata = defaultProcessedData.metadata + fields.map { it to TextNode("value for $it") }, + ), + ) + + fun withMissingRequiredField(sequenceId: Long = DefaultFiles.firstSequence, fields: List) = + defaultSuccessfulSubmittedData.withValues( + sequenceId = sequenceId, + data = defaultProcessedData.withValues( + metadata = defaultProcessedData.metadata.filterKeys { !fields.contains(it) }, + ), + ) + + fun withWrongTypeForFields(sequenceId: Long = DefaultFiles.firstSequence) = + defaultSuccessfulSubmittedData.withValues( + sequenceId = sequenceId, + data = defaultProcessedData.withValues( + metadata = defaultProcessedData.metadata + mapOf( + "region" to IntNode(5), + "age" to TextNode("not a number"), + ), + ), + ) + + fun withWrongDateFormat(sequenceId: Long = DefaultFiles.firstSequence) = + defaultSuccessfulSubmittedData.withValues( + sequenceId = sequenceId, + data = defaultProcessedData.withValues( + metadata = defaultProcessedData.metadata + mapOf( + "date" to TextNode("1.2.2021"), + ), + ), + ) + + fun withWrongPangoLineageFormat(sequenceId: Long = DefaultFiles.firstSequence) = + defaultSuccessfulSubmittedData.withValues( + sequenceId = sequenceId, + data = defaultProcessedData.withValues( + metadata = defaultProcessedData.metadata + mapOf( + "pangoLineage" to TextNode("A.5.invalid"), + ), + ), + ) + + fun withErrors(sequenceId: Long = DefaultFiles.firstSequence) = + defaultSuccessfulSubmittedData.withValues( + sequenceId = sequenceId, + errors = listOf( + PreprocessingAnnotation( + source = listOf( + PreprocessingAnnotationSource( + PreprocessingAnnotationSourceType.Metadata, + "host", + ), + ), + "Not this kind of host", + ), + PreprocessingAnnotation( + source = listOf( + PreprocessingAnnotationSource( + PreprocessingAnnotationSourceType.NucleotideSequence, + "main", + ), + ), + "dummy nucleotide sequence error", + ), + ), + ) + + fun withWarnings(sequenceId: Long = DefaultFiles.firstSequence) = + defaultSuccessfulSubmittedData.withValues( + sequenceId = sequenceId, + warnings = listOf( + PreprocessingAnnotation( + source = listOf( + PreprocessingAnnotationSource( + PreprocessingAnnotationSourceType.Metadata, + "host", + ), + ), + "Not this kind of host", + ), + PreprocessingAnnotation( + source = listOf( + PreprocessingAnnotationSource( + PreprocessingAnnotationSourceType.NucleotideSequence, + "main", + ), + ), + "dummy nucleotide sequence error", + ), + ), + ) +} + +fun SubmittedProcessedData.withValues( + sequenceId: Long? = null, + version: Long? = null, + data: ProcessedData? = null, + errors: List? = null, + warnings: List? = null, +) = SubmittedProcessedData( + sequenceId = sequenceId ?: this.sequenceId, + version = version ?: this.version, + data = data ?: this.data, + errors = errors ?: this.errors, + warnings = warnings ?: this.warnings, +) + +fun ProcessedData.withValues( + metadata: Map? = null, + unalignedNucleotideSequences: Map? = null, +) = ProcessedData( + metadata = metadata ?: this.metadata, + unalignedNucleotideSequences = unalignedNucleotideSequences ?: this.unalignedNucleotideSequences, +) diff --git a/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionControllerClient.kt b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionControllerClient.kt index e486cee37..f915e8469 100644 --- a/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionControllerClient.kt +++ b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionControllerClient.kt @@ -2,22 +2,19 @@ package org.pathoplexus.backend.controller import com.fasterxml.jackson.databind.ObjectMapper import org.pathoplexus.backend.service.OriginalData +import org.pathoplexus.backend.service.SubmittedProcessedData import org.pathoplexus.backend.service.UnprocessedData -import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.MediaType import org.springframework.mock.web.MockMultipartFile import org.springframework.test.web.servlet.MockMvc -import org.springframework.test.web.servlet.MvcResult import org.springframework.test.web.servlet.ResultActions import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post -import org.springframework.test.web.servlet.result.MockMvcResultMatchers const val USER_NAME = "testUser" -class SubmissionControllerClient(private val mockMvc: MockMvc, @Autowired val objectMapper: ObjectMapper) { - +class SubmissionControllerClient(private val mockMvc: MockMvc, private val objectMapper: ObjectMapper) { fun submit(username: String, metadataFile: MockMultipartFile, sequencesFile: MockMultipartFile): ResultActions = mockMvc.perform( multipart("/submit") @@ -26,33 +23,30 @@ class SubmissionControllerClient(private val mockMvc: MockMvc, @Autowired val ob .param("username", username), ) - fun submitDefaultFiles(username: String = USER_NAME): ResultActions = - submit(username, SubmitFiles.DefaultFiles.metadataFile, SubmitFiles.DefaultFiles.sequencesFile) - fun extractUnprocessedData(numberOfSequences: Int): ResultActions = mockMvc.perform( post("/extract-unprocessed-data") .param("numberOfSequences", numberOfSequences.toString()), ) - fun submitProcessedData(testData: String): ResultActions { - return mockMvc.perform( + fun submitProcessedData(vararg submittedProcessedData: SubmittedProcessedData): ResultActions { + val stringContent = submittedProcessedData.joinToString("\n") { objectMapper.writeValueAsString(it) } + + return submitProcessedDataRaw(stringContent) + } + + fun submitProcessedDataRaw(submittedProcessedData: String): ResultActions = + mockMvc.perform( post("/submit-processed-data") .contentType(MediaType.APPLICATION_NDJSON_VALUE) - .content(testData), + .content(submittedProcessedData), ) - .andExpect(MockMvcResultMatchers.status().isOk()) - } - fun getSequencesOfUser(): MvcResult { - return mockMvc.perform( + fun getSequencesOfUser(userName: String): ResultActions = + mockMvc.perform( get("/get-sequences-of-user") - .param("username", USER_NAME), + .param("username", userName), ) - .andExpect(MockMvcResultMatchers.status().isOk()) - .andExpect(MockMvcResultMatchers.content().contentType(MediaType.APPLICATION_JSON_VALUE)) - .andReturn() - } fun submitReviewedSequence(userName: String, reviewedData: UnprocessedData): ResultActions { return mockMvc.perform( diff --git a/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionControllerTest.kt b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionControllerTest.kt index 950b27c23..0d7fc438e 100644 --- a/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionControllerTest.kt +++ b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionControllerTest.kt @@ -3,17 +3,14 @@ package org.pathoplexus.backend.controller import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import org.assertj.core.api.Assertions.assertThat -import org.hamcrest.Matchers import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.MethodSource import org.pathoplexus.backend.controller.SubmitFiles.DefaultFiles import org.pathoplexus.backend.controller.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES -import org.pathoplexus.backend.service.SequenceVersion import org.pathoplexus.backend.service.SequenceVersionStatus import org.pathoplexus.backend.service.Status +import org.pathoplexus.backend.service.SubmittedProcessedData import org.pathoplexus.backend.service.UnprocessedData import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc @@ -28,11 +25,9 @@ import org.springframework.test.web.servlet.MvcResult import org.springframework.test.web.servlet.ResultActions import org.springframework.test.web.servlet.request.MockMvcRequestBuilders import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content -import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status import org.testcontainers.containers.PostgreSQLContainer import org.testcontainers.shaded.org.awaitility.Awaitility.await -import java.io.File @AutoConfigureMockMvc @SpringBootTest @@ -55,79 +50,32 @@ class SubmissionControllerTest( ) } - @ParameterizedTest(name = "{arguments}") - @MethodSource("provideValidationScenarios") - fun `validation of processed data`(scenario: Scenario) { - submitInitialData() - awaitResponse(queryUnprocessedSequences(NUMBER_OF_SEQUENCES)) - - val requestBuilder = submitProcessedData(scenario.inputData) - .andExpect(status().isOk) - .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) - .andExpect(jsonPath("\$").isArray()) - - if (scenario.expectedError == null) { - requestBuilder.andExpect(jsonPath("\$").isEmpty()) - } else { - val error = scenario.expectedError - - error.fieldsWithTypeMismatch.forEach { mismatch -> - requestBuilder.andExpect( - jsonPath( - "\$[0].fieldsWithTypeMismatch", - Matchers.hasItem( - Matchers.allOf( - Matchers.hasEntry("field", mismatch.field), - Matchers.hasEntry("shouldBeType", mismatch.shouldBeType), - Matchers.hasEntry("fieldValue", mismatch.fieldValue), - ), - ), - ), - ) - } - requestBuilder.andExpect( - jsonPath("\$[0].fieldsWithTypeMismatch", Matchers.hasSize(error.fieldsWithTypeMismatch.size)), - ) - - for (err in error.unknownFields) { - requestBuilder.andExpect(jsonPath("\$[0].unknownFields", Matchers.hasItem(err))) - } - requestBuilder.andExpect( - jsonPath("\$[0].unknownFields", Matchers.hasSize(error.unknownFields.size)), - ) - - for (err in error.missingRequiredFields) { - requestBuilder.andExpect(jsonPath("\$[0].missingRequiredFields", Matchers.hasItem(err))) - } - requestBuilder.andExpect( - jsonPath("\$[0].missingRequiredFields", Matchers.hasSize(error.missingRequiredFields.size)), - ) - - for (err in error.genericErrors) { - requestBuilder.andExpect(jsonPath("\$[0].genericErrors", Matchers.hasItem(err))) - } - requestBuilder.andExpect( - jsonPath("\$[0].genericErrors", Matchers.hasSize(error.genericErrors.size)), - ) - } - } - - @Test - fun `handling of errors in processed data`() { - submitInitialData() - awaitResponse(queryUnprocessedSequences(NUMBER_OF_SEQUENCES)) - - submitProcessedData(processedInputDataFromFile("error_feedback")) - - expectStatusInResponse(querySequenceList(), 1, Status.NEEDS_REVIEW.name) - } - @Test fun `approving of processed data`() { val sequencesThatAreProcessed = listOf(1) submitInitialData() awaitResponse(queryUnprocessedSequences(NUMBER_OF_SEQUENCES)) - submitProcessedData(processedInputDataFromFile("no_validation_errors")) + submitProcessedData( + """ + { + "sequenceId": 1, + "version": 1, + "errors": [], + "warnings": [], + "data": { + "metadata": { + "date": "2002-12-15", + "host": "Homo sapiens", + "region": "Europe", + "country": "Spain", + "division": "Schaffhausen", + "pangoLineage": "XBB.1.5" + }, + "unalignedNucleotideSequences": { "main": "NNNNNNNNNNNNNNNN" } + } + } + """.replace(Regex("\\s"), ""), + ) // TODO(#311) replace this JSON by a method in PreparedProcessedData approveProcessedSequences(sequencesThatAreProcessed) @@ -438,7 +386,7 @@ class SubmissionControllerTest( .filter { it.isNotBlank() } .map { objectMapper.readValue(it) } .map { - SequenceVersion( + SubmittedProcessedData( it.sequenceId, it.version, data = objectMapper.readValue(objectMapper.writeValueAsString(it.data)), @@ -500,107 +448,5 @@ class SubmissionControllerTest( fun afterAll() { postgres.stop() } - - @JvmStatic - fun provideValidationScenarios() = listOf( - Scenario( - name = "Happy Path", - inputData = processedInputDataFromFile("no_validation_errors"), - expectedError = null as ValidationError?, - ), - Scenario( - name = "Unknown field", - inputData = processedInputDataFromFile("unknown_field"), - expectedError = ValidationError( - id = 1, - missingRequiredFields = emptyList(), - fieldsWithTypeMismatch = emptyList(), - unknownFields = listOf("not_a_meta_data_field"), - genericErrors = emptyList(), - ), - ), - Scenario( - name = "Missing required field", - inputData = processedInputDataFromFile("missing_required_field"), - expectedError = ValidationError( - id = 1, - missingRequiredFields = listOf("date", "country"), - fieldsWithTypeMismatch = emptyList(), - unknownFields = emptyList(), - genericErrors = emptyList(), - ), - ), - Scenario( - name = "Wrong type field", - inputData = processedInputDataFromFile("wrong_type_field"), - expectedError = ValidationError( - id = 1, - missingRequiredFields = emptyList(), - fieldsWithTypeMismatch = listOf( - TypeMismatch(field = "date", shouldBeType = "date", fieldValue = "\"15.12.2002\""), - ), - unknownFields = emptyList(), - genericErrors = emptyList(), - ), - ), - Scenario( - name = "Invalid ID / Non-existing ID", - inputData = processedInputDataFromFile("invalid_id"), - expectedError = ValidationError( - id = 12, - missingRequiredFields = emptyList(), - fieldsWithTypeMismatch = emptyList(), - unknownFields = emptyList(), - genericErrors = listOf("SequenceId does not exist"), - ), - ), - Scenario( - name = "null in nullable field", - inputData = processedInputDataFromFile("null_value"), - expectedError = null, - ), - ) - - data class Scenario( - val name: String, - val inputData: String, - val expectedError: ErrorType?, - ) { - override fun toString() = name - } - - data class ValidationError( - val id: Int, - val missingRequiredFields: List, - val fieldsWithTypeMismatch: List, - val unknownFields: List, - val genericErrors: List, - ) - - data class TypeMismatch( - val field: String, - val shouldBeType: String, - val fieldValue: String, - ) - - fun processedInputDataFromFile(fileName: String): String = inputData[fileName] ?: error( - "$fileName.json not found", - ) - - private val inputData: Map by lazy { - val fileMap = mutableMapOf() - - val jsonResourceDirectory = "src/test/resources/processedInputData" - - val directory = File(jsonResourceDirectory) - - directory.listFiles { _, name -> name.endsWith(".json") }?.forEach { file -> - val fileName = file.nameWithoutExtension - val formattedJson = file.readText().replace("\n", "").replace("\r", "").replace(" ", "") - fileMap[fileName] = formattedJson - } - - fileMap - } } } diff --git a/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionConvenienceClient.kt b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionConvenienceClient.kt new file mode 100644 index 000000000..5c5344bd5 --- /dev/null +++ b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmissionConvenienceClient.kt @@ -0,0 +1,56 @@ +package org.pathoplexus.backend.controller + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import org.pathoplexus.backend.model.HeaderId +import org.pathoplexus.backend.service.SequenceVersionStatus +import org.pathoplexus.backend.service.UnprocessedData +import org.springframework.http.MediaType +import org.springframework.test.web.servlet.ResultActions +import org.springframework.test.web.servlet.result.MockMvcResultMatchers + +class SubmissionConvenienceClient( + private val client: SubmissionControllerClient, + private val objectMapper: ObjectMapper, +) { + fun submitDefaultFiles(username: String = USER_NAME): List { + val submit = client.submit( + username, + SubmitFiles.DefaultFiles.metadataFile, + SubmitFiles.DefaultFiles.sequencesFile, + ) + + return deserializeJsonResponse(submit) + } + + fun extractUnprocessedData(numberOfSequences: Int = SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES) = + client.extractUnprocessedData(numberOfSequences) + .expectNdjsonAndGetContent() + + fun getSequencesOfUser(userName: String = USER_NAME): List { + return deserializeJsonResponse(client.getSequencesOfUser(userName)) + } + + fun getSequenceVersionOfUser( + sequenceId: Long, + version: Long, + userName: String = USER_NAME, + ): SequenceVersionStatus { + val sequencesOfUser = getSequencesOfUser(userName) + + return sequencesOfUser.find { it.sequenceId == sequenceId && it.version == version } + ?: error("Did not find $sequenceId.$version for $userName") + } + + private inline fun deserializeJsonResponse(resultActions: ResultActions): T { + val content = + resultActions + .andExpect(MockMvcResultMatchers.status().isOk) + .andExpect(MockMvcResultMatchers.content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andReturn() + .response + .contentAsString + + return objectMapper.readValue(content) + } +} diff --git a/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmitProcessedDataEndpointTest.kt b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmitProcessedDataEndpointTest.kt new file mode 100644 index 000000000..27595e7e1 --- /dev/null +++ b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmitProcessedDataEndpointTest.kt @@ -0,0 +1,325 @@ +package org.pathoplexus.backend.controller + +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.containsString +import org.hamcrest.Matchers.`is` +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource +import org.pathoplexus.backend.controller.SubmitFiles.DefaultFiles.firstSequence +import org.pathoplexus.backend.service.Status +import org.pathoplexus.backend.service.SubmittedProcessedData +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.http.MediaType +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status + +@EndpointTest +class SubmitProcessedDataEndpointTest( + @Autowired val submissionControllerClient: SubmissionControllerClient, + @Autowired val convenienceClient: SubmissionConvenienceClient, +) { + + @Test + fun `WHEN I submit successfully preprocessed data THEN the sequence is in status processed`() { + prepareExtractedSequencesInDatabase() + + submissionControllerClient.submitProcessedData( + PreparedProcessedData.successfullyProcessed(sequenceId = 3), + PreparedProcessedData.successfullyProcessed(sequenceId = 4), + ) + .andExpect(status().isOk) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$[0].sequenceId").value(3)) + .andExpect(jsonPath("\$[0].validation.type").value("Ok")) + .andExpect(jsonPath("\$[1].sequenceId").value(4)) + .andExpect(jsonPath("\$[1].validation.type").value("Ok")) + + convenienceClient.getSequenceVersionOfUser(sequenceId = 3, version = 1).assertStatusIs(Status.PROCESSED) + } + + @Test + fun `WHEN I submit null for a non-required field THEN the sequence is in status processed`() { + prepareExtractedSequencesInDatabase() + + submissionControllerClient.submitProcessedData( + PreparedProcessedData.withNullForFields(fields = listOf("dateSubmitted")), + ) + .andExpect(status().isOk) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$[0].sequenceId").value(firstSequence)) + .andExpect(jsonPath("\$[0].validation.type").value("Ok")) + .andReturn() + + prepareExtractedSequencesInDatabase() + + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.PROCESSED) + } + + @Test + fun `WHEN I submit data with errors THEN the sequence is in status needs review`() { + prepareExtractedSequencesInDatabase() + + submissionControllerClient.submitProcessedData(PreparedProcessedData.withErrors(firstSequence)) + .andExpect(status().isOk) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$[0].sequenceId").value(firstSequence)) + .andExpect(jsonPath("\$[0].validation.type").value("Ok")) + + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.NEEDS_REVIEW) + } + + @Test + fun `WHEN I submit data with warnings THEN the sequence is in status processed`() { + prepareExtractedSequencesInDatabase() + + submissionControllerClient.submitProcessedData(PreparedProcessedData.withWarnings(firstSequence)) + .andExpect(status().isOk) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$[0].sequenceId").value(firstSequence)) + .andExpect(jsonPath("\$[0].validation.type").value("Ok")) + + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.PROCESSED) + } + + @ParameterizedTest(name = "{arguments}") + @MethodSource("provideInvalidDataScenarios") + fun `GIVEN invalid processed data THEN the response contains validation errors`( + invalidDataScenario: InvalidDataScenario, + ) { + prepareExtractedSequencesInDatabase() + + val response = submissionControllerClient.submitProcessedData(invalidDataScenario.processedData) + .andExpect(status().isOk) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$[0].sequenceId").value(invalidDataScenario.processedData.sequenceId)) + .andExpect(jsonPath("\$[0].validation.type").value("Error")) + + for ((index, expectedError) in invalidDataScenario.expectedValidationErrors.withIndex()) { + val (expectedType, fieldName, expectedMessage) = expectedError + response + .andExpect(jsonPath("\$[0].validation.validationErrors[$index].type").value(expectedType)) + .andExpect(jsonPath("\$[0].validation.validationErrors[$index].fieldName").value(fieldName)) + .andExpect( + jsonPath("\$[0].validation.validationErrors[$index].message") + .value(containsString(expectedMessage)), + ) + } + + val sequenceStatus = convenienceClient.getSequenceVersionOfUser( + sequenceId = invalidDataScenario.processedData.sequenceId, + version = 1, + ) + assertThat(sequenceStatus.status, `is`(Status.NEEDS_REVIEW)) + } + + @Test + fun `WHEN I submit data for a non-existent sequence id THEN refuses update with unprocessable entity`() { + prepareExtractedSequencesInDatabase() + + val nonExistentSequenceId = 999L + + submissionControllerClient.submitProcessedData( + PreparedProcessedData.successfullyProcessed(sequenceId = 1), + PreparedProcessedData.successfullyProcessed(sequenceId = nonExistentSequenceId), + ) + .andExpect(status().isUnprocessableEntity) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$.detail").value("Sequence version $nonExistentSequenceId.1 does not exist")) + + convenienceClient.getSequenceVersionOfUser(sequenceId = 1, version = 1).assertStatusIs(Status.PROCESSING) + } + + @Test + fun `WHEN I submit data for a non-existent sequence version THEN refuses update with unprocessable entity`() { + prepareExtractedSequencesInDatabase() + + val nonExistentVersion = 999L + + submissionControllerClient.submitProcessedData( + PreparedProcessedData.successfullyProcessed(sequenceId = firstSequence), + PreparedProcessedData.successfullyProcessed(sequenceId = firstSequence) + .withValues(version = nonExistentVersion), + ) + .andExpect(status().isUnprocessableEntity) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect( + jsonPath("\$.detail").value( + "Sequence version $firstSequence.$nonExistentVersion does not exist", + ), + ) + + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.PROCESSING) + } + + @Test + fun `WHEN I submit data for a sequence that is not in processing THEN refuses update with unprocessable entity`() { + convenienceClient.submitDefaultFiles() + convenienceClient.extractUnprocessedData(1) + + val sequenceIdNotInProcessing: Long = 2 + convenienceClient.getSequenceVersionOfUser(sequenceId = sequenceIdNotInProcessing, version = 1) + .assertStatusIs(Status.RECEIVED) + + submissionControllerClient.submitProcessedData( + PreparedProcessedData.successfullyProcessed(sequenceId = firstSequence), + PreparedProcessedData.successfullyProcessed(sequenceId = sequenceIdNotInProcessing), + ) + .andExpect(status().isUnprocessableEntity) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect( + jsonPath("\$.detail").value( + "Sequence version $sequenceIdNotInProcessing.1 is in not in state PROCESSING (was RECEIVED)", + ), + ) + + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.PROCESSING) + convenienceClient.getSequenceVersionOfUser(sequenceId = sequenceIdNotInProcessing, version = 1) + .assertStatusIs(Status.RECEIVED) + } + + @Test + fun `WHEN I submit a JSON entry with a missing field THEN returns bad request`() { + convenienceClient.submitDefaultFiles() + convenienceClient.extractUnprocessedData(1) + + submissionControllerClient.submitProcessedDataRaw( + """ + { + "sequenceId": 1, + "version": 1, + "data": { + "noMetadata": null, + "unalignedNucleotideSequences": { + "main": "NNNNNN" + } + } + } + """.replace(Regex("\\s"), ""), + ) + .andExpect(status().isBadRequest) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_VALUE)) + .andExpect(jsonPath("\$.detail").value(containsString("Failed to deserialize NDJSON"))) + .andExpect(jsonPath("\$.detail").value(containsString("failed for JSON property metadata"))) + } + + private fun prepareExtractedSequencesInDatabase() { + convenienceClient.submitDefaultFiles() + convenienceClient.extractUnprocessedData() + } + + companion object { + @JvmStatic + fun provideInvalidDataScenarios() = listOf( + InvalidDataScenario( + name = "data with unknown metadata fields", + processedData = PreparedProcessedData.withUnknownMetadataField( + fields = listOf( + "unknown field 1", + "unknown field 2", + ), + ), + expectedValidationErrors = listOf( + Triple( + "UnknownField", + "unknown field 1", + "Found unknown field 'unknown field 1' in processed data", + ), + Triple( + "UnknownField", + "unknown field 2", + "Found unknown field 'unknown field 2' in processed data", + ), + ), + ), + InvalidDataScenario( + name = "data with missing required fields", + processedData = PreparedProcessedData.withMissingRequiredField(fields = listOf("date", "region")), + expectedValidationErrors = listOf( + Triple( + "MissingRequiredField", + "date", + "Missing the required field 'date'", + ), + Triple( + "MissingRequiredField", + "region", + "Missing the required field 'region'", + ), + ), + ), + InvalidDataScenario( + name = "data with wrong type for fields", + processedData = PreparedProcessedData.withWrongTypeForFields(), + expectedValidationErrors = listOf( + Triple( + "TypeMismatch", + "region", + "Expected type 'string' for field 'region', found value '5'", + ), + Triple( + "TypeMismatch", + "age", + "Expected type 'integer' for field 'age', found value '\"not a number\"'", + ), + ), + ), + InvalidDataScenario( + name = "data with wrong date format", + processedData = PreparedProcessedData.withWrongDateFormat(), + expectedValidationErrors = listOf( + Triple( + "TypeMismatch", + "date", + "Expected type 'date' in format 'yyyy-MM-dd' for field 'date', found value '\"1.2.2021\"'", + ), + ), + ), + InvalidDataScenario( + name = "data with wrong pango lineage format", + processedData = PreparedProcessedData.withWrongPangoLineageFormat(), + expectedValidationErrors = listOf( + Triple( + "TypeMismatch", + "pangoLineage", + "Expected type 'pango_lineage' for field 'pangoLineage', found value '\"A.5.invalid\"'.", + ), + ), + ), + InvalidDataScenario( + name = "data with explicit null for required field", + processedData = PreparedProcessedData.withNullForFields(fields = listOf("date")), + expectedValidationErrors = listOf( + Triple( + "MissingRequiredField", + "date", + "Field 'date' is null, but a value is required.", + ), + ), + ), + ) + } +} + +data class InvalidDataScenario( + val name: String, + val processedData: SubmittedProcessedData, + val expectedValidationErrors: List>, +) { + override fun toString(): String { + val errorsDisplay = expectedValidationErrors.joinToString(" and ") { it.first } + val prefix = if (expectedValidationErrors.size > 1) { + "the validation errors" + } else { + "the validation error" + } + + return "GIVEN $name THEN the response contains $prefix $errorsDisplay" + } +} diff --git a/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmitReviewEndpoint.kt b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmitReviewEndpoint.kt index 148f41add..a9965e88b 100644 --- a/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmitReviewEndpoint.kt +++ b/backend/src/test/kotlin/org/pathoplexus/backend/controller/SubmitReviewEndpoint.kt @@ -2,6 +2,7 @@ package org.pathoplexus.backend.controller import org.junit.jupiter.api.Test import org.pathoplexus.backend.controller.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES +import org.pathoplexus.backend.controller.SubmitFiles.DefaultFiles.firstSequence import org.pathoplexus.backend.service.Status import org.pathoplexus.backend.service.UnprocessedData import org.springframework.beans.factory.annotation.Autowired @@ -11,79 +12,62 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status @EndpointTest class SubmitReviewEndpoint( @Autowired val client: SubmissionControllerClient, + @Autowired val convenienceClient: SubmissionConvenienceClient, ) { @Test fun `GIVEN a sequence needs review WHEN I submit a reviewed sequence THEN the status changes to REVIEWED`() { - client.submitDefaultFiles() + convenienceClient.submitDefaultFiles() awaitResponse(client.extractUnprocessedData(NUMBER_OF_SEQUENCES).andReturn()) - client.submitProcessedData(processedInputDataFromFile("error_feedback")) + client.submitProcessedData(PreparedProcessedData.withErrors()) - expectStatusInResponse( - client.getSequencesOfUser(), - 1, - Status.NEEDS_REVIEW.name, - ) - - val reviewedData = - UnprocessedData( - sequenceId = 1, - version = 1, - data = emptyOriginalData, - ) + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.NEEDS_REVIEW) + val reviewedData = UnprocessedData( + sequenceId = 1, + version = 1, + data = emptyOriginalData, + ) client.submitReviewedSequence(USER_NAME, reviewedData) .andExpect(status().isOk()) - expectStatusInResponse( - client.getSequencesOfUser(), - 1, - Status.REVIEWED.name, - ) + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.REVIEWED) } @Test fun `GIVEN a sequence is processed WHEN I submit a review to that sequence THEN the status changes to REVIEWED`() { - client.submitDefaultFiles() + convenienceClient.submitDefaultFiles() awaitResponse(client.extractUnprocessedData(NUMBER_OF_SEQUENCES).andReturn()) - client.submitProcessedData(processedInputDataFromFile("no_validation_errors")) + client.submitProcessedData(PreparedProcessedData.successfullyProcessed()) - expectStatusInResponse( - client.getSequencesOfUser(), - 1, - Status.PROCESSED.name, - ) + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.PROCESSED) - val reviewedData = - UnprocessedData( - sequenceId = 1, - version = 1, - data = emptyOriginalData, - ) + val reviewedData = UnprocessedData( + sequenceId = 1, + version = 1, + data = emptyOriginalData, + ) client.submitReviewedSequence(USER_NAME, reviewedData) .andExpect(status().isOk()) - expectStatusInResponse( - client.getSequencesOfUser(), - 1, - Status.REVIEWED.name, - ) + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.REVIEWED) } @Test fun `WHEN a version does not exist THEN it returns an unprocessable entity error`() { - client.submitDefaultFiles() + convenienceClient.submitDefaultFiles() awaitResponse(client.extractUnprocessedData(NUMBER_OF_SEQUENCES).andReturn()) - client.submitProcessedData(processedInputDataFromFile("error_feedback")) + client.submitProcessedData(PreparedProcessedData.withErrors()) - expectStatusInResponse( - client.getSequencesOfUser(), - 1, - Status.NEEDS_REVIEW.name, - ) + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.NEEDS_REVIEW) val reviewedDataWithNonExistingVersion = UnprocessedData( @@ -104,16 +88,13 @@ class SubmitReviewEndpoint( @Test fun `WHEN a sequenceId does not exist THEN it returns an unprocessable entity error`() { - client.submitDefaultFiles() + convenienceClient.submitDefaultFiles() awaitResponse(client.extractUnprocessedData(NUMBER_OF_SEQUENCES).andReturn()) - client.submitProcessedData(processedInputDataFromFile("error_feedback")) + client.submitProcessedData(PreparedProcessedData.withErrors()) - expectStatusInResponse( - client.getSequencesOfUser(), - 1, - Status.NEEDS_REVIEW.name, - ) + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.NEEDS_REVIEW) val reviewedDataWithNonExistingSequenceId = UnprocessedData( sequenceId = 2, @@ -131,25 +112,19 @@ class SubmitReviewEndpoint( ), ) - expectStatusInResponse( - client.getSequencesOfUser(), - 0, - Status.REVIEWED.name, - ) + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.NEEDS_REVIEW) } @Test fun `WHEN a sequenceId does not belong to a user THEN it returns an forbidden error`() { - client.submitDefaultFiles() + convenienceClient.submitDefaultFiles() awaitResponse(client.extractUnprocessedData(NUMBER_OF_SEQUENCES).andReturn()) - client.submitProcessedData(processedInputDataFromFile("error_feedback")) + client.submitProcessedData(PreparedProcessedData.withErrors()) - expectStatusInResponse( - client.getSequencesOfUser(), - 1, - Status.NEEDS_REVIEW.name, - ) + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.NEEDS_REVIEW) val reviewedDataFromWrongSubmitter = UnprocessedData( @@ -169,10 +144,7 @@ class SubmitReviewEndpoint( ), ) - expectStatusInResponse( - client.getSequencesOfUser(), - 0, - Status.REVIEWED.name, - ) + convenienceClient.getSequenceVersionOfUser(sequenceId = firstSequence, version = 1) + .assertStatusIs(Status.NEEDS_REVIEW) } } diff --git a/backend/src/test/kotlin/org/pathoplexus/backend/controller/TestHelpers.kt b/backend/src/test/kotlin/org/pathoplexus/backend/controller/TestHelpers.kt index 061513d53..2bd9b1ce2 100644 --- a/backend/src/test/kotlin/org/pathoplexus/backend/controller/TestHelpers.kt +++ b/backend/src/test/kotlin/org/pathoplexus/backend/controller/TestHelpers.kt @@ -3,25 +3,22 @@ package org.pathoplexus.backend.controller import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import org.assertj.core.api.Assertions +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.`is` +import org.pathoplexus.backend.service.SequenceVersionStatus +import org.pathoplexus.backend.service.Status import org.springframework.test.web.servlet.MvcResult import org.springframework.test.web.servlet.ResultActions import org.springframework.test.web.servlet.result.MockMvcResultMatchers import org.testcontainers.shaded.org.awaitility.Awaitility.await -import java.io.File val jacksonObjectMapper: ObjectMapper = jacksonObjectMapper().findAndRegisterModules() inline fun ResultActions.expectNdjsonAndGetContent(): List { andExpect(MockMvcResultMatchers.status().isOk) andExpect(MockMvcResultMatchers.content().contentType("application/x-ndjson")) - val response = andReturn() - await().until { - response.response.isCommitted - } - - val content = response.response.contentAsString + val content = awaitResponse(andReturn()) return content.lines().filter { it.isNotEmpty() }.map { jacksonObjectMapper.readValue(it) } } @@ -33,33 +30,6 @@ fun awaitResponse(result: MvcResult): String { return result.response.contentAsString } -fun expectStatusInResponse(result: MvcResult, numberOfSequences: Int, expectedStatus: String): String { - awaitResponse(result) - - val responseContent = result.response.contentAsString - val statusCount = responseContent.split(expectedStatus).size - 1 - - Assertions.assertThat(statusCount).isEqualTo(numberOfSequences) - - return responseContent -} - -fun processedInputDataFromFile(fileName: String): String = inputData[fileName] ?: error( - "$fileName.json not found", -) - -private val inputData: Map by lazy { - val fileMap = mutableMapOf() - - val jsonResourceDirectory = "src/test/resources/processedInputData" - - val directory = File(jsonResourceDirectory) - - directory.listFiles { _, name -> name.endsWith(".json") }?.forEach { file -> - val fileName = file.nameWithoutExtension - val formattedJson = file.readText().replace("\n", "").replace("\r", "").replace(" ", "") - fileMap[fileName] = formattedJson - } - - fileMap +fun SequenceVersionStatus.assertStatusIs(status: Status) { + assertThat(this.status, `is`(status)) } diff --git a/backend/src/test/resources/processedInputData/error_feedback.json b/backend/src/test/resources/processedInputData/error_feedback.json deleted file mode 100644 index 574fe529d..000000000 --- a/backend/src/test/resources/processedInputData/error_feedback.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "sequenceId": 1, - "version": 1, - "errors": [ - { - "source": [{ "type": "metadata", "name": "host" }], - "message": "Not this kind of host" - } - ], - "warnings": [], - "data": { - "metadata": { - "date": "2002-12-15", - "host": "google.com", - "region": "Europe", - "country": "Spain", - "division": "Schaffhausen", - "pangoLineage": "XBB.1.5" - }, - "unalignedNucleotideSequences": { "main": "NNNNNNNNNNNNNNNN" } - } -} diff --git a/backend/src/test/resources/processedInputData/error_feedback_with_id2.json b/backend/src/test/resources/processedInputData/error_feedback_with_id2.json deleted file mode 100644 index 54d9cc9ce..000000000 --- a/backend/src/test/resources/processedInputData/error_feedback_with_id2.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "sequenceId": 2, - "version": 1, - "errors": [ - { - "source": [{ "type": "metadata", "name": "host" }], - "message": "Not this kind of host" - } - ], - "warnings": [], - "data": { - "metadata": { - "date": "2002-12-15", - "host": "google.com", - "region": "Europe", - "country": "Spain", - "division": "Schaffhausen", - "pangoLineage": "XBB.1.5" - }, - "unalignedNucleotideSequences": { "main": "NNNNNNNNNNNNNNNN" } - } -} diff --git a/backend/src/test/resources/processedInputData/invalid_id.json b/backend/src/test/resources/processedInputData/invalid_id.json deleted file mode 100644 index c1165a64e..000000000 --- a/backend/src/test/resources/processedInputData/invalid_id.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "sequenceId": 12, - "version": 1, - "data": { - "metadata": { - "host": "Homo sapiens", - "region": "Europe", - "division": "Schaffhausen", - "date": "2023-01-01", - "country": "Switzerland" - }, - "nucleotideSequences": { - "main": "NNNNNNNNNNNNNNNN" - } - } -} diff --git a/backend/src/test/resources/processedInputData/missing_required_field.json b/backend/src/test/resources/processedInputData/missing_required_field.json deleted file mode 100644 index 22cff02b6..000000000 --- a/backend/src/test/resources/processedInputData/missing_required_field.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "sequenceId": 1, - "version": 1, - "data": { - "metadata": { - "host": "Homo sapiens", - "region": "Europe", - "division": "Schaffhausen" - }, - "nucleotideSequences": { "main": "NNNNNNNNNNNNNNNN" } - } -} diff --git a/backend/src/test/resources/processedInputData/no_validation_errors.json b/backend/src/test/resources/processedInputData/no_validation_errors.json deleted file mode 100644 index 9a6713e45..000000000 --- a/backend/src/test/resources/processedInputData/no_validation_errors.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "sequenceId": 1, - "version": 1, - "errors": [], - "warnings": [], - "data": { - "metadata": { - "date": "2002-12-15", - "host": "Homo sapiens", - "region": "Europe", - "country": "Spain", - "division": "Schaffhausen", - "pangoLineage": "XBB.1.5" - }, - "unalignedNucleotideSequences": { "main": "NNNNNNNNNNNNNNNN" } - } -} diff --git a/backend/src/test/resources/processedInputData/null_value.json b/backend/src/test/resources/processedInputData/null_value.json deleted file mode 100644 index 9a604b9ee..000000000 --- a/backend/src/test/resources/processedInputData/null_value.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "sequenceId": 1, - "version": 1, - "errors": [], - "warnings": [], - "data": { - "metadata": { - "date": "2002-12-15", - "host": "Homo sapiens", - "region": "Europe", - "country": "Spain", - "division": "Schaffhausen", - "pangoLineage": null - }, - "unalignedNucleotideSequences": { "main": "NNNNNNNNNNNNNNNN" } - } -} diff --git a/backend/src/test/resources/processedInputData/unknown_field.json b/backend/src/test/resources/processedInputData/unknown_field.json deleted file mode 100644 index 242bef380..000000000 --- a/backend/src/test/resources/processedInputData/unknown_field.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "sequenceId": 1, - "version": 1, - "data": { - "metadata": { - "date": "2002-12-15", - "not_a_meta_data_field": "not_important", - "host": "Homo sapiens", - "country": "Spain", - "region": "Europe", - "division": "Schaffhausen" - }, - "nucleotideSequences": { "main": "NNNNNNNNNNNNNNNN" } - } -} diff --git a/backend/src/test/resources/processedInputData/wrong_type_field.json b/backend/src/test/resources/processedInputData/wrong_type_field.json deleted file mode 100644 index b15efa34c..000000000 --- a/backend/src/test/resources/processedInputData/wrong_type_field.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "sequenceId": 1, - "version": 1, - "data": { - "metadata": { - "date": "15.12.2002", - "host": "Homo sapiens", - "region": "Europe", - "country": "Spain", - "division": "Schaffhausen" - }, - "nucleotideSequences": { "main": "NNNNNNNNNNNNNNNN" } - } -} diff --git a/preprocessing/specification.md b/preprocessing/specification.md index 34aaba26e..87912a5c0 100644 --- a/preprocessing/specification.md +++ b/preprocessing/specification.md @@ -43,6 +43,8 @@ Sequences without an error will be released. Sequences with an error will not be ## Technical specification +Also see the Swagger UI available in the backend at `/swagger-ui/index.html`. + ### Pulling unpreprocessed data To retrieve unpreprocessed data, the preprocessing pipeline sends a POST request to the backend's `/extract-unprocessed-data` with the request parameter `numberOfSequences` (integer). This returns a response in [NDJSON](http://ndjson.org/) containing at most the specified number of sequence entries. If there are no entries that require preprocessing, an empty file is returned. @@ -103,7 +105,7 @@ The `errors` and `warnings` fields contain an array of objects of the following ```js { source: { - type: "metadata" | "nucleotideSequence", + type: "Metadata" | "NucleotideSequence", name: string }[], message: string diff --git a/website/tests/util/preprocessingPipeline.ts b/website/tests/util/preprocessingPipeline.ts index 79a7d1c19..bfd4407d2 100644 --- a/website/tests/util/preprocessingPipeline.ts +++ b/website/tests/util/preprocessingPipeline.ts @@ -12,8 +12,8 @@ export const fakeProcessingPipeline = async ({ const body = { sequenceId, version, - errors: error ? [{ source: { fieldName: 'host', type: 'metadata' }, message: 'Not this kind of host' }] : [], - warnings: [{ source: { fieldName: 'all', type: 'all' }, message: '"There is no warning"-warning' }], + errors: error ? [{ source: [{ name: 'host', type: 'Metadata' }], message: 'Not this kind of host' }] : [], + warnings: [{ source: [{ name: 'date', type: 'Metadata' }], message: '"There is no warning"-warning' }], data: { metadata: { date: '2002-12-15', @@ -49,7 +49,8 @@ export const fakeProcessingPipeline = async ({ body: JSON.stringify(body), }); if (!response.ok) { - throw new Error(`Unexpected response: ${response.statusText}`); + const body = await response.text(); + throw new Error(`Unexpected response with status '${response.statusText}': ${body}`); } };