refactor(yki): add successful rows always to database

Opetushallitus · Dec 20, 2024 · 7d877b2 · 7d877b2
1 parent 63f08dd
commit 7d877b2
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 45 deletions.
diff --git a/server/src/main/kotlin/fi/oph/kitu/csvparsing/CsvExportError.kt b/server/src/main/kotlin/fi/oph/kitu/csvparsing/CsvExportError.kt
@@ -32,15 +32,9 @@ abstract class CsvExportError(
     }
 }
 
-class CsvExportException(
-    val errors: Iterable<CsvExportError>,
-    message: String? = "Unable to convert string to csv, because the string had ${errors.count()} error(s).",
-    cause: Throwable? = null,
-) : Throwable(message, cause)
-
-fun LoggingEventBuilder.addErrors(exception: CsvExportException) {
+fun LoggingEventBuilder.addErrors(errors: Iterable<CsvExportError>) {
     // add all errors to log
-    exception.errors.forEachIndexed { i, error ->
+    errors.forEachIndexed { i, error ->
         this.add("serialization.error[$i].index" to i)
         for (kvp in error.keyValues) {
             this.add("serialization.error[$i].${kvp.first}" to kvp.second)

diff --git a/server/src/main/kotlin/fi/oph/kitu/csvparsing/CsvParser.kt b/server/src/main/kotlin/fi/oph/kitu/csvparsing/CsvParser.kt
@@ -1,6 +1,5 @@
 package fi.oph.kitu.csvparsing
 
-import com.fasterxml.jackson.databind.MappingIterator
 import com.fasterxml.jackson.databind.exc.InvalidFormatException
 import com.fasterxml.jackson.databind.module.SimpleModule
 import com.fasterxml.jackson.dataformat.csv.CsvMapper
@@ -10,6 +9,7 @@ import fi.oph.kitu.logging.add
 import org.ietf.jgss.Oid
 import org.slf4j.spi.LoggingEventBuilder
 import java.io.ByteArrayOutputStream
+import java.lang.RuntimeException
 import kotlin.reflect.full.findAnnotation
 
 class CsvParser(
@@ -92,7 +92,7 @@ class CsvParser(
     /**
      * Converts retrieved String response into a list that is the type of Body.
      */
-    inline fun <reified T> convertCsvToData(csvString: String): List<T> {
+    inline fun <reified T> convertCsvtToResults(csvString: String): Iterable<Result<T>> {
         if (csvString.isBlank()) {
             event.add("serialization.isEmptyList" to true)
             return emptyList()
@@ -103,41 +103,47 @@ class CsvParser(
         val csvMapper = getCsvMapper<T>()
         val schema = getSchema<T>(csvMapper)
 
-        // the lines are needed to read line by line in order to distinguish all erroneous lines
-        val errors = mutableListOf<CsvExportError>()
-
         val iterator =
             csvMapper
                 .readerFor(T::class.java)
                 .with(schema)
                 .readValues<T?>(csvString)
 
-        val data =
-            iterator.toDataWithErrorHandling { index, e ->
-                when (e) {
-                    is InvalidFormatException -> errors.add(InvalidFormatCsvExportError(index, e))
-                    else -> errors.add(SimpleCsvExportError(index, e))
-                }
-            }
-
-        if (errors.isEmpty()) {
-            return data
+        // the lines are needed to read line by line in order to distinguish all erroneous lines
+        val data = mutableListOf<Result<T>>()
+        while (iterator.hasNext()) {
+            data.add(runCatching { iterator.nextValue() })
         }
 
-        throw CsvExportException(errors)
+        return data
     }
 }
 
-fun <T> MappingIterator<T>.toDataWithErrorHandling(
-    onFailure: (index: Int, exception: Throwable) -> Unit = { _, _ -> },
+fun <T> Iterable<Result<T>>.foldWithErrors(
+    continueOnError: Boolean,
+    action: (Iterable<CsvExportError>) -> Unit,
 ): List<T> {
+    val errors = mutableListOf<CsvExportError>()
     val data = mutableListOf<T>()
-    var index = 0
-    while (this.hasNext()) {
-        runCatching { this.nextValue() }
-            .onSuccess { d -> data.add(d) }
-            .onFailure { e -> onFailure(index, e) }
-            .also { index++ }
+    // Add errors to error list
+    this.forEachIndexed { index, result ->
+        result
+            .onFailure { e ->
+                when (e) {
+                    is InvalidFormatException -> errors.add(InvalidFormatCsvExportError(index, e))
+                    else -> errors.add(SimpleCsvExportError(index, e))
+                }
+            }.onSuccess { row -> data.add(row) }
+
+        if (errors.isNotEmpty()) {
+            action(errors)
+
+            if (!continueOnError) {
+                throw RuntimeException(
+                    "Unable to convert string to csv, because the string had ${errors.count()} error(s).",
+                )
+            }
+        }
     }
 
     return data

diff --git a/server/src/main/kotlin/fi/oph/kitu/yki/YkiService.kt b/server/src/main/kotlin/fi/oph/kitu/yki/YkiService.kt
@@ -1,9 +1,9 @@
 package fi.oph.kitu.yki
 
 import fi.oph.kitu.PeerService
-import fi.oph.kitu.csvparsing.CsvExportException
 import fi.oph.kitu.csvparsing.CsvParser
 import fi.oph.kitu.csvparsing.addErrors
+import fi.oph.kitu.csvparsing.foldWithErrors
 import fi.oph.kitu.logging.add
 import fi.oph.kitu.logging.addHttpResponse
 import fi.oph.kitu.logging.withEvent
@@ -58,17 +58,11 @@ class YkiService(
 
             event.addHttpResponse(PeerService.Solki, "suoritukset", response)
 
-            val suoritukset =
-                try {
-                    parser.convertCsvToData<YkiSuoritusCsv>(response.body ?: "")
-                } catch (e: CsvExportException) {
-                    event.addErrors(e)
-
-                    if (!importSuorituksetContinueOnError) {
-                        throw e
-                    } else {
-                        listOf()
-                    }
+            val suoritukset = mutableListOf<YkiSuoritusCsv>()
+            parser
+                .convertCsvtToResults<YkiSuoritusCsv>(response.body ?: "")
+                .foldWithErrors(importSuorituksetContinueOnError) {
+                    event.addErrors(it)
                 }
 
             if (dryRun != true) {
@@ -91,7 +85,12 @@ class YkiService(
             event.addHttpResponse(PeerService.Solki, "arvioijat", response)
 
             val arvioijat =
-                parser.convertCsvToData<SolkiArvioijaResponse>(response.body ?: throw Error.EmptyArvioijatResponse())
+                parser
+                    .convertCsvtToResults<SolkiArvioijaResponse>(
+                        response.body ?: throw Error.EmptyArvioijatResponse(),
+                    ).foldWithErrors(false) {
+                        event.addErrors(it)
+                    }
 
             event.add("yki.arvioijat.receivedCount" to arvioijat.size)
             if (arvioijat.isEmpty()) {