feat(kielitesti): fetch suoritukset

server/pom.xml

@@ -163,6 +163,10 @@
             <artifactId>db-scheduler-spring-boot-starter</artifactId>
             <version>14.1.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-jdbc</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

server/src/main/kotlin/fi/oph/kitu/kielitesti/KotoRepository.kt

@@ -0,0 +1,31 @@
+package fi.oph.kitu.kielitesti
+
+import org.springframework.data.annotation.Id
+import org.springframework.data.repository.CrudRepository
+import org.springframework.stereotype.Repository
+import java.time.Instant
+
+data class KotoSuoritus(
+    @Id
+    val id: Int? = null,
+    val first_name: String,
+    val last_name: String,
+    val oppija_oid: String,
+    val email: String,
+    val time_completed: Instant,
+    val courseid: Int,
+    val coursename: String,
+    val luetun_ymmartaminen_result_system: Double,
+    val luetun_ymmartaminen_result_teacher: Double,
+    val kuullun_ymmartaminen_result_system: Double,
+    val kuullun_ymmartaminen_result_teacher: Double,
+    val puhe_result_system: Double,
+    val puhe_result_teacher: Double,
+    val kirjoittaminen_result_system: Double,
+    val kirjottaminen_result_teacher: Double,
+    val total_evaluation_teacher: String,
+    val total_evaluation_system: String,
+)
+
+@Repository
+interface KotoRepository : CrudRepository<KotoSuoritus, Int>

server/src/main/kotlin/fi/oph/kitu/kielitesti/KotoScheduledTasks.kt

@@ -0,0 +1,24 @@
+package fi.oph.kitu.kielitesti
+
+import com.github.kagkarlsson.scheduler.task.Task
+import com.github.kagkarlsson.scheduler.task.helper.Tasks
+import com.github.kagkarlsson.scheduler.task.schedule.Schedules
+import org.springframework.beans.factory.annotation.Value
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+import java.time.Instant
+
+@Configuration
+@ConditionalOnProperty(name = ["kitu.kielitesti.scheduling.enabled"], matchIfMissing = false)
+class KotoScheduledTasks {
+    @Value("\${kitu.kielitesti.scheduling.import.schedule}")
+    lateinit var kielitestiImportSchedule: String
+
+    @Bean
+    fun dailyImportKotoSuoritukset(moodleService: MoodleService): Task<Instant> =
+        Tasks
+            .recurring("Kielitesti-import", Schedules.parseSchedule(kielitestiImportSchedule), Instant::class.java)
+            .initialData(Instant.EPOCH)
+            .executeStateful { taskInstance, _ -> moodleService.importSuoritukset(taskInstance.data) }
+}

server/src/main/kotlin/fi/oph/kitu/kielitesti/MoodleService.kt

@@ -1,39 +1,15 @@
 package fi.oph.kitu.kielitesti
 
+import com.fasterxml.jackson.core.JsonParser
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
-import com.fasterxml.jackson.module.kotlin.registerKotlinModule
-import fi.oph.kitu.oppija.Oppija
+import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.http.MediaType
 import org.springframework.stereotype.Service
 import org.springframework.web.client.RestClient
-
-data class MoodleUser(
-    val id: Long,
-    val fullname: String,
-    val extrafields: List<Any>,
-)
-
-data class MoodleUserList(
-    val list: List<MoodleUser>,
-    val maxusersperpage: Long?,
-    val overflow: Boolean?,
-) {
-    companion object {
-        private val mapper = ObjectMapper().registerKotlinModule()
-
-        fun tryParse(json: String): List<Oppija> {
-            try {
-                val body = mapper.readValue<MoodleUserList>(json)
-                return body.list.map { Oppija(it.id, it.fullname) } ?: listOf()
-            } catch (e: Exception) {
-                val moodleError = mapper.readValue<MoodleErrorMessage>(json)
-                throw MoodleException(moodleError)
-            }
-        }
-    }
-}
+import org.springframework.web.client.toEntity
+import java.time.Instant
 
 class MoodleException(
     val moodleErrorMessage: MoodleErrorMessage,
@@ -46,10 +22,41 @@ data class MoodleErrorMessage(
     val debuginfo: String?,
 )
 
+data class MoodleSuorituksetResponse(
+    val users: List<User>,
+) {
+    data class User(
+        val firstname: String,
+        val lastname: String,
+        val OIDnumber: String,
+        val email: String,
+        val completions: List<Completion>,
+    ) {
+        data class Completion(
+            val courseid: Int,
+            val coursename: String,
+            val results: List<Result>,
+            val timecompleted: Int,
+            val total_evaluation_teacher: String,
+            val total_evaluation_system: String,
+        ) {
+            data class Result(
+                val name: String,
+                val quiz_result_system: Double,
+                val quiz_result_teacher: Double,
+            )
+        }
+    }
+}
+
 @Service
 class MoodleService(
     private val restClientBuilder: RestClient.Builder,
+    private val kotoRepository: KotoRepository,
+    private val jacksonObjectMapper: ObjectMapper,
 ) {
+    private val logger = LoggerFactory.getLogger(javaClass)
+
     @Value("\${kitu.kielitesti.wstoken}")
     lateinit var moodleToken: String
 
@@ -58,21 +65,79 @@ class MoodleService(
 
     private val restClient by lazy { restClientBuilder.baseUrl(kielitestiBaseurl).build() }
 
-    fun getUsers(): List<Oppija> {
+    private inline fun <reified T> tryParseMoodleResponse(json: String): T {
+        try {
+            return jacksonObjectMapper.enable(JsonParser.Feature.INCLUDE_SOURCE_IN_LOCATION).readValue<T>(json)
+        } catch (e: Exception) {
+            val moodleError = jacksonObjectMapper.readValue<MoodleErrorMessage>(json)
+            throw MoodleException(moodleError)
+        }
+    }
+
+    fun importSuoritukset(from: Instant): Instant {
         val response =
             restClient
                 .get()
                 .uri(
-                    "/webservice/rest/server.php?wstoken={token}&wsfunction={function}&moodlewsrestformat=json&query=",
-                    mapOf(
+                    "/webservice/rest/server.php?wstoken={token}&wsfunction={function}&moodlewsrestformat=json&from={from}",
+                    mapOf<String?, Any>(
                         "token" to moodleToken,
-                        "function" to "core_user_search_identity",
+                        "function" to "local_completion_export_get_completions",
+                        "from" to from.epochSecond,
                     ),
                 ).accept(MediaType.APPLICATION_JSON)
                 .retrieve()
-                .body(String::class.java)
+                .toEntity<String>()
+
+        logger
+            .atInfo()
+            .addKeyValue("request.token", moodleToken)
+            .addKeyValue("response.body", response.body)
+            .addKeyValue("response.headers", response.headers)
+            .log("moodle response")
+
+        if (response.body == null) {
+            return from
+        }
+
+        val suorituksetResponse =
+            tryParseMoodleResponse<MoodleSuorituksetResponse>(response.body!!)
+
+        val suoritukset =
+            suorituksetResponse.users.flatMap { user ->
+                user.completions.map { completion ->
+                    val luetunYmmartaminen = completion.results.find { it.name == "luetun ymm\u00e4rt\u00e4minen" }!!
+                    val kuullunYmmartaminen = completion.results.find { it.name == "kuullun ymm\u00e4rt\u00e4minen" }!!
+                    val puhe = completion.results.find { it.name == "puhe" }!!
+                    val kirjoittaminen = completion.results.find { it.name == "kirjoittaminen" }!!
+                    KotoSuoritus(
+                        first_name = user.firstname,
+                        last_name = user.lastname,
+                        email = user.email,
+                        oppija_oid = user.OIDnumber,
+                        time_completed = Instant.ofEpochSecond(completion.timecompleted.toLong()),
+                        courseid = completion.courseid,
+                        coursename = completion.coursename,
+                        luetun_ymmartaminen_result_system = luetunYmmartaminen.quiz_result_system,
+                        luetun_ymmartaminen_result_teacher = luetunYmmartaminen.quiz_result_teacher,
+                        kuullun_ymmartaminen_result_system = kuullunYmmartaminen.quiz_result_system,
+                        kuullun_ymmartaminen_result_teacher = kuullunYmmartaminen.quiz_result_teacher,
+                        puhe_result_system = puhe.quiz_result_system,
+                        puhe_result_teacher = puhe.quiz_result_teacher,
+                        kirjoittaminen_result_system = kirjoittaminen.quiz_result_system,
+                        kirjottaminen_result_teacher = kirjoittaminen.quiz_result_teacher,
+                        total_evaluation_teacher = completion.total_evaluation_teacher,
+                        total_evaluation_system = completion.total_evaluation_system,
+                    )
+                }
+            }
+
+        val result = kotoRepository.saveAll(suoritukset)
+
+        logger.atInfo().addKeyValue("db.saved", result.count()).log("saved suoritukset")
+
+        val lastSeen = suoritukset.maxOfOrNull { it.time_completed }
 
-        if (response.isNullOrEmpty()) return listOf()
-        return MoodleUserList.tryParse(response)
+        return checkNotNull(lastSeen)
     }
 }

server/src/main/resources/application-e2e.properties

@@ -7,6 +7,7 @@ kitu.opintopolkuHostname=virkailija.untuvaopintopolku.fi
 
 kitu.kielitesti.wstoken=
 kitu.kielitesti.baseurl=
+kitu.kielitesti.scheduling.enabled=false
 
 kitu.oppijanumero.username=koto-rekisteri
 kitu.oppijanumero.password=

server/src/main/resources/application-local.properties

@@ -8,6 +8,8 @@ kitu.appUrl=http://localhost:8080
 kitu.opintopolkuHostname=virkailija.untuvaopintopolku.fi
 
 kitu.kielitesti.baseurl=https://kielitesti.mmg.fi
+kitu.kielitesti.scheduling.enabled=true
+kitu.kielitesti.scheduling.import.schedule=FIXED_DELAY|60s
 
 kitu.oppijanumero.username=koto-rekisteri
 kitu.oppijanumero.callerid=1.2.246.562.24.85478397072

server/src/main/resources/application.properties

@@ -8,6 +8,8 @@ springdoc.swagger-ui.csrf.enabled=true
 springdoc.api-docs.enabled=false
 
 kitu.kielitesti.wstoken=${KIELITESTI_TOKEN}
+kitu.kielitesti.scheduling.enabled=true
+kitu.kielitesti.scheduling.import.schedule=
 
 kitu.oppijanumero.username=
 kitu.oppijanumero.password=

server/src/main/resources/db/migration/V4__add_courseid_and_coursename_to_koto_suoritus.sql

@@ -0,0 +1,5 @@
+alter table koto_suoritus
+    add column courseid int not null default (-1);
+
+alter table koto_suoritus
+    add column coursename text not null default '';

server/src/main/resources/static/open-api.yaml

@@ -53,24 +53,6 @@ paths:
         400:
           description: "Virheellinen pyyntö"
 
-  /api/kielitesti/user:
-    get:
-      tags:
-        - "user-controller"
-      summary: "Hakee listan Moodlen käyttäjistä"
-      operationId: "getUsers"
-      responses:
-        200:
-          description: "Käyttäjien hakeminen onnistui"
-          content:
-            "application/json":
-              schema:
-                type: array
-                items:
-                  $ref: "#/components/schemas/Oppija"
-        503:
-          description: "Käyttäjien haku Moodlesta epäonnistui"
-
   /api/oppijanumero:
     get:
       tags: