Merge branch 'main' into isodates

backend/build.gradle

@@ -3,7 +3,7 @@ import org.gradle.api.tasks.testing.logging.TestLogEvent
 
 plugins {
     id 'org.springframework.boot' version '3.3.1'
-    id 'io.spring.dependency-management' version '1.1.5'
+    id 'io.spring.dependency-management' version '1.1.6'
     id 'org.jetbrains.kotlin.jvm' version '2.0.0'
     id 'org.jetbrains.kotlin.plugin.spring' version '2.0.0'
     id 'org.jlleitschuh.gradle.ktlint' version '12.1.1'
@@ -63,7 +63,7 @@ dependencies {
     testImplementation "com.ninja-squad:springmockk:4.0.2"
     testImplementation "org.testcontainers:postgresql:1.19.8"
     testImplementation "org.junit.platform:junit-platform-launcher:1.10.3"
-    ktlint("com.pinterest.ktlint:ktlint-cli:1.3.0") {
+    ktlint("com.pinterest.ktlint:ktlint-cli:1.3.1") {
         attributes {
             attribute(Bundling.BUNDLING_ATTRIBUTE, getObjects().named(Bundling, Bundling.EXTERNAL))
         }

backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt

@@ -213,10 +213,24 @@ data class SequenceEntryStatus(
     val dataUseTerms: DataUseTerms,
 ) : AccessionVersionInterface
 
+data class EditedSequenceEntryData(
+    @Schema(example = "LOC_000S01D") override val accession: Accession,
+    @Schema(example = "1") override val version: Version,
+    val data: OriginalData<GeneticSequence>,
+) : AccessionVersionInterface
+
 data class UnprocessedData(
-    @Schema(example = "123") override val accession: Accession,
+    @Schema(example = "LOC_000S01D") override val accession: Accession,
     @Schema(example = "1") override val version: Version,
     val data: OriginalData<GeneticSequence>,
+    @Schema(description = "The submission id that was used in the upload to link metadata and sequences")
+    val submissionId: String,
+    @Schema(description = "The username of the submitter")
+    val submitter: String,
+    @Schema(example = "42", description = "The id of the group that this sequence entry was submitted by")
+    val groupId: Int,
+    @Schema(example = "1720304713", description = "Unix timestamp in seconds")
+    val submittedAt: Long,
 ) : AccessionVersionInterface
 
 data class OriginalData<SequenceType>(

backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt

@@ -19,6 +19,7 @@ import org.loculus.backend.api.Accessions
 import org.loculus.backend.api.CompressionFormat
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DataUseTermsType
+import org.loculus.backend.api.EditedSequenceEntryData
 import org.loculus.backend.api.ExternalSubmittedData
 import org.loculus.backend.api.GetSequenceResponse
 import org.loculus.backend.api.Organism
@@ -284,8 +285,8 @@ class SubmissionController(
         @PathVariable @Valid
         organism: Organism,
         @HiddenParam authenticatedUser: AuthenticatedUser,
-        @RequestBody accessionVersion: UnprocessedData,
-    ) = submissionDatabaseService.submitEditedData(authenticatedUser, accessionVersion, organism)
+        @RequestBody editedSequenceEntryData: EditedSequenceEntryData,
+    ) = submissionDatabaseService.submitEditedData(authenticatedUser, editedSequenceEntryData, organism)
 
     @Operation(description = GET_SEQUENCES_DESCRIPTION)
     @GetMapping("/get-sequences", produces = [MediaType.APPLICATION_JSON_VALUE])

backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt

@@ -6,9 +6,7 @@ import com.fasterxml.jackson.databind.node.LongNode
 import com.fasterxml.jackson.databind.node.NullNode
 import com.fasterxml.jackson.databind.node.TextNode
 import kotlinx.datetime.Clock
-import kotlinx.datetime.LocalDateTime
 import kotlinx.datetime.TimeZone
-import kotlinx.datetime.toInstant
 import kotlinx.datetime.toLocalDateTime
 import mu.KotlinLogging
 import org.loculus.backend.api.DataUseTerms
@@ -21,6 +19,7 @@ import org.loculus.backend.service.submission.RawProcessedData
 import org.loculus.backend.service.submission.SubmissionDatabaseService
 import org.loculus.backend.utils.Accession
 import org.loculus.backend.utils.Version
+import org.loculus.backend.utils.toTimestamp
 import org.springframework.stereotype.Service
 import org.springframework.transaction.annotation.Transactional
 
@@ -121,11 +120,3 @@ class ReleasedDataModel(
     }
 }
 
-private fun LocalDateTime.toTimestamp() = this.toInstant(TimeZone.UTC).epochSeconds
-
-private fun LocalDateTime.toUtcDate(): String {
-    return this.toInstant(TimeZone.currentSystemDefault())
-        .toLocalDateTime(TimeZone.UTC)
-        .date
-        .toString()
-}

backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt

@@ -38,6 +38,7 @@ import org.loculus.backend.api.ApproveDataScope
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DataUseTermsType
 import org.loculus.backend.api.DeleteSequenceScope
+import org.loculus.backend.api.EditedSequenceEntryData
 import org.loculus.backend.api.ExternalSubmittedData
 import org.loculus.backend.api.GeneticSequence
 import org.loculus.backend.api.GetSequenceResponse
@@ -63,6 +64,7 @@ import org.loculus.backend.service.groupmanagement.GroupManagementDatabaseServic
 import org.loculus.backend.service.groupmanagement.GroupManagementPreconditionValidator
 import org.loculus.backend.utils.Accession
 import org.loculus.backend.utils.Version
+import org.loculus.backend.utils.toTimestamp
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Service
 import org.springframework.transaction.annotation.Transactional
@@ -126,7 +128,15 @@ class SubmissionDatabaseService(
         val preprocessing = SequenceEntriesPreprocessedDataTable
 
         return table
-            .select(table.accessionColumn, table.versionColumn, table.originalDataColumn)
+            .select(
+                table.accessionColumn,
+                table.versionColumn,
+                table.originalDataColumn,
+                table.submissionIdColumn,
+                table.submitterColumn,
+                table.groupIdColumn,
+                table.submittedAtColumn,
+            )
             .where {
                 table.organismIs(organism) and
                     not(table.isRevocationColumn) and
@@ -146,12 +156,16 @@ class SubmissionDatabaseService(
             .map { chunk ->
                 val chunkOfUnprocessedData = chunk.map {
                     UnprocessedData(
-                        it[table.accessionColumn],
-                        it[table.versionColumn],
-                        compressionService.decompressSequencesInOriginalData(
+                        accession = it[table.accessionColumn],
+                        version = it[table.versionColumn],
+                        data = compressionService.decompressSequencesInOriginalData(
                             it[table.originalDataColumn]!!,
                             organism,
                         ),
+                        submissionId = it[table.submissionIdColumn],
+                        submitter = it[table.submitterColumn],
+                        groupId = it[table.groupIdColumn],
+                        submittedAt = it[table.submittedAtColumn].toTimestamp(),
                     )
                 }
                 updateStatusToProcessing(chunkOfUnprocessedData, pipelineVersion)
@@ -782,35 +796,35 @@ class SubmissionDatabaseService(
 
     fun submitEditedData(
         authenticatedUser: AuthenticatedUser,
-        editedAccessionVersion: UnprocessedData,
+        editedSequenceEntryData: EditedSequenceEntryData,
         organism: Organism,
     ) {
-        log.info { "edited sequence entry submitted $editedAccessionVersion" }
+        log.info { "edited sequence entry submitted $editedSequenceEntryData" }
 
         accessionPreconditionValidator.validate {
-            thatAccessionVersionExists(editedAccessionVersion)
+            thatAccessionVersionExists(editedSequenceEntryData)
                 .andThatUserIsAllowedToEditSequenceEntries(authenticatedUser)
                 .andThatSequenceEntriesAreInStates(listOf(Status.AWAITING_APPROVAL, Status.HAS_ERRORS))
                 .andThatOrganismIs(organism)
         }
 
         SequenceEntriesTable.update(
             where = {
-                SequenceEntriesTable.accessionVersionIsIn(listOf(editedAccessionVersion))
+                SequenceEntriesTable.accessionVersionIsIn(listOf(editedSequenceEntryData))
             },
         ) {
             it[originalDataColumn] = compressionService
-                .compressSequencesInOriginalData(editedAccessionVersion.data, organism)
+                .compressSequencesInOriginalData(editedSequenceEntryData.data, organism)
         }
 
         SequenceEntriesPreprocessedDataTable.deleteWhere {
-            accessionVersionEquals(editedAccessionVersion)
+            accessionVersionEquals(editedSequenceEntryData)
         }
 
         auditLogger.log(
             authenticatedUser.username,
             "Edited sequence: " +
-                editedAccessionVersion.displayAccessionVersion(),
+                editedSequenceEntryData.displayAccessionVersion(),
         )
     }
 

backend/src/main/kotlin/org/loculus/backend/utils/LocalDateTimeExtensions.kt

@@ -0,0 +1,13 @@
+package org.loculus.backend.utils
+
+import kotlinx.datetime.LocalDateTime
+import kotlinx.datetime.TimeZone
+import kotlinx.datetime.toInstant
+import kotlinx.datetime.toLocalDateTime
+
+fun LocalDateTime.toTimestamp() = this.toInstant(TimeZone.UTC).epochSeconds
+
+fun LocalDateTime.toUtcDateString(): String = this.toInstant(TimeZone.currentSystemDefault())
+    .toLocalDateTime(TimeZone.UTC)
+    .date
+    .toString()

backend/src/test/kotlin/org/loculus/backend/controller/submission/ExtractUnprocessedDataEndpointTest.kt

@@ -4,16 +4,20 @@ import org.hamcrest.CoreMatchers.containsString
 import org.hamcrest.CoreMatchers.hasItem
 import org.hamcrest.CoreMatchers.`is`
 import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.allOf
 import org.hamcrest.Matchers.containsInAnyOrder
 import org.hamcrest.Matchers.empty
+import org.hamcrest.Matchers.greaterThan
 import org.hamcrest.Matchers.hasProperty
 import org.hamcrest.Matchers.hasSize
+import org.hamcrest.Matchers.matchesRegex
 import org.junit.jupiter.api.Test
 import org.loculus.backend.api.Status.IN_PROCESSING
 import org.loculus.backend.api.Status.RECEIVED
 import org.loculus.backend.api.UnprocessedData
 import org.loculus.backend.config.BackendSpringProperty
 import org.loculus.backend.controller.DEFAULT_ORGANISM
+import org.loculus.backend.controller.DEFAULT_USER_NAME
 import org.loculus.backend.controller.EndpointTest
 import org.loculus.backend.controller.OTHER_ORGANISM
 import org.loculus.backend.controller.assertStatusIs
@@ -65,15 +69,24 @@ class ExtractUnprocessedDataEndpointTest(
 
     @Test
     fun `WHEN extracting unprocessed data THEN only previously not extracted sequence entries are returned`() {
-        val accessionVersions = convenienceClient.submitDefaultFiles().submissionIdMappings
+        val submissionResult = convenienceClient.submitDefaultFiles()
+        val accessionVersions = submissionResult.submissionIdMappings
 
         val result7 = client.extractUnprocessedData(7)
         val responseBody7 = result7.expectNdjsonAndGetContent<UnprocessedData>()
         assertThat(responseBody7, hasSize(7))
         assertThat(
             responseBody7,
             hasItem(
-                UnprocessedData(accessionVersions.first().accession, 1, defaultOriginalData),
+                allOf(
+                    hasProperty<UnprocessedData>("accession", `is`(accessionVersions[0].accession)),
+                    hasProperty("version", `is`(1L)),
+                    hasProperty("data", `is`(defaultOriginalData)),
+                    hasProperty("submissionId", matchesRegex("custom[0-9]")),
+                    hasProperty("submitter", `is`(DEFAULT_USER_NAME)),
+                    hasProperty("groupId", `is`(submissionResult.groupId)),
+                    hasProperty("submittedAt", greaterThan(1_700_000_000L)),
+                ),
             ),
         )
 

backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt

@@ -3,6 +3,8 @@ package org.loculus.backend.controller.submission
 import org.hamcrest.CoreMatchers.containsString
 import org.hamcrest.CoreMatchers.hasItem
 import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.allOf
+import org.hamcrest.Matchers.hasProperty
 import org.hamcrest.Matchers.hasSize
 import org.hamcrest.Matchers.`is`
 import org.junit.jupiter.api.Test
@@ -102,10 +104,9 @@ class ReviseEndpointTest(
         assertThat(
             responseBody,
             hasItem(
-                UnprocessedData(
-                    accession = accessions.first(),
-                    version = 2,
-                    data = defaultOriginalData,
+                allOf(
+                    hasProperty<UnprocessedData>("accession", `is`(accessions.first())),
+                    hasProperty("version", `is`(2L)),
                 ),
             ),
         )

backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt

@@ -6,10 +6,10 @@ import org.loculus.backend.api.AccessionVersionInterface
 import org.loculus.backend.api.ApproveDataScope
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DeleteSequenceScope
+import org.loculus.backend.api.EditedSequenceEntryData
 import org.loculus.backend.api.ExternalSubmittedData
 import org.loculus.backend.api.Status
 import org.loculus.backend.api.SubmittedProcessedData
-import org.loculus.backend.api.UnprocessedData
 import org.loculus.backend.api.WarningsFilter
 import org.loculus.backend.controller.DEFAULT_EXTERNAL_METADATA_UPDATER
 import org.loculus.backend.controller.DEFAULT_GROUP_NAME
@@ -147,7 +147,7 @@ class SubmissionControllerClient(private val mockMvc: MockMvc, private val objec
     )
 
     fun submitEditedSequenceEntryVersion(
-        editedData: UnprocessedData,
+        editedData: EditedSequenceEntryData,
         organism: String = DEFAULT_ORGANISM,
         jwt: String? = jwtForDefaultUser,
     ): ResultActions = mockMvc.perform(

backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt

@@ -9,6 +9,7 @@ import org.loculus.backend.api.AccessionVersionInterface
 import org.loculus.backend.api.AccessionVersionOriginalMetadata
 import org.loculus.backend.api.ApproveDataScope
 import org.loculus.backend.api.DataUseTerms
+import org.loculus.backend.api.EditedSequenceEntryData
 import org.loculus.backend.api.GeneticSequence
 import org.loculus.backend.api.GetSequenceResponse
 import org.loculus.backend.api.Organism
@@ -284,7 +285,7 @@ class SubmissionConvenienceClient(
     fun submitDefaultEditedData(accessions: List<Accession>, userName: String = DEFAULT_USER_NAME) {
         accessions.forEach { accession ->
             client.submitEditedSequenceEntryVersion(
-                UnprocessedData(accession, 1L, defaultOriginalData),
+                EditedSequenceEntryData(accession, 1L, defaultOriginalData),
                 jwt = generateJwtFor(userName),
             )
         }