feat: add Loculus internal metadata to data that is sent to preproces…

…sing pipeline #2263

There are more Loculus internal metadata fields, but they are not filled with data at this point of the submission process

backend/src/main/kotlin/org/loculus/backend/api/SubmissionTypes.kt

@@ -213,10 +213,24 @@ data class SequenceEntryStatus(
     val dataUseTerms: DataUseTerms,
 ) : AccessionVersionInterface
 
+data class EditedSequenceEntryData(
+    @Schema(example = "LOC_000S01D") override val accession: Accession,
+    @Schema(example = "1") override val version: Version,
+    val data: OriginalData<GeneticSequence>,
+) : AccessionVersionInterface
+
 data class UnprocessedData(
-    @Schema(example = "123") override val accession: Accession,
+    @Schema(example = "LOC_000S01D") override val accession: Accession,
     @Schema(example = "1") override val version: Version,
     val data: OriginalData<GeneticSequence>,
+    @Schema(description = "The submission id that was used in the upload to link metadata and sequences")
+    val submissionId: String,
+    @Schema(description = "The username of the submitter")
+    val submitter: String,
+    @Schema(example = "42", description = "The id of the group that this sequence entry was submitted by")
+    val groupId: Int,
+    @Schema(example = "1720304713", description = "Unix timestamp in seconds")
+    val submittedAt: Long,
 ) : AccessionVersionInterface
 
 data class OriginalData<SequenceType>(

backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt

@@ -19,6 +19,7 @@ import org.loculus.backend.api.Accessions
 import org.loculus.backend.api.CompressionFormat
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DataUseTermsType
+import org.loculus.backend.api.EditedSequenceEntryData
 import org.loculus.backend.api.ExternalSubmittedData
 import org.loculus.backend.api.GetSequenceResponse
 import org.loculus.backend.api.Organism
@@ -284,8 +285,8 @@ class SubmissionController(
         @PathVariable @Valid
         organism: Organism,
         @HiddenParam authenticatedUser: AuthenticatedUser,
-        @RequestBody accessionVersion: UnprocessedData,
-    ) = submissionDatabaseService.submitEditedData(authenticatedUser, accessionVersion, organism)
+        @RequestBody editedSequenceEntryData: EditedSequenceEntryData,
+    ) = submissionDatabaseService.submitEditedData(authenticatedUser, editedSequenceEntryData, organism)
 
     @Operation(description = GET_SEQUENCES_DESCRIPTION)
     @GetMapping("/get-sequences", produces = [MediaType.APPLICATION_JSON_VALUE])

backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt

@@ -6,9 +6,7 @@ import com.fasterxml.jackson.databind.node.LongNode
 import com.fasterxml.jackson.databind.node.NullNode
 import com.fasterxml.jackson.databind.node.TextNode
 import kotlinx.datetime.Clock
-import kotlinx.datetime.LocalDateTime
 import kotlinx.datetime.TimeZone
-import kotlinx.datetime.toInstant
 import kotlinx.datetime.toLocalDateTime
 import mu.KotlinLogging
 import org.loculus.backend.api.DataUseTerms
@@ -21,6 +19,7 @@ import org.loculus.backend.service.submission.RawProcessedData
 import org.loculus.backend.service.submission.SubmissionDatabaseService
 import org.loculus.backend.utils.Accession
 import org.loculus.backend.utils.Version
+import org.loculus.backend.utils.toTimestamp
 import org.springframework.stereotype.Service
 import org.springframework.transaction.annotation.Transactional
 
@@ -118,5 +117,3 @@ class ReleasedDataModel(
         return SiloVersionStatus.REVISED
     }
 }
-
-private fun LocalDateTime.toTimestamp() = this.toInstant(TimeZone.UTC).epochSeconds

backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt

@@ -38,6 +38,7 @@ import org.loculus.backend.api.ApproveDataScope
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DataUseTermsType
 import org.loculus.backend.api.DeleteSequenceScope
+import org.loculus.backend.api.EditedSequenceEntryData
 import org.loculus.backend.api.ExternalSubmittedData
 import org.loculus.backend.api.GeneticSequence
 import org.loculus.backend.api.GetSequenceResponse
@@ -63,6 +64,7 @@ import org.loculus.backend.service.groupmanagement.GroupManagementDatabaseServic
 import org.loculus.backend.service.groupmanagement.GroupManagementPreconditionValidator
 import org.loculus.backend.utils.Accession
 import org.loculus.backend.utils.Version
+import org.loculus.backend.utils.toTimestamp
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.stereotype.Service
 import org.springframework.transaction.annotation.Transactional
@@ -126,7 +128,15 @@ class SubmissionDatabaseService(
         val preprocessing = SequenceEntriesPreprocessedDataTable
 
         return table
-            .select(table.accessionColumn, table.versionColumn, table.originalDataColumn)
+            .select(
+                table.accessionColumn,
+                table.versionColumn,
+                table.originalDataColumn,
+                table.submissionIdColumn,
+                table.submitterColumn,
+                table.groupIdColumn,
+                table.submittedAtColumn,
+            )
             .where {
                 table.organismIs(organism) and
                     not(table.isRevocationColumn) and
@@ -146,12 +156,16 @@ class SubmissionDatabaseService(
             .map { chunk ->
                 val chunkOfUnprocessedData = chunk.map {
                     UnprocessedData(
-                        it[table.accessionColumn],
-                        it[table.versionColumn],
-                        compressionService.decompressSequencesInOriginalData(
+                        accession = it[table.accessionColumn],
+                        version = it[table.versionColumn],
+                        data = compressionService.decompressSequencesInOriginalData(
                             it[table.originalDataColumn]!!,
                             organism,
                         ),
+                        submissionId = it[table.submissionIdColumn],
+                        submitter = it[table.submitterColumn],
+                        groupId = it[table.groupIdColumn],
+                        submittedAt = it[table.submittedAtColumn].toTimestamp(),
                     )
                 }
                 updateStatusToProcessing(chunkOfUnprocessedData, pipelineVersion)
@@ -782,35 +796,35 @@ class SubmissionDatabaseService(
 
     fun submitEditedData(
         authenticatedUser: AuthenticatedUser,
-        editedAccessionVersion: UnprocessedData,
+        editedSequenceEntryData: EditedSequenceEntryData,
         organism: Organism,
     ) {
-        log.info { "edited sequence entry submitted $editedAccessionVersion" }
+        log.info { "edited sequence entry submitted $editedSequenceEntryData" }
 
         accessionPreconditionValidator.validate {
-            thatAccessionVersionExists(editedAccessionVersion)
+            thatAccessionVersionExists(editedSequenceEntryData)
                 .andThatUserIsAllowedToEditSequenceEntries(authenticatedUser)
                 .andThatSequenceEntriesAreInStates(listOf(Status.AWAITING_APPROVAL, Status.HAS_ERRORS))
                 .andThatOrganismIs(organism)
         }
 
         SequenceEntriesTable.update(
             where = {
-                SequenceEntriesTable.accessionVersionIsIn(listOf(editedAccessionVersion))
+                SequenceEntriesTable.accessionVersionIsIn(listOf(editedSequenceEntryData))
             },
         ) {
             it[originalDataColumn] = compressionService
-                .compressSequencesInOriginalData(editedAccessionVersion.data, organism)
+                .compressSequencesInOriginalData(editedSequenceEntryData.data, organism)
         }
 
         SequenceEntriesPreprocessedDataTable.deleteWhere {
-            accessionVersionEquals(editedAccessionVersion)
+            accessionVersionEquals(editedSequenceEntryData)
         }
 
         auditLogger.log(
             authenticatedUser.username,
             "Edited sequence: " +
-                editedAccessionVersion.displayAccessionVersion(),
+                editedSequenceEntryData.displayAccessionVersion(),
         )
     }
 

backend/src/main/kotlin/org/loculus/backend/utils/LocalDateTimeExtensions.kt

@@ -0,0 +1,13 @@
+package org.loculus.backend.utils
+
+import kotlinx.datetime.LocalDateTime
+import kotlinx.datetime.TimeZone
+import kotlinx.datetime.toInstant
+import kotlinx.datetime.toLocalDateTime
+
+fun LocalDateTime.toTimestamp() = this.toInstant(TimeZone.UTC).epochSeconds
+
+fun LocalDateTime.toUtcDateString(): String = this.toInstant(TimeZone.currentSystemDefault())
+    .toLocalDateTime(TimeZone.UTC)
+    .date
+    .toString()

backend/src/test/kotlin/org/loculus/backend/controller/submission/ExtractUnprocessedDataEndpointTest.kt

@@ -4,16 +4,20 @@ import org.hamcrest.CoreMatchers.containsString
 import org.hamcrest.CoreMatchers.hasItem
 import org.hamcrest.CoreMatchers.`is`
 import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.allOf
 import org.hamcrest.Matchers.containsInAnyOrder
 import org.hamcrest.Matchers.empty
+import org.hamcrest.Matchers.greaterThan
 import org.hamcrest.Matchers.hasProperty
 import org.hamcrest.Matchers.hasSize
+import org.hamcrest.Matchers.matchesRegex
 import org.junit.jupiter.api.Test
 import org.loculus.backend.api.Status.IN_PROCESSING
 import org.loculus.backend.api.Status.RECEIVED
 import org.loculus.backend.api.UnprocessedData
 import org.loculus.backend.config.BackendSpringProperty
 import org.loculus.backend.controller.DEFAULT_ORGANISM
+import org.loculus.backend.controller.DEFAULT_USER_NAME
 import org.loculus.backend.controller.EndpointTest
 import org.loculus.backend.controller.OTHER_ORGANISM
 import org.loculus.backend.controller.assertStatusIs
@@ -65,15 +69,24 @@ class ExtractUnprocessedDataEndpointTest(
 
     @Test
     fun `WHEN extracting unprocessed data THEN only previously not extracted sequence entries are returned`() {
-        val accessionVersions = convenienceClient.submitDefaultFiles().submissionIdMappings
+        val submissionResult = convenienceClient.submitDefaultFiles()
+        val accessionVersions = submissionResult.submissionIdMappings
 
         val result7 = client.extractUnprocessedData(7)
         val responseBody7 = result7.expectNdjsonAndGetContent<UnprocessedData>()
         assertThat(responseBody7, hasSize(7))
         assertThat(
             responseBody7,
             hasItem(
-                UnprocessedData(accessionVersions.first().accession, 1, defaultOriginalData),
+                allOf(
+                    hasProperty<UnprocessedData>("accession", `is`(accessionVersions[0].accession)),
+                    hasProperty("version", `is`(1L)),
+                    hasProperty("data", `is`(defaultOriginalData)),
+                    hasProperty("submissionId", matchesRegex("custom[0-9]")),
+                    hasProperty("submitter", `is`(DEFAULT_USER_NAME)),
+                    hasProperty("groupId", `is`(submissionResult.groupId)),
+                    hasProperty("submittedAt", greaterThan(1_700_000_000L)),
+                )
             ),
         )
 

backend/src/test/kotlin/org/loculus/backend/controller/submission/ReviseEndpointTest.kt

@@ -3,6 +3,8 @@ package org.loculus.backend.controller.submission
 import org.hamcrest.CoreMatchers.containsString
 import org.hamcrest.CoreMatchers.hasItem
 import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.allOf
+import org.hamcrest.Matchers.hasProperty
 import org.hamcrest.Matchers.hasSize
 import org.hamcrest.Matchers.`is`
 import org.junit.jupiter.api.Test
@@ -102,10 +104,9 @@ class ReviseEndpointTest(
         assertThat(
             responseBody,
             hasItem(
-                UnprocessedData(
-                    accession = accessions.first(),
-                    version = 2,
-                    data = defaultOriginalData,
+                allOf(
+                    hasProperty<UnprocessedData>("accession", `is`(accessions.first())),
+                    hasProperty("version", `is`(2L)),
                 ),
             ),
         )

backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionControllerClient.kt

@@ -6,6 +6,7 @@ import org.loculus.backend.api.AccessionVersionInterface
 import org.loculus.backend.api.ApproveDataScope
 import org.loculus.backend.api.DataUseTerms
 import org.loculus.backend.api.DeleteSequenceScope
+import org.loculus.backend.api.EditedSequenceEntryData
 import org.loculus.backend.api.ExternalSubmittedData
 import org.loculus.backend.api.Status
 import org.loculus.backend.api.SubmittedProcessedData
@@ -147,7 +148,7 @@ class SubmissionControllerClient(private val mockMvc: MockMvc, private val objec
     )
 
     fun submitEditedSequenceEntryVersion(
-        editedData: UnprocessedData,
+        editedData: EditedSequenceEntryData,
         organism: String = DEFAULT_ORGANISM,
         jwt: String? = jwtForDefaultUser,
     ): ResultActions = mockMvc.perform(

backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt

@@ -9,6 +9,7 @@ import org.loculus.backend.api.AccessionVersionInterface
 import org.loculus.backend.api.AccessionVersionOriginalMetadata
 import org.loculus.backend.api.ApproveDataScope
 import org.loculus.backend.api.DataUseTerms
+import org.loculus.backend.api.EditedSequenceEntryData
 import org.loculus.backend.api.GeneticSequence
 import org.loculus.backend.api.GetSequenceResponse
 import org.loculus.backend.api.Organism
@@ -284,7 +285,7 @@ class SubmissionConvenienceClient(
     fun submitDefaultEditedData(accessions: List<Accession>, userName: String = DEFAULT_USER_NAME) {
         accessions.forEach { accession ->
             client.submitEditedSequenceEntryVersion(
-                UnprocessedData(accession, 1L, defaultOriginalData),
+                EditedSequenceEntryData(accession, 1L, defaultOriginalData),
                 jwt = generateJwtFor(userName),
             )
         }

backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEditedSequenceEntryVersionEndpointTest.kt

@@ -6,8 +6,8 @@ import org.hamcrest.Matchers.containsString
 import org.hamcrest.Matchers.`is`
 import org.hamcrest.Matchers.not
 import org.junit.jupiter.api.Test
+import org.loculus.backend.api.EditedSequenceEntryData
 import org.loculus.backend.api.Status
-import org.loculus.backend.api.UnprocessedData
 import org.loculus.backend.controller.DEFAULT_USER_NAME
 import org.loculus.backend.controller.EndpointTest
 import org.loculus.backend.controller.OTHER_ORGANISM
@@ -29,7 +29,7 @@ class SubmitEditedSequenceEntryVersionEndpointTest(
     fun `GIVEN invalid authorization token THEN returns 401 Unauthorized`() {
         expectUnauthorizedResponse(isModifyingRequest = true) {
             client.submitEditedSequenceEntryVersion(
-                generateUnprocessedData("1"),
+                generateEditedData("1"),
                 jwt = it,
             )
         }
@@ -42,7 +42,7 @@ class SubmitEditedSequenceEntryVersionEndpointTest(
         convenienceClient.getSequenceEntry(accession = accessions.first(), version = 1)
             .assertStatusIs(Status.HAS_ERRORS)
 
-        val editedData = generateUnprocessedData(accessions.first())
+        val editedData = generateEditedData(accessions.first())
         client.submitEditedSequenceEntryVersion(editedData)
             .andExpect(status().isNoContent)
 
@@ -57,7 +57,7 @@ class SubmitEditedSequenceEntryVersionEndpointTest(
         convenienceClient.getSequenceEntry(accession = accessions.first(), version = 1)
             .assertStatusIs(Status.AWAITING_APPROVAL)
 
-        val editedData = generateUnprocessedData(accessions.first())
+        val editedData = generateEditedData(accessions.first())
 
         client.submitEditedSequenceEntryVersion(editedData)
             .andExpect(status().isNoContent)
@@ -76,7 +76,7 @@ class SubmitEditedSequenceEntryVersionEndpointTest(
             .find { it.accession == firstAccession && it.version == 1L }!!
         assertThat(entryBeforeEdit.originalMetadata, `is`(not(anEmptyMap())))
 
-        val editedData = generateUnprocessedData(firstAccession)
+        val editedData = generateEditedData(firstAccession)
 
         client.submitEditedSequenceEntryVersion(editedData)
             .andExpect(status().isNoContent)
@@ -93,7 +93,7 @@ class SubmitEditedSequenceEntryVersionEndpointTest(
         convenienceClient.getSequenceEntry(accession = accessions.first(), version = 1)
             .assertStatusIs(Status.HAS_ERRORS)
 
-        val editedDataWithNonExistingVersion = generateUnprocessedData(accessions.first(), version = 2)
+        val editedDataWithNonExistingVersion = generateEditedData(accessions.first(), version = 2)
         val sequenceString = editedDataWithNonExistingVersion.displayAccessionVersion()
 
         client.submitEditedSequenceEntryVersion(editedDataWithNonExistingVersion)
@@ -113,7 +113,7 @@ class SubmitEditedSequenceEntryVersionEndpointTest(
 
         val nonExistingAccession = "nonExistingAccession"
 
-        val editedDataWithNonExistingAccession = generateUnprocessedData(nonExistingAccession)
+        val editedDataWithNonExistingAccession = generateEditedData(nonExistingAccession)
 
         client.submitEditedSequenceEntryVersion(editedDataWithNonExistingAccession)
             .andExpect(status().isUnprocessableEntity)
@@ -134,7 +134,7 @@ class SubmitEditedSequenceEntryVersionEndpointTest(
         convenienceClient.getSequenceEntry(accession = accessions.first(), version = 1)
             .assertStatusIs(Status.HAS_ERRORS)
 
-        val editedData = generateUnprocessedData(accessions.first())
+        val editedData = generateEditedData(accessions.first())
 
         client.submitEditedSequenceEntryVersion(editedData, organism = OTHER_ORGANISM)
             .andExpect(status().isUnprocessableEntity)
@@ -156,7 +156,7 @@ class SubmitEditedSequenceEntryVersionEndpointTest(
         convenienceClient.getSequenceEntry(accession = accessions.first(), version = 1)
             .assertStatusIs(Status.HAS_ERRORS)
 
-        val editedDataFromWrongSubmitter = generateUnprocessedData(accessions.first())
+        val editedDataFromWrongSubmitter = generateEditedData(accessions.first())
         val nonExistingUser = "whoseNameMayNotBeMentioned"
 
         client.submitEditedSequenceEntryVersion(editedDataFromWrongSubmitter, jwt = generateJwtFor(nonExistingUser))
@@ -175,15 +175,15 @@ class SubmitEditedSequenceEntryVersionEndpointTest(
             .prepareDataTo(Status.HAS_ERRORS, username = DEFAULT_USER_NAME)
             .first()
 
-        val editedData = generateUnprocessedData(accessionVersion.accession, accessionVersion.version)
+        val editedData = generateEditedData(accessionVersion.accession, accessionVersion.version)
         client.submitEditedSequenceEntryVersion(editedData, jwt = jwtForSuperUser)
             .andExpect(status().isNoContent)
 
         convenienceClient.getSequenceEntry(accession = accessionVersion.accession, version = accessionVersion.version)
             .assertStatusIs(Status.RECEIVED)
     }
 
-    private fun generateUnprocessedData(accession: String, version: Long = 1) = UnprocessedData(
+    private fun generateEditedData(accession: String, version: Long = 1) = EditedSequenceEntryData(
         accession = accession,
         version = version,
         data = emptyOriginalData,