From d117cf12ac516f83412bf5394efbc686a7e67879 Mon Sep 17 00:00:00 2001 From: Fabian Engelniederhammer Date: Mon, 7 Oct 2024 12:48:05 +0200 Subject: [PATCH 1/3] fix(backend): return open data use terms url in `get-released-data` when data is open resolves #2953 --- .../backend/config/BackendSpringConfig.kt | 14 +- .../backend/model/ReleasedDataModel.kt | 14 +- .../org/loculus/backend/model/SubmitModel.kt | 7 +- .../DataUseTermsDatabaseService.kt | 7 +- .../DataUseTermsPreconditionValidator.kt | 8 +- .../debug/DeleteSequenceDataService.kt | 8 +- .../SeqSetCitationsDatabaseService.kt | 18 +- .../submission/SubmissionDatabaseService.kt | 36 ++-- .../org/loculus/backend/utils/DateProvider.kt | 15 ++ .../loculus/backend/controller/TestHelpers.kt | 7 + .../DataUseTermsControllerTest.kt | 8 +- .../submission/GetReleasedDataEndpointTest.kt | 167 ++++++++++++++++++ .../submission/SubmissionConvenienceClient.kt | 29 ++- 13 files changed, 262 insertions(+), 76 deletions(-) create mode 100644 backend/src/main/kotlin/org/loculus/backend/utils/DateProvider.kt diff --git a/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt b/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt index 2e7824031..0071c904c 100644 --- a/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt +++ b/backend/src/main/kotlin/org/loculus/backend/config/BackendSpringConfig.kt @@ -72,13 +72,15 @@ class BackendSpringConfig { fun backendConfig( objectMapper: ObjectMapper, @Value("\${${BackendSpringProperty.BACKEND_CONFIG_PATH}}") configPath: String, - ): BackendConfig { - val config = objectMapper.readValue(File(configPath)) - logger.info { "Loaded backend config from $configPath" } - logger.info { "Config: $config" } - return objectMapper.readValue(File(configPath)) - } + ): BackendConfig = readBackendConfig(objectMapper, configPath) @Bean fun openApi(backendConfig: BackendConfig) = buildOpenApiSchema(backendConfig) } + +fun readBackendConfig(objectMapper: ObjectMapper, configPath: String): BackendConfig { + val config = objectMapper.readValue(File(configPath)) + logger.info { "Loaded backend config from $configPath" } + logger.info { "Config: $config" } + return objectMapper.readValue(File(configPath)) +} diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index 59f05a39b..751ec493d 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -5,9 +5,6 @@ import com.fasterxml.jackson.databind.node.IntNode import com.fasterxml.jackson.databind.node.LongNode import com.fasterxml.jackson.databind.node.NullNode import com.fasterxml.jackson.databind.node.TextNode -import kotlinx.datetime.Clock -import kotlinx.datetime.TimeZone -import kotlinx.datetime.toLocalDateTime import mu.KotlinLogging import org.loculus.backend.api.DataUseTerms import org.loculus.backend.api.GeneticSequence @@ -19,6 +16,7 @@ import org.loculus.backend.service.submission.RawProcessedData import org.loculus.backend.service.submission.SubmissionDatabaseService import org.loculus.backend.service.submission.UpdateTrackerTable import org.loculus.backend.utils.Accession +import org.loculus.backend.utils.DateProvider import org.loculus.backend.utils.Version import org.loculus.backend.utils.toTimestamp import org.loculus.backend.utils.toUtcDateString @@ -41,6 +39,7 @@ val RELEASED_DATA_RELATED_TABLES: List = open class ReleasedDataModel( private val submissionDatabaseService: SubmissionDatabaseService, private val backendConfig: BackendConfig, + private val dateProvider: DateProvider, ) { @Transactional(readOnly = true) open fun getReleasedData(organism: Organism): Sequence> { @@ -103,10 +102,9 @@ open class ReleasedDataModel( ("versionComment" to TextNode(rawProcessedData.versionComment)) if (backendConfig.dataUseTermsUrls != null) { - val url = if (rawProcessedData.dataUseTerms == DataUseTerms.Open) { - backendConfig.dataUseTermsUrls.open - } else { - backendConfig.dataUseTermsUrls.restricted + val url = when (currentDataUseTerms) { + DataUseTerms.Open -> backendConfig.dataUseTermsUrls.open + is DataUseTerms.Restricted -> backendConfig.dataUseTermsUrls.restricted } metadata += ("dataUseTermsUrl" to TextNode(url)) } @@ -123,7 +121,7 @@ open class ReleasedDataModel( private fun computeDataUseTerm(rawProcessedData: RawProcessedData): DataUseTerms = if ( rawProcessedData.dataUseTerms is DataUseTerms.Restricted && - rawProcessedData.dataUseTerms.restrictedUntil > Clock.System.now().toLocalDateTime(TimeZone.UTC).date + rawProcessedData.dataUseTerms.restrictedUntil > dateProvider.getCurrentDate() ) { DataUseTerms.Restricted(rawProcessedData.dataUseTerms.restrictedUntil) } else { diff --git a/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt index 8b2105896..3c17f0157 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/SubmitModel.kt @@ -1,8 +1,5 @@ package org.loculus.backend.model -import kotlinx.datetime.Clock -import kotlinx.datetime.TimeZone -import kotlinx.datetime.toLocalDateTime import mu.KotlinLogging import org.apache.commons.compress.archivers.zip.ZipFile import org.apache.commons.compress.compressors.CompressorStreamFactory @@ -20,6 +17,7 @@ import org.loculus.backend.service.submission.CompressionAlgorithm import org.loculus.backend.service.submission.MetadataUploadAuxTable import org.loculus.backend.service.submission.SequenceUploadAuxTable import org.loculus.backend.service.submission.UploadDatabaseService +import org.loculus.backend.utils.DateProvider import org.loculus.backend.utils.FastaReader import org.loculus.backend.utils.metadataEntryStreamAsSequence import org.loculus.backend.utils.revisionEntryStreamAsSequence @@ -77,6 +75,7 @@ class SubmitModel( private val uploadDatabaseService: UploadDatabaseService, private val groupManagementPreconditionValidator: GroupManagementPreconditionValidator, private val dataUseTermsPreconditionValidator: DataUseTermsPreconditionValidator, + private val dateProvider: DateProvider, ) { companion object AcceptedFileTypes { @@ -212,7 +211,7 @@ class SubmitModel( "intermediate storing uploaded metadata of type ${submissionParams.uploadType.name} " + "from $submissionParams.submitter with UploadId $uploadId" } - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) + val now = dateProvider.getCurrentDateTime() try { when (submissionParams) { is SubmissionParams.OriginalSubmissionParams -> { diff --git a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsDatabaseService.kt index 54acad99c..124c135c9 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsDatabaseService.kt @@ -1,8 +1,5 @@ package org.loculus.backend.service.datauseterms -import kotlinx.datetime.Clock -import kotlinx.datetime.TimeZone -import kotlinx.datetime.toLocalDateTime import org.jetbrains.exposed.sql.batchInsert import org.jetbrains.exposed.sql.selectAll import org.loculus.backend.api.DataUseTerms @@ -13,6 +10,7 @@ import org.loculus.backend.controller.NotFoundException import org.loculus.backend.log.AuditLogger import org.loculus.backend.service.submission.AccessionPreconditionValidator import org.loculus.backend.utils.Accession +import org.loculus.backend.utils.DateProvider import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional @@ -22,6 +20,7 @@ class DataUseTermsDatabaseService( private val accessionPreconditionValidator: AccessionPreconditionValidator, private val dataUseTermsPreconditionValidator: DataUseTermsPreconditionValidator, private val auditLogger: AuditLogger, + private val dateProvider: DateProvider, ) { fun setNewDataUseTerms( @@ -29,7 +28,7 @@ class DataUseTermsDatabaseService( accessions: List, newDataUseTerms: DataUseTerms, ) { - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) + val now = dateProvider.getCurrentDateTime() accessionPreconditionValidator.validate { thatAccessionsExist(accessions) diff --git a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsPreconditionValidator.kt b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsPreconditionValidator.kt index 58c7f136c..7bd753ed3 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsPreconditionValidator.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/datauseterms/DataUseTermsPreconditionValidator.kt @@ -1,10 +1,7 @@ package org.loculus.backend.service.datauseterms -import kotlinx.datetime.Clock import kotlinx.datetime.DateTimeUnit -import kotlinx.datetime.TimeZone import kotlinx.datetime.plus -import kotlinx.datetime.toLocalDateTime import mu.KotlinLogging import org.jetbrains.exposed.sql.and import org.loculus.backend.api.DataUseTerms @@ -12,12 +9,13 @@ import org.loculus.backend.api.DataUseTermsType import org.loculus.backend.controller.BadRequestException import org.loculus.backend.controller.UnprocessableEntityException import org.loculus.backend.utils.Accession +import org.loculus.backend.utils.DateProvider import org.springframework.stereotype.Component private val logger = KotlinLogging.logger { } @Component -class DataUseTermsPreconditionValidator { +class DataUseTermsPreconditionValidator(private val dateProvider: DateProvider) { fun checkThatTransitionIsAllowed(accessions: List, newDataUseTerms: DataUseTerms) { val dataUseTerms = DataUseTermsTable @@ -54,7 +52,7 @@ class DataUseTermsPreconditionValidator { fun checkThatRestrictedUntilIsAllowed(dataUseTerms: DataUseTerms) { if (dataUseTerms is DataUseTerms.Restricted) { - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC).date + val now = dateProvider.getCurrentDate() val oneYearFromNow = now.plus(1, DateTimeUnit.YEAR) if (dataUseTerms.restrictedUntil < now) { diff --git a/backend/src/main/kotlin/org/loculus/backend/service/debug/DeleteSequenceDataService.kt b/backend/src/main/kotlin/org/loculus/backend/service/debug/DeleteSequenceDataService.kt index 05549c189..24dbee382 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/debug/DeleteSequenceDataService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/debug/DeleteSequenceDataService.kt @@ -1,8 +1,5 @@ package org.loculus.backend.service.debug -import kotlinx.datetime.Clock -import kotlinx.datetime.TimeZone -import kotlinx.datetime.toLocalDateTime import org.jetbrains.exposed.sql.deleteAll import org.jetbrains.exposed.sql.insert import org.loculus.backend.service.datauseterms.DataUseTermsTable @@ -11,11 +8,12 @@ import org.loculus.backend.service.submission.MetadataUploadAuxTable import org.loculus.backend.service.submission.SequenceEntriesPreprocessedDataTable import org.loculus.backend.service.submission.SequenceEntriesTable import org.loculus.backend.service.submission.SequenceUploadAuxTable +import org.loculus.backend.utils.DateProvider import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional @Component -class DeleteSequenceDataService { +class DeleteSequenceDataService(private val dateProvider: DateProvider) { @Transactional fun deleteAllSequenceData() { SequenceEntriesTable.deleteAll() @@ -26,7 +24,7 @@ class DeleteSequenceDataService { CurrentProcessingPipelineTable.deleteAll() CurrentProcessingPipelineTable.insert { it[versionColumn] = 1 - it[startedUsingAtColumn] = Clock.System.now().toLocalDateTime(TimeZone.UTC) + it[startedUsingAtColumn] = dateProvider.getCurrentDateTime() } } } diff --git a/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt index e52259ff0..ef27dfe44 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt @@ -1,8 +1,8 @@ package org.loculus.backend.service.seqsetcitations -import kotlinx.datetime.Clock -import kotlinx.datetime.LocalDateTime +import kotlinx.datetime.DateTimeUnit import kotlinx.datetime.TimeZone +import kotlinx.datetime.minus import kotlinx.datetime.toJavaLocalDateTime import kotlinx.datetime.toLocalDateTime import mu.KotlinLogging @@ -35,6 +35,7 @@ import org.loculus.backend.controller.UnprocessableEntityException import org.loculus.backend.service.crossref.CrossRefService import org.loculus.backend.service.crossref.DoiEntry import org.loculus.backend.service.submission.AccessionPreconditionValidator +import org.loculus.backend.utils.DateProvider import org.loculus.backend.utils.getNextSequenceNumber import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional @@ -50,6 +51,7 @@ class SeqSetCitationsDatabaseService( private val accessionPreconditionValidator: AccessionPreconditionValidator, private val backendConfig: BackendConfig, private val crossRefService: CrossRefService, + private val dateProvider: DateProvider, pool: DataSource, ) { init { @@ -69,8 +71,6 @@ class SeqSetCitationsDatabaseService( validateSeqSetName(seqSetName) validateSeqSetRecords(seqSetRecords) - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) - val seqsetIdNumber = getNextSequenceNumber("seqset_id_sequence") val insertedSet = SeqSetsTable .insert { @@ -78,7 +78,7 @@ class SeqSetCitationsDatabaseService( it[name] = seqSetName it[description] = seqSetDescription ?: "" it[seqSetVersion] = 1 - it[createdAt] = now + it[createdAt] = dateProvider.getCurrentDateTime() it[createdBy] = authenticatedUser.username } @@ -116,8 +116,6 @@ class SeqSetCitationsDatabaseService( validateSeqSetName(seqSetName) validateSeqSetRecords(seqSetRecords) - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) - val maxVersion = SeqSetsTable .select(SeqSetsTable.seqSetVersion.max()) .where { SeqSetsTable.seqSetId eq seqSetId and (SeqSetsTable.createdBy eq username) } @@ -144,7 +142,7 @@ class SeqSetCitationsDatabaseService( it[SeqSetsTable.name] = seqSetName it[SeqSetsTable.description] = seqSetDescription ?: "" it[SeqSetsTable.seqSetVersion] = newVersion - it[SeqSetsTable.createdAt] = now + it[SeqSetsTable.createdAt] = dateProvider.getCurrentDateTime() it[SeqSetsTable.createdBy] = username } @@ -314,8 +312,8 @@ class SeqSetCitationsDatabaseService( throw NotFoundException("SeqSet $seqSetId, version $version does not exist") } - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC).toJavaLocalDateTime() - val sevenDaysAgo = LocalDateTime.parse(now.minusDays(7).toString()) + val now = dateProvider.getCurrentInstant() + val sevenDaysAgo = now.minus(7, DateTimeUnit.DAY, TimeZone.UTC).toLocalDateTime(TimeZone.UTC) val count = SeqSetsTable .selectAll() .where { diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt index 7af62c7e6..2ef933e6d 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt @@ -3,10 +3,10 @@ package org.loculus.backend.service.submission import com.fasterxml.jackson.core.JacksonException import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import kotlinx.datetime.Clock -import kotlinx.datetime.Instant +import kotlinx.datetime.DateTimeUnit import kotlinx.datetime.LocalDateTime import kotlinx.datetime.TimeZone +import kotlinx.datetime.minus import kotlinx.datetime.toLocalDateTime import mu.KotlinLogging import org.jetbrains.exposed.sql.Database @@ -68,6 +68,7 @@ import org.loculus.backend.service.groupmanagement.GroupEntity import org.loculus.backend.service.groupmanagement.GroupManagementDatabaseService import org.loculus.backend.service.groupmanagement.GroupManagementPreconditionValidator import org.loculus.backend.utils.Accession +import org.loculus.backend.utils.DateProvider import org.loculus.backend.utils.Version import org.loculus.backend.utils.toTimestamp import org.springframework.beans.factory.annotation.Value @@ -94,6 +95,7 @@ open class SubmissionDatabaseService( private val emptyProcessedDataProvider: EmptyProcessedDataProvider, private val compressionService: CompressionService, private val auditLogger: AuditLogger, + private val dateProvider: DateProvider, @Value("\${${BackendSpringProperty.STREAM_BATCH_SIZE}}") private val streamBatchSize: Int, ) { @@ -181,7 +183,6 @@ open class SubmissionDatabaseService( } private fun updateStatusToProcessing(sequenceEntries: List, pipelineVersion: Long) { - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) log.info { "updating status to processing. Number of sequence entries: ${sequenceEntries.size}" } SequenceEntriesPreprocessedDataTable.batchInsert(sequenceEntries) { @@ -189,7 +190,7 @@ open class SubmissionDatabaseService( this[SequenceEntriesPreprocessedDataTable.versionColumn] = it.version this[SequenceEntriesPreprocessedDataTable.pipelineVersionColumn] = pipelineVersion this[SequenceEntriesPreprocessedDataTable.processingStatusColumn] = IN_PROCESSING.name - this[SequenceEntriesPreprocessedDataTable.startedProcessingAtColumn] = now + this[SequenceEntriesPreprocessedDataTable.startedProcessingAtColumn] = dateProvider.getCurrentDateTime() } } @@ -269,8 +270,6 @@ open class SubmissionDatabaseService( organism: Organism, externalMetadataUpdater: String, ) { - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) - accessionPreconditionValidator.validate { thatAccessionVersionExists(submittedExternalMetadata) .andThatSequenceEntriesAreInStates( @@ -296,7 +295,7 @@ open class SubmissionDatabaseService( it[versionColumn] = submittedExternalMetadata.version it[updaterIdColumn] = externalMetadataUpdater it[externalMetadataColumn] = submittedExternalMetadata.externalMetadata - it[updatedAtColumn] = now + it[updatedAtColumn] = dateProvider.getCurrentDateTime() } if (numberInserted != 1) { @@ -305,7 +304,7 @@ open class SubmissionDatabaseService( it[versionColumn] = submittedExternalMetadata.version it[updaterIdColumn] = externalMetadataUpdater it[externalMetadataColumn] = submittedExternalMetadata.externalMetadata - it[updatedAtColumn] = now + it[updatedAtColumn] = dateProvider.getCurrentDateTime() } } } @@ -315,8 +314,6 @@ open class SubmissionDatabaseService( organism: Organism, pipelineVersion: Long, ): PreprocessingStatus { - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) - val submittedErrors = submittedProcessedData.errors.orEmpty() val submittedWarnings = submittedProcessedData.warnings.orEmpty() @@ -342,7 +339,7 @@ open class SubmissionDatabaseService( it[processedDataColumn] = compressionService.compressSequencesInProcessedData(processedData, organism) it[errorsColumn] = submittedErrors it[warningsColumn] = submittedWarnings - it[finishedProcessingAtColumn] = now + it[finishedProcessingAtColumn] = dateProvider.getCurrentDateTime() } if (numberInserted != 1) { @@ -464,8 +461,6 @@ open class SubmissionDatabaseService( log.info { "approving ${accessionVersionsFilter.size} sequences by ${authenticatedUser.username}" } } - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) - if (accessionVersionsFilter != null) { accessionPreconditionValidator.validate { thatAccessionVersionsExist(accessionVersionsFilter) @@ -506,6 +501,7 @@ open class SubmissionDatabaseService( return emptyList() } + val now = dateProvider.getCurrentDateTime() for (accessionVersionsChunk in accessionVersionsToUpdate.chunked(1000)) { SequenceEntriesTable.update( where = { @@ -721,7 +717,6 @@ open class SubmissionDatabaseService( .andThatSequenceEntriesAreInStates(listOf(Status.APPROVED_FOR_RELEASE)) .andThatOrganismIs(organism) } - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) SequenceEntriesTable.insert( SequenceEntriesTable.select( @@ -733,9 +728,7 @@ open class SubmissionDatabaseService( SequenceEntriesTable.submissionIdColumn, SequenceEntriesTable.submitterColumn, SequenceEntriesTable.groupIdColumn, - dateTimeParam( - now, - ), + dateTimeParam(dateProvider.getCurrentDateTime()), booleanParam(true), SequenceEntriesTable.organismColumn, ).where { ( @@ -1026,9 +1019,9 @@ open class SubmissionDatabaseService( } fun cleanUpStaleSequencesInProcessing(timeToStaleInSeconds: Long) { - val staleDateTime = Instant.fromEpochMilliseconds( - Clock.System.now().toEpochMilliseconds() - timeToStaleInSeconds * 1000, - ).toLocalDateTime(TimeZone.UTC) + val staleDateTime = dateProvider.getCurrentInstant() + .minus(timeToStaleInSeconds, DateTimeUnit.SECOND, TimeZone.UTC) + .toLocalDateTime(TimeZone.UTC) // Check if there are any stale sequences before attempting to delete val staleSequencesExist = SequenceEntriesPreprocessedDataTable @@ -1064,14 +1057,13 @@ open class SubmissionDatabaseService( if (pipelineNeedsUpdate) { log.info { "Updating current processing pipeline to newer version: $newVersion" } - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC) CurrentProcessingPipelineTable.update( where = { CurrentProcessingPipelineTable.versionColumn neq newVersion }, ) { it[versionColumn] = newVersion - it[startedUsingAtColumn] = now + it[startedUsingAtColumn] = dateProvider.getCurrentDateTime() } } newVersion diff --git a/backend/src/main/kotlin/org/loculus/backend/utils/DateProvider.kt b/backend/src/main/kotlin/org/loculus/backend/utils/DateProvider.kt new file mode 100644 index 000000000..72dcd409e --- /dev/null +++ b/backend/src/main/kotlin/org/loculus/backend/utils/DateProvider.kt @@ -0,0 +1,15 @@ +package org.loculus.backend.utils + +import kotlinx.datetime.Clock +import kotlinx.datetime.TimeZone +import kotlinx.datetime.toLocalDateTime +import org.springframework.stereotype.Component + +@Component +class DateProvider { + fun getCurrentInstant() = Clock.System.now() + + fun getCurrentDateTime() = getCurrentInstant().toLocalDateTime(TimeZone.UTC) + + fun getCurrentDate() = getCurrentDateTime().date +} diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt b/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt index acc066d15..54dbe3cb7 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt @@ -3,6 +3,11 @@ package org.loculus.backend.controller import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue +import kotlinx.datetime.Clock +import kotlinx.datetime.DateTimeUnit.Companion.MONTH +import kotlinx.datetime.TimeZone +import kotlinx.datetime.plus +import kotlinx.datetime.toLocalDateTime import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers import org.hamcrest.Matchers.`is` @@ -22,6 +27,8 @@ const val OTHER_ORGANISM = "otherOrganism" const val DEFAULT_PIPELINE_VERSION = 1L const val DEFAULT_EXTERNAL_METADATA_UPDATER = "ena" +fun dateMonthsFromNow(months: Int) = Clock.System.now().toLocalDateTime(TimeZone.UTC).date.plus(months, MONTH) + fun AccessionVersionInterface.toAccessionVersion() = AccessionVersion(this.accession, this.version) fun List.getAccessionVersions() = map { it.toAccessionVersion() } diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerTest.kt index e05e66534..024e04e2f 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/datauseterms/DataUseTermsControllerTest.kt @@ -1,10 +1,5 @@ package org.loculus.backend.controller.datauseterms -import kotlinx.datetime.Clock -import kotlinx.datetime.DateTimeUnit.Companion.MONTH -import kotlinx.datetime.TimeZone -import kotlinx.datetime.plus -import kotlinx.datetime.toLocalDateTime import org.hamcrest.CoreMatchers.containsString import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest @@ -14,6 +9,7 @@ import org.loculus.backend.api.DataUseTermsChangeRequest import org.loculus.backend.api.DataUseTermsType import org.loculus.backend.controller.DEFAULT_USER_NAME import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.dateMonthsFromNow import org.loculus.backend.controller.expectUnauthorizedResponse import org.loculus.backend.controller.generateJwtFor import org.loculus.backend.controller.jwtForSuperUser @@ -26,8 +22,6 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status -private fun dateMonthsFromNow(months: Int) = Clock.System.now().toLocalDateTime(TimeZone.UTC).date.plus(months, MONTH) - @EndpointTest class DataUseTermsControllerTest( @Autowired private val client: DataUseTermsControllerClient, diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt index 7e32926e6..9e854253c 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt @@ -1,15 +1,24 @@ package org.loculus.backend.controller.submission import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.node.BooleanNode import com.fasterxml.jackson.databind.node.IntNode import com.fasterxml.jackson.databind.node.NullNode import com.fasterxml.jackson.databind.node.TextNode import com.fasterxml.jackson.module.kotlin.readValue import com.github.luben.zstd.ZstdInputStream +import com.ninjasquad.springmockk.MockkBean +import io.mockk.every import kotlinx.datetime.Clock +import kotlinx.datetime.DateTimeUnit import kotlinx.datetime.Instant +import kotlinx.datetime.LocalDate +import kotlinx.datetime.LocalDateTime +import kotlinx.datetime.LocalTime import kotlinx.datetime.TimeZone +import kotlinx.datetime.plus +import kotlinx.datetime.toInstant import kotlinx.datetime.toLocalDateTime import org.hamcrest.CoreMatchers.`is` import org.hamcrest.MatcherAssert.assertThat @@ -18,23 +27,42 @@ import org.hamcrest.Matchers.hasSize import org.hamcrest.Matchers.matchesPattern import org.hamcrest.Matchers.not import org.hamcrest.Matchers.notNullValue +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import org.loculus.backend.api.AccessionVersionInterface +import org.loculus.backend.api.DataUseTerms +import org.loculus.backend.api.DataUseTermsChangeRequest import org.loculus.backend.api.GeneticSequence import org.loculus.backend.api.ProcessedData import org.loculus.backend.api.Status import org.loculus.backend.api.VersionStatus +import org.loculus.backend.config.BackendConfig +import org.loculus.backend.config.BackendSpringProperty +import org.loculus.backend.config.DataUseTermsUrls +import org.loculus.backend.config.readBackendConfig import org.loculus.backend.controller.DEFAULT_GROUP_NAME import org.loculus.backend.controller.DEFAULT_USER_NAME import org.loculus.backend.controller.EndpointTest +import org.loculus.backend.controller.datauseterms.DataUseTermsControllerClient +import org.loculus.backend.controller.dateMonthsFromNow import org.loculus.backend.controller.expectNdjsonAndGetContent import org.loculus.backend.controller.jacksonObjectMapper +import org.loculus.backend.controller.jwtForDefaultUser +import org.loculus.backend.controller.submission.GetReleasedDataEndpointWithDataUseTermsUrlTest.ConfigWithModifiedDataUseTermsUrlSpringConfig import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES import org.loculus.backend.utils.Accession +import org.loculus.backend.utils.DateProvider import org.loculus.backend.utils.Version import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.test.context.TestConfiguration +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Import +import org.springframework.context.annotation.Primary import org.springframework.http.HttpHeaders import org.springframework.http.HttpHeaders.ETAG import org.springframework.http.MediaType +import org.springframework.test.context.TestPropertySource import org.springframework.test.web.servlet.MvcResult import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header @@ -331,6 +359,145 @@ class GetReleasedDataEndpointTest( } } +private const val OPEN_DATA_USE_TERMS_URL = "openUrl" +private const val RESTRICTED_DATA_USE_TERMS_URL = "restrictedUrl" + +@EndpointTest +@Import(ConfigWithModifiedDataUseTermsUrlSpringConfig::class) +@TestPropertySource(properties = ["spring.main.allow-bean-definition-overriding=true"]) +class GetReleasedDataEndpointWithDataUseTermsUrlTest( + @Autowired val convenienceClient: SubmissionConvenienceClient, + @Autowired val dataUseTermsClient: DataUseTermsControllerClient, + @Autowired val submissionControllerClient: SubmissionControllerClient, +) { + @MockkBean + private lateinit var dateProvider: DateProvider + + @BeforeEach + fun setup() { + every { dateProvider.getCurrentDateTime() } answers { callOriginal() } + every { dateProvider.getCurrentDate() } answers { callOriginal() } + } + + @Test + fun `GIVEN sequence entry WHEN I change data use terms THEN returns updated data use terms`() { + every { dateProvider.getCurrentInstant() } answers { callOriginal() } + + val threeMonthsFromNow = dateMonthsFromNow(3) + + var accessionVersion = convenienceClient.prepareDataTo( + status = Status.APPROVED_FOR_RELEASE, + dataUseTerms = DataUseTerms.Restricted(threeMonthsFromNow), + )[0] + + assertAccessionVersionIsRestrictedUntil(accessionVersion, threeMonthsFromNow) + + val oneMonthFromNow = dateMonthsFromNow(1) + dataUseTermsClient.changeDataUseTerms( + newDataUseTerms = DataUseTermsChangeRequest( + accessions = listOf(accessionVersion.accession), + newDataUseTerms = DataUseTerms.Restricted(oneMonthFromNow), + ), + jwt = jwtForDefaultUser, + ) + + assertAccessionVersionIsRestrictedUntil(accessionVersion, oneMonthFromNow) + + dataUseTermsClient.changeDataUseTerms( + newDataUseTerms = DataUseTermsChangeRequest( + accessions = listOf(accessionVersion.accession), + newDataUseTerms = DataUseTerms.Open, + ), + jwt = jwtForDefaultUser, + ) + + assertAccessionVersionIsOpen(accessionVersion) + } + + @Test + fun `GIVEN sequence entry with expired restricted data use terms THEN returns open data use terms`() { + every { dateProvider.getCurrentInstant() } answers { callOriginal() } + + val threeMonthsFromNow = dateMonthsFromNow(3) + + var accessionVersion = convenienceClient.prepareDataTo( + status = Status.APPROVED_FOR_RELEASE, + dataUseTerms = DataUseTerms.Restricted(threeMonthsFromNow), + )[0] + + assertAccessionVersionIsRestrictedUntil(accessionVersion, threeMonthsFromNow) + + val threeMonthsAndADayFromNow = LocalDateTime( + date = dateMonthsFromNow(3).plus(1, DateTimeUnit.DAY), + time = LocalTime.fromSecondOfDay(0), + ).toInstant(TimeZone.UTC) + every { dateProvider.getCurrentInstant() } answers { threeMonthsAndADayFromNow } + + assertAccessionVersionIsOpen(accessionVersion) + } + + private fun assertAccessionVersionIsOpen(accessionVersion: AccessionVersionInterface) { + assertAccessionVersionHasReleasedDataValues( + accessionVersion = accessionVersion, + dataUseTerms = "OPEN", + restrictedUntilDate = null, + dataUseTermsUrl = OPEN_DATA_USE_TERMS_URL, + ) + } + + private fun assertAccessionVersionIsRestrictedUntil( + accessionVersion: AccessionVersionInterface, + restrictedUntilDate: LocalDate, + ) { + assertAccessionVersionHasReleasedDataValues( + accessionVersion = accessionVersion, + dataUseTerms = "RESTRICTED", + restrictedUntilDate = restrictedUntilDate, + dataUseTermsUrl = RESTRICTED_DATA_USE_TERMS_URL, + ) + } + + private fun assertAccessionVersionHasReleasedDataValues( + accessionVersion: AccessionVersionInterface, + dataUseTerms: String, + restrictedUntilDate: LocalDate?, + dataUseTermsUrl: String, + ) { + val releasedData = submissionControllerClient.getReleasedData() + .expectNdjsonAndGetContent>() + .find { it.metadata["accessionVersion"]?.textValue() == accessionVersion.displayAccessionVersion() }!! + + assertThat(releasedData.metadata["dataUseTerms"]?.textValue(), `is`(dataUseTerms)) + when (restrictedUntilDate) { + null -> assertThat(releasedData.metadata["dataUseTermsRestrictedUntil"], `is`(NullNode.instance)) + else -> assertThat( + releasedData.metadata["dataUseTermsRestrictedUntil"]?.textValue(), + `is`(restrictedUntilDate.toString()), + ) + } + assertThat(releasedData.metadata["dataUseTermsUrl"]?.textValue(), `is`(dataUseTermsUrl)) + } + + @TestConfiguration + class ConfigWithModifiedDataUseTermsUrlSpringConfig { + @Bean + @Primary + fun backendConfig( + objectMapper: ObjectMapper, + @Value("\${${BackendSpringProperty.BACKEND_CONFIG_PATH}}") configPath: String, + ): BackendConfig { + val originalConfig = readBackendConfig(objectMapper = objectMapper, configPath = configPath) + + return originalConfig.copy( + dataUseTermsUrls = DataUseTermsUrls( + open = OPEN_DATA_USE_TERMS_URL, + restricted = RESTRICTED_DATA_USE_TERMS_URL, + ), + ) + } + } +} + private fun List>.findAccessionVersionStatus( accession: Accession, version: Version, diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt index 155656c6f..bf82c6eaf 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmissionConvenienceClient.kt @@ -87,8 +87,9 @@ class SubmissionConvenienceClient( organism: String = DEFAULT_ORGANISM, username: String = DEFAULT_USER_NAME, groupId: Int? = null, + dataUseTerms: DataUseTerms = DataUseTerms.Open, ): List { - submitDefaultFiles(organism = organism, username = username, groupId = groupId) + submitDefaultFiles(organism = organism, username = username, groupId = groupId, dataUseTerms = dataUseTerms) return extractUnprocessedData(organism = organism).getAccessionVersions() } @@ -117,9 +118,14 @@ class SubmissionConvenienceClient( organism: String = DEFAULT_ORGANISM, username: String = DEFAULT_USER_NAME, groupId: Int? = null, + dataUseTerms: DataUseTerms = DataUseTerms.Open, ): List { - val accessionVersions = - prepareDefaultSequenceEntriesToInProcessing(organism = organism, username = username, groupId = groupId) + val accessionVersions = prepareDefaultSequenceEntriesToInProcessing( + organism = organism, + username = username, + groupId = groupId, + dataUseTerms = dataUseTerms, + ) submitProcessedData( accessionVersions.map { PreparedProcessedData.withErrors(accession = it.accession) @@ -133,9 +139,14 @@ class SubmissionConvenienceClient( organism: String = DEFAULT_ORGANISM, username: String = DEFAULT_USER_NAME, groupId: Int? = null, + dataUseTerms: DataUseTerms = DataUseTerms.Open, ): List { - val accessionVersions = - prepareDefaultSequenceEntriesToInProcessing(organism = organism, username = username, groupId = groupId) + val accessionVersions = prepareDefaultSequenceEntriesToInProcessing( + organism = organism, + username = username, + groupId = groupId, + dataUseTerms = dataUseTerms, + ) submitProcessedData( *accessionVersions.map { when (organism) { @@ -156,11 +167,13 @@ class SubmissionConvenienceClient( organism: String = DEFAULT_ORGANISM, username: String = DEFAULT_USER_NAME, groupId: Int? = null, + dataUseTerms: DataUseTerms = DataUseTerms.Open, ): List { val accessionVersions = prepareDefaultSequenceEntriesToAwaitingApproval( organism = organism, username = username, groupId = groupId, + dataUseTerms = dataUseTerms, ) approveProcessedSequenceEntries( @@ -345,35 +358,41 @@ class SubmissionConvenienceClient( organism: String = DEFAULT_ORGANISM, username: String = DEFAULT_USER_NAME, groupId: Int? = null, + dataUseTerms: DataUseTerms = DataUseTerms.Open, ): List = when (status) { Status.RECEIVED -> submitDefaultFiles( organism = organism, username = username, groupId = groupId, + dataUseTerms = dataUseTerms, ).submissionIdMappings Status.IN_PROCESSING -> prepareDefaultSequenceEntriesToInProcessing( organism = organism, username = username, groupId = groupId, + dataUseTerms = dataUseTerms, ) Status.HAS_ERRORS -> prepareDefaultSequenceEntriesToHasErrors( organism = organism, username = username, groupId = groupId, + dataUseTerms = dataUseTerms, ) Status.AWAITING_APPROVAL -> prepareDefaultSequenceEntriesToAwaitingApproval( organism = organism, username = username, groupId = groupId, + dataUseTerms = dataUseTerms, ) Status.APPROVED_FOR_RELEASE -> prepareDefaultSequenceEntriesToApprovedForRelease( organism = organism, username = username, groupId = groupId, + dataUseTerms = dataUseTerms, ) else -> throw Exception("Test issue: No data preparation defined for status $status") From ccc288ff3c061f4127b4ee707ea26a2926d0b3e3 Mon Sep 17 00:00:00 2001 From: Fabian Engelniederhammer Date: Mon, 7 Oct 2024 12:52:56 +0200 Subject: [PATCH 2/3] refactor(backend): avoid creating too many nested read-only maps This is probably a bit more efficient --- .../backend/model/ReleasedDataModel.kt | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt index 751ec493d..45ef1976f 100644 --- a/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt +++ b/backend/src/main/kotlin/org/loculus/backend/model/ReleasedDataModel.kt @@ -84,30 +84,35 @@ open class ReleasedDataModel( } var metadata = rawProcessedData.processedData.metadata + - ("accession" to TextNode(rawProcessedData.accession)) + - ("version" to LongNode(rawProcessedData.version)) + - (HEADER_TO_CONNECT_METADATA_AND_SEQUENCES to TextNode(rawProcessedData.submissionId)) + - ("accessionVersion" to TextNode(rawProcessedData.displayAccessionVersion())) + - ("isRevocation" to BooleanNode.valueOf(rawProcessedData.isRevocation)) + - ("submitter" to TextNode(rawProcessedData.submitter)) + - ("groupId" to IntNode(rawProcessedData.groupId)) + - ("groupName" to TextNode(rawProcessedData.groupName)) + - ("submittedDate" to TextNode(rawProcessedData.submittedAtTimestamp.toUtcDateString())) + - ("submittedAtTimestamp" to LongNode(rawProcessedData.submittedAtTimestamp.toTimestamp())) + - ("releasedAtTimestamp" to LongNode(rawProcessedData.releasedAtTimestamp.toTimestamp())) + - ("releasedDate" to TextNode(rawProcessedData.releasedAtTimestamp.toUtcDateString())) + - ("versionStatus" to TextNode(versionStatus.name)) + - ("dataUseTerms" to TextNode(currentDataUseTerms.type.name)) + - ("dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil) + - ("versionComment" to TextNode(rawProcessedData.versionComment)) - - if (backendConfig.dataUseTermsUrls != null) { - val url = when (currentDataUseTerms) { - DataUseTerms.Open -> backendConfig.dataUseTermsUrls.open - is DataUseTerms.Restricted -> backendConfig.dataUseTermsUrls.restricted + mapOf( + ("accession" to TextNode(rawProcessedData.accession)), + ("version" to LongNode(rawProcessedData.version)), + (HEADER_TO_CONNECT_METADATA_AND_SEQUENCES to TextNode(rawProcessedData.submissionId)), + ("accessionVersion" to TextNode(rawProcessedData.displayAccessionVersion())), + ("isRevocation" to BooleanNode.valueOf(rawProcessedData.isRevocation)), + ("submitter" to TextNode(rawProcessedData.submitter)), + ("groupId" to IntNode(rawProcessedData.groupId)), + ("groupName" to TextNode(rawProcessedData.groupName)), + ("submittedDate" to TextNode(rawProcessedData.submittedAtTimestamp.toUtcDateString())), + ("submittedAtTimestamp" to LongNode(rawProcessedData.submittedAtTimestamp.toTimestamp())), + ("releasedAtTimestamp" to LongNode(rawProcessedData.releasedAtTimestamp.toTimestamp())), + ("releasedDate" to TextNode(rawProcessedData.releasedAtTimestamp.toUtcDateString())), + ("versionStatus" to TextNode(versionStatus.name)), + ("dataUseTerms" to TextNode(currentDataUseTerms.type.name)), + ("dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil), + ("versionComment" to TextNode(rawProcessedData.versionComment)), + ).let { + when (backendConfig.dataUseTermsUrls) { + null -> it + else -> { + val url = when (currentDataUseTerms) { + DataUseTerms.Open -> backendConfig.dataUseTermsUrls.open + is DataUseTerms.Restricted -> backendConfig.dataUseTermsUrls.restricted + } + it + ("dataUseTermsUrl" to TextNode(url)) + } + } } - metadata += ("dataUseTermsUrl" to TextNode(url)) - } return ProcessedData( metadata = metadata, From 84b78c11c31fa67177ac781942456cb69aa5c4e8 Mon Sep 17 00:00:00 2001 From: Fabian Engelniederhammer Date: Mon, 7 Oct 2024 16:40:59 +0200 Subject: [PATCH 3/3] refactor(backend): move time zone usages to `DateProvider` --- .../SeqSetCitationsDatabaseService.kt | 3 +-- .../submission/SubmissionDatabaseService.kt | 5 ++-- .../org/loculus/backend/utils/DateProvider.kt | 6 ++++- .../backend/utils/LocalDateTimeExtensions.kt | 7 +++--- .../loculus/backend/controller/TestHelpers.kt | 4 ++-- .../DeleteAllSequenceDataEndpointTest.kt | 4 ++-- .../submission/GetReleasedDataEndpointTest.kt | 24 +++++++------------ .../submission/SubmitEndpointTest.kt | 5 ++-- 8 files changed, 25 insertions(+), 33 deletions(-) diff --git a/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt index ef27dfe44..02d2d3712 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/seqsetcitations/SeqSetCitationsDatabaseService.kt @@ -1,7 +1,6 @@ package org.loculus.backend.service.seqsetcitations import kotlinx.datetime.DateTimeUnit -import kotlinx.datetime.TimeZone import kotlinx.datetime.minus import kotlinx.datetime.toJavaLocalDateTime import kotlinx.datetime.toLocalDateTime @@ -313,7 +312,7 @@ class SeqSetCitationsDatabaseService( } val now = dateProvider.getCurrentInstant() - val sevenDaysAgo = now.minus(7, DateTimeUnit.DAY, TimeZone.UTC).toLocalDateTime(TimeZone.UTC) + val sevenDaysAgo = now.minus(7, DateTimeUnit.DAY, DateProvider.timeZone).toLocalDateTime(DateProvider.timeZone) val count = SeqSetsTable .selectAll() .where { diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt index 2ef933e6d..cbc6cf8c4 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import kotlinx.datetime.DateTimeUnit import kotlinx.datetime.LocalDateTime -import kotlinx.datetime.TimeZone import kotlinx.datetime.minus import kotlinx.datetime.toLocalDateTime import mu.KotlinLogging @@ -1020,8 +1019,8 @@ open class SubmissionDatabaseService( fun cleanUpStaleSequencesInProcessing(timeToStaleInSeconds: Long) { val staleDateTime = dateProvider.getCurrentInstant() - .minus(timeToStaleInSeconds, DateTimeUnit.SECOND, TimeZone.UTC) - .toLocalDateTime(TimeZone.UTC) + .minus(timeToStaleInSeconds, DateTimeUnit.SECOND, DateProvider.timeZone) + .toLocalDateTime(DateProvider.timeZone) // Check if there are any stale sequences before attempting to delete val staleSequencesExist = SequenceEntriesPreprocessedDataTable diff --git a/backend/src/main/kotlin/org/loculus/backend/utils/DateProvider.kt b/backend/src/main/kotlin/org/loculus/backend/utils/DateProvider.kt index 72dcd409e..6472db083 100644 --- a/backend/src/main/kotlin/org/loculus/backend/utils/DateProvider.kt +++ b/backend/src/main/kotlin/org/loculus/backend/utils/DateProvider.kt @@ -9,7 +9,11 @@ import org.springframework.stereotype.Component class DateProvider { fun getCurrentInstant() = Clock.System.now() - fun getCurrentDateTime() = getCurrentInstant().toLocalDateTime(TimeZone.UTC) + fun getCurrentDateTime() = getCurrentInstant().toLocalDateTime(timeZone) fun getCurrentDate() = getCurrentDateTime().date + + companion object { + val timeZone = TimeZone.UTC + } } diff --git a/backend/src/main/kotlin/org/loculus/backend/utils/LocalDateTimeExtensions.kt b/backend/src/main/kotlin/org/loculus/backend/utils/LocalDateTimeExtensions.kt index e4ecea19c..f3323e032 100644 --- a/backend/src/main/kotlin/org/loculus/backend/utils/LocalDateTimeExtensions.kt +++ b/backend/src/main/kotlin/org/loculus/backend/utils/LocalDateTimeExtensions.kt @@ -1,13 +1,12 @@ package org.loculus.backend.utils import kotlinx.datetime.LocalDateTime -import kotlinx.datetime.TimeZone import kotlinx.datetime.toInstant import kotlinx.datetime.toLocalDateTime -fun LocalDateTime.toTimestamp() = this.toInstant(TimeZone.UTC).epochSeconds +fun LocalDateTime.toTimestamp() = this.toInstant(DateProvider.timeZone).epochSeconds -fun LocalDateTime.toUtcDateString(): String = this.toInstant(TimeZone.currentSystemDefault()) - .toLocalDateTime(TimeZone.UTC) +fun LocalDateTime.toUtcDateString(): String = this.toInstant(DateProvider.timeZone) + .toLocalDateTime(DateProvider.timeZone) .date .toString() diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt b/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt index 54dbe3cb7..b847ffa2e 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/TestHelpers.kt @@ -5,7 +5,6 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import kotlinx.datetime.Clock import kotlinx.datetime.DateTimeUnit.Companion.MONTH -import kotlinx.datetime.TimeZone import kotlinx.datetime.plus import kotlinx.datetime.toLocalDateTime import org.hamcrest.MatcherAssert.assertThat @@ -15,6 +14,7 @@ import org.loculus.backend.api.AccessionVersion import org.loculus.backend.api.AccessionVersionInterface import org.loculus.backend.api.SequenceEntryStatus import org.loculus.backend.api.Status +import org.loculus.backend.utils.DateProvider import org.springframework.test.web.servlet.MvcResult import org.springframework.test.web.servlet.ResultActions import org.springframework.test.web.servlet.result.MockMvcResultMatchers @@ -27,7 +27,7 @@ const val OTHER_ORGANISM = "otherOrganism" const val DEFAULT_PIPELINE_VERSION = 1L const val DEFAULT_EXTERNAL_METADATA_UPDATER = "ena" -fun dateMonthsFromNow(months: Int) = Clock.System.now().toLocalDateTime(TimeZone.UTC).date.plus(months, MONTH) +fun dateMonthsFromNow(months: Int) = Clock.System.now().toLocalDateTime(DateProvider.timeZone).date.plus(months, MONTH) fun AccessionVersionInterface.toAccessionVersion() = AccessionVersion(this.accession, this.version) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt index e59cc384e..7e66f1dec 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/debug/DeleteAllSequenceDataEndpointTest.kt @@ -2,7 +2,6 @@ package org.loculus.backend.controller.debug import kotlinx.datetime.Clock import kotlinx.datetime.DateTimeUnit.Companion.MONTH -import kotlinx.datetime.TimeZone import kotlinx.datetime.plus import kotlinx.datetime.toLocalDateTime import org.hamcrest.MatcherAssert.assertThat @@ -28,6 +27,7 @@ import org.loculus.backend.controller.submission.SubmissionConvenienceClient import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.NUMBER_OF_SEQUENCES import org.loculus.backend.controller.withAuth import org.loculus.backend.service.submission.UseNewerProcessingPipelineVersionTask +import org.loculus.backend.utils.DateProvider import org.springframework.beans.factory.annotation.Autowired import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post @@ -97,7 +97,7 @@ class DeleteAllSequenceDataEndpointTest( @Test fun `GIVEN accession with data user terms history WHEN deleting all sequences THEN history is deleted`() { val restrictedDataUseTerms = DataUseTerms.Restricted( - Clock.System.now().toLocalDateTime(TimeZone.UTC).date.plus(1, MONTH), + Clock.System.now().toLocalDateTime(DateProvider.timeZone).date.plus(1, MONTH), ) val accession = submissionConvenienceClient.submitDefaultFiles(dataUseTerms = restrictedDataUseTerms) .submissionIdMappings diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt index 9e854253c..6ad5e5347 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/GetReleasedDataEndpointTest.kt @@ -16,7 +16,6 @@ import kotlinx.datetime.Instant import kotlinx.datetime.LocalDate import kotlinx.datetime.LocalDateTime import kotlinx.datetime.LocalTime -import kotlinx.datetime.TimeZone import kotlinx.datetime.plus import kotlinx.datetime.toInstant import kotlinx.datetime.toLocalDateTime @@ -81,7 +80,8 @@ class GetReleasedDataEndpointTest( @Autowired val convenienceClient: SubmissionConvenienceClient, @Autowired val submissionControllerClient: SubmissionControllerClient, ) { - val currentYear = Clock.System.now().toLocalDateTime(TimeZone.UTC).year + private val currentYear = Clock.System.now().toLocalDateTime(DateProvider.timeZone).year + private val currentDate = Clock.System.now().toLocalDateTime(DateProvider.timeZone).date.toString() @Test fun `GIVEN no sequence entries in database THEN returns empty response & etag in header`() { @@ -120,8 +120,8 @@ class GetReleasedDataEndpointTest( "groupName" to TextNode(DEFAULT_GROUP_NAME), "versionStatus" to TextNode("LATEST_VERSION"), "dataUseTerms" to TextNode("OPEN"), - "releasedDate" to TextNode(Clock.System.now().toLocalDateTime(TimeZone.UTC).date.toString()), - "submittedDate" to TextNode(Clock.System.now().toLocalDateTime(TimeZone.UTC).date.toString()), + "releasedDate" to TextNode(currentDate), + "submittedDate" to TextNode(currentDate), "dataUseTermsRestrictedUntil" to NullNode.getInstance(), "versionComment" to NullNode.getInstance(), "booleanColumn" to BooleanNode.TRUE, @@ -261,16 +261,8 @@ class GetReleasedDataEndpointTest( "groupId" -> assertThat(value.intValue(), `is`(greaterThan(0))) "accession", "version", "accessionVersion", "submissionId" -> {} "dataUseTerms" -> assertThat(value, `is`(TextNode("OPEN"))) - "submittedDate" -> assertThat( - value, - `is`(TextNode(Clock.System.now().toLocalDateTime(TimeZone.UTC).date.toString())), - ) - - "releasedDate" -> assertThat( - value, - `is`(TextNode(Clock.System.now().toLocalDateTime(TimeZone.UTC).date.toString())), - ) - + "submittedDate" -> assertThat(value, `is`(TextNode(currentDate))) + "releasedDate" -> assertThat(value, `is`(TextNode(currentDate))) "versionComment" -> assertThat( value, `is`(TextNode("This is a test revocation")), @@ -354,7 +346,7 @@ class GetReleasedDataEndpointTest( } private fun expectIsTimestampWithCurrentYear(value: JsonNode) { - val dateTime = Instant.fromEpochSeconds(value.asLong()).toLocalDateTime(TimeZone.UTC) + val dateTime = Instant.fromEpochSeconds(value.asLong()).toLocalDateTime(DateProvider.timeZone) assertThat(dateTime.year, `is`(currentYear)) } } @@ -430,7 +422,7 @@ class GetReleasedDataEndpointWithDataUseTermsUrlTest( val threeMonthsAndADayFromNow = LocalDateTime( date = dateMonthsFromNow(3).plus(1, DateTimeUnit.DAY), time = LocalTime.fromSecondOfDay(0), - ).toInstant(TimeZone.UTC) + ).toInstant(DateProvider.timeZone) every { dateProvider.getCurrentInstant() } answers { threeMonthsAndADayFromNow } assertAccessionVersionIsOpen(accessionVersion) diff --git a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt index 5d72cd109..ecc5b55c2 100644 --- a/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/controller/submission/SubmitEndpointTest.kt @@ -3,7 +3,6 @@ package org.loculus.backend.controller.submission import kotlinx.datetime.Clock import kotlinx.datetime.DateTimeUnit.Companion.DAY import kotlinx.datetime.DateTimeUnit.Companion.YEAR -import kotlinx.datetime.TimeZone import kotlinx.datetime.minus import kotlinx.datetime.plus import kotlinx.datetime.toLocalDateTime @@ -29,6 +28,7 @@ import org.loculus.backend.controller.submission.SubmitFiles.DefaultFiles.NUMBER import org.loculus.backend.model.SubmitModel.AcceptedFileTypes.metadataFileTypes import org.loculus.backend.model.SubmitModel.AcceptedFileTypes.sequenceFileTypes import org.loculus.backend.service.submission.CompressionAlgorithm +import org.loculus.backend.utils.DateProvider import org.springframework.beans.factory.annotation.Autowired import org.springframework.http.MediaType.APPLICATION_JSON_VALUE import org.springframework.mock.web.MockMultipartFile @@ -225,7 +225,7 @@ class SubmitEndpointTest( @JvmStatic fun badRequestForSubmit(): List { - val now = Clock.System.now().toLocalDateTime(TimeZone.UTC).date + val now = Clock.System.now().toLocalDateTime(DateProvider.timeZone).date return listOf( Arguments.of( @@ -415,7 +415,6 @@ class SubmitEndpointTest( "The date 'restrictedUntil' must be in the future, up to a maximum of 1 year from now.", DEFAULT_ORGANISM, DataUseTerms.Restricted(now.minus(1, DAY)), - ), Arguments.of( "restricted use data with until date further than 1 year",