Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(backend): add iso dates to metadata #2235

Merged
merged 12 commits into from
Jul 10, 2024
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
theosanderson marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,10 @@ class ReleasedDataModel(
("submitter" to TextNode(rawProcessedData.submitter)) +
("groupId" to IntNode(rawProcessedData.groupId)) +
("groupName" to TextNode(rawProcessedData.groupName)) +
("submittedAt" to LongNode(rawProcessedData.submittedAt.toTimestamp())) +
("releasedAt" to LongNode(rawProcessedData.releasedAt.toTimestamp())) +
("submittedDate" to TextNode(rawProcessedData.submittedAtTimestamp.toUtcDate())) +
("submittedAtTimestamp" to LongNode(rawProcessedData.submittedAtTimestamp.toTimestamp())) +
("releasedAtTimestamp" to LongNode(rawProcessedData.releasedAtTimestamp.toTimestamp())) +
("releasedDate" to TextNode(rawProcessedData.releasedAtTimestamp.toUtcDate())) +
("versionStatus" to TextNode(siloVersionStatus.name)) +
("dataUseTerms" to TextNode(currentDataUseTerms.type.name)) +
("dataUseTermsRestrictedUntil" to restrictedDataUseTermsUntil)
Expand Down Expand Up @@ -120,3 +122,10 @@ class ReleasedDataModel(
}

private fun LocalDateTime.toTimestamp() = this.toInstant(TimeZone.UTC).epochSeconds

private fun LocalDateTime.toUtcDate(): String {
theosanderson marked this conversation as resolved.
Show resolved Hide resolved
return this.toInstant(TimeZone.currentSystemDefault())
.toLocalDateTime(TimeZone.UTC)
.date
.toString()
}
theosanderson marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ object SequenceEntriesTable : Table(SEQUENCE_ENTRIES_TABLE_NAME) {
val submitterColumn = varchar("submitter", 255)
val approverColumn = varchar("approver", 255)
val groupIdColumn = integer("group_id")
val submittedAtColumn = datetime("submitted_at")
val releasedAtColumn = datetime("released_at").nullable()
val submittedAtTimestampColumn = datetime("submitted_at")
val releasedAtTimestampColumn = datetime("released_at").nullable()
val isRevocationColumn = bool("is_revocation").default(false)

override val primaryKey = PrimaryKey(accessionColumn, versionColumn)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ object SequenceEntriesView : Table(SEQUENCE_ENTRIES_VIEW_NAME) {
val submissionIdColumn = varchar("submission_id", 255)
val submitterColumn = varchar("submitter", 255)
val groupIdColumn = integer("group_id")
val submittedAtColumn = datetime("submitted_at")
val submittedAtTimestampColumn = datetime("submitted_at")
val startedProcessingAtColumn = datetime("started_processing_at").nullable()
val finishedProcessingAtColumn = datetime("finished_processing_at").nullable()
val releasedAtColumn = datetime("released_at").nullable()
val releasedAtTimestampColumn = datetime("released_at").nullable()
val statusColumn = varchar("status", 255)
val isRevocationColumn = bool("is_revocation").default(false)
val errorsColumn = jacksonSerializableJsonb<List<PreprocessingAnnotation>>("errors").nullable()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ class SubmissionDatabaseService(
SequenceEntriesTable.accessionVersionIsIn(accessionVersionsChunk)
},
) {
it[releasedAtColumn] = now
it[releasedAtTimestampColumn] = now
it[approverColumn] = authenticatedUser.username
}
}
Expand Down Expand Up @@ -506,8 +506,8 @@ class SubmissionDatabaseService(
SequenceEntriesView.jointDataColumn,
SequenceEntriesView.submitterColumn,
SequenceEntriesView.groupIdColumn,
SequenceEntriesView.submittedAtColumn,
SequenceEntriesView.releasedAtColumn,
SequenceEntriesView.submittedAtTimestampColumn,
SequenceEntriesView.releasedAtTimestampColumn,
SequenceEntriesView.submissionIdColumn,
DataUseTermsTable.dataUseTermsTypeColumn,
DataUseTermsTable.restrictedUntilColumn,
Expand Down Expand Up @@ -536,8 +536,8 @@ class SubmissionDatabaseService(
null -> emptyProcessedDataProvider.provide(organism)
else -> compressionService.decompressSequencesInProcessedData(processedData, organism)
},
submittedAt = it[SequenceEntriesView.submittedAtColumn],
releasedAt = it[SequenceEntriesView.releasedAtColumn]!!,
submittedAtTimestamp = it[SequenceEntriesView.submittedAtTimestampColumn],
releasedAtTimestamp = it[SequenceEntriesView.releasedAtTimestampColumn]!!,
dataUseTerms = DataUseTerms.fromParameters(
DataUseTermsType.fromString(it[DataUseTermsTable.dataUseTermsTypeColumn]),
it[DataUseTermsTable.restrictedUntilColumn],
Expand Down Expand Up @@ -582,7 +582,7 @@ class SubmissionDatabaseService(
SequenceEntriesView.groupIdColumn,
SequenceEntriesView.submitterColumn,
SequenceEntriesView.organismColumn,
SequenceEntriesView.submittedAtColumn,
SequenceEntriesView.submittedAtTimestampColumn,
DataUseTermsTable.dataUseTermsTypeColumn,
DataUseTermsTable.restrictedUntilColumn,
)
Expand Down Expand Up @@ -671,7 +671,7 @@ class SubmissionDatabaseService(
SequenceEntriesTable.submissionIdColumn,
SequenceEntriesTable.submitterColumn,
SequenceEntriesTable.groupIdColumn,
SequenceEntriesTable.submittedAtColumn,
SequenceEntriesTable.submittedAtTimestampColumn,
SequenceEntriesTable.isRevocationColumn,
SequenceEntriesTable.organismColumn,
),
Expand Down Expand Up @@ -983,8 +983,8 @@ data class RawProcessedData(
val submitter: String,
val groupId: Int,
val groupName: String,
val submittedAt: LocalDateTime,
val releasedAt: LocalDateTime,
val submittedAtTimestamp: LocalDateTime,
val releasedAtTimestamp: LocalDateTime,
val submissionId: String,
val processedData: ProcessedData<GeneticSequence>,
val dataUseTerms: DataUseTerms,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status
import org.testcontainers.shaded.org.awaitility.Awaitility.await

private val ADDED_FIELDS_WITH_UNKNOWN_VALUES_FOR_RELEASE = listOf(
"releasedAt",
"releasedAtTimestamp",
"submissionId",
"submittedAt",
"submittedAtTimestamp",
"groupId",
)

Expand Down Expand Up @@ -103,6 +103,8 @@ class GetReleasedDataEndpointTest(
"groupName" to TextNode(DEFAULT_GROUP_NAME),
"versionStatus" to TextNode("LATEST_VERSION"),
"dataUseTerms" to TextNode("OPEN"),
"releasedDate" to TextNode(Clock.System.now().toLocalDateTime(TimeZone.UTC).date.toString()),
"submittedDate" to TextNode(Clock.System.now().toLocalDateTime(TimeZone.UTC).date.toString()),
"dataUseTermsRestrictedUntil" to NullNode.getInstance(),
"booleanColumn" to BooleanNode.TRUE,
)
Expand All @@ -114,8 +116,8 @@ class GetReleasedDataEndpointTest(
)
for ((key, value) in it.metadata) {
when (key) {
"submittedAt" -> expectIsTimestampWithCurrentYear(value)
"releasedAt" -> expectIsTimestampWithCurrentYear(value)
"submittedAtTimestamp" -> expectIsTimestampWithCurrentYear(value)
"releasedAtTimestamp" -> expectIsTimestampWithCurrentYear(value)
"submissionId" -> assertThat(value.textValue(), matchesPattern("^custom\\d$"))
"groupId" -> assertThat(value.intValue(), greaterThan(0))
else -> assertThat(value, `is`(expectedMetadata[key]))
Expand Down Expand Up @@ -201,13 +203,14 @@ class GetReleasedDataEndpointTest(
when (key) {
"isRevocation" -> assertThat(value, `is`(BooleanNode.TRUE))
"versionStatus" -> assertThat(value, `is`(TextNode("LATEST_VERSION")))
"submittedAt" -> expectIsTimestampWithCurrentYear(value)
"releasedAt" -> expectIsTimestampWithCurrentYear(value)
"submittedAtTimestamp" -> expectIsTimestampWithCurrentYear(value)
"releasedAtTimestamp" -> expectIsTimestampWithCurrentYear(value)
"submitter" -> assertThat(value, `is`(TextNode(DEFAULT_USER_NAME)))
"groupName" -> assertThat(value, `is`(TextNode(DEFAULT_GROUP_NAME)))
"groupId" -> assertThat(value.intValue(), `is`(greaterThan(0)))
"accession", "version", "accessionVersion", "submissionId" -> {}
"dataUseTerms" -> assertThat(value, `is`(TextNode("OPEN")))
"submittedDate" to TextNode(Clock.System.now().toLocalDateTime(TimeZone.UTC).date.toString()),
else -> assertThat("value for $key", value, `is`(NullNode.instance))
}
}
Expand Down
12 changes: 10 additions & 2 deletions kubernetes/loculus/templates/_common-metadata.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,20 @@ fields:
generateIndex: true
autocomplete: true
header: Submission details
- name: submittedAt
- name: submittedAtTimestamp
type: timestamp
displayName: Date submitted (timestamp)
header: Submission details
- name: submittedDate
type: string
displayName: Date submitted
header: Submission details
- name: releasedAt
- name: releasedAtTimestamp
type: timestamp
displayName: Date released (timestamp)
header: Submission details
- name: releasedDate
type: string
displayName: Date released
header: Submission details
- name: dataUseTerms
Expand Down
2 changes: 1 addition & 1 deletion website/src/pages/seq/[accessionVersion]/versions.astro
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const { organism, versionListResult } = await getVersionsData(accession);
return list.map((version) => (
<li class='mb-4'>
<div class='mb-2'>
<div class='font-semibold'>{version.submittedAt}</div>
<div class='font-semibold'>{version.submittedAtTimestamp}</div>
<div class='flex flex-row gap-3 justify-start'>
<a
href={routes.sequencesDetailsPage(
Expand Down
4 changes: 2 additions & 2 deletions website/src/settings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ export const ACCESSION_FIELD = 'accession';
export const VERSION_FIELD = 'version';
export const VERSION_STATUS_FIELD = 'versionStatus';
export const IS_REVOCATION_FIELD = 'isRevocation';
export const SUBMITTED_AT_FIELD = 'submittedAt';
export const RELEASED_AT_FIELD = 'releasedAt';
export const SUBMITTED_AT_FIELD = 'submittedAtTimestamp';
export const RELEASED_AT_FIELD = 'releasedAtTimestamp';
export const SUBMITTER_FIELD = 'submitter';
export const GROUP_NAME_FIELD = 'groupName';
export const GROUP_ID_FIELD = 'groupId';
Expand Down
4 changes: 2 additions & 2 deletions website/src/types/lapis.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,12 @@ export const sequenceEntryHistoryEntry = accessionVersion
accessionVersion: z.string(),
versionStatus: siloVersionStatusSchema,
isRevocation: z.boolean(),
submittedAt: z.number(),
submittedAtTimestamp: z.number(),
}),
)
.transform((raw) => ({
...raw,
submittedAt: parseUnixTimestamp(raw.submittedAt),
submittedAtTimestamp: parseUnixTimestamp(raw.submittedAtTimestamp),
}));

export type SequenceEntryHistoryEntry = z.infer<typeof sequenceEntryHistoryEntry>;
Expand Down
Loading