Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cornelius' edits of the header count PR #2774

Closed
wants to merge 1 commit into from
Closed

Cornelius' edits of the header count PR #2774

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ private val log = KotlinLogging.logger { }
@RequestMapping("/{organism}")
@Validated
@SecurityRequirement(name = "bearerAuth")
class SubmissionController(
open class SubmissionController(
private val submitModel: SubmitModel,
private val releasedDataModel: ReleasedDataModel,
private val submissionDatabaseService: SubmissionDatabaseService,
Expand Down Expand Up @@ -260,7 +260,7 @@ class SubmissionController(
)
@GetMapping("/get-released-data", produces = [MediaType.APPLICATION_NDJSON_VALUE])
@Transactional(isolation = Isolation.REPEATABLE_READ) // All operations will be performed on the same snapshot
fun getReleasedData(
open fun getReleasedData(
@PathVariable @Valid organism: Organism,
@RequestParam compression: CompressionFormat?,
): ResponseEntity<StreamingResponseBody> {
Expand All @@ -269,9 +269,13 @@ class SubmissionController(
if (compression != null) {
headers.add(HttpHeaders.CONTENT_ENCODING, compression.compressionName)
}

val totalRecords = submissionDatabaseService.countReleasedSubmissions(organism)
headers.add("x-total-records", totalRecords.toString())

// There's a possibility that the totalRecords change between the count and the actual query
// this is not too bad, if the client ends up with a few more records than expected
// We just need to make sure the etag used is from before the count
// Alternatively, we could read once to file while counting and then stream the file
val streamBody = streamTransactioned(compression) { releasedDataModel.getReleasedData(organism) }

return ResponseEntity(streamBody, headers, HttpStatus.OK)
Expand Down
24 changes: 8 additions & 16 deletions backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ private val log = KotlinLogging.logger { }

@Service
@Transactional
class SubmissionDatabaseService(
open class SubmissionDatabaseService(
private val processedSequenceEntryValidatorFactory: ProcessedSequenceEntryValidatorFactory,
private val externalMetadataValidatorFactory: ExternalMetadataValidatorFactory,
private val accessionPreconditionValidator: AccessionPreconditionValidator,
Expand Down Expand Up @@ -550,21 +550,13 @@ class SubmissionDatabaseService(
.associate { it[SequenceEntriesView.accessionColumn] to it[maxVersionExpression]!! }
}

fun countReleasedSubmissions(organism: Organism): Long = SequenceEntriesView.join(
DataUseTermsTable,
JoinType.LEFT,
additionalConstraint = {
(SequenceEntriesView.accessionColumn eq DataUseTermsTable.accessionColumn) and
(DataUseTermsTable.isNewestDataUseTerms)
},
)
.select(
SequenceEntriesView.accessionColumn,
).where {
SequenceEntriesView.statusIs(Status.APPROVED_FOR_RELEASE) and SequenceEntriesView.organismIs(
organism,
)
}.count()
fun countReleasedSubmissions(organism: Organism): Long = SequenceEntriesView.select(
SequenceEntriesView.accessionColumn,
).where {
SequenceEntriesView.statusIs(Status.APPROVED_FOR_RELEASE) and SequenceEntriesView.organismIs(
organism,
)
}.count()

fun streamReleasedSubmissions(organism: Organism): Sequence<RawProcessedData> = SequenceEntriesView.join(
DataUseTermsTable,
Expand Down
Loading