From 3d4e389cbb9be3728e743dc1e25fc4d8f4fa2a52 Mon Sep 17 00:00:00 2001
From: Cornelius Roemer <cornelius.roemer@gmail.com>
Date: Thu, 12 Sep 2024 20:56:32 +0200
Subject: [PATCH] Slight refactors, remove unnecessary left-join (count is not
 affected by left join

---
 .../controller/SubmissionController.kt        | 10 +++++---
 .../submission/SubmissionDatabaseService.kt   | 24 +++++++------------
 2 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt
index 45b6170ad..b6bfee395 100644
--- a/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt
+++ b/backend/src/main/kotlin/org/loculus/backend/controller/SubmissionController.kt
@@ -66,7 +66,7 @@ private val log = KotlinLogging.logger { }
 @RequestMapping("/{organism}")
 @Validated
 @SecurityRequirement(name = "bearerAuth")
-class SubmissionController(
+open class SubmissionController(
     private val submitModel: SubmitModel,
     private val releasedDataModel: ReleasedDataModel,
     private val submissionDatabaseService: SubmissionDatabaseService,
@@ -260,7 +260,7 @@ class SubmissionController(
     )
     @GetMapping("/get-released-data", produces = [MediaType.APPLICATION_NDJSON_VALUE])
     @Transactional(isolation = Isolation.REPEATABLE_READ) // All operations will be performed on the same snapshot
-    fun getReleasedData(
+    open fun getReleasedData(
         @PathVariable @Valid organism: Organism,
         @RequestParam compression: CompressionFormat?,
     ): ResponseEntity<StreamingResponseBody> {
@@ -269,9 +269,13 @@ class SubmissionController(
         if (compression != null) {
             headers.add(HttpHeaders.CONTENT_ENCODING, compression.compressionName)
         }
+
         val totalRecords = submissionDatabaseService.countReleasedSubmissions(organism)
         headers.add("x-total-records", totalRecords.toString())
-
+        // There's a possibility that the totalRecords change between the count and the actual query
+        // this is not too bad, if the client ends up with a few more records than expected
+        // We just need to make sure the etag used is from before the count
+        // Alternatively, we could read once to file while counting and then stream the file
         val streamBody = streamTransactioned(compression) { releasedDataModel.getReleasedData(organism) }
 
         return ResponseEntity(streamBody, headers, HttpStatus.OK)
diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt
index e95382bf8..59214ad7e 100644
--- a/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt
+++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/SubmissionDatabaseService.kt
@@ -82,7 +82,7 @@ private val log = KotlinLogging.logger { }
 
 @Service
 @Transactional
-class SubmissionDatabaseService(
+open class SubmissionDatabaseService(
     private val processedSequenceEntryValidatorFactory: ProcessedSequenceEntryValidatorFactory,
     private val externalMetadataValidatorFactory: ExternalMetadataValidatorFactory,
     private val accessionPreconditionValidator: AccessionPreconditionValidator,
@@ -550,21 +550,13 @@ class SubmissionDatabaseService(
             .associate { it[SequenceEntriesView.accessionColumn] to it[maxVersionExpression]!! }
     }
 
-    fun countReleasedSubmissions(organism: Organism): Long = SequenceEntriesView.join(
-        DataUseTermsTable,
-        JoinType.LEFT,
-        additionalConstraint = {
-            (SequenceEntriesView.accessionColumn eq DataUseTermsTable.accessionColumn) and
-                (DataUseTermsTable.isNewestDataUseTerms)
-        },
-    )
-        .select(
-            SequenceEntriesView.accessionColumn,
-        ).where {
-            SequenceEntriesView.statusIs(Status.APPROVED_FOR_RELEASE) and SequenceEntriesView.organismIs(
-                organism,
-            )
-        }.count()
+    fun countReleasedSubmissions(organism: Organism): Long = SequenceEntriesView.select(
+        SequenceEntriesView.accessionColumn,
+    ).where {
+        SequenceEntriesView.statusIs(Status.APPROVED_FOR_RELEASE) and SequenceEntriesView.organismIs(
+            organism,
+        )
+    }.count()
 
     fun streamReleasedSubmissions(organism: Organism): Sequence<RawProcessedData> = SequenceEntriesView.join(
         DataUseTermsTable,