add option in cli for ranked relevance score directory

ArDoCo · Aug 21, 2024 · 09e828a · 09e828a
1 parent 6e11081
commit 09e828a
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 10 deletions.
diff --git a/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/RankCommand.kt b/cli/src/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/cli/commands/RankCommand.kt
@@ -27,6 +27,12 @@ class RankCommand(private val outputFileOption: SingleNullableOption<String>) :
         fullName = "ground-truth"
     ).required()
     private val fileHeaderOption by option(ArgType.Boolean, description = "Whether the files have a header", fullName = "header").default(false)
+    private val rankedRelevanceListDirectoryOption by option(
+        ArgType.String,
+        shortName = "rrl",
+        description = "The directory of the ranked relevance list files",
+        fullName = "ranked-relevance-list-directory"
+    ).default("")
 
     override fun execute() {
         println("Calculating rank metrics")
@@ -42,8 +48,7 @@ class RankCommand(private val outputFileOption: SingleNullableOption<String>) :
             return
         }
         val rankedResults: List<List<String>> =
-            rankedListDirectory.listFiles()?.filter {
-                    file ->
+            rankedListDirectory.listFiles()?.filter { file ->
                 file.isFile
             }?.map { file -> file.readLines().filter { it.isNotBlank() }.drop(if (fileHeaderOption) 1 else 0) } ?: emptyList()
         if (rankedResults.isEmpty()) {
@@ -52,12 +57,28 @@ class RankCommand(private val outputFileOption: SingleNullableOption<String>) :
         }
         val groundTruth = groundTruthFile.readLines().filter { it.isNotBlank() }.drop(if (fileHeaderOption) 1 else 0).toSet()
 
+        var rankedRelevances: List<List<Double>>? = null
+        if (rankedRelevanceListDirectoryOption != "") {
+            val rankedRelevanceListDirectory = File(rankedRelevanceListDirectoryOption)
+            if (!rankedRelevanceListDirectory.exists() || !rankedRelevanceListDirectory.isDirectory) {
+                println("The directory of the ranked relevance list files does not exist or is not a directory")
+                return
+            }
+            rankedRelevances = rankedRelevanceListDirectory.listFiles()?.filter { file ->
+                file.isFile
+            }?.map { file -> file.readLines().filter { it.isNotBlank() }.map { it.toDouble() }.drop(if (fileHeaderOption) 1 else 0) } ?: emptyList()
+            if (rankedRelevances.isEmpty()) {
+                println("No relevance scores found")
+                return
+            }
+        }
         val rankMetrics = RankMetricsCalculator.Instance
 
         val result =
             rankMetrics.calculateMetrics(
                 rankedResults,
-                groundTruth
+                groundTruth,
+                rankedRelevances
             )
         result.prettyPrint()
 

diff --git a/...c/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/rest/controller/RankMetricsController.kt b/...c/main/kotlin/edu/kit/kastel/mcse/ardoco/metrics/rest/controller/RankMetricsController.kt
@@ -21,11 +21,11 @@ class RankMetricsController {
 
     @Operation(summary = "Calculate rank metrics for one project")
     @PostMapping
-    fun calculateClassificationMetrics(
+    fun calculateRankMetrics(
         @RequestBody body: RankMetricsRequest
     ): SingleRankMetricsResult {
         val rankMetricsCalculator = RankMetricsCalculator.Instance
-        val result = rankMetricsCalculator.calculateMetrics(body.rankedResults, body.groundTruth)
+        val result = rankMetricsCalculator.calculateMetrics(body.rankedResults, body.groundTruth, body.rankedRelevances)
         return result
     }
 
@@ -36,10 +36,10 @@ class RankMetricsController {
     ): AverageRankMetricsResponse {
         val rankMetricsCalculator = RankMetricsCalculator.Instance
 
-        val requests = body.classificationMetricsRequests
+        val requests = body.rankMetricsRequests
         val results =
             requests.map {
-                rankMetricsCalculator.calculateMetrics(it.rankedResults, it.groundTruth)
+                rankMetricsCalculator.calculateMetrics(it.rankedResults, it.groundTruth, it.rankedRelevances)
             }
 
         val averages = rankMetricsCalculator.calculateAverages(results, body.weights)
@@ -48,16 +48,17 @@ class RankMetricsController {
     }
 
     data class AverageRankMetricsRequest(
-        val classificationMetricsRequests: List<RankMetricsRequest>,
+        val rankMetricsRequests: List<RankMetricsRequest>,
         val weights: List<Int>? = null
     )
 
     data class AverageRankMetricsResponse(
-        val classificationResults: List<AggregatedRankMetricsResult>
+        val rankResults: List<AggregatedRankMetricsResult>
     )
 
     data class RankMetricsRequest(
         val rankedResults: List<List<String>>,
-        val groundTruth: Set<String>
+        val groundTruth: Set<String>,
+        val rankedRelevances: List<List<Double>>?
     )
 }