feat: SKFP-1341 add way to customize dataset ids (#257)

Ferlab-Ste-Justine · Jan 8, 2025 · 430715c · 430715c
1 parent 9600f59
commit 430715c
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 9 deletions.
diff --git a/...e-spark3/src/main/scala/bio/ferlab/datalake/spark3/genomics/prepared/VariantCentric.scala b/...e-spark3/src/main/scala/bio/ferlab/datalake/spark3/genomics/prepared/VariantCentric.scala
@@ -85,11 +85,17 @@ import java.time.LocalDateTime
  *    }
  * }}}
  * @param rc the etl context
+ * @param destinationDatasetId custom destination dataset id
+ * @param enrichedVariantsDatasetId custom enrich_variants dataset id
+ * @param enrichedConsequencesDatasetId custom enrich_consequences dataset id
  */
-case class VariantCentric(rc: RuntimeETLContext) extends SimpleSingleETL(rc) {
-  override val mainDestination: DatasetConf = conf.getDataset("es_index_variant_centric")
-  private val enriched_variants: DatasetConf = conf.getDataset("enriched_variants")
-  private val enriched_consequences: DatasetConf = conf.getDataset("enriched_consequences")
+case class VariantCentric(rc: RuntimeETLContext,
+                          destinationDatasetId: String = "es_index_variant_centric",
+                          enrichedVariantsDatasetId: String = "enriched_variants",
+                          enrichedConsequencesDatasetId: String = "enriched_consequences") extends SimpleSingleETL(rc) {
+  override val mainDestination: DatasetConf = conf.getDataset(destinationDatasetId)
+  private val enriched_variants: DatasetConf = conf.getDataset(enrichedVariantsDatasetId)
+  private val enriched_consequences: DatasetConf = conf.getDataset(enrichedConsequencesDatasetId)
 
   override def extract(lastRunValue: LocalDateTime = minValue,
                        currentRunValue: LocalDateTime = LocalDateTime.now()): Map[String, DataFrame] = {

diff --git a/...rk3/src/main/scala/bio/ferlab/datalake/spark3/genomics/prepared/VariantsSuggestions.scala b/...rk3/src/main/scala/bio/ferlab/datalake/spark3/genomics/prepared/VariantsSuggestions.scala
@@ -12,11 +12,14 @@ import org.apache.spark.sql.functions._
 
 import java.time.LocalDateTime
 
-case class VariantsSuggestions(rc: RuntimeETLContext) extends SimpleSingleETL(rc) {
-
-  override val mainDestination: DatasetConf = conf.getDataset("es_index_variant_suggestions")
-  private val enriched_variants: DatasetConf = conf.getDataset("enriched_variants")
-  private val enriched_consequences: DatasetConf = conf.getDataset("enriched_consequences")
+case class VariantsSuggestions(rc: RuntimeETLContext,
+                               destinationDatasetId: String = "es_index_variant_suggestions",
+                               enrichedVariantsDatasetId: String = "enriched_variants",
+                               enrichedConsequencesDatasetId: String = "enriched_consequences") extends SimpleSingleETL(rc) {
+
+  override val mainDestination: DatasetConf = conf.getDataset(destinationDatasetId)
+  private val enriched_variants: DatasetConf = conf.getDataset(enrichedVariantsDatasetId)
+  private val enriched_consequences: DatasetConf = conf.getDataset(enrichedConsequencesDatasetId)
 
   final val variantSymbolAaChangeWeight = 4
   final val variantSymbolWeight = 2