From fbd3218a165921211ff7dde6820d4dde7491e7ac Mon Sep 17 00:00:00 2001
From: Ian Streeter <ian@snowplowanalytics.com>
Date: Tue, 7 Jan 2025 19:31:29 +0000
Subject: [PATCH] With AQE

---
 modules/core/src/main/resources/reference.conf                 | 2 +-
 .../processing/SparkUtils.scala                                | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/resources/reference.conf b/modules/core/src/main/resources/reference.conf
index 33d8526..1a91488 100644
--- a/modules/core/src/main/resources/reference.conf
+++ b/modules/core/src/main/resources/reference.conf
@@ -128,7 +128,7 @@
       "spark.memory.storageFraction": "0"
       "spark.databricks.delta.autoCompact.enabled": "false"
       "spark.scheduler.mode": "FAIR"
-      "spark.sql.adaptive.enabled": "false"
+      "spark.sql.adaptive.enabled": "true"
     }
     "gcpUserAgent": ${gcpUserAgent}
   }
diff --git a/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala b/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala
index 2dbe4bf..31c437d 100644
--- a/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala
+++ b/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala
@@ -119,5 +119,6 @@ private[processing] object SparkUtils {
    * the intermediate dataframes.
    */
   private def chooseWriterParallelism(): Int =
-    (Runtime.getRuntime.availableProcessors - 1).max(1)
+    200
+  // (Runtime.getRuntime.availableProcessors - 1).max(1)
 }