Adding connection information when a prepare fails.

service-self-backfill/src/main/kotlin/app/cash/backfila/service/selfbackfill/LocalClientServiceClient.kt

@@ -24,4 +24,6 @@ internal class LocalClientServiceClient @Inject internal constructor(
   override suspend fun runBatch(request: RunBatchRequest): RunBatchResponse {
     return backfilaClientServiceHandler.runBatch(request)
   }
+
+  override fun connectionLogData() = "LocalClientServiceClient so no connection"
 }

service-self-backfill/src/test/kotlin/app/cash/backfila/service/selfbackfill/BackfilaSelfBackfillDevelopmentService.kt

@@ -85,6 +85,8 @@ internal fun main(args: Array<String>) {
                 override suspend fun runBatch(request: RunBatchRequest): RunBatchResponse {
                   TODO("Not yet implemented")
                 }
+
+                override fun connectionLogData() = "Fake development service client"
               }
             }
           },

service/src/main/kotlin/app/cash/backfila/BackfillCreator.kt

@@ -126,24 +126,38 @@ class BackfillCreator @Inject constructor(
       )
     } catch (e: Exception) {
       logger.info(e) { "PrepareBackfill on `$service` failed" }
-      throw BadRequestException("PrepareBackfill on `$service` failed: ${e.message}", e)
+      throw BadRequestException(
+        "PrepareBackfill on `$service` failed: ${e.message}. " +
+          "connectionData: ${client.connectionLogData()}",
+        e,
+      )
     }
 
     prepareBackfillResponse.error_message?.let {
-      throw BadRequestException("PrepareBackfill on `$service` failed: $it")
+      throw BadRequestException(
+        "PrepareBackfill on `$service` failed: $it. " +
+          "connectionData: ${client.connectionLogData()}",
+      )
     }
 
     val partitions = prepareBackfillResponse.partitions
     if (partitions.isEmpty()) {
-      throw BadRequestException("PrepareBackfill returned no partitions")
+      throw BadRequestException(
+        "PrepareBackfill returned no partitions. " +
+          "connectionData: ${client.connectionLogData()}",
+      )
     }
     if (partitions.any { it.partition_name == null }) {
-      throw BadRequestException("PrepareBackfill returned unnamed partitions")
+      throw BadRequestException(
+        "PrepareBackfill returned unnamed partitions. " +
+          "connectionData: ${client.connectionLogData()}",
+      )
     }
     if (partitions.distinctBy { it.partition_name }.size != partitions.size) {
       throw BadRequestException(
         "PrepareBackfill did not return distinct partition names:" +
-          " ${partitions.map { it.partition_name }}",
+          " ${partitions.map { it.partition_name }}. " +
+          "connectionData: ${client.connectionLogData()}",
       )
     }
 

service/src/main/kotlin/app/cash/backfila/client/BackfilaClientServiceClient.kt

@@ -13,4 +13,9 @@ interface BackfilaClientServiceClient {
   suspend fun getNextBatchRange(request: GetNextBatchRangeRequest): GetNextBatchRangeResponse
 
   suspend fun runBatch(request: RunBatchRequest): RunBatchResponse
+
+  /**
+   * Gives us a way to probe for connection information when something fails to help with debugging.
+   */
+  fun connectionLogData(): String
 }

service/src/main/kotlin/app/cash/backfila/client/EnvoyClientServiceClient.kt

@@ -11,6 +11,7 @@ import retrofit2.Response
 
 internal class EnvoyClientServiceClient internal constructor(
   private val api: EnvoyClientServiceApi,
+  private val connectionLogData: String,
 ) : BackfilaClientServiceClient {
 
   override fun prepareBackfill(request: PrepareBackfillRequest): PrepareBackfillResponse {
@@ -26,6 +27,8 @@ internal class EnvoyClientServiceClient internal constructor(
     return api.runBatch(request)
   }
 
+  override fun connectionLogData() = connectionLogData
+
   private fun <T> Response<T>.getOrThrow(): T {
     if (!this.isSuccessful) {
       throw HttpException(this)

service/src/main/kotlin/app/cash/backfila/client/EnvoyClientServiceClientProvider.kt

@@ -71,7 +71,10 @@ class EnvoyClientServiceClientProvider @Inject constructor(
       .addCallAdapterFactory(GuavaCallAdapterFactory.create())
       .build()
     val api = retrofit.create(EnvoyClientServiceApi::class.java)
-    return EnvoyClientServiceClient(api)
+    val logData = "envoyConfig: ${httpClientEndpointConfig.envoy}, " +
+      "url: ${httpClientEndpointConfig.url}, " +
+      "headers: $headers"
+    return EnvoyClientServiceClient(api, logData)
   }
 
   private fun adapter() = moshi.adapter<EnvoyConnectorData>()

service/src/main/kotlin/app/cash/backfila/client/HttpClientServiceClient.kt

@@ -11,6 +11,7 @@ import retrofit2.Response
 
 internal class HttpClientServiceClient internal constructor(
   private val api: HttpClientServiceApi,
+  private val connectionLogData: String,
 ) : BackfilaClientServiceClient {
 
   override fun prepareBackfill(request: PrepareBackfillRequest): PrepareBackfillResponse {
@@ -26,6 +27,8 @@ internal class HttpClientServiceClient internal constructor(
     return api.runBatch(request)
   }
 
+  override fun connectionLogData() = connectionLogData
+
   private fun <T> Response<T>.getOrThrow(): T {
     if (!this.isSuccessful) {
       throw HttpException(this)

service/src/main/kotlin/app/cash/backfila/client/HttpClientServiceClientProvider.kt

@@ -67,7 +67,9 @@ class HttpClientServiceClientProvider @Inject constructor(
       .addCallAdapterFactory(GuavaCallAdapterFactory.create())
       .build()
     val api = retrofit.create(HttpClientServiceApi::class.java)
-    return HttpClientServiceClient(api)
+    val logData = "url: ${httpClientEndpointConfig.url}, " +
+      "headers: $headers"
+    return HttpClientServiceClient(api, logData)
   }
 
   private fun adapter() = moshi.adapter<HttpConnectorData>()

service/src/test/kotlin/app/cash/backfila/actions/CreateBackfillActionTest.kt

@@ -406,7 +406,7 @@ class CreateBackfillActionTest {
             .build(),
         )
       }.isInstanceOf(BadRequestException::class.java)
-        .hasMessage("PrepareBackfill on `deep-fryer` failed: We're out of chicken")
+        .hasMessage("PrepareBackfill on `deep-fryer` failed: We're out of chicken. connectionData: FakeBackfilaClientServiceClient so no connection")
 
       transacter.transaction { session ->
         val runs = queryFactory.newQuery<BackfillRunQuery>().list(session)

service/src/test/kotlin/app/cash/backfila/client/FakeBackfilaClientServiceClient.kt

@@ -93,4 +93,6 @@ class FakeBackfilaClientServiceClient @Inject constructor() : BackfilaClientServ
     runBatchRequests.send(request)
     return runBatchResponses.receive().getOrThrow()
   }
+
+  override fun connectionLogData() = "FakeBackfilaClientServiceClient so no connection"
 }

service/src/test/kotlin/app/cash/backfila/development/BackfilaDevelopmentService.kt

@@ -90,6 +90,8 @@ fun main(args: Array<String>) {
                 override suspend fun runBatch(request: RunBatchRequest): RunBatchResponse {
                   TODO("Not yet implemented")
                 }
+
+                override fun connectionLogData() = "Fake development service client"
               }
             }
           },

service/src/test/kotlin/app/cash/backfila/development/finedining/ClumsyMealsBackfill.kt

@@ -0,0 +1,46 @@
+package app.cash.backfila.development.finedining
+
+import app.cash.backfila.client.BackfillConfig
+import app.cash.backfila.client.Description
+import app.cash.backfila.client.PrepareBackfillConfig
+import app.cash.backfila.client.stat.StaticDatasourceBackfill
+import javax.inject.Inject
+import wisp.logging.getLogger
+
+class ClumsyMealsBackfill @Inject constructor() : StaticDatasourceBackfill<String, ClumsyMealsBackfill.ClumsyMealsAttributes>() {
+  var servedPlates = 0 // Keeps track of the number of successfully served plates.
+  var brokenPlatesSoFar = 0 // When plates break it keeps track of how many have broken so far.
+  override fun validate(config: PrepareBackfillConfig<ClumsyMealsAttributes>) {
+    if (config.parameters.blockedEntrance) {
+      throw RuntimeException("Entrance is BLOCKED. No customers can be served!")
+    }
+  }
+
+  override fun runOne(item: String, config: BackfillConfig<ClumsyMealsAttributes>) {
+    Thread.sleep(100L) // Sleep for half a second
+    // We potentially break on every 25th plate.
+    if (servedPlates % 25 == 0 && brokenPlatesSoFar < config.parameters.brokenPlates) {
+      brokenPlatesSoFar++
+      logger.info { "Broke a plate!" }
+      throw RuntimeException("Failed to serve: Broke a plate!!!")
+    }
+    logger.info { "Poorly served $item" }
+    servedPlates++
+    brokenPlatesSoFar = 0
+  }
+
+  @Description("Adaptable clumsiness.")
+  data class ClumsyMealsAttributes(
+    @Description("Whether the entrance is blocked or not. Can be used to test Prepare failures.")
+    val blockedEntrance: Boolean = true,
+    @Description("How many plates break before a successful plate is served. Can be used to test retry logic.")
+    val brokenPlates: Int = 1,
+  )
+
+  // Generate the meal place settings for the backfill.
+  override val staticDatasource: List<String> = (1..5000).map { i -> "place setting $i" }
+
+  companion object {
+    private val logger = getLogger<ClumsyMealsBackfill>()
+  }
+}

service/src/test/kotlin/app/cash/backfila/development/finedining/FineDiningServiceModule.kt

@@ -19,5 +19,6 @@ internal class FineDiningServiceModule : KAbstractModule() {
       ),
     )
     install(StaticDatasourceBackfillModule.create<SlowMealsBackfill>())
+    install(StaticDatasourceBackfillModule.create<ClumsyMealsBackfill>())
   }
 }

service/src/test/kotlin/app/cash/backfila/development/finedining/SlowMealsBackfill.kt

@@ -12,11 +12,11 @@ class SlowMealsBackfill @Inject constructor() : StaticDatasourceBackfill<String,
   }
 
   data class SlowMealsAttributes(
-    val mealDelayMs: Long = 1000L,
+    val mealDelayMs: Long = 100L,
   )
 
   // Generate the meal place settings for the backfill.
-  override val staticDatasource: List<String> = (1..1000).map { i -> "place setting $i" }
+  override val staticDatasource: List<String> = (1..5000).map { i -> "place setting $i" }
 
   companion object {
     private val logger = getLogger<SlowMealsBackfill>()