feat(jaeger): add Jaeger for Distributed Tracing

server/app/build.gradle.kts

@@ -68,6 +68,8 @@ dependencies {
     implementation("org.springframework.boot:spring-boot-starter-data-jdbc")
     implementation("org.jooq:jooq:3.19.11")
     jooqGenerator("org.jooq:jooq-meta-extensions:3.19.11")
+    implementation("io.micrometer:micrometer-tracing-bridge-otel")
+    implementation("io.opentelemetry:opentelemetry-exporter-otlp")
 }
 
 tasks.named<Test>("test") {
@@ -107,6 +109,7 @@ sourceSets {
     main {
         kotlin {
             srcDir(layout.buildDirectory.dir("openapi/src/main").get().toString())
+            srcDir(layout.buildDirectory.dir("generated-src/jooq/main").get().toString())
         }
     }
 }

server/app/src/main/kotlin/mu/muse/application/ApplicationConfiguration.kt

@@ -3,6 +3,7 @@ package mu.muse.application
 import mu.muse.application.muse.PersistenceConfiguration
 import mu.muse.application.muse.RestConfiguration
 import mu.muse.application.muse.SecurityConfiguration
+import mu.muse.application.muse.TracingConfiguration
 import mu.muse.application.muse.UseCaseConfiguration
 import org.springframework.context.annotation.Configuration
 import org.springframework.context.annotation.Import
@@ -14,5 +15,6 @@ import org.springframework.context.annotation.Import
     RestConfiguration::class,
     SecurityConfiguration::class,
     MvcConfiguration::class,
+    TracingConfiguration::class,
 )
 class ApplicationConfiguration

server/app/src/main/kotlin/mu/muse/application/muse/JaegerDbSpanCreator.kt

@@ -0,0 +1,33 @@
+package mu.muse.application.muse
+
+import io.opentelemetry.api.trace.Span
+import io.opentelemetry.api.trace.Tracer
+import org.jooq.ExecuteContext
+import org.jooq.ExecuteListener
+import java.util.concurrent.ConcurrentHashMap
+
+class JaegerDbSpanCreator(
+    private val tracer: Tracer,
+) : ExecuteListener {
+
+    companion object {
+        const val CONTEXT_KEY_SPAN_ID = "spanId"
+    }
+
+    private val spanIdToSpan = ConcurrentHashMap<String, Span>()
+
+    override fun start(ctx: ExecuteContext) {
+        val span = tracer.spanBuilder("PostgreSQL Query").startSpan()
+        span.setAttribute("sql", ctx.query().toString())
+        val spanId = span.spanContext.spanId
+        spanIdToSpan[spanId] = span
+        ctx.data(CONTEXT_KEY_SPAN_ID, spanId)
+    }
+
+    override fun end(ctx: ExecuteContext) {
+        val spanId = ctx.data(CONTEXT_KEY_SPAN_ID)
+        val span = spanIdToSpan.remove(spanId)
+            ?: throw RuntimeException("No span for Span ID=`${spanId}`") // fail-fast in case of breaking invariant
+        span.end()
+    }
+}

server/app/src/main/kotlin/mu/muse/application/muse/PersistenceConfiguration.kt

@@ -2,13 +2,15 @@ package mu.muse.application.muse
 
 import com.zaxxer.hikari.HikariConfig
 import com.zaxxer.hikari.HikariDataSource
+import io.opentelemetry.api.trace.Tracer
 import mu.muse.persistence.instrument.jooq.JooqPostgresInstrumentIdGenerator
 import mu.muse.persistence.instrument.jooq.JooqPostgresInstrumentRepository
 import mu.muse.persistence.user.jooq.JooqPostgresUserIdGenerator
 import mu.muse.persistence.user.jooq.JooqPostgresUserRepository
 import org.jooq.DSLContext
 import org.jooq.SQLDialect
 import org.jooq.impl.DSL
+import org.jooq.impl.DefaultConfiguration
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
@@ -47,7 +49,16 @@ class PersistenceConfiguration {
         }
 
     @Bean
-    fun dslContext(dataSource: DataSource): DSLContext = DSL.using(dataSource, SQLDialect.POSTGRES)
+    fun jooqConfiguration(dataSource: DataSource, otelTracer: Tracer) =
+        with(DefaultConfiguration()) {
+            set(dataSource)
+            set(SQLDialect.POSTGRES)
+            set(JaegerDbSpanCreator(otelTracer))
+            this
+        }
+
+    @Bean
+    fun dslContext(jooqConfiguration: org.jooq.Configuration) = DSL.using(jooqConfiguration)
 
     @Bean
     fun userIdGenerator(dslContext: DSLContext) = JooqPostgresUserIdGenerator(dslContext)

server/app/src/main/kotlin/mu/muse/application/muse/TracingConfiguration.kt

@@ -0,0 +1,20 @@
+package mu.muse.application.muse
+
+import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter
+import org.springframework.beans.factory.annotation.Value
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+
+@Configuration
+class TracingConfiguration(
+    @Value("\${tracing.url}") private val tracingUrl: String,
+) {
+
+    @Bean
+    fun otlpHttpSpanExporter(): OtlpGrpcSpanExporter {
+        return OtlpGrpcSpanExporter
+            .builder()
+            .setEndpoint(tracingUrl)
+            .build()
+    }
+}

server/app/src/main/resources/application.properties

@@ -21,3 +21,5 @@ spring.sql.init.mode=always
 spring.sql.init.schema-locations=classpath:db/schema.sql
 spring.sql.init.data-locations=classpath:db/data.sql
 
+management.tracing.sampling.probability=1.0
+tracing.url=${OPEN_TELEMETERY_URL}  # there is no autoconfiguration for default exporter, so it requires to set up it explicitly

tools/docker/docker-compose.yml

@@ -46,6 +46,7 @@ services:
       - POSTGRES_DB
       - POSTGRES_DSN
       - CLIENT_PORT
+      - OPEN_TELEMETERY_URL
     networks:
       - muse
     profiles:
@@ -88,6 +89,23 @@ services:
       - muse
     profiles:
       - local
+
+  muse-jaeger:
+    image: jaegertracing/all-in-one:1.60
+    container_name: "muse-jaeger"
+    ports:
+      - "${OPEN_TELEMETERY_COLLECTOR_PORT}:4317"
+      - "${JAEGER_UI_PORT}:16686"
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+    networks:
+      - muse
+    profiles:
+      - local
+      - dev
+      - staging
+      - prod
+
 ##  portainer:
 #    image: portainer/portainer-ce:linux-amd64-2.20.3-alpine
 #    container_name: portainer

tools/docker/env/dev.env

@@ -12,4 +12,10 @@ POSTGRES_PORT=50009
 POSTGRES_USER=muse
 POSTGRES_PASSWORD=muse
 POSTGRES_DB=muse
-POSTGRES_DSN=jdbc:postgresql://muse-postgres:5432/${POSTGRES_DB} # used default postgresql port due to bridge driver
+POSTGRES_DSN=jdbc:postgresql://muse-postgres:5432/${POSTGRES_DB} # used default postgresql port due to bridge driver
+
+OPEN_TELEMETERY_COLLECTOR_PORT=50023
+OPEN_TELEMETERY_URL=http://muse-jaeger:4317
+
+JAEGER_UI_PORT=50028
+

tools/docker/env/local.env

@@ -13,3 +13,8 @@ POSTGRES_USER=muse
 POSTGRES_PASSWORD=muse
 POSTGRES_DB=muse
 POSTGRES_DSN=jdbc:postgresql://muse-postgres:5432/${POSTGRES_DB} # used default postgresql port due to bridge driver
+
+OPEN_TELEMETERY_COLLECTOR_PORT=50021
+OPEN_TELEMETERY_URL=http://muse-jaeger:4317
+
+JAEGER_UI_PORT=50026

tools/docker/env/prod.env

@@ -13,3 +13,8 @@ POSTGRES_USER=muse
 POSTGRES_PASSWORD=muse
 POSTGRES_DB=muse
 POSTGRES_DSN=jdbc:postgresql://muse-postgres:5432/${POSTGRES_DB} # used default postgresql port due to bridge driver
+
+OPEN_TELEMETERY_COLLECTOR_PORT=50024
+OPEN_TELEMETERY_URL=http://muse-jaeger:4317
+
+JAEGER_UI_PORT=50025

tools/docker/env/staging.env

@@ -12,4 +12,10 @@ POSTGRES_PORT=50015
 POSTGRES_USER=muse
 POSTGRES_PASSWORD=muse
 POSTGRES_DB=muse
-POSTGRES_DSN=jdbc:postgresql://muse-postgres:5432/${POSTGRES_DB} # used default postgresql port due to bridge driver
+POSTGRES_DSN=jdbc:postgresql://muse-postgres:5432/${POSTGRES_DB} # used default postgresql port due to bridge driver
+
+OPEN_TELEMETERY_COLLECTOR_PORT=50022
+OPEN_TELEMETERY_URL=http://muse-jaeger:4317
+
+JAEGER_UI_PORT=50027
+