From 131b7d2494d5f1cf49e0a8639af0973f047d10ce Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Tue, 17 Oct 2023 17:17:29 -0700 Subject: [PATCH 01/20] update to v2 evaluation --- build.gradle.kts | 2 +- core/build.gradle.kts | 5 +- core/src/main/kotlin/Config.kt | 2 +- core/src/main/kotlin/EvaluationProxy.kt | 42 ++-- core/src/main/kotlin/assignment/Assignment.kt | 18 +- .../kotlin/assignment/AssignmentTracker.kt | 36 ++- core/src/main/kotlin/cohort/CohortApi.kt | 1 - core/src/main/kotlin/cohort/CohortStorage.kt | 1 - .../main/kotlin/deployment/DeploymentApi.kt | 13 +- .../kotlin/deployment/DeploymentStorage.kt | 25 +- core/src/main/kotlin/project/ProjectProxy.kt | 70 +++--- core/src/main/kotlin/project/ProjectRunner.kt | 4 +- .../src/main/kotlin/util/EvaluationContext.kt | 55 +++++ core/src/main/kotlin/util/FlagConfig.kt | 39 ++-- core/src/main/kotlin/util/Redis.kt | 2 +- core/src/test/kotlin/Utils.kt | 15 ++ .../kotlin/assignment/AssignmentFilterTest.kt | 83 ++++--- .../assignment/AssignmentServiceTest.kt | 38 ++-- core/src/test/kotlin/assignment/Utils.kt | 15 -- core/src/test/kotlin/util/CacheTest.kt | 2 - gradle.properties | 10 +- service/build.gradle.kts | 5 +- service/src/main/kotlin/Server.kt | 215 +++++++++++------- 23 files changed, 420 insertions(+), 278 deletions(-) create mode 100644 core/src/main/kotlin/util/EvaluationContext.kt create mode 100644 core/src/test/kotlin/Utils.kt delete mode 100644 core/src/test/kotlin/assignment/Utils.kt diff --git a/build.gradle.kts b/build.gradle.kts index fcb0877..ca7559e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - kotlin("jvm") version "1.8.10" + kotlin("jvm") version "1.9.10" id("io.github.gradle-nexus.publish-plugin") version "1.1.0" } diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 59e81d1..adf4029 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -1,8 +1,8 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { - kotlin("jvm") version "1.8.10" - kotlin("plugin.serialization") version "1.8.0" + kotlin("jvm") version "1.9.10" + kotlin("plugin.serialization") version "1.9.0" `maven-publish` signing id("org.jlleitschuh.gradle.ktlint") version "11.3.1" @@ -29,7 +29,6 @@ val kaml: String by project dependencies { implementation("com.amplitude:evaluation-core:$experimentEvaluationVersion") - implementation("com.amplitude:evaluation-serialization:$experimentEvaluationVersion") implementation("com.amplitude:java-sdk:$amplitudeAnalytics") implementation("org.json:json:$amplitudeAnalyticsJson") implementation("io.lettuce:lettuce-core:$lettuce") diff --git a/core/src/main/kotlin/Config.kt b/core/src/main/kotlin/Config.kt index f843169..4da74b9 100644 --- a/core/src/main/kotlin/Config.kt +++ b/core/src/main/kotlin/Config.kt @@ -185,7 +185,7 @@ object EnvKey { object Default { const val PORT = 3546 - const val SERVER_URL = "https://api.lab.amplitude.com" + const val SERVER_URL = "https://flag.lab.amplitude.com" const val COHORT_SERVER_URL = "https://cohort.lab.amplitude.com" const val FLAG_SYNC_INTERVAL_MILLIS = 10 * 1000L const val COHORT_SYNC_INTERVAL_MILLIS = 60 * 1000L diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index 5741f14..e29abba 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -1,11 +1,8 @@ package com.amplitude import com.amplitude.deployment.getDeploymentStorage -import com.amplitude.experiment.evaluation.FlagConfig -import com.amplitude.experiment.evaluation.SkylabUser -import com.amplitude.experiment.evaluation.Variant -import com.amplitude.experiment.evaluation.serialization.SerialFlagConfig -import com.amplitude.experiment.evaluation.serialization.SerialVariant +import com.amplitude.experiment.evaluation.EvaluationFlag +import com.amplitude.experiment.evaluation.EvaluationVariant import com.amplitude.project.ProjectProxy import com.amplitude.project.getProjectStorage import com.amplitude.util.json @@ -68,7 +65,7 @@ class EvaluationProxy( log.info("Evaluation proxy shut down.") } - suspend fun getFlagConfigs(deploymentKey: String?): List { + suspend fun getFlagConfigs(deploymentKey: String?): List { val project = deploymentProxies[deploymentKey] ?: throw HttpErrorResponseException(404, "Deployment not found.") val projectProxy = projectProxies[project] ?: throw HttpErrorResponseException(404, "Project not found.") return projectProxy.getFlagConfigs(deploymentKey) @@ -82,28 +79,41 @@ class EvaluationProxy( suspend fun evaluate( deploymentKey: String?, - user: SkylabUser?, + user: Map?, flagKeys: Set? = null - ): Map { + ): Map { val project = deploymentProxies[deploymentKey] ?: throw HttpErrorResponseException(404, "Deployment not found.") val projectProxy = projectProxies[project] ?: throw HttpErrorResponseException(404, "Project not found.") return projectProxy.evaluate(deploymentKey, user, flagKeys) } + + suspend fun evaluateV1( + deploymentKey: String?, + user: Map?, + flagKeys: Set? = null + ): Map { + val project = deploymentProxies[deploymentKey] ?: throw HttpErrorResponseException(404, "Deployment not found.") + val projectProxy = projectProxies[project] ?: throw HttpErrorResponseException(404, "Project not found.") + return projectProxy.evaluateV1(deploymentKey, user, flagKeys) + } } +// Serialized Proxy Calls + suspend fun EvaluationProxy.getSerializedFlagConfigs(deploymentKey: String?): String = - getFlagConfigs(deploymentKey).encodeToJsonString() + json.encodeToString(getFlagConfigs(deploymentKey)) suspend fun EvaluationProxy.getSerializedCohortMembershipsForUser(deploymentKey: String?, userId: String?): String = - getCohortMembershipsForUser(deploymentKey, userId).encodeToJsonString() + json.encodeToString(getCohortMembershipsForUser(deploymentKey, userId)) suspend fun EvaluationProxy.serializedEvaluate( deploymentKey: String?, - user: SkylabUser?, + user: Map?, flagKeys: Set? = null -) = evaluate(deploymentKey, user, flagKeys).encodeToJsonString() +): String = json.encodeToString(evaluate(deploymentKey, user, flagKeys)) -private fun List.encodeToJsonString(): String = json.encodeToString(map { SerialFlagConfig(it) }) -private fun Set.encodeToJsonString(): String = json.encodeToString(this) -private fun Map.encodeToJsonString(): String = - json.encodeToString(mapValues { SerialVariant(it.value) }) +suspend fun EvaluationProxy.serializedEvaluateV1( + deploymentKey: String?, + user: Map?, + flagKeys: Set? = null +): String = json.encodeToString(evaluateV1(deploymentKey, user, flagKeys)) diff --git a/core/src/main/kotlin/assignment/Assignment.kt b/core/src/main/kotlin/assignment/Assignment.kt index e8b43d6..e3e3559 100644 --- a/core/src/main/kotlin/assignment/Assignment.kt +++ b/core/src/main/kotlin/assignment/Assignment.kt @@ -1,21 +1,25 @@ package com.amplitude.assignment -import com.amplitude.experiment.evaluation.FlagResult -import com.amplitude.experiment.evaluation.SkylabUser +import com.amplitude.experiment.evaluation.EvaluationContext +import com.amplitude.experiment.evaluation.EvaluationVariant +import com.amplitude.util.deviceId +import com.amplitude.util.userId const val DAY_MILLIS: Long = 24 * 60 * 60 * 1000 data class Assignment( - val user: SkylabUser, - val results: Map, + val context: EvaluationContext, + val results: Map, val timestamp: Long = System.currentTimeMillis() ) fun Assignment.canonicalize(): String { - val sb = StringBuilder().append(this.user.userId?.trim(), " ", this.user.deviceId?.trim(), " ") + val sb = StringBuilder().append(this.context.userId()?.trim(), " ", this.context.deviceId()?.trim(), " ") for (key in this.results.keys.sorted()) { - val value = this.results[key] - sb.append(key.trim(), " ", value?.variant?.key?.trim(), " ") + val variant = this.results[key] + sb.append(key.trim(), " ", variant?.key?.trim(), " ") } return sb.toString() } + + diff --git a/core/src/main/kotlin/assignment/AssignmentTracker.kt b/core/src/main/kotlin/assignment/AssignmentTracker.kt index 8d2d164..8710c38 100644 --- a/core/src/main/kotlin/assignment/AssignmentTracker.kt +++ b/core/src/main/kotlin/assignment/AssignmentTracker.kt @@ -3,9 +3,18 @@ package com.amplitude.assignment import com.amplitude.Amplitude import com.amplitude.AssignmentConfiguration import com.amplitude.Event -import com.amplitude.experiment.evaluation.FLAG_TYPE_MUTUAL_EXCLUSION_GROUP +import com.amplitude.util.deviceId +import com.amplitude.util.userId import org.json.JSONObject +object FlagType { + const val RELEASE = "release" + const val EXPERIMENT = "experiment" + const val MUTUAL_EXCLUSION_GROUP = "mutual-exclusion-group" + const val HOLDOUT_GROUP = "holdout-group" + const val RELEASE_GROUP = "release-group" +} + interface AssignmentTracker { suspend fun track(assignment: Assignment) } @@ -38,31 +47,36 @@ class AmplitudeAssignmentTracker( internal fun Assignment.toAmplitudeEvent(): Event { val event = Event( "[Experiment] Assignment", - this.user.userId, - this.user.deviceId + this.context.userId(), + this.context.deviceId() ) event.eventProperties = JSONObject().apply { - for ((flagKey, result) in this@toAmplitudeEvent.results) { - put("$flagKey.variant", result.variant.key) - put("$flagKey.details", result.description) + for ((flagKey, variant) in this@toAmplitudeEvent.results) { + val version = variant.metadata?.get("version") + val segmentName = variant.metadata?.get("segmentName") + val details = "v$version rule:$segmentName" + put("$flagKey.variant", variant.key) + put("$flagKey.details", details) } } event.userProperties = JSONObject().apply { val set = JSONObject() val unset = JSONObject() - for ((flagKey, result) in this@toAmplitudeEvent.results) { - if (result.type == FLAG_TYPE_MUTUAL_EXCLUSION_GROUP) { + for ((flagKey, variant) in this@toAmplitudeEvent.results) { + val flagType = variant.metadata?.get("flagType") as? String + val default = variant.metadata?.get("default") as? Boolean ?: false + if (flagType == FlagType.MUTUAL_EXCLUSION_GROUP) { // Dont set user properties for mutual exclusion groups. continue - } else if (result.isDefaultVariant) { + } else if (default) { unset.put("[Experiment] $flagKey", "-") } else { - set.put("[Experiment] $flagKey", result.variant.key) + set.put("[Experiment] $flagKey", variant.key) } } put("\$set", set) put("\$unset", unset) } - event.insertId = "${this.user.userId} ${this.user.deviceId} ${this.canonicalize().hashCode()} ${this.timestamp / DAY_MILLIS}" + event.insertId = "${this.context.userId()} ${this.context.deviceId()} ${this.canonicalize().hashCode()} ${this.timestamp / DAY_MILLIS}" return event } diff --git a/core/src/main/kotlin/cohort/CohortApi.kt b/core/src/main/kotlin/cohort/CohortApi.kt index 54ac19f..4a0adfa 100644 --- a/core/src/main/kotlin/cohort/CohortApi.kt +++ b/core/src/main/kotlin/cohort/CohortApi.kt @@ -17,7 +17,6 @@ import io.ktor.utils.io.jvm.javaio.toInputStream import kotlinx.coroutines.delay import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import kotlinx.serialization.decodeFromString import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser import java.util.Base64 diff --git a/core/src/main/kotlin/cohort/CohortStorage.kt b/core/src/main/kotlin/cohort/CohortStorage.kt index 7037e66..5aa8098 100644 --- a/core/src/main/kotlin/cohort/CohortStorage.kt +++ b/core/src/main/kotlin/cohort/CohortStorage.kt @@ -6,7 +6,6 @@ import com.amplitude.util.RedisKey import com.amplitude.util.json import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import kotlin.time.Duration diff --git a/core/src/main/kotlin/deployment/DeploymentApi.kt b/core/src/main/kotlin/deployment/DeploymentApi.kt index 7762f3f..28accb9 100644 --- a/core/src/main/kotlin/deployment/DeploymentApi.kt +++ b/core/src/main/kotlin/deployment/DeploymentApi.kt @@ -1,8 +1,7 @@ package com.amplitude.deployment import com.amplitude.VERSION -import com.amplitude.experiment.evaluation.FlagConfig -import com.amplitude.experiment.evaluation.serialization.SerialFlagConfig +import com.amplitude.experiment.evaluation.EvaluationFlag import com.amplitude.util.get import com.amplitude.util.json import com.amplitude.util.logger @@ -11,10 +10,9 @@ import io.ktor.client.HttpClient import io.ktor.client.call.body import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.request.headers -import kotlinx.serialization.decodeFromString interface DeploymentApi { - suspend fun getFlagConfigs(deploymentKey: String): List + suspend fun getFlagConfigs(deploymentKey: String): List } class DeploymentApiV1( @@ -27,18 +25,17 @@ class DeploymentApiV1( private val client = HttpClient(OkHttp) - override suspend fun getFlagConfigs(deploymentKey: String): List { + override suspend fun getFlagConfigs(deploymentKey: String): List { log.debug("getFlagConfigs: start - deploymentKey=$deploymentKey") val response = retry(onFailure = { e -> log.info("Get flag configs failed: $e") }) { - client.get(serverUrl, "/sdk/v1/flags") { + client.get(serverUrl, "/sdk/v2/flags?v=0") { headers { set("Authorization", "Api-Key $deploymentKey") set("X-Amp-Exp-Library", "experiment-local-proxy/$VERSION") } } } - val body = json.decodeFromString>(response.body()) - return body.map { it.convert() }.also { + return json.decodeFromString>(response.body()).also { log.debug("getFlagConfigs: end - deploymentKey=$deploymentKey") } } diff --git a/core/src/main/kotlin/deployment/DeploymentStorage.kt b/core/src/main/kotlin/deployment/DeploymentStorage.kt index c786d05..42bcfa3 100644 --- a/core/src/main/kotlin/deployment/DeploymentStorage.kt +++ b/core/src/main/kotlin/deployment/DeploymentStorage.kt @@ -1,8 +1,7 @@ package com.amplitude.deployment import com.amplitude.RedisConfiguration -import com.amplitude.experiment.evaluation.FlagConfig -import com.amplitude.experiment.evaluation.serialization.SerialFlagConfig +import com.amplitude.experiment.evaluation.EvaluationFlag import com.amplitude.util.RedisConnection import com.amplitude.util.RedisKey import com.amplitude.util.json @@ -11,7 +10,6 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString interface DeploymentStorage { @@ -19,8 +17,8 @@ interface DeploymentStorage { suspend fun getDeployments(): Set suspend fun putDeployment(deploymentKey: String) suspend fun removeDeployment(deploymentKey: String) - suspend fun getFlagConfigs(deploymentKey: String): List? - suspend fun putFlagConfigs(deploymentKey: String, flagConfigs: List) + suspend fun getFlagConfigs(deploymentKey: String): List? + suspend fun putFlagConfigs(deploymentKey: String, flagConfigs: List) suspend fun removeFlagConfigs(deploymentKey: String) } @@ -43,7 +41,7 @@ class InMemoryDeploymentStorage : DeploymentStorage { ) private val lock = Mutex() - private val deploymentStorage = mutableMapOf?>() + private val deploymentStorage = mutableMapOf?>() override suspend fun getDeployments(): Set { return lock.withLock { @@ -65,13 +63,13 @@ class InMemoryDeploymentStorage : DeploymentStorage { } } - override suspend fun getFlagConfigs(deploymentKey: String): List? { + override suspend fun getFlagConfigs(deploymentKey: String): List? { return lock.withLock { deploymentStorage[deploymentKey] } } - override suspend fun putFlagConfigs(deploymentKey: String, flagConfigs: List) { + override suspend fun putFlagConfigs(deploymentKey: String, flagConfigs: List) { lock.withLock { deploymentStorage[deploymentKey] = flagConfigs } @@ -101,7 +99,7 @@ class RedisDeploymentStorage( ) private val mutex = Mutex() - private val flagConfigCache: MutableList = mutableListOf() + private val flagConfigCache: MutableList = mutableListOf() override suspend fun getDeployments(): Set { return redis.smembers(RedisKey.Deployments(projectId)) ?: emptySet() @@ -117,13 +115,14 @@ class RedisDeploymentStorage( deployments.emit(getDeployments()) } - override suspend fun getFlagConfigs(deploymentKey: String): List? { + // TODO Add in memory caching w/ invalidation + override suspend fun getFlagConfigs(deploymentKey: String): List? { // High volume, use read only redis val jsonEncodedFlags = readOnlyRedis.get(RedisKey.FlagConfigs(projectId, deploymentKey)) ?: return null - return json.decodeFromString>(jsonEncodedFlags).map { it.convert() } + return json.decodeFromString(jsonEncodedFlags) } - override suspend fun putFlagConfigs(deploymentKey: String, flagConfigs: List) { + override suspend fun putFlagConfigs(deploymentKey: String, flagConfigs: List) { // Optimization so repeat puts don't update the data to the same value in redis. val changed = mutex.withLock { if (flagConfigs != flagConfigCache) { @@ -135,7 +134,7 @@ class RedisDeploymentStorage( } } if (changed) { - val jsonEncodedFlags = json.encodeToString(flagConfigs.map { SerialFlagConfig(it) }) + val jsonEncodedFlags = json.encodeToString(flagConfigs) redis.set(RedisKey.FlagConfigs(projectId, deploymentKey), jsonEncodedFlags) } } diff --git a/core/src/main/kotlin/project/ProjectProxy.kt b/core/src/main/kotlin/project/ProjectProxy.kt index 663e4c4..db366d4 100644 --- a/core/src/main/kotlin/project/ProjectProxy.kt +++ b/core/src/main/kotlin/project/ProjectProxy.kt @@ -10,12 +10,12 @@ import com.amplitude.cohort.getCohortStorage import com.amplitude.deployment.DeploymentApiV1 import com.amplitude.deployment.getDeploymentStorage import com.amplitude.experiment.evaluation.EvaluationEngineImpl -import com.amplitude.experiment.evaluation.FlagConfig -import com.amplitude.experiment.evaluation.FlagResult -import com.amplitude.experiment.evaluation.SkylabUser -import com.amplitude.experiment.evaluation.Variant +import com.amplitude.experiment.evaluation.EvaluationFlag +import com.amplitude.experiment.evaluation.EvaluationVariant +import com.amplitude.experiment.evaluation.topologicalSort import com.amplitude.util.getCohortIds import com.amplitude.util.logger +import com.amplitude.util.toEvaluationContext import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch import kotlin.time.DurationUnit @@ -69,8 +69,8 @@ class ProjectProxy( projectRunner.stop() } - suspend fun getFlagConfigs(deploymentKey: String?): List { - if (deploymentKey.isNullOrEmpty() || !deploymentKey.startsWith("server-")) { + suspend fun getFlagConfigs(deploymentKey: String?): List { + if (deploymentKey.isNullOrEmpty()) { throw HttpErrorResponseException(status = 401, message = "Invalid deployment.") } return deploymentStorage.getFlagConfigs(deploymentKey) @@ -78,7 +78,7 @@ class ProjectProxy( } suspend fun getCohortMembershipsForUser(deploymentKey: String?, userId: String?): Set { - if (deploymentKey.isNullOrEmpty() || !deploymentKey.startsWith("server-")) { + if (deploymentKey.isNullOrEmpty()) { throw HttpErrorResponseException(status = 401, message = "Invalid deployment.") } if (userId.isNullOrEmpty()) { @@ -91,45 +91,51 @@ class ProjectProxy( suspend fun evaluate( deploymentKey: String?, - user: SkylabUser?, + user: Map?, flagKeys: Set? = null - ): Map { - if (deploymentKey.isNullOrEmpty() || !deploymentKey.startsWith("server-")) { + ): Map { + if (deploymentKey.isNullOrEmpty()) { throw HttpErrorResponseException(status = 401, message = "Invalid deployment.") } - // Get flag configs for the deployment from storage. - val flagConfigs = deploymentStorage.getFlagConfigs(deploymentKey) - if (flagConfigs == null || flagConfigs.isEmpty()) { + // Get flag configs for the deployment from storage and topo sort. + val storageFlags = deploymentStorage.getFlagConfigs(deploymentKey) + if (storageFlags.isNullOrEmpty()) { return mapOf() } - // Enrich user with cohort IDs. - val enrichedUser = user?.userId?.let { userId -> - user.copy(cohortIds = cohortStorage.getCohortMembershipsForUser(userId)) + val flags = topologicalSort(storageFlags, flagKeys ?: setOf()) + if (flags.isEmpty()) { + return mapOf() } + // Enrich user with cohort IDs and build the evaluation context + val userId = user?.get("user_id") as? String + val enrichedUser = if (userId != null) { + user.toMutableMap().apply { + put("cohort_ids", cohortStorage.getCohortMembershipsForUser(userId)) + } + } else null + val evaluationContext = enrichedUser.toEvaluationContext() // Evaluate results - log.debug("evaluate - user=$enrichedUser") - val result = engine.evaluate(flagConfigs, enrichedUser) + log.debug("evaluate - context={}", evaluationContext) + val result = engine.evaluate(evaluationContext, flags) if (enrichedUser != null) { coroutineScope { launch { - assignmentTracker.track(Assignment(enrichedUser, result)) + assignmentTracker.track(Assignment(evaluationContext, result)) } } } - return result.filterDeployedVariants(flagKeys) + return result } - /** - * Filter only non-default, deployed variants from the results that are included if flag keys (if not empty). - */ - private fun Map.filterDeployedVariants(flagKeys: Set?): Map { - return filter { entry -> - val isVariant = !entry.value.isDefaultVariant - val isIncluded = (flagKeys.isNullOrEmpty() || flagKeys.contains(entry.key)) - val isDeployed = entry.value.deployed - isVariant && isIncluded && isDeployed - }.mapValues { entry -> - entry.value.variant - }.toMap() + suspend fun evaluateV1( + deploymentKey: String?, + user: Map?, + flagKeys: Set? = null + ): Map { + return evaluate(deploymentKey, user, flagKeys).filter { entry -> + val default = entry.value.metadata?.get("default") as? Boolean ?: false + val deployed = entry.value.metadata?.get("deployed") as? Boolean ?: true + (!default && deployed) + } } } diff --git a/core/src/main/kotlin/project/ProjectRunner.kt b/core/src/main/kotlin/project/ProjectRunner.kt index c4a50a2..547601e 100644 --- a/core/src/main/kotlin/project/ProjectRunner.kt +++ b/core/src/main/kotlin/project/ProjectRunner.kt @@ -7,7 +7,7 @@ import com.amplitude.cohort.CohortStorage import com.amplitude.deployment.DeploymentApi import com.amplitude.deployment.DeploymentRunner import com.amplitude.deployment.DeploymentStorage -import com.amplitude.experiment.evaluation.FlagConfig +import com.amplitude.experiment.evaluation.EvaluationFlag import com.amplitude.util.getCohortIds import com.amplitude.util.logger import kotlinx.coroutines.CoroutineScope @@ -108,7 +108,7 @@ class ProjectRunner( } private suspend fun removeUnusedCohorts(deploymentKeys: Set) { - val allFlagConfigs = mutableListOf() + val allFlagConfigs = mutableListOf() for (deploymentKey in deploymentKeys) { allFlagConfigs += deploymentStorage.getFlagConfigs(deploymentKey) ?: continue } diff --git a/core/src/main/kotlin/util/EvaluationContext.kt b/core/src/main/kotlin/util/EvaluationContext.kt new file mode 100644 index 0000000..ef7a840 --- /dev/null +++ b/core/src/main/kotlin/util/EvaluationContext.kt @@ -0,0 +1,55 @@ +package com.amplitude.util + +import com.amplitude.experiment.evaluation.EvaluationContext + +internal fun EvaluationContext.userId(): String? { + return (this["user"] as? Map<*, *>)?.get("user_id") as? String +} +internal fun EvaluationContext.deviceId(): String? { + return (this["user"] as? Map<*, *>)?.get("device_id") as? String +} + +internal fun MutableMap?.toEvaluationContext(): EvaluationContext { + val context = EvaluationContext() + if (this == null) { + return context + } + val groups = mutableMapOf>() + val userGroups = this["groups"] as? Map<*, *> + if (!userGroups.isNullOrEmpty()) { + for (entry in userGroups) { + val groupType = entry.key as? String ?: continue + val groupNames = entry.value as? Collection<*> ?: continue + if (groupNames.isNotEmpty()) { + val groupName = groupNames.first() as? String ?: continue + val groupNameMap = mutableMapOf().apply { put("group_name", groupName) } + val groupProperties = this.select("group_properties", groupType, groupName) as? Map<*, *> + if (!groupProperties.isNullOrEmpty()) { + groupNameMap["group_properties"] = groupProperties + } + groups[groupType] = groupNameMap + } + } + context["groups"] = groups + } + remove("groups") + remove("group_properties") + context["user"] = this + return context +} + + +private fun Map<*, *>.select(vararg selector: Any?): Any? { + var map: Map<*, *> = this + var result: Any? + for (i in 0 until selector.size - 1) { + val select = selector[i] + result = map[select] + if (result is Map<*, *>) { + map = result + } else { + return null + } + } + return map[selector.last()] +} diff --git a/core/src/main/kotlin/util/FlagConfig.kt b/core/src/main/kotlin/util/FlagConfig.kt index 81b99f8..fa88e8f 100644 --- a/core/src/main/kotlin/util/FlagConfig.kt +++ b/core/src/main/kotlin/util/FlagConfig.kt @@ -1,11 +1,11 @@ package com.amplitude.util -import com.amplitude.experiment.evaluation.FlagConfig -import com.amplitude.experiment.evaluation.UserPropertyFilter +import com.amplitude.experiment.evaluation.EvaluationCondition +import com.amplitude.experiment.evaluation.EvaluationFlag +import com.amplitude.experiment.evaluation.EvaluationOperator +import com.amplitude.experiment.evaluation.EvaluationSegment -private const val COHORT_PROP_KEY = "userdata_cohort" - -fun Collection.getCohortIds(): Set { +fun Collection.getCohortIds(): Set { val cohortIds = mutableSetOf() for (flag in this) { cohortIds += flag.getCohortIds() @@ -13,22 +13,29 @@ fun Collection.getCohortIds(): Set { return cohortIds } -private fun FlagConfig.getCohortIds(): Set { +private fun EvaluationFlag.getCohortIds(): Set { val cohortIds = mutableSetOf() - for (filter in this.allUsersTargetingConfig.conditions) { - if (filter.isCohortFilter()) { - cohortIds += filter.values - } + for (segment in this.segments) { + cohortIds += segment.getCohortConditionIds() + } + return cohortIds +} + +private fun EvaluationSegment.getCohortConditionIds(): Set { + val cohortIds = mutableSetOf() + if (conditions == null) { + return cohortIds } - val customSegments = this.customSegmentTargetingConfigs ?: listOf() - for (segment in customSegments) { - for (filter in segment.conditions) { - if (filter.isCohortFilter()) { - cohortIds += filter.values + for (outer in conditions!!) { + for (condition in outer) { + if (condition.isCohortFilter()) { + cohortIds += condition.values } } } return cohortIds } -private fun UserPropertyFilter.isCohortFilter(): Boolean = this.prop == COHORT_PROP_KEY +// Only cohort filters use these operators. +private fun EvaluationCondition.isCohortFilter(): Boolean = + this.op == EvaluationOperator.SET_CONTAINS_ANY || this.op == EvaluationOperator.SET_DOES_NOT_CONTAIN_ANY diff --git a/core/src/main/kotlin/util/Redis.kt b/core/src/main/kotlin/util/Redis.kt index 96711dd..52e80e0 100644 --- a/core/src/main/kotlin/util/Redis.kt +++ b/core/src/main/kotlin/util/Redis.kt @@ -15,7 +15,7 @@ private const val STORAGE_PROTOCOL_VERSION = "v1" internal sealed class RedisKey(val value: String) { - object Projects : RedisKey("projects") + data object Projects : RedisKey("projects") data class Deployments( val projectId: String diff --git a/core/src/test/kotlin/Utils.kt b/core/src/test/kotlin/Utils.kt new file mode 100644 index 0000000..eee8bd9 --- /dev/null +++ b/core/src/test/kotlin/Utils.kt @@ -0,0 +1,15 @@ +fun user( + userId: String? = null, + deviceId: String? = null, + userProperties: Map? = null, + groups: Map>? = null, + groupProperties: Map>>? = null, +): MutableMap { + return mutableMapOf( + "user_id" to userId, + "device_id" to deviceId, + "user_properties" to userProperties, + "groups" to groups, + "group_properties" to groupProperties, + ) +} diff --git a/core/src/test/kotlin/assignment/AssignmentFilterTest.kt b/core/src/test/kotlin/assignment/AssignmentFilterTest.kt index 8d9901e..2aeb57d 100644 --- a/core/src/test/kotlin/assignment/AssignmentFilterTest.kt +++ b/core/src/test/kotlin/assignment/AssignmentFilterTest.kt @@ -1,8 +1,7 @@ -package com.amplitude.experiment.assignment - import com.amplitude.assignment.Assignment import com.amplitude.assignment.InMemoryAssignmentFilter -import com.amplitude.experiment.evaluation.SkylabUser +import com.amplitude.experiment.evaluation.EvaluationVariant +import com.amplitude.util.toEvaluationContext import kotlinx.coroutines.runBlocking import org.junit.Assert import org.junit.Test @@ -13,10 +12,10 @@ class AssignmentFilterTest { fun `test single assignment`() = runBlocking { val filter = InMemoryAssignmentFilter(100) val assignment = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) Assert.assertTrue(filter.shouldTrack(assignment)) @@ -26,18 +25,18 @@ class AssignmentFilterTest { fun `test duplicate assignments`() = runBlocking { val filter = InMemoryAssignmentFilter(100) val assignment1 = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) filter.shouldTrack(assignment1) val assignment2 = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) Assert.assertFalse(filter.shouldTrack(assignment2)) @@ -47,18 +46,18 @@ class AssignmentFilterTest { fun `test same user different results`() = runBlocking { val filter = InMemoryAssignmentFilter(100) val assignment1 = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) Assert.assertTrue(filter.shouldTrack(assignment1)) val assignment2 = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("control"), - "flag-key-2" to flagResult("on") + "flag-key-1" to EvaluationVariant(key = "control"), + "flag-key-2" to EvaluationVariant(key = "on") ) ) Assert.assertTrue(filter.shouldTrack(assignment2)) @@ -68,18 +67,18 @@ class AssignmentFilterTest { fun `test same results for different users`() = runBlocking { val filter = InMemoryAssignmentFilter(100) val assignment1 = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) Assert.assertTrue(filter.shouldTrack(assignment1)) val assignment2 = Assignment( - SkylabUser(userId = "different user"), + user(userId = "different user").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) Assert.assertTrue(filter.shouldTrack(assignment2)) @@ -89,17 +88,17 @@ class AssignmentFilterTest { fun `test empty results`() = runBlocking { val filter = InMemoryAssignmentFilter(100) val assignment1 = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), mapOf() ) Assert.assertTrue(filter.shouldTrack(assignment1)) val assignment2 = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), mapOf() ) Assert.assertFalse(filter.shouldTrack(assignment2)) val assignment3 = Assignment( - SkylabUser(userId = "different user"), + user(userId = "different user").toEvaluationContext(), mapOf() ) Assert.assertTrue(filter.shouldTrack(assignment3)) @@ -109,18 +108,18 @@ class AssignmentFilterTest { fun `test duplicate assignments with different result ordering`() = runBlocking { val filter = InMemoryAssignmentFilter(100) val assignment1 = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), linkedMapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) Assert.assertTrue(filter.shouldTrack(assignment1)) val assignment2 = Assignment( - SkylabUser(userId = "user"), + user(userId = "user").toEvaluationContext(), linkedMapOf( - "flag-key-2" to flagResult("control"), - "flag-key-1" to flagResult("on") + "flag-key-2" to EvaluationVariant(key = "control"), + "flag-key-1" to EvaluationVariant(key = "on") ) ) Assert.assertFalse(filter.shouldTrack(assignment2)) @@ -130,26 +129,26 @@ class AssignmentFilterTest { fun `test lru replacement`() = runBlocking { val filter = InMemoryAssignmentFilter(2) val assignment1 = Assignment( - SkylabUser(userId = "user1"), + user(userId = "user").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) Assert.assertTrue(filter.shouldTrack(assignment1)) val assignment2 = Assignment( - SkylabUser(userId = "user2"), + user(userId = "user2").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) Assert.assertTrue(filter.shouldTrack(assignment2)) val assignment3 = Assignment( - SkylabUser(userId = "user3"), + user(userId = "user3").toEvaluationContext(), mapOf( - "flag-key-1" to flagResult("on"), - "flag-key-2" to flagResult("control") + "flag-key-1" to EvaluationVariant(key = "on"), + "flag-key-2" to EvaluationVariant(key = "control") ) ) Assert.assertTrue(filter.shouldTrack(assignment3)) diff --git a/core/src/test/kotlin/assignment/AssignmentServiceTest.kt b/core/src/test/kotlin/assignment/AssignmentServiceTest.kt index 6dd9e15..2829105 100644 --- a/core/src/test/kotlin/assignment/AssignmentServiceTest.kt +++ b/core/src/test/kotlin/assignment/AssignmentServiceTest.kt @@ -1,9 +1,10 @@ -package com.amplitude.experiment.assignment - import com.amplitude.assignment.Assignment import com.amplitude.assignment.DAY_MILLIS import com.amplitude.assignment.toAmplitudeEvent -import com.amplitude.experiment.evaluation.SkylabUser +import com.amplitude.experiment.evaluation.EvaluationVariant +import com.amplitude.util.deviceId +import com.amplitude.util.toEvaluationContext +import com.amplitude.util.userId import kotlinx.coroutines.runBlocking import org.junit.Assert import org.junit.Test @@ -12,30 +13,35 @@ class AssignmentServiceTest { @Test fun `test assignment to amplitude event`() = runBlocking { - val user = SkylabUser(userId = "user", deviceId = "device") + val user = user(userId = "user", deviceId = "device").toEvaluationContext() val results = mapOf( - "flag-key-1" to flagResult( - variant = "on", - description = "description-1", - isDefaultVariant = false + "flag-key-1" to EvaluationVariant( + key = "on", + metadata = mapOf( + "version" to 1, + "segmentName" to "Segment 1" + ) ), - "flag-key-2" to flagResult( - variant = "off", - description = "description-2", - isDefaultVariant = true + "flag-key-2" to EvaluationVariant( + key = "off", + metadata = mapOf( + "default" to true, + "version" to 1, + "segmentName" to "All Other Users", + ) ) ) val assignment = Assignment(user, results) val event = assignment.toAmplitudeEvent() - Assert.assertEquals(user.userId, event.userId) - Assert.assertEquals(user.deviceId, event.deviceId) + Assert.assertEquals(user.userId(), event.userId) + Assert.assertEquals(user.deviceId(), event.deviceId) Assert.assertEquals("[Experiment] Assignment", event.eventType) val eventProperties = event.eventProperties Assert.assertEquals(4, eventProperties.length()) Assert.assertEquals("on", eventProperties.get("flag-key-1.variant")) - Assert.assertEquals("description-1", eventProperties.get("flag-key-1.details")) + Assert.assertEquals("v1 rule:Segment 1", eventProperties.get("flag-key-1.details")) Assert.assertEquals("off", eventProperties.get("flag-key-2.variant")) - Assert.assertEquals("description-2", eventProperties.get("flag-key-2.details")) + Assert.assertEquals("v1 rule:All Other Users", eventProperties.get("flag-key-2.details")) val userProperties = event.userProperties Assert.assertEquals(2, userProperties.length()) Assert.assertEquals(1, userProperties.getJSONObject("\$set").length()) diff --git a/core/src/test/kotlin/assignment/Utils.kt b/core/src/test/kotlin/assignment/Utils.kt deleted file mode 100644 index 008f4f6..0000000 --- a/core/src/test/kotlin/assignment/Utils.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.amplitude.experiment.assignment - -import com.amplitude.experiment.evaluation.FlagResult -import com.amplitude.experiment.evaluation.Variant - -internal fun flagResult( - variant: String, - description: String = "description", - isDefaultVariant: Boolean = false, - expKey: String? = null, - deployed: Boolean = true, - type: String = "release" -): FlagResult { - return FlagResult(Variant(variant), description, isDefaultVariant, expKey, deployed, type) -} diff --git a/core/src/test/kotlin/util/CacheTest.kt b/core/src/test/kotlin/util/CacheTest.kt index 7598a70..db7f5e1 100644 --- a/core/src/test/kotlin/util/CacheTest.kt +++ b/core/src/test/kotlin/util/CacheTest.kt @@ -1,5 +1,3 @@ -package com.amplituide.util - import com.amplitude.util.Cache import kotlinx.coroutines.Job import kotlinx.coroutines.joinAll diff --git a/gradle.properties b/gradle.properties index 747a8e4..1ee7abd 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,16 +1,16 @@ # kotlin kotlin.code.style=official -ktorVersion=2.2.4 -kotlinVersion=1.8.10 -serialziationVersion=1.8.0 +ktorVersion=2.3.5 +kotlinVersion=1.9.10 +serialziationVersion=1.9.0 # logging & metrics logbackVersion=1.4.6 prometheusVersion=1.10.5 # amplitude -experimentEvaluationVersion = 1.1.1 -amplitudeAnalytics = 1.10.3 +experimentEvaluationVersion = 2.0.0-beta.2 +amplitudeAnalytics = 1.12.0 amplitudeAnalyticsJson = 20230227 # redis diff --git a/service/build.gradle.kts b/service/build.gradle.kts index 14c5fda..60dc1b1 100644 --- a/service/build.gradle.kts +++ b/service/build.gradle.kts @@ -3,8 +3,8 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { application id("io.ktor.plugin") version "2.2.4" - kotlin("jvm") version "1.8.10" - kotlin("plugin.serialization") version "1.8.0" + kotlin("jvm") version "1.9.10" + kotlin("plugin.serialization") version "1.9.0" id("org.jlleitschuh.gradle.ktlint") version "11.3.1" } @@ -34,7 +34,6 @@ val serializationVersion: String by project dependencies { implementation(project(":core")) implementation("com.amplitude:evaluation-core:$experimentEvaluationVersion") - implementation("com.amplitude:evaluation-serialization:$experimentEvaluationVersion") implementation("io.ktor:ktor-server-call-logging-jvm:$ktorVersion") implementation("io.ktor:ktor-server-core-jvm:$ktorVersion") implementation("io.ktor:ktor-server-metrics-micrometer-jvm:$ktorVersion") diff --git a/service/src/main/kotlin/Server.kt b/service/src/main/kotlin/Server.kt index 008ea33..2eb99c6 100644 --- a/service/src/main/kotlin/Server.kt +++ b/service/src/main/kotlin/Server.kt @@ -1,7 +1,6 @@ package com.amplitude -import com.amplitude.experiment.evaluation.SkylabUser -import com.amplitude.experiment.evaluation.serialization.SerialExperimentUser +import io.ktor.server.application.Application import com.amplitude.plugins.configureLogging import com.amplitude.plugins.configureMetrics import com.amplitude.util.json @@ -18,6 +17,7 @@ import io.ktor.server.engine.embeddedServer import io.ktor.server.netty.Netty import io.ktor.server.plugins.contentnegotiation.ContentNegotiation import io.ktor.server.request.ApplicationRequest +import io.ktor.server.request.header import io.ktor.server.request.uri import io.ktor.server.response.respond import io.ktor.server.routing.get @@ -25,10 +25,13 @@ import io.ktor.server.routing.post import io.ktor.server.routing.routing import io.ktor.util.toByteArray import kotlinx.coroutines.runBlocking -import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive import java.io.FileNotFoundException import java.util.Base64 +private lateinit var evaluationProxy: EvaluationProxy + fun main() { val log = logger("Service") @@ -59,91 +62,118 @@ fun main() { ConfigurationFile.fromEnv() } + /* + * Initialize and start the evaluation proxy. + */ + evaluationProxy = EvaluationProxy( + projectsFile.projects, + configFile.configuration + ) + /* * Start the server. */ - embeddedServer(Netty, port = configFile.configuration.port, host = "0.0.0.0") { - /* - * Initialize and start the evaluation proxy. - */ - val evaluationProxy = EvaluationProxy( - projectsFile.projects, - configFile.configuration - ) - runBlocking { - evaluationProxy.start() - } + embeddedServer( + factory = Netty, + port = configFile.configuration.port, + host = "0.0.0.0", + module = Application::proxyServer + ).start(wait = true) +} - /* - * Configure ktor plugins. - */ - configureLogging() - configureMetrics() - install(ContentNegotiation) { - json() - } - install( - createApplicationPlugin("shutdown") { - val plugin = ShutDownUrl("/shutdown") { 0 } - onCall { call -> - if (call.request.uri == plugin.url) { - evaluationProxy.shutdown() - plugin.doShutdown(call) - } - } - } - ) - - /* - * Configure endpoints. - */ - routing { - get("/sdk/v1/deployments/{deployment}/flags") { - val deployment = this.call.parameters["deployment"] - val result = try { - evaluationProxy.getSerializedFlagConfigs(deployment) - } catch (e: HttpErrorResponseException) { - call.respond(HttpStatusCode.fromValue(e.status), e.message) - return@get - } - call.respond(result) - } +fun Application.proxyServer() { + + runBlocking { + evaluationProxy.start() + } - get("/sdk/v1/deployments/{deployment}/users/{userId}/cohorts") { - val deployment = this.call.parameters["deployment"] - val userId = this.call.parameters["userId"] - val result = try { - evaluationProxy.getSerializedCohortMembershipsForUser(deployment, userId) - } catch (e: HttpErrorResponseException) { - call.respond(HttpStatusCode.fromValue(e.status), e.message) - return@get + /* + * Configure ktor plugins. + */ + configureLogging() + configureMetrics() + install(ContentNegotiation) { + json() + } + install( + createApplicationPlugin("shutdown") { + val plugin = ShutDownUrl("/shutdown") { 0 } + onCall { call -> + if (call.request.uri == plugin.url) { + evaluationProxy.shutdown() + plugin.doShutdown(call) } - call.respond(result) } + } + ) - get("/sdk/vardata") { - call.evaluate(evaluationProxy, ApplicationRequest::getUserFromHeader) + /* + * Configure endpoints. + */ + routing { + get("/sdk/v2/flags") { + val deployment = this.call.request.getDeploymentKey() + val result = try { + evaluationProxy.getSerializedFlagConfigs(deployment) + } catch (e: HttpErrorResponseException) { + call.respond(HttpStatusCode.fromValue(e.status), e.message) + return@get } + call.respond(result) + } - post("/sdk/vardata") { - call.evaluate(evaluationProxy, ApplicationRequest::getUserFromBody) + get("/sdk/v2/users/{userId}/cohorts") { + val deployment = this.call.request.getDeploymentKey() + val userId = this.call.parameters["userId"] + val result = try { + evaluationProxy.getSerializedCohortMembershipsForUser(deployment, userId) + } catch (e: HttpErrorResponseException) { + call.respond(HttpStatusCode.fromValue(e.status), e.message) + return@get } + call.respond(result) + } - get("/v1/vardata") { - call.evaluate(evaluationProxy, ApplicationRequest::getUserFromQuery) - } + // V2 evaluation endpoints - post("/v1/vardata") { - call.evaluate(evaluationProxy, ApplicationRequest::getUserFromBody) - } - get("/status") { - call.respond("OK") - } + get("/sdk/v2/vardata") { + call.evaluate(evaluationProxy, ApplicationRequest::getUserFromHeader) + } + + post("/sdk/v2/vardata") { + call.evaluate(evaluationProxy, ApplicationRequest::getUserFromBody) } - }.start(wait = true) + + // V1 evaluation endpoints + + get("/sdk/vardata") { + call.evaluateV1(evaluationProxy, ApplicationRequest::getUserFromHeader) + } + + post("/sdk/vardata") { + call.evaluateV1(evaluationProxy, ApplicationRequest::getUserFromBody) + } + + get("/v1/vardata") { + call.evaluateV1(evaluationProxy, ApplicationRequest::getUserFromQuery) + } + + post("/v1/vardata") { + call.evaluateV1(evaluationProxy, ApplicationRequest::getUserFromBody) + } + + // Health check + + get("/status") { + call.respond("OK") + } + } } -suspend fun ApplicationCall.evaluate(evaluationProxy: EvaluationProxy, userProvider: suspend ApplicationRequest.() -> SkylabUser) { +suspend fun ApplicationCall.evaluate( + evaluationProxy: EvaluationProxy, + userProvider: suspend ApplicationRequest.() -> Map +) { val result = try { // Deployment key is included in Authorization header with prefix "Api-Key " val deploymentKey = request.getDeploymentKey() @@ -157,6 +187,23 @@ suspend fun ApplicationCall.evaluate(evaluationProxy: EvaluationProxy, userProvi respond(result) } +suspend fun ApplicationCall.evaluateV1( + evaluationProxy: EvaluationProxy, + userProvider: suspend ApplicationRequest.() -> Map +) { + val result = try { + // Deployment key is included in Authorization header with prefix "Api-Key " + val deploymentKey = request.getDeploymentKey() + val user = request.userProvider() + val flagKeys = request.getFlagKeys() + evaluationProxy.serializedEvaluateV1(deploymentKey, user, flagKeys) + } catch (e: HttpErrorResponseException) { + respond(HttpStatusCode.fromValue(e.status), e.message) + return + } + respond(result) +} + /** * Get the deployment key from the request, included in Authorization header with prefix "Api-Key " */ @@ -188,37 +235,41 @@ private fun ApplicationRequest.getFlagKeys(): Set { /** * Get the user from the header. Used for SDK GET requests. */ -private fun ApplicationRequest.getUserFromHeader(): SkylabUser { +private fun ApplicationRequest.getUserFromHeader(): Map { val b64User = this.headers["X-Amp-Exp-User"] val userJson = Base64.getDecoder().decode(b64User).toString(Charsets.UTF_8) - return json.decodeFromString(userJson).convert() + return json.decodeFromString(userJson) } /** * Get the user from the body. Used for SDK/REST POST requests. */ -private suspend fun ApplicationRequest.getUserFromBody(): SkylabUser { +private suspend fun ApplicationRequest.getUserFromBody(): Map { val userJson = this.receiveChannel().toByteArray().toString(Charsets.UTF_8) - return json.decodeFromString(userJson).convert() + return json.decodeFromString(userJson) } /** * Get the user from the query. Used for REST GET requests. */ -private fun ApplicationRequest.getUserFromQuery(): SkylabUser { +private fun ApplicationRequest.getUserFromQuery(): JsonObject { val userId = this.queryParameters["user_id"] val deviceId = this.queryParameters["device_id"] val context = this.queryParameters["context"] - var user = if (context != null) { - json.decodeFromString(context).convert() + var user: JsonObject = if (context != null) { + json.decodeFromString(context) } else { - SkylabUser() + JsonObject(emptyMap()) } if (userId != null) { - user = user.copy(userId = userId) + user = JsonObject(user.toMutableMap().apply { + put("user_id", JsonPrimitive(userId)) + }) } if (deviceId != null) { - user = user.copy(deviceId = deviceId) + user = JsonObject(user.toMutableMap().apply { + put("device_id", JsonPrimitive(userId)) + }) } return user } From b6b2db8c766ccef6e5c18d2e81b531e9bbb076a6 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Tue, 24 Oct 2023 16:52:16 -0700 Subject: [PATCH 02/20] mgmt api to manage deployments; per flag cohort downloads on update --- core/src/main/kotlin/Config.kt | 26 +-- core/src/main/kotlin/EvaluationProxy.kt | 150 +++++++++++++++--- core/src/main/kotlin/cohort/CohortApi.kt | 28 ++-- core/src/main/kotlin/cohort/CohortLoader.kt | 43 ++--- .../kotlin/deployment/DeploymentLoader.kt | 22 +-- .../kotlin/deployment/DeploymentRunner.kt | 6 +- .../kotlin/deployment/DeploymentStorage.kt | 120 +++++++------- core/src/main/kotlin/project/Project.kt | 8 + core/src/main/kotlin/project/ProjectApi.kt | 54 +++++++ core/src/main/kotlin/project/ProjectProxy.kt | 55 +++---- core/src/main/kotlin/project/ProjectRunner.kt | 33 ++-- core/src/main/kotlin/util/FlagConfig.kt | 2 +- 12 files changed, 352 insertions(+), 195 deletions(-) create mode 100644 core/src/main/kotlin/project/Project.kt create mode 100644 core/src/main/kotlin/project/ProjectApi.kt diff --git a/core/src/main/kotlin/Config.kt b/core/src/main/kotlin/Config.kt index 4da74b9..8874f68 100644 --- a/core/src/main/kotlin/Config.kt +++ b/core/src/main/kotlin/Config.kt @@ -12,11 +12,11 @@ import java.io.File @Serializable data class ProjectsFile( - val projects: List + val projects: List ) { companion object { fun fromEnv(): ProjectsFile { - val project = Project.fromEnv() + val project = ProjectConfiguration.fromEnv() return ProjectsFile(listOf(project)) } @@ -58,19 +58,17 @@ data class ConfigurationFile( } @Serializable -data class Project( - val id: String, +data class ProjectConfiguration( val apiKey: String, val secretKey: String, - val deploymentKeys: Set + val managementKey: String ) { companion object { - fun fromEnv(): Project { - val id = checkNotNull(stringEnv(EnvKey.PROJECT_ID)) { "${EnvKey.PROJECT_ID} environment variable must be set." } + fun fromEnv(): ProjectConfiguration { val apiKey = checkNotNull(stringEnv(EnvKey.API_KEY)) { "${EnvKey.API_KEY} environment variable must be set." } val secretKey = checkNotNull(stringEnv(EnvKey.SECRET_KEY)) { "${EnvKey.SECRET_KEY} environment variable must be set." } - val deploymentKey = checkNotNull(stringEnv(EnvKey.EXPERIMENT_DEPLOYMENT_KEY)) { "${EnvKey.SECRET_KEY} environment variable must be set." } - return Project(id, apiKey, secretKey, setOf(deploymentKey)) + val managementKey = checkNotNull(stringEnv(EnvKey.EXPERIMENT_MANAGEMENT_KEY)) { "${EnvKey.EXPERIMENT_MANAGEMENT_KEY} environment variable must be set." } + return ProjectConfiguration(apiKey, secretKey, managementKey) } } } @@ -80,6 +78,7 @@ data class Configuration( val port: Int = Default.PORT, val serverUrl: String = Default.SERVER_URL, val cohortServerUrl: String = Default.COHORT_SERVER_URL, + val deploymentSyncIntervalMillis: Long = Default.DEPLOYMENT_SYNC_INTERVAL_MILLIS, val flagSyncIntervalMillis: Long = Default.FLAG_SYNC_INTERVAL_MILLIS, val cohortSyncIntervalMillis: Long = Default.COHORT_SYNC_INTERVAL_MILLIS, val maxCohortSize: Int = Default.MAX_COHORT_SIZE, @@ -91,6 +90,10 @@ data class Configuration( port = intEnv(EnvKey.PORT, Default.PORT)!!, serverUrl = stringEnv(EnvKey.SERVER_URL, Default.SERVER_URL)!!, cohortServerUrl = stringEnv(EnvKey.COHORT_SERVER_URL, Default.COHORT_SERVER_URL)!!, + deploymentSyncIntervalMillis = longEnv( + EnvKey.DEPLOYMENT_SYNC_INTERVAL_MILLIS, + Default.DEPLOYMENT_SYNC_INTERVAL_MILLIS + )!!, flagSyncIntervalMillis = longEnv( EnvKey.FLAG_SYNC_INTERVAL_MILLIS, Default.FLAG_SYNC_INTERVAL_MILLIS @@ -164,11 +167,11 @@ object EnvKey { const val SERVER_URL = "AMPLITUDE_SERVER_URL" const val COHORT_SERVER_URL = "AMPLITUDE_COHORT_SERVER_URL" - const val PROJECT_ID = "AMPLITUDE_PROJECT_ID" const val API_KEY = "AMPLITUDE_API_KEY" const val SECRET_KEY = "AMPLITUDE_SECRET_KEY" - const val EXPERIMENT_DEPLOYMENT_KEY = "AMPLITUDE_EXPERIMENT_DEPLOYMENT_KEY" + const val EXPERIMENT_MANAGEMENT_KEY = "AMPLITUDE_EXPERIMENT_MANAGEMENT_API_KEY" + const val DEPLOYMENT_SYNC_INTERVAL_MILLIS = "AMPLITUDE_DEPLOYMENT_SYNC_INTERVAL_MILLIS" const val FLAG_SYNC_INTERVAL_MILLIS = "AMPLITUDE_FLAG_SYNC_INTERVAL_MILLIS" const val COHORT_SYNC_INTERVAL_MILLIS = "AMPLITUDE_COHORT_SYNC_INTERVAL_MILLIS" const val MAX_COHORT_SIZE = "AMPLITUDE_MAX_COHORT_SIZE" @@ -187,6 +190,7 @@ object Default { const val PORT = 3546 const val SERVER_URL = "https://flag.lab.amplitude.com" const val COHORT_SERVER_URL = "https://cohort.lab.amplitude.com" + const val DEPLOYMENT_SYNC_INTERVAL_MILLIS = 60 * 1000L const val FLAG_SYNC_INTERVAL_MILLIS = 10 * 1000L const val COHORT_SYNC_INTERVAL_MILLIS = 60 * 1000L const val MAX_COHORT_SIZE = Int.MAX_VALUE diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index e29abba..5be7093 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -1,16 +1,28 @@ package com.amplitude +import com.amplitude.assignment.AmplitudeAssignmentTracker +import com.amplitude.cohort.getCohortStorage import com.amplitude.deployment.getDeploymentStorage import com.amplitude.experiment.evaluation.EvaluationFlag import com.amplitude.experiment.evaluation.EvaluationVariant +import com.amplitude.project.Project +import com.amplitude.project.ProjectApiV1 import com.amplitude.project.ProjectProxy import com.amplitude.project.getProjectStorage import com.amplitude.util.json import com.amplitude.util.logger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import kotlinx.serialization.encodeToString +import kotlin.time.DurationUnit +import kotlin.time.toDuration const val VERSION = "0.3.2" @@ -21,7 +33,7 @@ class HttpErrorResponseException( ) : Exception(message, cause) class EvaluationProxy( - private val projects: List, + private val projectConfigurations: List, private val configuration: Configuration = Configuration() ) { @@ -29,51 +41,134 @@ class EvaluationProxy( val log by logger() } - private val projectProxies = projects.associateWith { ProjectProxy(it, configuration) } - private val deploymentProxies = projects - .map { project -> project.deploymentKeys.associateWith { project }.toMutableMap() } - .reduce { acc, map -> acc.apply { putAll(map) } } + private val supervisor = SupervisorJob() + private val scope = CoroutineScope(supervisor) + + private val projectProxies = mutableMapOf() + private val apiKeysToProject = mutableMapOf() + private val secretKeysToProject = mutableMapOf() + private val deploymentKeysToProject = mutableMapOf() + private val mutex = Mutex() private val projectStorage = getProjectStorage(configuration.redis) - suspend fun start() = coroutineScope { - log.info("Starting evaluation proxy. projects=${projectProxies.keys.map { it.id }}") - for (project in projects) { - projectStorage.putProject(project.id) + suspend fun start() { + log.info("Starting evaluation proxy.") + /* + * Fetch deployments, setup initial mappings for each project + * configuration, and create the project proxy. + */ + log.info("Setting up ${projectConfigurations.size} project(s)") + for (projectConfiguration in projectConfigurations) { + val projectApi = ProjectApiV1(projectConfiguration.managementKey) + val deployments = projectApi.getDeployments() + if (deployments.isEmpty()) { + continue + } + val projectId = deployments.first().projectId + log.info("Fetched ${deployments.size} deployments for project $projectId") + // Add the project to local mappings. + val project = Project( + id = projectId, + apiKey = projectConfiguration.apiKey, + secretKey = projectConfiguration.secretKey, + managementKey = projectConfiguration.managementKey, + ) + apiKeysToProject[project.apiKey] = project + secretKeysToProject[project.secretKey] = project + for (deployment in deployments) { + deploymentKeysToProject[deployment.key] = project + } + + // Create a project proxy and add the project to storage. + val assignmentTracker = AmplitudeAssignmentTracker(project.apiKey, configuration.assignment) + val deploymentStorage = getDeploymentStorage(project.id, configuration.redis) + val cohortStorage = getCohortStorage( + project.id, + configuration.redis, + configuration.cohortSyncIntervalMillis.toDuration(DurationUnit.MILLISECONDS) + ) + val projectProxy = ProjectProxy( + project, + configuration, + assignmentTracker, + deploymentStorage, + cohortStorage + ) + projectProxies[project] = projectProxy + } + + /* + * Update project storage with configured projects, and clean up + * projects that have been removed. + */ + // Add all configured projects to storage + val projectIds = projectProxies.map { it.key.id }.toSet() + for (projectId in projectIds) { + log.info("Adding project $projectId") + projectStorage.putProject(projectId) } // Remove all non-configured projects and associated data val storageProjectIds = projectStorage.getProjects() - val projectIds = projects.map { it.id }.toSet() for (projectId in storageProjectIds - projectIds) { log.info("Removing project $projectId") - val storage = getDeploymentStorage(projectId, configuration.redis) - val deployments = storage.getDeployments() + val deploymentStorage = getDeploymentStorage(projectId, configuration.redis) + val cohortStorage = getCohortStorage( + projectId, + configuration.redis, + configuration.cohortSyncIntervalMillis.toDuration(DurationUnit.MILLISECONDS) + ) + // Remove all deployments for project + val deployments = deploymentStorage.getDeployments() for (deployment in deployments) { - log.info("Removing deployment $deployment for project $projectId") - storage.removeDeployment(deployment) - storage.removeFlagConfigs(deployment) + log.info("Removing deployment and flag configs for deployment $deployment for project $projectId") + deploymentStorage.removeDeployment(deployment) + deploymentStorage.removeAllFlags(deployment) + } + // Remove all cohorts for project + val cohortDescriptions = cohortStorage.getCohortDescriptions().values + for (cohortDescription in cohortDescriptions) { + cohortStorage.removeCohort(cohortDescription) } projectStorage.removeProject(projectId) } - projectProxies.map { launch { it.value.start() } }.joinAll() + + /* + * Start all project proxies. + */ + projectProxies.map { scope.launch { it.value.start() } }.joinAll() + + /* + * Periodically update the local cache of deployments to project values. + */ + scope.launch { + while (true) { + delay(configuration.deploymentSyncIntervalMillis) + for ((project, projectProxy) in projectProxies) { + val deployments = projectProxy.getDeployments().associateWith { project } + mutex.withLock { deploymentKeysToProject.putAll(deployments) } + } + } + } log.info("Evaluation proxy started.") } suspend fun shutdown() = coroutineScope { log.info("Shutting down evaluation proxy.") projectProxies.map { launch { it.value.shutdown() } }.joinAll() + supervisor.cancelAndJoin() log.info("Evaluation proxy shut down.") } + // Apis + suspend fun getFlagConfigs(deploymentKey: String?): List { - val project = deploymentProxies[deploymentKey] ?: throw HttpErrorResponseException(404, "Deployment not found.") - val projectProxy = projectProxies[project] ?: throw HttpErrorResponseException(404, "Project not found.") + val projectProxy = getProjectProxy(deploymentKey) return projectProxy.getFlagConfigs(deploymentKey) } suspend fun getCohortMembershipsForUser(deploymentKey: String?, userId: String?): Set { - val project = deploymentProxies[deploymentKey] ?: throw HttpErrorResponseException(404, "Deployment not found.") - val projectProxy = projectProxies[project] ?: throw HttpErrorResponseException(404, "Project not found.") + val projectProxy = getProjectProxy(deploymentKey) return projectProxy.getCohortMembershipsForUser(deploymentKey, userId) } @@ -82,8 +177,7 @@ class EvaluationProxy( user: Map?, flagKeys: Set? = null ): Map { - val project = deploymentProxies[deploymentKey] ?: throw HttpErrorResponseException(404, "Deployment not found.") - val projectProxy = projectProxies[project] ?: throw HttpErrorResponseException(404, "Project not found.") + val projectProxy = getProjectProxy(deploymentKey) return projectProxy.evaluate(deploymentKey, user, flagKeys) } @@ -92,10 +186,18 @@ class EvaluationProxy( user: Map?, flagKeys: Set? = null ): Map { - val project = deploymentProxies[deploymentKey] ?: throw HttpErrorResponseException(404, "Deployment not found.") - val projectProxy = projectProxies[project] ?: throw HttpErrorResponseException(404, "Project not found.") + val projectProxy = getProjectProxy(deploymentKey) return projectProxy.evaluateV1(deploymentKey, user, flagKeys) } + + // Private + + private suspend fun getProjectProxy(deploymentKey: String?): ProjectProxy { + val cachedProject = mutex.withLock { + deploymentKeysToProject[deploymentKey] + } ?: throw HttpErrorResponseException(401, "Invalid deployment key.") + return projectProxies[cachedProject] ?: throw HttpErrorResponseException(404, "Project not found.") + } } // Serialized Proxy Calls diff --git a/core/src/main/kotlin/cohort/CohortApi.kt b/core/src/main/kotlin/cohort/CohortApi.kt index 4a0adfa..f4bd042 100644 --- a/core/src/main/kotlin/cohort/CohortApi.kt +++ b/core/src/main/kotlin/cohort/CohortApi.kt @@ -51,6 +51,8 @@ data class GetCohortAsyncResponse( ) interface CohortApi { + + suspend fun getCohortDescription(cohortId: String): CohortDescription suspend fun getCohortDescriptions(cohortIds: Set): List suspend fun getCohortMembers(cohortDescription: CohortDescription): Set } @@ -72,20 +74,24 @@ class CohortApiV5( } } + override suspend fun getCohortDescription(cohortId: String): CohortDescription { + val response = retry(onFailure = { e -> log.info("Get cohort descriptions failed: $e") }) { + client.get(serverUrl, "/api/3/cohorts/info/$cohortId") { + headers { set("Authorization", "Basic $basicAuth") } + } + } + val serialDescription = json.decodeFromString(response.body()) + return CohortDescription( + id = serialDescription.cohortId, + lastComputed = serialDescription.lastComputed, + size = serialDescription.size + ) + } + override suspend fun getCohortDescriptions(cohortIds: Set): List { log.debug("getCohortDescriptions: start") return cohortIds.map { cohortId -> - val response = retry(onFailure = { e -> log.info("Get cohort descriptions failed: $e") }) { - client.get(serverUrl, "/api/3/cohorts/info/$cohortId") { - headers { set("Authorization", "Basic $basicAuth") } - } - } - val serialDescription = json.decodeFromString(response.body()) - CohortDescription( - id = serialDescription.cohortId, - lastComputed = serialDescription.lastComputed, - size = serialDescription.size - ) + getCohortDescription(cohortId) }.toList().also { log.debug("getCohortDescriptions: end - result=$it") } } diff --git a/core/src/main/kotlin/cohort/CohortLoader.kt b/core/src/main/kotlin/cohort/CohortLoader.kt index 66753bb..5ee2d3b 100644 --- a/core/src/main/kotlin/cohort/CohortLoader.kt +++ b/core/src/main/kotlin/cohort/CohortLoader.kt @@ -19,38 +19,23 @@ class CohortLoader( private val jobsMutex = Mutex() private val jobs = mutableMapOf() - suspend fun loadCohorts(cohortIds: Set, state: Set = cohortIds) = coroutineScope { - log.debug("loadCohorts: start - cohortIds=$cohortIds") - - // Get cohort descriptions from storage and network. - val networkCohortDescriptions = cohortApi.getCohortDescriptions(state) - - // Filter cohorts received from network. Removes cohorts which are: - // 1. Not requested for management by this function. - // 2. Larger than the max size. - // 3. Are equal to what has been downloaded already. - val cohorts = networkCohortDescriptions.filter { networkCohortDescription -> - val storageDescription = cohortStorage.getCohortDescription(networkCohortDescription.id) - cohortIds.contains(networkCohortDescription.id) && - networkCohortDescription.size <= maxCohortSize && - networkCohortDescription.lastComputed > (storageDescription?.lastComputed ?: -1) - } - log.debug("loadCohorts: filtered network descriptions - $cohorts") - - // Download and store each cohort if a download job has not already been started. - for (cohort in cohorts) { - val job = jobsMutex.withLock { - jobs.getOrPut(cohort.id) { + suspend fun loadCohort(cohortId: String) = coroutineScope { + log.debug("loadCohort: start - cohortId={}", cohortId) + val networkCohort = cohortApi.getCohortDescription(cohortId) + val storageCohort = cohortStorage.getCohortDescription(cohortId) + val shouldDownloadCohort = networkCohort.size <= maxCohortSize && + networkCohort.lastComputed > (storageCohort?.lastComputed ?: -1) + if (shouldDownloadCohort) { + jobsMutex.withLock { + jobs.getOrPut(cohortId) { launch { - log.info("Downloading cohort. $cohort") - val cohortMembers = cohortApi.getCohortMembers(cohort) - cohortStorage.putCohort(cohort, cohortMembers) - jobsMutex.withLock { jobs.remove(cohort.id) } + log.info("Downloading cohort. $networkCohort") + val cohortMembers = cohortApi.getCohortMembers(networkCohort) + cohortStorage.putCohort(networkCohort, cohortMembers) + jobsMutex.withLock { jobs.remove(cohortId) } } } - } - job.join() + }.join() } - log.debug("loadCohorts: end - cohortIds=$cohortIds") } } diff --git a/core/src/main/kotlin/deployment/DeploymentLoader.kt b/core/src/main/kotlin/deployment/DeploymentLoader.kt index 3994d55..5d03f1c 100644 --- a/core/src/main/kotlin/deployment/DeploymentLoader.kt +++ b/core/src/main/kotlin/deployment/DeploymentLoader.kt @@ -27,16 +27,20 @@ class DeploymentLoader( jobsMutex.withLock { jobs.getOrPut(deploymentKey) { launch { - val networkFlagConfigs = deploymentApi.getFlagConfigs(deploymentKey) - val storageFlagConfigs = deploymentStorage.getFlagConfigs(deploymentKey) ?: listOf() - val networkCohortIds = networkFlagConfigs.getCohortIds() - val storageCohortIds = storageFlagConfigs.getCohortIds() - val addedCohortIds = networkCohortIds - storageCohortIds - if (addedCohortIds.isNotEmpty()) { - cohortLoader.loadCohorts(addedCohortIds, networkCohortIds) + val networkFlags = deploymentApi.getFlagConfigs(deploymentKey) + for (flag in networkFlags) { + val cohortIds = flag.getCohortIds() + if (cohortIds.isNotEmpty()) { + launch { + for (cohortId in cohortIds) { + cohortLoader.loadCohort(cohortId) + } + deploymentStorage.putFlag(deploymentKey, flag) + } + } else { + deploymentStorage.putFlag(deploymentKey, flag) + } } - deploymentStorage.putFlagConfigs(deploymentKey, networkFlagConfigs) - jobs.remove(deploymentKey) } } }.join() diff --git a/core/src/main/kotlin/deployment/DeploymentRunner.kt b/core/src/main/kotlin/deployment/DeploymentRunner.kt index 3f1a7f2..04623a1 100644 --- a/core/src/main/kotlin/deployment/DeploymentRunner.kt +++ b/core/src/main/kotlin/deployment/DeploymentRunner.kt @@ -40,9 +40,9 @@ class DeploymentRunner( scope.launch { while (true) { delay(configuration.cohortSyncIntervalMillis) - val cohortIds = deploymentStorage.getFlagConfigs(deploymentKey)?.getCohortIds() - if (!cohortIds.isNullOrEmpty()) { - cohortLoader.loadCohorts(cohortIds) + val cohortIds = deploymentStorage.getAllFlags(deploymentKey).values.getCohortIds() + for (cohortId in cohortIds) { + cohortLoader.loadCohort(cohortId) } } } diff --git a/core/src/main/kotlin/deployment/DeploymentStorage.kt b/core/src/main/kotlin/deployment/DeploymentStorage.kt index 42bcfa3..2d58c2a 100644 --- a/core/src/main/kotlin/deployment/DeploymentStorage.kt +++ b/core/src/main/kotlin/deployment/DeploymentStorage.kt @@ -5,21 +5,21 @@ import com.amplitude.experiment.evaluation.EvaluationFlag import com.amplitude.util.RedisConnection import com.amplitude.util.RedisKey import com.amplitude.util.json -import kotlinx.coroutines.channels.BufferOverflow -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString interface DeploymentStorage { - val deployments: Flow> suspend fun getDeployments(): Set suspend fun putDeployment(deploymentKey: String) suspend fun removeDeployment(deploymentKey: String) - suspend fun getFlagConfigs(deploymentKey: String): List? - suspend fun putFlagConfigs(deploymentKey: String, flagConfigs: List) - suspend fun removeFlagConfigs(deploymentKey: String) + suspend fun getFlag(deploymentKey: String, flagKey: String): EvaluationFlag? + suspend fun getAllFlags(deploymentKey: String): Map + suspend fun putFlag(deploymentKey: String, flag: EvaluationFlag) + suspend fun putAllFlags(deploymentKey: String, flags: List) + suspend fun removeFlag(deploymentKey: String, flagKey: String) + suspend fun removeAllFlags(deploymentKey: String) } fun getDeploymentStorage(projectId: String, redisConfiguration: RedisConfiguration?): DeploymentStorage { @@ -35,13 +35,8 @@ fun getDeploymentStorage(projectId: String, redisConfiguration: RedisConfigurati class InMemoryDeploymentStorage : DeploymentStorage { - override val deployments = MutableSharedFlow>( - extraBufferCapacity = 1, - onBufferOverflow = BufferOverflow.DROP_OLDEST - ) - private val lock = Mutex() - private val deploymentStorage = mutableMapOf?>() + private val deploymentStorage = mutableMapOf?>() override suspend fun getDeployments(): Set { return lock.withLock { @@ -51,33 +46,49 @@ class InMemoryDeploymentStorage : DeploymentStorage { override suspend fun putDeployment(deploymentKey: String) { return lock.withLock { - deploymentStorage[deploymentKey] = null - deployments.emit(deploymentStorage.keys) + deploymentStorage.putIfAbsent(deploymentKey, null) } } override suspend fun removeDeployment(deploymentKey: String) { return lock.withLock { deploymentStorage.remove(deploymentKey) - deployments.emit(deploymentStorage.keys) } } - override suspend fun getFlagConfigs(deploymentKey: String): List? { + override suspend fun getFlag(deploymentKey: String, flagKey: String): EvaluationFlag? { return lock.withLock { - deploymentStorage[deploymentKey] + deploymentStorage[deploymentKey]?.get(flagKey) } } - override suspend fun putFlagConfigs(deploymentKey: String, flagConfigs: List) { - lock.withLock { - deploymentStorage[deploymentKey] = flagConfigs + override suspend fun getAllFlags(deploymentKey: String): Map { + return lock.withLock { + deploymentStorage[deploymentKey]?.toMap() ?: mapOf() } } - override suspend fun removeFlagConfigs(deploymentKey: String) { - lock.withLock { - deploymentStorage.remove(deploymentKey) + override suspend fun putFlag(deploymentKey: String, flag: EvaluationFlag) { + return lock.withLock { + deploymentStorage[deploymentKey]?.put(flag.key, flag) + } + } + + override suspend fun putAllFlags(deploymentKey: String, flags: List) { + return lock.withLock { + deploymentStorage[deploymentKey]?.putAll(flags.associateBy { it.key }) + } + } + + override suspend fun removeFlag(deploymentKey: String, flagKey: String) { + return lock.withLock { + deploymentStorage[deploymentKey]?.remove(flagKey) + } + } + + override suspend fun removeAllFlags(deploymentKey: String) { + return lock.withLock { + deploymentStorage[deploymentKey] = null } } } @@ -92,57 +103,50 @@ class RedisDeploymentStorage( private val redis = RedisConnection(uri, prefix) private val readOnlyRedis = RedisConnection(readOnlyUri, prefix) - // TODO Could be optimized w/ pub sub - override val deployments = MutableSharedFlow>( - extraBufferCapacity = 1, - onBufferOverflow = BufferOverflow.DROP_OLDEST - ) - - private val mutex = Mutex() - private val flagConfigCache: MutableList = mutableListOf() - override suspend fun getDeployments(): Set { return redis.smembers(RedisKey.Deployments(projectId)) ?: emptySet() } override suspend fun putDeployment(deploymentKey: String) { redis.sadd(RedisKey.Deployments(projectId), setOf(deploymentKey)) - deployments.emit(getDeployments()) } override suspend fun removeDeployment(deploymentKey: String) { redis.srem(RedisKey.Deployments(projectId), deploymentKey) - deployments.emit(getDeployments()) + } + + override suspend fun getFlag(deploymentKey: String, flagKey: String): EvaluationFlag? { + val flagJson = redis.hget(RedisKey.FlagConfigs(projectId, deploymentKey), flagKey) ?: return null + return json.decodeFromString(flagJson) } // TODO Add in memory caching w/ invalidation - override suspend fun getFlagConfigs(deploymentKey: String): List? { + override suspend fun getAllFlags(deploymentKey: String): Map { // High volume, use read only redis - val jsonEncodedFlags = readOnlyRedis.get(RedisKey.FlagConfigs(projectId, deploymentKey)) ?: return null - return json.decodeFromString(jsonEncodedFlags) - } - - override suspend fun putFlagConfigs(deploymentKey: String, flagConfigs: List) { - // Optimization so repeat puts don't update the data to the same value in redis. - val changed = mutex.withLock { - if (flagConfigs != flagConfigCache) { - flagConfigCache.clear() - flagConfigCache += flagConfigs - true - } else { - false - } - } - if (changed) { - val jsonEncodedFlags = json.encodeToString(flagConfigs) - redis.set(RedisKey.FlagConfigs(projectId, deploymentKey), jsonEncodedFlags) + return readOnlyRedis.hgetall(RedisKey.FlagConfigs(projectId, deploymentKey)) + ?.mapValues { json.decodeFromString(it.value) } ?: mapOf() + } + + override suspend fun putFlag(deploymentKey: String, flag: EvaluationFlag) { + val flagJson = json.encodeToString(flag) + redis.hset(RedisKey.FlagConfigs(projectId, deploymentKey), mapOf(flag.key to flagJson)) + } + + override suspend fun putAllFlags(deploymentKey: String, flags: List) { + for (flag in flags) { + putFlag(deploymentKey, flag) } } - override suspend fun removeFlagConfigs(deploymentKey: String) { - redis.del(RedisKey.FlagConfigs(projectId, deploymentKey)) - mutex.withLock { - flagConfigCache.clear() + override suspend fun removeFlag(deploymentKey: String, flagKey: String) { + redis.hdel(RedisKey.FlagConfigs(projectId, deploymentKey), flagKey) + } + + override suspend fun removeAllFlags(deploymentKey: String) { + val redisKey = RedisKey.FlagConfigs(projectId, deploymentKey) + val flags = redis.hgetall(RedisKey.FlagConfigs(projectId, deploymentKey)) ?: return + for (key in flags.keys) { + redis.hdel(redisKey, key) } } } diff --git a/core/src/main/kotlin/project/Project.kt b/core/src/main/kotlin/project/Project.kt new file mode 100644 index 0000000..ebae10a --- /dev/null +++ b/core/src/main/kotlin/project/Project.kt @@ -0,0 +1,8 @@ +package com.amplitude.project + +data class Project( + val id: String, + val apiKey: String, + val secretKey: String, + val managementKey: String, +) diff --git a/core/src/main/kotlin/project/ProjectApi.kt b/core/src/main/kotlin/project/ProjectApi.kt new file mode 100644 index 0000000..f8a923d --- /dev/null +++ b/core/src/main/kotlin/project/ProjectApi.kt @@ -0,0 +1,54 @@ +package com.amplitude.project + +import com.amplitude.util.get +import com.amplitude.util.json +import com.amplitude.util.logger +import com.amplitude.util.retry +import io.ktor.client.HttpClient +import io.ktor.client.call.body +import io.ktor.client.engine.okhttp.OkHttp +import io.ktor.client.plugins.HttpTimeout +import io.ktor.client.request.headers +import kotlinx.serialization.Serializable + +private const val MANAGEMENT_SERVER_URL = "https://experiment.amplitude.com" + +@Serializable +internal data class Deployment( + val id: String, + val projectId: String, + val label: String, + val key: String, + val deleted: Boolean, +) + +internal interface ProjectApi { + suspend fun getDeployments(): List +} + +internal class ProjectApiV1(private val managementKey: String): ProjectApi { + + companion object { + val log by logger() + } + + private val client = HttpClient(OkHttp) { + install(HttpTimeout) { + socketTimeoutMillis = 30000 + } + } + override suspend fun getDeployments(): List { + log.debug("getDeployments: start") + val response = retry(onFailure = { e -> log.error("Get deployments failed: $e") }) { + client.get(MANAGEMENT_SERVER_URL, "/api/1/deployments") { + headers { + set("Authorization", "Bearer $managementKey") + set("Accept", "application/json") + } + } + } + return json.decodeFromString>(response.body()) + .filter { !it.deleted } + .also { log.debug("getDeployments: end") } + } +} diff --git a/core/src/main/kotlin/project/ProjectProxy.kt b/core/src/main/kotlin/project/ProjectProxy.kt index db366d4..574937c 100644 --- a/core/src/main/kotlin/project/ProjectProxy.kt +++ b/core/src/main/kotlin/project/ProjectProxy.kt @@ -2,13 +2,12 @@ package com.amplitude.project import com.amplitude.Configuration import com.amplitude.HttpErrorResponseException -import com.amplitude.Project -import com.amplitude.assignment.AmplitudeAssignmentTracker import com.amplitude.assignment.Assignment +import com.amplitude.assignment.AssignmentTracker import com.amplitude.cohort.CohortApiV5 -import com.amplitude.cohort.getCohortStorage +import com.amplitude.cohort.CohortStorage import com.amplitude.deployment.DeploymentApiV1 -import com.amplitude.deployment.getDeploymentStorage +import com.amplitude.deployment.DeploymentStorage import com.amplitude.experiment.evaluation.EvaluationEngineImpl import com.amplitude.experiment.evaluation.EvaluationFlag import com.amplitude.experiment.evaluation.EvaluationVariant @@ -18,12 +17,13 @@ import com.amplitude.util.logger import com.amplitude.util.toEvaluationContext import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch -import kotlin.time.DurationUnit -import kotlin.time.toDuration -class ProjectProxy( +internal class ProjectProxy( private val project: Project, - configuration: Configuration = Configuration() + configuration: Configuration, + private val assignmentTracker: AssignmentTracker, + private val deploymentStorage: DeploymentStorage, + private val cohortStorage: CohortStorage, ) { companion object { @@ -32,17 +32,12 @@ class ProjectProxy( private val engine = EvaluationEngineImpl() - private val assignmentTracker = AmplitudeAssignmentTracker(project.apiKey, configuration.assignment) + private val projectApi = ProjectApiV1(project.managementKey) private val deploymentApi = DeploymentApiV1(configuration.serverUrl) - private val deploymentStorage = getDeploymentStorage(project.id, configuration.redis) private val cohortApi = CohortApiV5(configuration.cohortServerUrl, project.apiKey, project.secretKey) - private val cohortStorage = getCohortStorage( - project.id, - configuration.redis, - configuration.cohortSyncIntervalMillis.toDuration(DurationUnit.MILLISECONDS) - ) private val projectRunner = ProjectRunner( configuration, + projectApi, deploymentApi, deploymentStorage, cohortApi, @@ -50,22 +45,12 @@ class ProjectProxy( ) suspend fun start() { - log.info("Starting project. projectId=${project.id} deploymentKeys=${project.deploymentKeys}") - // Add deployments to storage - for (deploymentKey in project.deploymentKeys) { - deploymentStorage.putDeployment(deploymentKey) - } - // Remove deployments which are no longer being managed - val storageDeploymentKeys = deploymentStorage.getDeployments() - for (storageDeploymentKey in storageDeploymentKeys - project.deploymentKeys) { - deploymentStorage.removeDeployment(storageDeploymentKey) - deploymentStorage.removeFlagConfigs(storageDeploymentKey) - } + log.info("Starting project. projectId=${project.id}") projectRunner.start() } suspend fun shutdown() { - log.info("Shutting down project. projectId=${project.id}") + log.info("Shutting down project. project.id=${project.id}") projectRunner.stop() } @@ -73,8 +58,7 @@ class ProjectProxy( if (deploymentKey.isNullOrEmpty()) { throw HttpErrorResponseException(status = 401, message = "Invalid deployment.") } - return deploymentStorage.getFlagConfigs(deploymentKey) - ?: throw HttpErrorResponseException(status = 404, message = "Unknown deployment.") + return deploymentStorage.getAllFlags(deploymentKey).values.toList() } suspend fun getCohortMembershipsForUser(deploymentKey: String?, userId: String?): Set { @@ -84,8 +68,7 @@ class ProjectProxy( if (userId.isNullOrEmpty()) { throw HttpErrorResponseException(status = 400, message = "Invalid user ID.") } - val cohortIds = deploymentStorage.getFlagConfigs(deploymentKey)?.getCohortIds() - ?: throw HttpErrorResponseException(status = 404, message = "Unknown deployment.") + val cohortIds = deploymentStorage.getAllFlags(deploymentKey).values.getCohortIds() return cohortStorage.getCohortMembershipsForUser(userId, cohortIds) } @@ -98,8 +81,8 @@ class ProjectProxy( throw HttpErrorResponseException(status = 401, message = "Invalid deployment.") } // Get flag configs for the deployment from storage and topo sort. - val storageFlags = deploymentStorage.getFlagConfigs(deploymentKey) - if (storageFlags.isNullOrEmpty()) { + val storageFlags = deploymentStorage.getAllFlags(deploymentKey) + if (storageFlags.isEmpty()) { return mapOf() } val flags = topologicalSort(storageFlags, flagKeys ?: setOf()) @@ -138,4 +121,10 @@ class ProjectProxy( (!default && deployed) } } + + // Internal + + internal suspend fun getDeployments(): Set { + return deploymentStorage.getDeployments() + } } diff --git a/core/src/main/kotlin/project/ProjectRunner.kt b/core/src/main/kotlin/project/ProjectRunner.kt index 547601e..89637cc 100644 --- a/core/src/main/kotlin/project/ProjectRunner.kt +++ b/core/src/main/kotlin/project/ProjectRunner.kt @@ -20,8 +20,9 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -class ProjectRunner( +internal class ProjectRunner( private val configuration: Configuration, + private val projectApi: ProjectApi, private val deploymentApi: DeploymentApi, private val deploymentStorage: DeploymentStorage, cohortApi: CohortApi, @@ -40,18 +41,17 @@ class ProjectRunner( private val cohortLoader = CohortLoader(configuration.maxCohortSize, cohortApi, cohortStorage) suspend fun start() { - refresh(deploymentStorage.getDeployments()) - // Collect deployment updates from the storage - scope.launch { - deploymentStorage.deployments.collect { deployments -> - refresh(deployments) - } - } - // Periodic deployment refresher + refresh() + // Periodic deployment update and refresher scope.launch { while (true) { - delay(configuration.flagSyncIntervalMillis) - refresh(deploymentStorage.getDeployments()) + delay(configuration.deploymentSyncIntervalMillis) + // Get deployments from API, update storage, and refresh + val deployments = projectApi.getDeployments() + for (deployment in deployments) { + deploymentStorage.putDeployment(deployment.key) + } + refresh() } } } @@ -65,8 +65,9 @@ class ProjectRunner( supervisor.cancelAndJoin() } - private suspend fun refresh(deploymentKeys: Set) { - log.debug("refresh: start - deploymentKeys=$deploymentKeys") + private suspend fun refresh() { + log.debug("refresh: start") + val deploymentKeys = deploymentStorage.getDeployments() lock.withLock { val jobs = mutableListOf() val runningDeployments = deploymentRunners.keys.toSet() @@ -82,7 +83,7 @@ class ProjectRunner( // Keep cohorts which are targeted by all stored deployments. removeUnusedCohorts(deploymentKeys) } - log.debug("refresh: end - deploymentKeys=$deploymentKeys") + log.debug("refresh: end") } // Must be run within lock @@ -103,14 +104,14 @@ class ProjectRunner( private suspend fun removeDeploymentInternal(deploymentKey: String) { log.info("Removing deployment $deploymentKey") deploymentRunners.remove(deploymentKey)?.stop() - deploymentStorage.removeFlagConfigs(deploymentKey) + deploymentStorage.removeAllFlags(deploymentKey) deploymentStorage.removeDeployment(deploymentKey) } private suspend fun removeUnusedCohorts(deploymentKeys: Set) { val allFlagConfigs = mutableListOf() for (deploymentKey in deploymentKeys) { - allFlagConfigs += deploymentStorage.getFlagConfigs(deploymentKey) ?: continue + allFlagConfigs += deploymentStorage.getAllFlags(deploymentKey).values } val allTargetedCohortIds = allFlagConfigs.getCohortIds() val allStoredCohortDescriptions = cohortStorage.getCohortDescriptions().values diff --git a/core/src/main/kotlin/util/FlagConfig.kt b/core/src/main/kotlin/util/FlagConfig.kt index fa88e8f..1e1f243 100644 --- a/core/src/main/kotlin/util/FlagConfig.kt +++ b/core/src/main/kotlin/util/FlagConfig.kt @@ -13,7 +13,7 @@ fun Collection.getCohortIds(): Set { return cohortIds } -private fun EvaluationFlag.getCohortIds(): Set { +fun EvaluationFlag.getCohortIds(): Set { val cohortIds = mutableSetOf() for (segment in this.segments) { cohortIds += segment.getCohortConditionIds() From 9eb59d51f95c08e0020408f8e270f57aecf67f27 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Tue, 24 Oct 2023 21:39:35 -0700 Subject: [PATCH 03/20] works with light manual testing --- compose-config.yaml | 6 +- core/src/main/kotlin/Config.kt | 4 +- core/src/main/kotlin/EvaluationProxy.kt | 4 +- core/src/main/kotlin/cohort/CohortApi.kt | 6 +- core/src/main/kotlin/cohort/CohortLoader.kt | 3 +- .../main/kotlin/deployment/DeploymentApi.kt | 10 ++-- .../kotlin/deployment/DeploymentLoader.kt | 4 +- .../kotlin/deployment/DeploymentRunner.kt | 2 +- .../kotlin/deployment/DeploymentStorage.kt | 7 +-- core/src/main/kotlin/project/ProjectApi.kt | 12 +++- core/src/main/kotlin/project/ProjectRunner.kt | 57 +++++++++++-------- core/src/main/kotlin/util/Http.kt | 11 ++-- core/src/main/kotlin/util/Json.kt | 4 ++ service/src/main/kotlin/Server.kt | 11 ++-- 14 files changed, 82 insertions(+), 59 deletions(-) diff --git a/compose-config.yaml b/compose-config.yaml index dc777dc..3a8febb 100644 --- a/compose-config.yaml +++ b/compose-config.yaml @@ -1,9 +1,7 @@ projects: - - id: "TODO" - apiKey: "TODO" + - apiKey: "TODO" secretKey: "TODO" - deploymentKeys: - - "TODO" + managementKey: "TODO" configuration: redis: diff --git a/core/src/main/kotlin/Config.kt b/core/src/main/kotlin/Config.kt index 8874f68..04af042 100644 --- a/core/src/main/kotlin/Config.kt +++ b/core/src/main/kotlin/Config.kt @@ -5,7 +5,9 @@ import com.amplitude.util.intEnv import com.amplitude.util.json import com.amplitude.util.longEnv import com.amplitude.util.stringEnv +import com.amplitude.util.yaml import com.charleskorn.kaml.Yaml +import com.charleskorn.kaml.YamlConfiguration import kotlinx.serialization.Serializable import kotlinx.serialization.decodeFromString import java.io.File @@ -23,7 +25,7 @@ data class ProjectsFile( fun fromFile(path: String): ProjectsFile { val data = File(path).readText() return if (path.endsWith(".yaml") || path.endsWith(".yml")) { - Yaml.default.decodeFromString(data) + yaml.decodeFromString(data) } else if (path.endsWith(".json")) { json.decodeFromString(data) } else { diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index 5be7093..23512bd 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -105,7 +105,7 @@ class EvaluationProxy( // Add all configured projects to storage val projectIds = projectProxies.map { it.key.id }.toSet() for (projectId in projectIds) { - log.info("Adding project $projectId") + log.debug("Adding project $projectId") projectStorage.putProject(projectId) } // Remove all non-configured projects and associated data @@ -122,7 +122,7 @@ class EvaluationProxy( val deployments = deploymentStorage.getDeployments() for (deployment in deployments) { log.info("Removing deployment and flag configs for deployment $deployment for project $projectId") - deploymentStorage.removeDeployment(deployment) + deploymentStorage.removeDeploymentInternal(deployment) deploymentStorage.removeAllFlags(deployment) } // Remove all cohorts for project diff --git a/core/src/main/kotlin/cohort/CohortApi.kt b/core/src/main/kotlin/cohort/CohortApi.kt index f4bd042..3a860f7 100644 --- a/core/src/main/kotlin/cohort/CohortApi.kt +++ b/core/src/main/kotlin/cohort/CohortApi.kt @@ -1,6 +1,6 @@ package com.amplitude.cohort -import com.amplitude.util.HttpErrorResponseException +import com.amplitude.util.HttpErrorException import com.amplitude.util.get import com.amplitude.util.json import com.amplitude.util.logger @@ -114,9 +114,9 @@ class CohortApiV5( if (statusResponse.status == HttpStatusCode.OK) { break } else if (statusResponse.status != HttpStatusCode.Accepted) { - throw HttpErrorResponseException(statusResponse.status) + throw HttpErrorException(statusResponse.status, statusResponse) } - delay(1000) + delay(5000) } // Download the cohort val downloadResponse = diff --git a/core/src/main/kotlin/cohort/CohortLoader.kt b/core/src/main/kotlin/cohort/CohortLoader.kt index 5ee2d3b..af90192 100644 --- a/core/src/main/kotlin/cohort/CohortLoader.kt +++ b/core/src/main/kotlin/cohort/CohortLoader.kt @@ -20,7 +20,7 @@ class CohortLoader( private val jobs = mutableMapOf() suspend fun loadCohort(cohortId: String) = coroutineScope { - log.debug("loadCohort: start - cohortId={}", cohortId) + log.trace("loadCohort: start - cohortId={}", cohortId) val networkCohort = cohortApi.getCohortDescription(cohortId) val storageCohort = cohortStorage.getCohortDescription(cohortId) val shouldDownloadCohort = networkCohort.size <= maxCohortSize && @@ -37,5 +37,6 @@ class CohortLoader( } }.join() } + log.trace("loadCohort: end - cohortId={}", cohortId) } } diff --git a/core/src/main/kotlin/deployment/DeploymentApi.kt b/core/src/main/kotlin/deployment/DeploymentApi.kt index 28accb9..9906862 100644 --- a/core/src/main/kotlin/deployment/DeploymentApi.kt +++ b/core/src/main/kotlin/deployment/DeploymentApi.kt @@ -10,6 +10,7 @@ import io.ktor.client.HttpClient import io.ktor.client.call.body import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.request.headers +import io.ktor.client.request.parameter interface DeploymentApi { suspend fun getFlagConfigs(deploymentKey: String): List @@ -26,9 +27,10 @@ class DeploymentApiV1( private val client = HttpClient(OkHttp) override suspend fun getFlagConfigs(deploymentKey: String): List { - log.debug("getFlagConfigs: start - deploymentKey=$deploymentKey") - val response = retry(onFailure = { e -> log.info("Get flag configs failed: $e") }) { - client.get(serverUrl, "/sdk/v2/flags?v=0") { + log.trace("getFlagConfigs: start - deploymentKey=$deploymentKey") + val response = retry(onFailure = { e -> log.error("Get flag configs failed: $e") }) { + client.get(serverUrl, "/sdk/v2/flags") { + parameter("v", "0") headers { set("Authorization", "Api-Key $deploymentKey") set("X-Amp-Exp-Library", "experiment-local-proxy/$VERSION") @@ -36,7 +38,7 @@ class DeploymentApiV1( } } return json.decodeFromString>(response.body()).also { - log.debug("getFlagConfigs: end - deploymentKey=$deploymentKey") + log.trace("getFlagConfigs: end - deploymentKey=$deploymentKey") } } } diff --git a/core/src/main/kotlin/deployment/DeploymentLoader.kt b/core/src/main/kotlin/deployment/DeploymentLoader.kt index 5d03f1c..e673cd1 100644 --- a/core/src/main/kotlin/deployment/DeploymentLoader.kt +++ b/core/src/main/kotlin/deployment/DeploymentLoader.kt @@ -23,7 +23,7 @@ class DeploymentLoader( private val jobs = mutableMapOf() suspend fun loadDeployment(deploymentKey: String) = coroutineScope { - log.debug("loadDeployment: - deploymentKey=$deploymentKey") + log.trace("loadDeployment: - deploymentKey=$deploymentKey") jobsMutex.withLock { jobs.getOrPut(deploymentKey) { launch { @@ -44,6 +44,6 @@ class DeploymentLoader( } } }.join() - log.debug("loadDeployment: end - deploymentKey=$deploymentKey") + log.trace("loadDeployment: end - deploymentKey=$deploymentKey") } } diff --git a/core/src/main/kotlin/deployment/DeploymentRunner.kt b/core/src/main/kotlin/deployment/DeploymentRunner.kt index 04623a1..970de7b 100644 --- a/core/src/main/kotlin/deployment/DeploymentRunner.kt +++ b/core/src/main/kotlin/deployment/DeploymentRunner.kt @@ -27,7 +27,7 @@ class DeploymentRunner( private val deploymentLoader = DeploymentLoader(deploymentApi, deploymentStorage, cohortLoader) suspend fun start() { - log.debug("start: - deploymentKey=$deploymentKey") + log.trace("start: - deploymentKey=$deploymentKey") deploymentLoader.loadDeployment(deploymentKey) // Periodic flag config loader scope.launch { diff --git a/core/src/main/kotlin/deployment/DeploymentStorage.kt b/core/src/main/kotlin/deployment/DeploymentStorage.kt index 2d58c2a..84c37b5 100644 --- a/core/src/main/kotlin/deployment/DeploymentStorage.kt +++ b/core/src/main/kotlin/deployment/DeploymentStorage.kt @@ -7,13 +7,12 @@ import com.amplitude.util.RedisKey import com.amplitude.util.json import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString interface DeploymentStorage { suspend fun getDeployments(): Set suspend fun putDeployment(deploymentKey: String) - suspend fun removeDeployment(deploymentKey: String) + suspend fun removeDeploymentInternal(deploymentKey: String) suspend fun getFlag(deploymentKey: String, flagKey: String): EvaluationFlag? suspend fun getAllFlags(deploymentKey: String): Map suspend fun putFlag(deploymentKey: String, flag: EvaluationFlag) @@ -50,7 +49,7 @@ class InMemoryDeploymentStorage : DeploymentStorage { } } - override suspend fun removeDeployment(deploymentKey: String) { + override suspend fun removeDeploymentInternal(deploymentKey: String) { return lock.withLock { deploymentStorage.remove(deploymentKey) } @@ -111,7 +110,7 @@ class RedisDeploymentStorage( redis.sadd(RedisKey.Deployments(projectId), setOf(deploymentKey)) } - override suspend fun removeDeployment(deploymentKey: String) { + override suspend fun removeDeploymentInternal(deploymentKey: String) { redis.srem(RedisKey.Deployments(projectId), deploymentKey) } diff --git a/core/src/main/kotlin/project/ProjectApi.kt b/core/src/main/kotlin/project/ProjectApi.kt index f8a923d..aec10ab 100644 --- a/core/src/main/kotlin/project/ProjectApi.kt +++ b/core/src/main/kotlin/project/ProjectApi.kt @@ -13,6 +13,11 @@ import kotlinx.serialization.Serializable private const val MANAGEMENT_SERVER_URL = "https://experiment.amplitude.com" + +@Serializable +private data class DeploymentsResponse( + val deployments: List +) @Serializable internal data class Deployment( val id: String, @@ -38,7 +43,7 @@ internal class ProjectApiV1(private val managementKey: String): ProjectApi { } } override suspend fun getDeployments(): List { - log.debug("getDeployments: start") + log.trace("getDeployments: start") val response = retry(onFailure = { e -> log.error("Get deployments failed: $e") }) { client.get(MANAGEMENT_SERVER_URL, "/api/1/deployments") { headers { @@ -47,8 +52,9 @@ internal class ProjectApiV1(private val managementKey: String): ProjectApi { } } } - return json.decodeFromString>(response.body()) + return json.decodeFromString(response.body()) + .deployments .filter { !it.deleted } - .also { log.debug("getDeployments: end") } + .also { log.trace("getDeployments: end") } } } diff --git a/core/src/main/kotlin/project/ProjectRunner.kt b/core/src/main/kotlin/project/ProjectRunner.kt index 89637cc..7c0ed74 100644 --- a/core/src/main/kotlin/project/ProjectRunner.kt +++ b/core/src/main/kotlin/project/ProjectRunner.kt @@ -46,11 +46,6 @@ internal class ProjectRunner( scope.launch { while (true) { delay(configuration.deploymentSyncIntervalMillis) - // Get deployments from API, update storage, and refresh - val deployments = projectApi.getDeployments() - for (deployment in deployments) { - deploymentStorage.putDeployment(deployment.key) - } refresh() } } @@ -65,30 +60,44 @@ internal class ProjectRunner( supervisor.cancelAndJoin() } - private suspend fun refresh() { - log.debug("refresh: start") - val deploymentKeys = deploymentStorage.getDeployments() - lock.withLock { - val jobs = mutableListOf() - val runningDeployments = deploymentRunners.keys.toSet() - val addedDeployments = deploymentKeys - runningDeployments - val removedDeployments = runningDeployments - deploymentKeys - addedDeployments.forEach { deployment -> - jobs += scope.launch { addDeploymentInternal(deployment) } + private suspend fun refresh() = lock.withLock { + log.trace("refresh: start") + // Get deployments from API and update the storage. + val networkDeployments = projectApi.getDeployments().map { it.key }.toSet() + val storageDeployments = deploymentStorage.getDeployments() + val addedDeployments = networkDeployments - storageDeployments + val removedDeployments = storageDeployments - networkDeployments + val jobs = mutableListOf() + for (addedDeployment in addedDeployments) { + log.info("Adding deployment $addedDeployment") + deploymentStorage.putDeployment(addedDeployment) + if (!deploymentRunners.contains(addedDeployment)) { + jobs += scope.launch { addDeploymentInternal(addedDeployment) } } - removedDeployments.forEach { deployment -> - jobs += scope.launch { removeDeploymentInternal(deployment) } + } + for (removedDeployment in removedDeployments) { + log.info("Removing deployment $removedDeployment") + deploymentStorage.removeAllFlags(removedDeployment) + deploymentStorage.removeDeploymentInternal(removedDeployment) + if (deploymentRunners.contains(removedDeployment)) { + jobs += scope.launch { removeDeploymentInternal(removedDeployment) } } - jobs.joinAll() - // Keep cohorts which are targeted by all stored deployments. - removeUnusedCohorts(deploymentKeys) } - log.debug("refresh: end") + jobs.joinAll() + // Keep cohorts which are targeted by all stored deployments. + removeUnusedCohorts(networkDeployments) + log.debug( + "Project refresh finished: addedDeployments={}, removedDeployments={}", + addedDeployments, + removedDeployments + ) + log.trace("refresh: end") } + // Must be run within lock private suspend fun addDeploymentInternal(deploymentKey: String) { - log.info("Adding deployment $deploymentKey") + log.debug("Adding and starting deployment runner for $deploymentKey") val deploymentRunner = DeploymentRunner( configuration, deploymentKey, @@ -102,10 +111,8 @@ internal class ProjectRunner( // Must be run within lock private suspend fun removeDeploymentInternal(deploymentKey: String) { - log.info("Removing deployment $deploymentKey") + log.debug("Removing and stopping deployment runner for $deploymentKey") deploymentRunners.remove(deploymentKey)?.stop() - deploymentStorage.removeAllFlags(deploymentKey) - deploymentStorage.removeDeployment(deploymentKey) } private suspend fun removeUnusedCohorts(deploymentKeys: Set) { diff --git a/core/src/main/kotlin/util/Http.kt b/core/src/main/kotlin/util/Http.kt index b03e334..0fb6bde 100644 --- a/core/src/main/kotlin/util/Http.kt +++ b/core/src/main/kotlin/util/Http.kt @@ -10,9 +10,10 @@ import io.ktor.http.HttpStatusCode import io.ktor.http.path import kotlinx.coroutines.delay -internal class HttpErrorResponseException( - val statusCode: HttpStatusCode -) : Exception("HTTP error response: code=$statusCode, message=${statusCode.description}") +internal class HttpErrorException( + val statusCode: HttpStatusCode, + response: HttpResponse? = null, +) : Exception("HTTP error response: code=$statusCode, message=${statusCode.description}, response=$response") internal data class RetryConfig( val times: Int = 8, @@ -34,9 +35,9 @@ internal suspend fun retry( if (response.status.value in 200..299) { return response } else { - throw HttpErrorResponseException(response.status) + throw HttpErrorException(response.status, response) } - } catch (e: HttpErrorResponseException) { + } catch (e: HttpErrorException) { val code = e.statusCode.value onFailure(e) if (code != 429 && code in 400..499) { diff --git a/core/src/main/kotlin/util/Json.kt b/core/src/main/kotlin/util/Json.kt index 6bf68f5..65cd77f 100644 --- a/core/src/main/kotlin/util/Json.kt +++ b/core/src/main/kotlin/util/Json.kt @@ -1,7 +1,11 @@ package com.amplitude.util +import com.charleskorn.kaml.Yaml +import com.charleskorn.kaml.YamlConfiguration import kotlinx.serialization.json.Json val json = Json { ignoreUnknownKeys = true } + +val yaml = Yaml(configuration = YamlConfiguration(strictMode = false)) diff --git a/service/src/main/kotlin/Server.kt b/service/src/main/kotlin/Server.kt index 2eb99c6..d6e9489 100644 --- a/service/src/main/kotlin/Server.kt +++ b/service/src/main/kotlin/Server.kt @@ -93,7 +93,7 @@ fun Application.proxyServer() { configureLogging() configureMetrics() install(ContentNegotiation) { - json() + json(json) } install( createApplicationPlugin("shutdown") { @@ -111,6 +111,9 @@ fun Application.proxyServer() { * Configure endpoints. */ routing { + + // Local Evaluation + get("/sdk/v2/flags") { val deployment = this.call.request.getDeploymentKey() val result = try { @@ -134,7 +137,7 @@ fun Application.proxyServer() { call.respond(result) } - // V2 evaluation endpoints + // Remote Evaluation V2 Endpoints get("/sdk/v2/vardata") { call.evaluate(evaluationProxy, ApplicationRequest::getUserFromHeader) @@ -144,7 +147,7 @@ fun Application.proxyServer() { call.evaluate(evaluationProxy, ApplicationRequest::getUserFromBody) } - // V1 evaluation endpoints + // Remote Evaluation V1 endpoints get("/sdk/vardata") { call.evaluateV1(evaluationProxy, ApplicationRequest::getUserFromHeader) @@ -162,7 +165,7 @@ fun Application.proxyServer() { call.evaluateV1(evaluationProxy, ApplicationRequest::getUserFromBody) } - // Health check + // Health Check get("/status") { call.respond("OK") From 67aae1fee6949e5bc0bea787d33658cc386d9e0c Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 25 Oct 2023 11:28:52 -0700 Subject: [PATCH 04/20] add cohort desc and member download endpoints; fix cohort flag & cohort loader --- core/src/main/kotlin/EvaluationProxy.kt | 17 +++++++++++++ core/src/main/kotlin/cohort/CohortApi.kt | 7 +++--- core/src/main/kotlin/cohort/CohortLoader.kt | 12 ++++++++- core/src/main/kotlin/cohort/CohortStorage.kt | 9 +++++++ .../kotlin/deployment/DeploymentLoader.kt | 4 +-- .../kotlin/deployment/DeploymentRunner.kt | 4 +-- core/src/main/kotlin/project/ProjectProxy.kt | 19 ++++++++++++++ core/src/main/kotlin/project/ProjectRunner.kt | 21 ++++++++++------ service/src/main/kotlin/Server.kt | 25 +++++++++++++++++++ 9 files changed, 100 insertions(+), 18 deletions(-) diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index 23512bd..84454b9 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -1,6 +1,7 @@ package com.amplitude import com.amplitude.assignment.AmplitudeAssignmentTracker +import com.amplitude.cohort.CohortDescription import com.amplitude.cohort.getCohortStorage import com.amplitude.deployment.getDeploymentStorage import com.amplitude.experiment.evaluation.EvaluationFlag @@ -167,6 +168,16 @@ class EvaluationProxy( return projectProxy.getFlagConfigs(deploymentKey) } + suspend fun getCohortDescription(deploymentKey: String?, cohortId: String?): CohortDescription { + val projectProxy = getProjectProxy(deploymentKey) + return projectProxy.getCohortDescription(cohortId) + } + + suspend fun getCohortMembers(deploymentKey: String?, cohortId: String?): Set { + val projectProxy = getProjectProxy(deploymentKey) + return projectProxy.getCohortMembers(cohortId) + } + suspend fun getCohortMembershipsForUser(deploymentKey: String?, userId: String?): Set { val projectProxy = getProjectProxy(deploymentKey) return projectProxy.getCohortMembershipsForUser(deploymentKey, userId) @@ -202,6 +213,12 @@ class EvaluationProxy( // Serialized Proxy Calls +suspend fun EvaluationProxy.getSerializedCohortDescription(deploymentKey: String?, cohortId: String?): String = + json.encodeToString(getCohortDescription(deploymentKey, cohortId)) + +suspend fun EvaluationProxy.getSerializedCohortMembers(deploymentKey: String?, cohortId: String?): String = + json.encodeToString(getCohortMembers(deploymentKey, cohortId)) + suspend fun EvaluationProxy.getSerializedFlagConfigs(deploymentKey: String?): String = json.encodeToString(getFlagConfigs(deploymentKey)) diff --git a/core/src/main/kotlin/cohort/CohortApi.kt b/core/src/main/kotlin/cohort/CohortApi.kt index 3a860f7..919c05a 100644 --- a/core/src/main/kotlin/cohort/CohortApi.kt +++ b/core/src/main/kotlin/cohort/CohortApi.kt @@ -96,21 +96,21 @@ class CohortApiV5( } override suspend fun getCohortMembers(cohortDescription: CohortDescription): Set { - log.debug("getCohortMembers: start - cohortDescription=$cohortDescription") + log.debug("getCohortMembers: start - cohortDescription={}", cohortDescription) // Initiate async cohort download val initialResponse = client.get(serverUrl, "/api/5/cohorts/request/${cohortDescription.id}") { headers { set("Authorization", "Basic $basicAuth") } parameter("lastComputed", cohortDescription.lastComputed) } val getCohortResponse = json.decodeFromString(initialResponse.body()) - log.debug("getCohortMembers: cohortId=${cohortDescription.id}, requestId=${getCohortResponse.requestId}") + log.debug("getCohortMembers: poll for status - cohortId=${cohortDescription.id}, requestId=${getCohortResponse.requestId}") // Poll until the cohort is ready for download while (true) { val statusResponse = client.get(serverUrl, "/api/5/cohorts/request-status/${getCohortResponse.requestId}") { headers { set("Authorization", "Basic $basicAuth") } } - log.debug("getCohortMembers: cohortId=${cohortDescription.id}, status=${statusResponse.status}") + log.trace("getCohortMembers: cohortId={}, status={}", cohortDescription.id, statusResponse.status) if (statusResponse.status == HttpStatusCode.OK) { break } else if (statusResponse.status != HttpStatusCode.Accepted) { @@ -119,6 +119,7 @@ class CohortApiV5( delay(5000) } // Download the cohort + log.debug("getCohortMembers: download cohort - cohortId=${cohortDescription.id}, requestId=${getCohortResponse.requestId}") val downloadResponse = client.get(serverUrl, "/api/5/cohorts/request/${getCohortResponse.requestId}/file") { headers { set("Authorization", "Basic $basicAuth") } diff --git a/core/src/main/kotlin/cohort/CohortLoader.kt b/core/src/main/kotlin/cohort/CohortLoader.kt index af90192..e8fdc42 100644 --- a/core/src/main/kotlin/cohort/CohortLoader.kt +++ b/core/src/main/kotlin/cohort/CohortLoader.kt @@ -3,6 +3,7 @@ package com.amplitude.cohort import com.amplitude.util.logger import kotlinx.coroutines.Job import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -19,7 +20,15 @@ class CohortLoader( private val jobsMutex = Mutex() private val jobs = mutableMapOf() - suspend fun loadCohort(cohortId: String) = coroutineScope { + suspend fun loadCohorts(cohortIds: Set) = coroutineScope { + val jobs = mutableListOf() + for (cohortId in cohortIds) { + jobs += launch { loadCohort(cohortId) } + } + jobs.joinAll() + } + + private suspend fun loadCohort(cohortId: String) = coroutineScope { log.trace("loadCohort: start - cohortId={}", cohortId) val networkCohort = cohortApi.getCohortDescription(cohortId) val storageCohort = cohortStorage.getCohortDescription(cohortId) @@ -33,6 +42,7 @@ class CohortLoader( val cohortMembers = cohortApi.getCohortMembers(networkCohort) cohortStorage.putCohort(networkCohort, cohortMembers) jobsMutex.withLock { jobs.remove(cohortId) } + log.info("Cohort download complete. $networkCohort") } } }.join() diff --git a/core/src/main/kotlin/cohort/CohortStorage.kt b/core/src/main/kotlin/cohort/CohortStorage.kt index 5aa8098..41c69a4 100644 --- a/core/src/main/kotlin/cohort/CohortStorage.kt +++ b/core/src/main/kotlin/cohort/CohortStorage.kt @@ -12,6 +12,7 @@ import kotlin.time.Duration interface CohortStorage { suspend fun getCohortDescription(cohortId: String): CohortDescription? suspend fun getCohortDescriptions(): Map + suspend fun getCohortMembers(cohortDescription: CohortDescription): Set? suspend fun getCohortMembershipsForUser(userId: String, cohortIds: Set? = null): Set suspend fun putCohort(description: CohortDescription, members: Set) suspend fun removeCohort(cohortDescription: CohortDescription) @@ -46,6 +47,10 @@ class InMemoryCohortStorage : CohortStorage { return lock.withLock { cohorts.mapValues { it.value.description } } } + override suspend fun getCohortMembers(cohortDescription: CohortDescription): Set? { + return lock.withLock { cohorts[cohortDescription.id]?.members } + } + override suspend fun putCohort(description: CohortDescription, members: Set) { return lock.withLock { cohorts[description.id] = Cohort(description, members) } } @@ -87,6 +92,10 @@ class RedisCohortStorage( return jsonEncodedDescriptions?.mapValues { json.decodeFromString(it.value) } ?: mapOf() } + override suspend fun getCohortMembers(cohortDescription: CohortDescription): Set? { + return redis.smembers(RedisKey.CohortMembers(projectId, cohortDescription)) + } + override suspend fun getCohortMembershipsForUser(userId: String, cohortIds: Set?): Set { val descriptions = getCohortDescriptions() val memberships = mutableSetOf() diff --git a/core/src/main/kotlin/deployment/DeploymentLoader.kt b/core/src/main/kotlin/deployment/DeploymentLoader.kt index e673cd1..1a098b6 100644 --- a/core/src/main/kotlin/deployment/DeploymentLoader.kt +++ b/core/src/main/kotlin/deployment/DeploymentLoader.kt @@ -32,9 +32,7 @@ class DeploymentLoader( val cohortIds = flag.getCohortIds() if (cohortIds.isNotEmpty()) { launch { - for (cohortId in cohortIds) { - cohortLoader.loadCohort(cohortId) - } + cohortLoader.loadCohorts(cohortIds) deploymentStorage.putFlag(deploymentKey, flag) } } else { diff --git a/core/src/main/kotlin/deployment/DeploymentRunner.kt b/core/src/main/kotlin/deployment/DeploymentRunner.kt index 970de7b..7742c1e 100644 --- a/core/src/main/kotlin/deployment/DeploymentRunner.kt +++ b/core/src/main/kotlin/deployment/DeploymentRunner.kt @@ -41,9 +41,7 @@ class DeploymentRunner( while (true) { delay(configuration.cohortSyncIntervalMillis) val cohortIds = deploymentStorage.getAllFlags(deploymentKey).values.getCohortIds() - for (cohortId in cohortIds) { - cohortLoader.loadCohort(cohortId) - } + cohortLoader.loadCohorts(cohortIds) } } } diff --git a/core/src/main/kotlin/project/ProjectProxy.kt b/core/src/main/kotlin/project/ProjectProxy.kt index 574937c..fb1e615 100644 --- a/core/src/main/kotlin/project/ProjectProxy.kt +++ b/core/src/main/kotlin/project/ProjectProxy.kt @@ -5,6 +5,7 @@ import com.amplitude.HttpErrorResponseException import com.amplitude.assignment.Assignment import com.amplitude.assignment.AssignmentTracker import com.amplitude.cohort.CohortApiV5 +import com.amplitude.cohort.CohortDescription import com.amplitude.cohort.CohortStorage import com.amplitude.deployment.DeploymentApiV1 import com.amplitude.deployment.DeploymentStorage @@ -61,6 +62,24 @@ internal class ProjectProxy( return deploymentStorage.getAllFlags(deploymentKey).values.toList() } + suspend fun getCohortDescription(cohortId: String?): CohortDescription { + if (cohortId.isNullOrEmpty()) { + throw HttpErrorResponseException(status = 404, message = "Cohort not found.") + } + return cohortStorage.getCohortDescription(cohortId) + ?: throw HttpErrorResponseException(status = 404, message = "Cohort not found.") + } + + suspend fun getCohortMembers(cohortId: String?): Set { + if (cohortId.isNullOrEmpty()) { + throw HttpErrorResponseException(status = 404, message = "Cohort not found.") + } + val cohortDescription = cohortStorage.getCohortDescription(cohortId) + ?: throw HttpErrorResponseException(status = 404, message = "Cohort not found.") + return cohortStorage.getCohortMembers(cohortDescription) + ?: throw HttpErrorResponseException(status = 404, message = "Cohort not found.") + } + suspend fun getCohortMembershipsForUser(deploymentKey: String?, userId: String?): Set { if (deploymentKey.isNullOrEmpty()) { throw HttpErrorResponseException(status = 401, message = "Invalid deployment.") diff --git a/core/src/main/kotlin/project/ProjectRunner.kt b/core/src/main/kotlin/project/ProjectRunner.kt index 7c0ed74..a8c27af 100644 --- a/core/src/main/kotlin/project/ProjectRunner.kt +++ b/core/src/main/kotlin/project/ProjectRunner.kt @@ -65,31 +65,33 @@ internal class ProjectRunner( // Get deployments from API and update the storage. val networkDeployments = projectApi.getDeployments().map { it.key }.toSet() val storageDeployments = deploymentStorage.getDeployments() + val runningDeployments = deploymentRunners.keys + // Determine added and removed deployments val addedDeployments = networkDeployments - storageDeployments val removedDeployments = storageDeployments - networkDeployments + val startingDeployments = networkDeployments - runningDeployments val jobs = mutableListOf() for (addedDeployment in addedDeployments) { log.info("Adding deployment $addedDeployment") deploymentStorage.putDeployment(addedDeployment) - if (!deploymentRunners.contains(addedDeployment)) { - jobs += scope.launch { addDeploymentInternal(addedDeployment) } - } + } + for (deployment in startingDeployments) { + jobs += scope.launch { addDeploymentInternal(deployment) } } for (removedDeployment in removedDeployments) { log.info("Removing deployment $removedDeployment") deploymentStorage.removeAllFlags(removedDeployment) deploymentStorage.removeDeploymentInternal(removedDeployment) - if (deploymentRunners.contains(removedDeployment)) { - jobs += scope.launch { removeDeploymentInternal(removedDeployment) } - } + jobs += scope.launch { removeDeploymentInternal(removedDeployment) } } jobs.joinAll() // Keep cohorts which are targeted by all stored deployments. removeUnusedCohorts(networkDeployments) log.debug( - "Project refresh finished: addedDeployments={}, removedDeployments={}", + "Project refresh finished: addedDeployments={}, removedDeployments={}, startedDeployments={}", addedDeployments, - removedDeployments + removedDeployments, + startingDeployments ) log.trace("refresh: end") } @@ -97,6 +99,9 @@ internal class ProjectRunner( // Must be run within lock private suspend fun addDeploymentInternal(deploymentKey: String) { + if (deploymentRunners.contains(deploymentKey)) { + return + } log.debug("Adding and starting deployment runner for $deploymentKey") val deploymentRunner = DeploymentRunner( configuration, diff --git a/service/src/main/kotlin/Server.kt b/service/src/main/kotlin/Server.kt index d6e9489..e408efd 100644 --- a/service/src/main/kotlin/Server.kt +++ b/service/src/main/kotlin/Server.kt @@ -7,6 +7,7 @@ import com.amplitude.util.json import com.amplitude.util.logger import com.amplitude.util.stringEnv import io.ktor.http.HttpStatusCode +import io.ktor.http.parameters import io.ktor.serialization.kotlinx.json.json import io.ktor.server.application.ApplicationCall import io.ktor.server.application.call @@ -125,6 +126,30 @@ fun Application.proxyServer() { call.respond(result) } + get("/sdk/v2/cohorts/{cohortId}/description") { + val deployment = this.call.request.getDeploymentKey() + val cohortId = this.call.parameters["cohortId"] + val result = try { + evaluationProxy.getSerializedCohortDescription(deployment, cohortId) + } catch (e: HttpErrorResponseException) { + call.respond(HttpStatusCode.fromValue(e.status), e.message) + return@get + } + call.respond(result) + } + + get("/sdk/v2/cohorts/{cohortId}/members") { + val deployment = this.call.request.getDeploymentKey() + val cohortId = this.call.parameters["cohortId"] + val result = try { + evaluationProxy.getSerializedCohortMembers(deployment, cohortId) + } catch (e: HttpErrorResponseException) { + call.respond(HttpStatusCode.fromValue(e.status), e.message) + return@get + } + call.respond(result) + } + get("/sdk/v2/users/{userId}/cohorts") { val deployment = this.call.request.getDeploymentKey() val userId = this.call.parameters["userId"] From 378bfa5f4560f72e1099d67d29749ee3fdd03724 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 25 Oct 2023 11:35:14 -0700 Subject: [PATCH 05/20] reduce public access --- core/src/main/kotlin/assignment/Assignment.kt | 6 +++--- .../kotlin/assignment/AssignmentFilter.kt | 4 ++-- .../kotlin/assignment/AssignmentTracker.kt | 6 +++--- core/src/main/kotlin/cohort/CohortApi.kt | 21 ++++--------------- core/src/main/kotlin/cohort/CohortLoader.kt | 2 +- core/src/main/kotlin/cohort/CohortStorage.kt | 8 +++---- .../main/kotlin/deployment/DeploymentApi.kt | 4 ++-- .../kotlin/deployment/DeploymentLoader.kt | 2 +- .../kotlin/deployment/DeploymentRunner.kt | 2 +- .../kotlin/deployment/DeploymentStorage.kt | 8 +++---- core/src/main/kotlin/project/Project.kt | 2 +- .../src/main/kotlin/project/ProjectStorage.kt | 8 +++---- core/src/main/kotlin/util/Cache.kt | 2 +- core/src/main/kotlin/util/FlagConfig.kt | 4 ++-- core/src/main/kotlin/util/Http.kt | 4 ++-- 15 files changed, 35 insertions(+), 48 deletions(-) diff --git a/core/src/main/kotlin/assignment/Assignment.kt b/core/src/main/kotlin/assignment/Assignment.kt index e3e3559..065ab7d 100644 --- a/core/src/main/kotlin/assignment/Assignment.kt +++ b/core/src/main/kotlin/assignment/Assignment.kt @@ -5,15 +5,15 @@ import com.amplitude.experiment.evaluation.EvaluationVariant import com.amplitude.util.deviceId import com.amplitude.util.userId -const val DAY_MILLIS: Long = 24 * 60 * 60 * 1000 +internal const val DAY_MILLIS: Long = 24 * 60 * 60 * 1000 -data class Assignment( +internal data class Assignment( val context: EvaluationContext, val results: Map, val timestamp: Long = System.currentTimeMillis() ) -fun Assignment.canonicalize(): String { +internal fun Assignment.canonicalize(): String { val sb = StringBuilder().append(this.context.userId()?.trim(), " ", this.context.deviceId()?.trim(), " ") for (key in this.results.keys.sorted()) { val variant = this.results[key] diff --git a/core/src/main/kotlin/assignment/AssignmentFilter.kt b/core/src/main/kotlin/assignment/AssignmentFilter.kt index 03f9706..9090434 100644 --- a/core/src/main/kotlin/assignment/AssignmentFilter.kt +++ b/core/src/main/kotlin/assignment/AssignmentFilter.kt @@ -2,11 +2,11 @@ package com.amplitude.assignment import com.amplitude.util.Cache -interface AssignmentFilter { +internal interface AssignmentFilter { suspend fun shouldTrack(assignment: Assignment): Boolean } -class InMemoryAssignmentFilter(size: Int) : AssignmentFilter { +internal class InMemoryAssignmentFilter(size: Int) : AssignmentFilter { // Cache of canonical assignment to the last sent timestamp. private val cache = Cache(size, DAY_MILLIS) diff --git a/core/src/main/kotlin/assignment/AssignmentTracker.kt b/core/src/main/kotlin/assignment/AssignmentTracker.kt index 8710c38..4f8b177 100644 --- a/core/src/main/kotlin/assignment/AssignmentTracker.kt +++ b/core/src/main/kotlin/assignment/AssignmentTracker.kt @@ -7,7 +7,7 @@ import com.amplitude.util.deviceId import com.amplitude.util.userId import org.json.JSONObject -object FlagType { +private object FlagType { const val RELEASE = "release" const val EXPERIMENT = "experiment" const val MUTUAL_EXCLUSION_GROUP = "mutual-exclusion-group" @@ -15,11 +15,11 @@ object FlagType { const val RELEASE_GROUP = "release-group" } -interface AssignmentTracker { +internal interface AssignmentTracker { suspend fun track(assignment: Assignment) } -class AmplitudeAssignmentTracker( +internal class AmplitudeAssignmentTracker( private val amplitude: Amplitude, private val assignmentFilter: AssignmentFilter ) : AssignmentTracker { diff --git a/core/src/main/kotlin/cohort/CohortApi.kt b/core/src/main/kotlin/cohort/CohortApi.kt index 919c05a..f5ddb76 100644 --- a/core/src/main/kotlin/cohort/CohortApi.kt +++ b/core/src/main/kotlin/cohort/CohortApi.kt @@ -23,18 +23,6 @@ import java.util.Base64 @Serializable private data class SerialCohortDescription( - @SerialName("lastComputed") val lastComputed: Long, - @SerialName("published") val published: Boolean, - @SerialName("archived") val archived: Boolean, - @SerialName("appId") val appId: Int, - @SerialName("lastMod") val lastMod: Long, - @SerialName("type") val type: String, - @SerialName("id") val id: String, - @SerialName("size") val size: Int -) - -@Serializable -private data class SerialSingleCohortDescription( @SerialName("cohort_id") val cohortId: String, @SerialName("app_id") val appId: Int = 0, @SerialName("org_id") val orgId: Int = 0, @@ -45,19 +33,18 @@ private data class SerialSingleCohortDescription( ) @Serializable -data class GetCohortAsyncResponse( +private data class GetCohortAsyncResponse( @SerialName("cohort_id") val cohortId: String, @SerialName("request_id") val requestId: String ) -interface CohortApi { - +internal interface CohortApi { suspend fun getCohortDescription(cohortId: String): CohortDescription suspend fun getCohortDescriptions(cohortIds: Set): List suspend fun getCohortMembers(cohortDescription: CohortDescription): Set } -class CohortApiV5( +internal class CohortApiV5( private val serverUrl: String, apiKey: String, secretKey: String @@ -80,7 +67,7 @@ class CohortApiV5( headers { set("Authorization", "Basic $basicAuth") } } } - val serialDescription = json.decodeFromString(response.body()) + val serialDescription = json.decodeFromString(response.body()) return CohortDescription( id = serialDescription.cohortId, lastComputed = serialDescription.lastComputed, diff --git a/core/src/main/kotlin/cohort/CohortLoader.kt b/core/src/main/kotlin/cohort/CohortLoader.kt index e8fdc42..7891218 100644 --- a/core/src/main/kotlin/cohort/CohortLoader.kt +++ b/core/src/main/kotlin/cohort/CohortLoader.kt @@ -8,7 +8,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -class CohortLoader( +internal class CohortLoader( @Volatile var maxCohortSize: Int, private val cohortApi: CohortApi, private val cohortStorage: CohortStorage diff --git a/core/src/main/kotlin/cohort/CohortStorage.kt b/core/src/main/kotlin/cohort/CohortStorage.kt index 41c69a4..c68b710 100644 --- a/core/src/main/kotlin/cohort/CohortStorage.kt +++ b/core/src/main/kotlin/cohort/CohortStorage.kt @@ -9,7 +9,7 @@ import kotlinx.coroutines.sync.withLock import kotlinx.serialization.encodeToString import kotlin.time.Duration -interface CohortStorage { +internal interface CohortStorage { suspend fun getCohortDescription(cohortId: String): CohortDescription? suspend fun getCohortDescriptions(): Map suspend fun getCohortMembers(cohortDescription: CohortDescription): Set? @@ -18,7 +18,7 @@ interface CohortStorage { suspend fun removeCohort(cohortDescription: CohortDescription) } -fun getCohortStorage(projectId: String, redisConfiguration: RedisConfiguration?, ttl: Duration): CohortStorage { +internal fun getCohortStorage(projectId: String, redisConfiguration: RedisConfiguration?, ttl: Duration): CohortStorage { val uri = redisConfiguration?.uri val readOnlyUri = redisConfiguration?.readOnlyUri ?: uri val prefix = redisConfiguration?.prefix @@ -29,7 +29,7 @@ fun getCohortStorage(projectId: String, redisConfiguration: RedisConfiguration?, } } -class InMemoryCohortStorage : CohortStorage { +internal class InMemoryCohortStorage : CohortStorage { private class Cohort( val description: CohortDescription, @@ -71,7 +71,7 @@ class InMemoryCohortStorage : CohortStorage { } } -class RedisCohortStorage( +internal class RedisCohortStorage( uri: String, readOnlyUri: String, prefix: String, diff --git a/core/src/main/kotlin/deployment/DeploymentApi.kt b/core/src/main/kotlin/deployment/DeploymentApi.kt index 9906862..a0b3ab8 100644 --- a/core/src/main/kotlin/deployment/DeploymentApi.kt +++ b/core/src/main/kotlin/deployment/DeploymentApi.kt @@ -12,11 +12,11 @@ import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.request.headers import io.ktor.client.request.parameter -interface DeploymentApi { +internal interface DeploymentApi { suspend fun getFlagConfigs(deploymentKey: String): List } -class DeploymentApiV1( +internal class DeploymentApiV1( private val serverUrl: String ) : DeploymentApi { diff --git a/core/src/main/kotlin/deployment/DeploymentLoader.kt b/core/src/main/kotlin/deployment/DeploymentLoader.kt index 1a098b6..7b2a9b3 100644 --- a/core/src/main/kotlin/deployment/DeploymentLoader.kt +++ b/core/src/main/kotlin/deployment/DeploymentLoader.kt @@ -9,7 +9,7 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -class DeploymentLoader( +internal class DeploymentLoader( private val deploymentApi: DeploymentApi, private val deploymentStorage: DeploymentStorage, private val cohortLoader: CohortLoader diff --git a/core/src/main/kotlin/deployment/DeploymentRunner.kt b/core/src/main/kotlin/deployment/DeploymentRunner.kt index 7742c1e..b76e9d6 100644 --- a/core/src/main/kotlin/deployment/DeploymentRunner.kt +++ b/core/src/main/kotlin/deployment/DeploymentRunner.kt @@ -10,7 +10,7 @@ import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.delay import kotlinx.coroutines.launch -class DeploymentRunner( +internal class DeploymentRunner( @Volatile var configuration: Configuration, private val deploymentKey: String, private val deploymentApi: DeploymentApi, diff --git a/core/src/main/kotlin/deployment/DeploymentStorage.kt b/core/src/main/kotlin/deployment/DeploymentStorage.kt index 84c37b5..3b85c31 100644 --- a/core/src/main/kotlin/deployment/DeploymentStorage.kt +++ b/core/src/main/kotlin/deployment/DeploymentStorage.kt @@ -9,7 +9,7 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.serialization.encodeToString -interface DeploymentStorage { +internal interface DeploymentStorage { suspend fun getDeployments(): Set suspend fun putDeployment(deploymentKey: String) suspend fun removeDeploymentInternal(deploymentKey: String) @@ -21,7 +21,7 @@ interface DeploymentStorage { suspend fun removeAllFlags(deploymentKey: String) } -fun getDeploymentStorage(projectId: String, redisConfiguration: RedisConfiguration?): DeploymentStorage { +internal fun getDeploymentStorage(projectId: String, redisConfiguration: RedisConfiguration?): DeploymentStorage { val uri = redisConfiguration?.uri val readOnlyUri = redisConfiguration?.readOnlyUri ?: uri val prefix = redisConfiguration?.prefix @@ -32,7 +32,7 @@ fun getDeploymentStorage(projectId: String, redisConfiguration: RedisConfigurati } } -class InMemoryDeploymentStorage : DeploymentStorage { +internal class InMemoryDeploymentStorage : DeploymentStorage { private val lock = Mutex() private val deploymentStorage = mutableMapOf?>() @@ -92,7 +92,7 @@ class InMemoryDeploymentStorage : DeploymentStorage { } } -class RedisDeploymentStorage( +internal class RedisDeploymentStorage( uri: String, readOnlyUri: String, prefix: String, diff --git a/core/src/main/kotlin/project/Project.kt b/core/src/main/kotlin/project/Project.kt index ebae10a..e6f991c 100644 --- a/core/src/main/kotlin/project/Project.kt +++ b/core/src/main/kotlin/project/Project.kt @@ -1,6 +1,6 @@ package com.amplitude.project -data class Project( +internal data class Project( val id: String, val apiKey: String, val secretKey: String, diff --git a/core/src/main/kotlin/project/ProjectStorage.kt b/core/src/main/kotlin/project/ProjectStorage.kt index 6d7462d..289517a 100644 --- a/core/src/main/kotlin/project/ProjectStorage.kt +++ b/core/src/main/kotlin/project/ProjectStorage.kt @@ -9,14 +9,14 @@ import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -interface ProjectStorage { +internal interface ProjectStorage { val projects: Flow> suspend fun getProjects(): Set suspend fun putProject(projectId: String) suspend fun removeProject(projectId: String) } -fun getProjectStorage(redisConfiguration: RedisConfiguration?): ProjectStorage { +internal fun getProjectStorage(redisConfiguration: RedisConfiguration?): ProjectStorage { val uri = redisConfiguration?.uri val prefix = redisConfiguration?.prefix return if (uri == null || prefix == null) { @@ -26,7 +26,7 @@ fun getProjectStorage(redisConfiguration: RedisConfiguration?): ProjectStorage { } } -class InMemoryProjectStorage : ProjectStorage { +internal class InMemoryProjectStorage : ProjectStorage { override val projects = MutableSharedFlow>( extraBufferCapacity = 1, @@ -51,7 +51,7 @@ class InMemoryProjectStorage : ProjectStorage { } } -class RedisProjectStorage( +internal class RedisProjectStorage( uri: String, prefix: String ) : ProjectStorage { diff --git a/core/src/main/kotlin/util/Cache.kt b/core/src/main/kotlin/util/Cache.kt index fb0f07e..ded1bca 100644 --- a/core/src/main/kotlin/util/Cache.kt +++ b/core/src/main/kotlin/util/Cache.kt @@ -7,7 +7,7 @@ import java.util.HashMap /** * Least recently used (LRU) cache with TTL for cache entries. */ -class Cache( +internal class Cache( private val capacity: Int, private val ttlMillis: Long = 0 ) { diff --git a/core/src/main/kotlin/util/FlagConfig.kt b/core/src/main/kotlin/util/FlagConfig.kt index 1e1f243..6552785 100644 --- a/core/src/main/kotlin/util/FlagConfig.kt +++ b/core/src/main/kotlin/util/FlagConfig.kt @@ -5,7 +5,7 @@ import com.amplitude.experiment.evaluation.EvaluationFlag import com.amplitude.experiment.evaluation.EvaluationOperator import com.amplitude.experiment.evaluation.EvaluationSegment -fun Collection.getCohortIds(): Set { +internal fun Collection.getCohortIds(): Set { val cohortIds = mutableSetOf() for (flag in this) { cohortIds += flag.getCohortIds() @@ -13,7 +13,7 @@ fun Collection.getCohortIds(): Set { return cohortIds } -fun EvaluationFlag.getCohortIds(): Set { +internal fun EvaluationFlag.getCohortIds(): Set { val cohortIds = mutableSetOf() for (segment in this.segments) { cohortIds += segment.getCohortConditionIds() diff --git a/core/src/main/kotlin/util/Http.kt b/core/src/main/kotlin/util/Http.kt index 0fb6bde..0fb6d8c 100644 --- a/core/src/main/kotlin/util/Http.kt +++ b/core/src/main/kotlin/util/Http.kt @@ -53,7 +53,7 @@ internal suspend fun retry( throw error!! } -suspend fun HttpClient.get( +internal suspend fun HttpClient.get( url: String, path: String, block: HttpRequestBuilder.() -> Unit @@ -61,7 +61,7 @@ suspend fun HttpClient.get( return request(HttpMethod.Get, url, path, block) } -suspend fun HttpClient.request( +internal suspend fun HttpClient.request( method: HttpMethod, url: String, path: String, From dde95256662276567794d2faec9fa488ee773378 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 25 Oct 2023 11:48:59 -0700 Subject: [PATCH 06/20] fix up redis storage classes --- core/src/main/kotlin/cohort/CohortStorage.kt | 43 +++++++++--------- .../kotlin/deployment/DeploymentStorage.kt | 42 ++++++++--------- .../src/main/kotlin/project/ProjectStorage.kt | 17 +++---- core/src/main/kotlin/util/Redis.kt | 45 +++++++++---------- 4 files changed, 74 insertions(+), 73 deletions(-) diff --git a/core/src/main/kotlin/cohort/CohortStorage.kt b/core/src/main/kotlin/cohort/CohortStorage.kt index c68b710..9e6681f 100644 --- a/core/src/main/kotlin/cohort/CohortStorage.kt +++ b/core/src/main/kotlin/cohort/CohortStorage.kt @@ -1,6 +1,8 @@ package com.amplitude.cohort import com.amplitude.RedisConfiguration +import com.amplitude.deployment.InMemoryDeploymentStorage +import com.amplitude.deployment.RedisDeploymentStorage import com.amplitude.util.RedisConnection import com.amplitude.util.RedisKey import com.amplitude.util.json @@ -20,12 +22,16 @@ internal interface CohortStorage { internal fun getCohortStorage(projectId: String, redisConfiguration: RedisConfiguration?, ttl: Duration): CohortStorage { val uri = redisConfiguration?.uri - val readOnlyUri = redisConfiguration?.readOnlyUri ?: uri - val prefix = redisConfiguration?.prefix - return if (uri == null || readOnlyUri == null || prefix == null) { + return if (uri == null) { InMemoryCohortStorage() } else { - RedisCohortStorage(uri, readOnlyUri, prefix, projectId, ttl) + val redis = RedisConnection(uri) + val readOnlyRedis = if (redisConfiguration.readOnlyUri != null) { + RedisConnection(redisConfiguration.readOnlyUri) + } else { + redis + } + RedisCohortStorage(projectId, ttl, redisConfiguration.prefix, redis, readOnlyRedis) } } @@ -72,28 +78,25 @@ internal class InMemoryCohortStorage : CohortStorage { } internal class RedisCohortStorage( - uri: String, - readOnlyUri: String, - prefix: String, private val projectId: String, - private val ttl: Duration + private val ttl: Duration, + private val prefix: String, + private val redis: RedisConnection, + private val readOnlyRedis: RedisConnection, ) : CohortStorage { - private val redis = RedisConnection(uri, prefix) - private val readOnlyRedis = RedisConnection(readOnlyUri, prefix) - override suspend fun getCohortDescription(cohortId: String): CohortDescription? { - val jsonEncodedDescription = redis.hget(RedisKey.CohortDescriptions(projectId), cohortId) ?: return null + val jsonEncodedDescription = redis.hget(RedisKey.CohortDescriptions(prefix, projectId), cohortId) ?: return null return json.decodeFromString(jsonEncodedDescription) } override suspend fun getCohortDescriptions(): Map { - val jsonEncodedDescriptions = redis.hgetall(RedisKey.CohortDescriptions(projectId)) + val jsonEncodedDescriptions = redis.hgetall(RedisKey.CohortDescriptions(prefix, projectId)) return jsonEncodedDescriptions?.mapValues { json.decodeFromString(it.value) } ?: mapOf() } override suspend fun getCohortMembers(cohortDescription: CohortDescription): Set? { - return redis.smembers(RedisKey.CohortMembers(projectId, cohortDescription)) + return redis.smembers(RedisKey.CohortMembers(prefix, projectId, cohortDescription)) } override suspend fun getCohortMembershipsForUser(userId: String, cohortIds: Set?): Set { @@ -101,7 +104,7 @@ internal class RedisCohortStorage( val memberships = mutableSetOf() for (description in descriptions.values) { // High volume, use read connection - val isMember = readOnlyRedis.sismember(RedisKey.CohortMembers(projectId, description), userId) + val isMember = readOnlyRedis.sismember(RedisKey.CohortMembers(prefix, projectId, description), userId) if (isMember) { memberships += description.id } @@ -113,16 +116,16 @@ internal class RedisCohortStorage( val jsonEncodedDescription = json.encodeToString(description) val existingDescription = getCohortDescription(description.id) if ((existingDescription?.lastComputed ?: 0L) < description.lastComputed) { - redis.sadd(RedisKey.CohortMembers(projectId, description), members) - redis.hset(RedisKey.CohortDescriptions(projectId), mapOf(description.id to jsonEncodedDescription)) + redis.sadd(RedisKey.CohortMembers(prefix, projectId, description), members) + redis.hset(RedisKey.CohortDescriptions(prefix, projectId), mapOf(description.id to jsonEncodedDescription)) if (existingDescription != null) { - redis.expire(RedisKey.CohortMembers(projectId, existingDescription), ttl) + redis.expire(RedisKey.CohortMembers(prefix, projectId, existingDescription), ttl) } } } override suspend fun removeCohort(cohortDescription: CohortDescription) { - redis.hdel(RedisKey.CohortDescriptions(projectId), cohortDescription.id) - redis.del(RedisKey.CohortMembers(projectId, cohortDescription)) + redis.hdel(RedisKey.CohortDescriptions(prefix, projectId), cohortDescription.id) + redis.del(RedisKey.CohortMembers(prefix, projectId, cohortDescription)) } } diff --git a/core/src/main/kotlin/deployment/DeploymentStorage.kt b/core/src/main/kotlin/deployment/DeploymentStorage.kt index 3b85c31..89e281d 100644 --- a/core/src/main/kotlin/deployment/DeploymentStorage.kt +++ b/core/src/main/kotlin/deployment/DeploymentStorage.kt @@ -2,6 +2,7 @@ package com.amplitude.deployment import com.amplitude.RedisConfiguration import com.amplitude.experiment.evaluation.EvaluationFlag +import com.amplitude.util.Redis import com.amplitude.util.RedisConnection import com.amplitude.util.RedisKey import com.amplitude.util.json @@ -23,12 +24,16 @@ internal interface DeploymentStorage { internal fun getDeploymentStorage(projectId: String, redisConfiguration: RedisConfiguration?): DeploymentStorage { val uri = redisConfiguration?.uri - val readOnlyUri = redisConfiguration?.readOnlyUri ?: uri - val prefix = redisConfiguration?.prefix - return if (uri == null || readOnlyUri == null || prefix == null) { + return if (uri == null) { InMemoryDeploymentStorage() } else { - RedisDeploymentStorage(uri, readOnlyUri, prefix, projectId) + val redis = RedisConnection(uri) + val readOnlyRedis = if (redisConfiguration.readOnlyUri != null) { + RedisConnection(redisConfiguration.readOnlyUri) + } else { + redis + } + RedisDeploymentStorage(projectId, redisConfiguration.prefix, redis, readOnlyRedis) } } @@ -93,42 +98,39 @@ internal class InMemoryDeploymentStorage : DeploymentStorage { } internal class RedisDeploymentStorage( - uri: String, - readOnlyUri: String, - prefix: String, - private val projectId: String + private val prefix: String, + private val projectId: String, + private val redis: Redis, + private val readOnlyRedis: Redis, ) : DeploymentStorage { - private val redis = RedisConnection(uri, prefix) - private val readOnlyRedis = RedisConnection(readOnlyUri, prefix) - override suspend fun getDeployments(): Set { - return redis.smembers(RedisKey.Deployments(projectId)) ?: emptySet() + return redis.smembers(RedisKey.Deployments(prefix, projectId)) ?: emptySet() } override suspend fun putDeployment(deploymentKey: String) { - redis.sadd(RedisKey.Deployments(projectId), setOf(deploymentKey)) + redis.sadd(RedisKey.Deployments(prefix, projectId), setOf(deploymentKey)) } override suspend fun removeDeploymentInternal(deploymentKey: String) { - redis.srem(RedisKey.Deployments(projectId), deploymentKey) + redis.srem(RedisKey.Deployments(prefix, projectId), deploymentKey) } override suspend fun getFlag(deploymentKey: String, flagKey: String): EvaluationFlag? { - val flagJson = redis.hget(RedisKey.FlagConfigs(projectId, deploymentKey), flagKey) ?: return null + val flagJson = redis.hget(RedisKey.FlagConfigs(prefix, projectId, deploymentKey), flagKey) ?: return null return json.decodeFromString(flagJson) } // TODO Add in memory caching w/ invalidation override suspend fun getAllFlags(deploymentKey: String): Map { // High volume, use read only redis - return readOnlyRedis.hgetall(RedisKey.FlagConfigs(projectId, deploymentKey)) + return readOnlyRedis.hgetall(RedisKey.FlagConfigs(prefix, projectId, deploymentKey)) ?.mapValues { json.decodeFromString(it.value) } ?: mapOf() } override suspend fun putFlag(deploymentKey: String, flag: EvaluationFlag) { val flagJson = json.encodeToString(flag) - redis.hset(RedisKey.FlagConfigs(projectId, deploymentKey), mapOf(flag.key to flagJson)) + redis.hset(RedisKey.FlagConfigs(prefix, projectId, deploymentKey), mapOf(flag.key to flagJson)) } override suspend fun putAllFlags(deploymentKey: String, flags: List) { @@ -138,12 +140,12 @@ internal class RedisDeploymentStorage( } override suspend fun removeFlag(deploymentKey: String, flagKey: String) { - redis.hdel(RedisKey.FlagConfigs(projectId, deploymentKey), flagKey) + redis.hdel(RedisKey.FlagConfigs(prefix, projectId, deploymentKey), flagKey) } override suspend fun removeAllFlags(deploymentKey: String) { - val redisKey = RedisKey.FlagConfigs(projectId, deploymentKey) - val flags = redis.hgetall(RedisKey.FlagConfigs(projectId, deploymentKey)) ?: return + val redisKey = RedisKey.FlagConfigs(prefix, projectId, deploymentKey) + val flags = redis.hgetall(RedisKey.FlagConfigs(prefix, projectId, deploymentKey)) ?: return for (key in flags.keys) { redis.hdel(redisKey, key) } diff --git a/core/src/main/kotlin/project/ProjectStorage.kt b/core/src/main/kotlin/project/ProjectStorage.kt index 289517a..8e4a73c 100644 --- a/core/src/main/kotlin/project/ProjectStorage.kt +++ b/core/src/main/kotlin/project/ProjectStorage.kt @@ -18,11 +18,10 @@ internal interface ProjectStorage { internal fun getProjectStorage(redisConfiguration: RedisConfiguration?): ProjectStorage { val uri = redisConfiguration?.uri - val prefix = redisConfiguration?.prefix - return if (uri == null || prefix == null) { + return if (uri == null) { InMemoryProjectStorage() } else { - RedisProjectStorage(uri, prefix) + RedisProjectStorage(redisConfiguration.prefix, RedisConnection(uri)) } } @@ -52,28 +51,26 @@ internal class InMemoryProjectStorage : ProjectStorage { } internal class RedisProjectStorage( - uri: String, - prefix: String + private val prefix: String, + private val redis: RedisConnection, ) : ProjectStorage { - private val redis = RedisConnection(uri, prefix) - override val projects = MutableSharedFlow>( extraBufferCapacity = 1, onBufferOverflow = BufferOverflow.DROP_OLDEST ) override suspend fun getProjects(): Set { - return redis.smembers(RedisKey.Projects) ?: emptySet() + return redis.smembers(RedisKey.Projects(prefix)) ?: emptySet() } override suspend fun putProject(projectId: String) { - redis.sadd(RedisKey.Projects, setOf(projectId)) + redis.sadd(RedisKey.Projects(prefix), setOf(projectId)) projects.emit(getProjects()) } override suspend fun removeProject(projectId: String) { - redis.srem(RedisKey.Projects, projectId) + redis.srem(RedisKey.Projects(prefix), projectId) projects.emit(getProjects()) } } diff --git a/core/src/main/kotlin/util/Redis.kt b/core/src/main/kotlin/util/Redis.kt index 52e80e0..06bae47 100644 --- a/core/src/main/kotlin/util/Redis.kt +++ b/core/src/main/kotlin/util/Redis.kt @@ -11,29 +11,33 @@ import kotlinx.coroutines.Deferred import kotlinx.coroutines.future.asDeferred import kotlin.time.Duration -private const val STORAGE_PROTOCOL_VERSION = "v1" +private const val STORAGE_PROTOCOL_VERSION = "v2" internal sealed class RedisKey(val value: String) { - data object Projects : RedisKey("projects") + data class Projects(val prefix: String) : RedisKey("$prefix:$STORAGE_PROTOCOL_VERSION:projects") data class Deployments( + val prefix: String, val projectId: String - ) : RedisKey("projects:$projectId:deployments") + ) : RedisKey("$prefix:$STORAGE_PROTOCOL_VERSION:projects:$projectId:deployments") data class FlagConfigs( + val prefix: String, val projectId: String, val deploymentKey: String - ) : RedisKey("projects:$projectId:deployments:$deploymentKey:flags") + ) : RedisKey("$prefix:$STORAGE_PROTOCOL_VERSION:projects:$projectId:deployments:$deploymentKey:flags") data class CohortDescriptions( + val prefix: String, val projectId: String - ) : RedisKey("projects:$projectId:cohorts") + ) : RedisKey("$prefix:$STORAGE_PROTOCOL_VERSION:projects:$projectId:cohorts") data class CohortMembers( + val prefix: String, val projectId: String, val cohortDescription: CohortDescription - ) : RedisKey("projects:$projectId:cohorts:${cohortDescription.id}:users:${cohortDescription.lastComputed}") + ) : RedisKey("$prefix:$STORAGE_PROTOCOL_VERSION:projects:$projectId:cohorts:${cohortDescription.id}:users:${cohortDescription.lastComputed}") } internal interface Redis { @@ -53,7 +57,6 @@ internal interface Redis { internal class RedisConnection( redisUri: String, - private val redisPrefix: String ) : Redis { private val connection: Deferred> @@ -65,73 +68,73 @@ internal class RedisConnection( override suspend fun get(key: RedisKey): String? { return connection.run { - get(key.getPrefixedKey()) + get(key.value) } } override suspend fun set(key: RedisKey, value: String) { connection.run { - set(key.getPrefixedKey(), value) + set(key.value, value) } } override suspend fun del(key: RedisKey) { connection.run { - del(key.getPrefixedKey()) + del(key.value) } } override suspend fun sadd(key: RedisKey, values: Set) { connection.run { - sadd(key.getPrefixedKey(), *values.toTypedArray()) + sadd(key.value, *values.toTypedArray()) } } override suspend fun srem(key: RedisKey, value: String) { connection.run { - srem(key.getPrefixedKey(), value) + srem(key.value, value) } } override suspend fun smembers(key: RedisKey): Set? { return connection.run { - smembers(key.getPrefixedKey()) + smembers(key.value) } } override suspend fun sismember(key: RedisKey, value: String): Boolean { return connection.run { - sismember(key.getPrefixedKey(), value) + sismember(key.value, value) } } override suspend fun hget(key: RedisKey, field: String): String? { return connection.run { - hget(key.getPrefixedKey(), field) + hget(key.value, field) } } override suspend fun hgetall(key: RedisKey): Map? { return connection.run { - hgetall(key.getPrefixedKey()) + hgetall(key.value) } } override suspend fun hset(key: RedisKey, values: Map) { connection.run { - hset(key.getPrefixedKey(), values) + hset(key.value, values) } } override suspend fun hdel(key: RedisKey, field: String) { connection.run { - hdel(key.getPrefixedKey(), field) + hdel(key.value, field) } } override suspend fun expire(key: RedisKey, ttl: Duration) { connection.run { - expire(key.getPrefixedKey(), ttl.inWholeSeconds) + expire(key.value, ttl.inWholeSeconds) } } @@ -140,8 +143,4 @@ internal class RedisConnection( ): R { return await().async().action().asDeferred().await() } - - private fun RedisKey.getPrefixedKey(): String { - return "$redisPrefix:$STORAGE_PROTOCOL_VERSION:${this.value}" - } } From 2c7a43902ba22191515e2e806bec2e1f7b28154a Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 25 Oct 2023 14:10:11 -0700 Subject: [PATCH 07/20] store additional deployment info --- core/src/main/kotlin/EvaluationProxy.kt | 8 +- core/src/main/kotlin/deployment/Deployment.kt | 11 +++ .../kotlin/deployment/DeploymentStorage.kt | 77 +++++++++++-------- core/src/main/kotlin/project/ProjectApi.kt | 12 ++- core/src/main/kotlin/project/ProjectProxy.kt | 2 +- core/src/main/kotlin/project/ProjectRunner.kt | 31 ++++---- 6 files changed, 86 insertions(+), 55 deletions(-) create mode 100644 core/src/main/kotlin/deployment/Deployment.kt diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index 84454b9..d19e7cf 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -121,10 +121,10 @@ class EvaluationProxy( ) // Remove all deployments for project val deployments = deploymentStorage.getDeployments() - for (deployment in deployments) { - log.info("Removing deployment and flag configs for deployment $deployment for project $projectId") - deploymentStorage.removeDeploymentInternal(deployment) - deploymentStorage.removeAllFlags(deployment) + for ((deploymentKey, _) in deployments) { + log.info("Removing deployment and flag configs for deployment $deploymentKey for project $projectId") + deploymentStorage.removeDeployment(deploymentKey) + deploymentStorage.removeAllFlags(deploymentKey) } // Remove all cohorts for project val cohortDescriptions = cohortStorage.getCohortDescriptions().values diff --git a/core/src/main/kotlin/deployment/Deployment.kt b/core/src/main/kotlin/deployment/Deployment.kt new file mode 100644 index 0000000..1c4f732 --- /dev/null +++ b/core/src/main/kotlin/deployment/Deployment.kt @@ -0,0 +1,11 @@ +package com.amplitude.deployment + +import kotlinx.serialization.Serializable + +@Serializable +internal data class Deployment( + val id: String, + val projectId: String, + val label: String, + val key: String, +) diff --git a/core/src/main/kotlin/deployment/DeploymentStorage.kt b/core/src/main/kotlin/deployment/DeploymentStorage.kt index 89e281d..3fcad47 100644 --- a/core/src/main/kotlin/deployment/DeploymentStorage.kt +++ b/core/src/main/kotlin/deployment/DeploymentStorage.kt @@ -11,9 +11,10 @@ import kotlinx.coroutines.sync.withLock import kotlinx.serialization.encodeToString internal interface DeploymentStorage { - suspend fun getDeployments(): Set - suspend fun putDeployment(deploymentKey: String) - suspend fun removeDeploymentInternal(deploymentKey: String) + suspend fun getDeployment(deploymentKey: String): Deployment? + suspend fun getDeployments(): Map + suspend fun putDeployment(deployment: Deployment) + suspend fun removeDeployment(deploymentKey: String) suspend fun getFlag(deploymentKey: String, flagKey: String): EvaluationFlag? suspend fun getAllFlags(deploymentKey: String): Map suspend fun putFlag(deploymentKey: String, flag: EvaluationFlag) @@ -39,60 +40,68 @@ internal fun getDeploymentStorage(projectId: String, redisConfiguration: RedisCo internal class InMemoryDeploymentStorage : DeploymentStorage { - private val lock = Mutex() - private val deploymentStorage = mutableMapOf?>() + private val mutex = Mutex() - override suspend fun getDeployments(): Set { - return lock.withLock { - deploymentStorage.keys.toSet() + private val deploymentStorage = mutableMapOf() + private val flagStorage = mutableMapOf>() + override suspend fun getDeployment(deploymentKey: String): Deployment? { + return mutex.withLock { + deploymentStorage[deploymentKey] } } - override suspend fun putDeployment(deploymentKey: String) { - return lock.withLock { - deploymentStorage.putIfAbsent(deploymentKey, null) + override suspend fun getDeployments(): Map { + return mutex.withLock { + deploymentStorage.toMap() } } - override suspend fun removeDeploymentInternal(deploymentKey: String) { - return lock.withLock { + override suspend fun putDeployment(deployment: Deployment) { + mutex.withLock { + deploymentStorage[deployment.key] = deployment + } + } + + override suspend fun removeDeployment(deploymentKey: String) { + return mutex.withLock { deploymentStorage.remove(deploymentKey) + flagStorage.remove(deploymentKey) } } override suspend fun getFlag(deploymentKey: String, flagKey: String): EvaluationFlag? { - return lock.withLock { - deploymentStorage[deploymentKey]?.get(flagKey) + return mutex.withLock { + flagStorage[deploymentKey]?.get(flagKey) } } override suspend fun getAllFlags(deploymentKey: String): Map { - return lock.withLock { - deploymentStorage[deploymentKey]?.toMap() ?: mapOf() + return mutex.withLock { + flagStorage[deploymentKey] ?: mapOf() } } override suspend fun putFlag(deploymentKey: String, flag: EvaluationFlag) { - return lock.withLock { - deploymentStorage[deploymentKey]?.put(flag.key, flag) + return mutex.withLock { + flagStorage.getOrPut(deploymentKey) { mutableMapOf() }[flag.key] = flag } } override suspend fun putAllFlags(deploymentKey: String, flags: List) { - return lock.withLock { - deploymentStorage[deploymentKey]?.putAll(flags.associateBy { it.key }) + return mutex.withLock { + flagStorage.getOrPut(deploymentKey) { mutableMapOf() }.putAll(flags.associateBy { it.key }) } } override suspend fun removeFlag(deploymentKey: String, flagKey: String) { - return lock.withLock { - deploymentStorage[deploymentKey]?.remove(flagKey) + return mutex.withLock { + flagStorage[deploymentKey]?.remove(flagKey) } } override suspend fun removeAllFlags(deploymentKey: String) { - return lock.withLock { - deploymentStorage[deploymentKey] = null + return mutex.withLock { + flagStorage.remove(deploymentKey) } } } @@ -103,17 +112,23 @@ internal class RedisDeploymentStorage( private val redis: Redis, private val readOnlyRedis: Redis, ) : DeploymentStorage { + override suspend fun getDeployment(deploymentKey: String): Deployment? { + val deploymentJson = redis.hget(RedisKey.Deployments(prefix, projectId), deploymentKey) ?: return null + return json.decodeFromString(deploymentJson) + } - override suspend fun getDeployments(): Set { - return redis.smembers(RedisKey.Deployments(prefix, projectId)) ?: emptySet() + override suspend fun getDeployments(): Map { + return redis.hgetall(RedisKey.Deployments(prefix, projectId)) + ?.mapValues { json.decodeFromString(it.value) } ?: mapOf() } - override suspend fun putDeployment(deploymentKey: String) { - redis.sadd(RedisKey.Deployments(prefix, projectId), setOf(deploymentKey)) + override suspend fun putDeployment(deployment: Deployment) { + val deploymentJson = json.encodeToString(deployment) + redis.hset(RedisKey.Deployments(prefix, projectId), mapOf(deployment.key to deploymentJson)) } - override suspend fun removeDeploymentInternal(deploymentKey: String) { - redis.srem(RedisKey.Deployments(prefix, projectId), deploymentKey) + override suspend fun removeDeployment(deploymentKey: String) { + redis.hdel(RedisKey.Deployments(prefix, projectId), deploymentKey) } override suspend fun getFlag(deploymentKey: String, flagKey: String): EvaluationFlag? { diff --git a/core/src/main/kotlin/project/ProjectApi.kt b/core/src/main/kotlin/project/ProjectApi.kt index aec10ab..b4d6909 100644 --- a/core/src/main/kotlin/project/ProjectApi.kt +++ b/core/src/main/kotlin/project/ProjectApi.kt @@ -1,5 +1,6 @@ package com.amplitude.project +import com.amplitude.deployment.Deployment import com.amplitude.util.get import com.amplitude.util.json import com.amplitude.util.logger @@ -16,10 +17,10 @@ private const val MANAGEMENT_SERVER_URL = "https://experiment.amplitude.com" @Serializable private data class DeploymentsResponse( - val deployments: List + val deployments: List ) @Serializable -internal data class Deployment( +internal data class SerialDeployment( val id: String, val projectId: String, val label: String, @@ -27,6 +28,11 @@ internal data class Deployment( val deleted: Boolean, ) +private fun SerialDeployment.toDeployment(): Deployment? { + if (deleted) return null + return Deployment(id, projectId, label, key) +} + internal interface ProjectApi { suspend fun getDeployments(): List } @@ -54,7 +60,7 @@ internal class ProjectApiV1(private val managementKey: String): ProjectApi { } return json.decodeFromString(response.body()) .deployments - .filter { !it.deleted } + .mapNotNull { it.toDeployment() } .also { log.trace("getDeployments: end") } } } diff --git a/core/src/main/kotlin/project/ProjectProxy.kt b/core/src/main/kotlin/project/ProjectProxy.kt index fb1e615..fcd3f86 100644 --- a/core/src/main/kotlin/project/ProjectProxy.kt +++ b/core/src/main/kotlin/project/ProjectProxy.kt @@ -144,6 +144,6 @@ internal class ProjectProxy( // Internal internal suspend fun getDeployments(): Set { - return deploymentStorage.getDeployments() + return deploymentStorage.getDeployments().keys } } diff --git a/core/src/main/kotlin/project/ProjectRunner.kt b/core/src/main/kotlin/project/ProjectRunner.kt index a8c27af..2b84b4f 100644 --- a/core/src/main/kotlin/project/ProjectRunner.kt +++ b/core/src/main/kotlin/project/ProjectRunner.kt @@ -63,35 +63,34 @@ internal class ProjectRunner( private suspend fun refresh() = lock.withLock { log.trace("refresh: start") // Get deployments from API and update the storage. - val networkDeployments = projectApi.getDeployments().map { it.key }.toSet() + val networkDeployments = projectApi.getDeployments().associateBy { it.key } val storageDeployments = deploymentStorage.getDeployments() - val runningDeployments = deploymentRunners.keys // Determine added and removed deployments - val addedDeployments = networkDeployments - storageDeployments - val removedDeployments = storageDeployments - networkDeployments - val startingDeployments = networkDeployments - runningDeployments + val addedDeployments = networkDeployments - storageDeployments.keys + val removedDeployments = storageDeployments - networkDeployments.keys + val startingDeployments = networkDeployments - deploymentRunners.keys val jobs = mutableListOf() - for (addedDeployment in addedDeployments) { + for ((_, addedDeployment) in addedDeployments) { log.info("Adding deployment $addedDeployment") deploymentStorage.putDeployment(addedDeployment) } - for (deployment in startingDeployments) { - jobs += scope.launch { addDeploymentInternal(deployment) } + for ((_, deployment) in startingDeployments) { + jobs += scope.launch { addDeploymentInternal(deployment.key) } } - for (removedDeployment in removedDeployments) { + for ((_, removedDeployment) in removedDeployments) { log.info("Removing deployment $removedDeployment") - deploymentStorage.removeAllFlags(removedDeployment) - deploymentStorage.removeDeploymentInternal(removedDeployment) - jobs += scope.launch { removeDeploymentInternal(removedDeployment) } + deploymentStorage.removeAllFlags(removedDeployment.key) + deploymentStorage.removeDeployment(removedDeployment.key) + jobs += scope.launch { removeDeploymentInternal(removedDeployment.key) } } jobs.joinAll() // Keep cohorts which are targeted by all stored deployments. - removeUnusedCohorts(networkDeployments) + removeUnusedCohorts(networkDeployments.keys) log.debug( "Project refresh finished: addedDeployments={}, removedDeployments={}, startedDeployments={}", - addedDeployments, - removedDeployments, - startingDeployments + addedDeployments.keys, + removedDeployments.keys, + startingDeployments.keys ) log.trace("refresh: end") } From f0a59cdf8ada0eeef4b2448691ba882531339df0 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 25 Oct 2023 14:22:24 -0700 Subject: [PATCH 08/20] add retries to cohort download requests --- core/src/main/kotlin/cohort/CohortApi.kt | 24 ++++++++++-------------- core/src/main/kotlin/util/Http.kt | 2 +- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/core/src/main/kotlin/cohort/CohortApi.kt b/core/src/main/kotlin/cohort/CohortApi.kt index f5ddb76..1e06a15 100644 --- a/core/src/main/kotlin/cohort/CohortApi.kt +++ b/core/src/main/kotlin/cohort/CohortApi.kt @@ -40,7 +40,6 @@ private data class GetCohortAsyncResponse( internal interface CohortApi { suspend fun getCohortDescription(cohortId: String): CohortDescription - suspend fun getCohortDescriptions(cohortIds: Set): List suspend fun getCohortMembers(cohortDescription: CohortDescription): Set } @@ -57,7 +56,7 @@ internal class CohortApiV5( private val basicAuth = Base64.getEncoder().encodeToString("$apiKey:$secretKey".toByteArray(Charsets.UTF_8)) private val client = HttpClient(OkHttp) { install(HttpTimeout) { - socketTimeoutMillis = 360000 + socketTimeoutMillis = 30000 } } @@ -75,28 +74,24 @@ internal class CohortApiV5( ) } - override suspend fun getCohortDescriptions(cohortIds: Set): List { - log.debug("getCohortDescriptions: start") - return cohortIds.map { cohortId -> - getCohortDescription(cohortId) - }.toList().also { log.debug("getCohortDescriptions: end - result=$it") } - } - override suspend fun getCohortMembers(cohortDescription: CohortDescription): Set { log.debug("getCohortMembers: start - cohortDescription={}", cohortDescription) // Initiate async cohort download - val initialResponse = client.get(serverUrl, "/api/5/cohorts/request/${cohortDescription.id}") { - headers { set("Authorization", "Basic $basicAuth") } - parameter("lastComputed", cohortDescription.lastComputed) + val initialResponse = retry(onFailure = { e -> log.error("Cohort download request failed: $e")}) { + client.get(serverUrl, "/api/5/cohorts/request/${cohortDescription.id}") { + headers { set("Authorization", "Basic $basicAuth") } + parameter("lastComputed", cohortDescription.lastComputed) + } } val getCohortResponse = json.decodeFromString(initialResponse.body()) log.debug("getCohortMembers: poll for status - cohortId=${cohortDescription.id}, requestId=${getCohortResponse.requestId}") // Poll until the cohort is ready for download while (true) { - val statusResponse = + val statusResponse = retry(onFailure = { e -> log.error("Cohort request status failed: $e")}) { client.get(serverUrl, "/api/5/cohorts/request-status/${getCohortResponse.requestId}") { headers { set("Authorization", "Basic $basicAuth") } } + } log.trace("getCohortMembers: cohortId={}, status={}", cohortDescription.id, statusResponse.status) if (statusResponse.status == HttpStatusCode.OK) { break @@ -107,10 +102,11 @@ internal class CohortApiV5( } // Download the cohort log.debug("getCohortMembers: download cohort - cohortId=${cohortDescription.id}, requestId=${getCohortResponse.requestId}") - val downloadResponse = + val downloadResponse = retry(onFailure = { e -> log.error("Cohort file download failed: $e")}) { client.get(serverUrl, "/api/5/cohorts/request/${getCohortResponse.requestId}/file") { headers { set("Authorization", "Basic $basicAuth") } } + } val csv = CSVParser.parse(downloadResponse.bodyAsChannel().toInputStream(), Charsets.UTF_8, csvFormat) return csv.map { it.get("user_id") }.filterNot { it.isNullOrEmpty() }.toSet() .also { log.debug("getCohortMembers: end - cohortId=${cohortDescription.id}, resultSize=${it.size}") } diff --git a/core/src/main/kotlin/util/Http.kt b/core/src/main/kotlin/util/Http.kt index 0fb6d8c..ff0445d 100644 --- a/core/src/main/kotlin/util/Http.kt +++ b/core/src/main/kotlin/util/Http.kt @@ -32,7 +32,7 @@ internal suspend fun retry( for (i in 0..config.times) { try { val response = block() - if (response.status.value in 200..299) { + if (response.status.value in 100..399) { return response } else { throw HttpErrorException(response.status, response) From 308f5e53536cc8534c3e963142b5a1fec2a6ac4c Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Thu, 26 Oct 2023 14:06:51 -0700 Subject: [PATCH 09/20] metrics --- core/src/main/kotlin/EvaluationProxy.kt | 15 ++++-- core/src/main/kotlin/Metrics.kt | 48 +++++++++++++++++++ .../kotlin/assignment/AssignmentTracker.kt | 12 ++++- core/src/main/kotlin/cohort/CohortLoader.kt | 16 ++++++- .../kotlin/deployment/DeploymentLoader.kt | 7 ++- core/src/main/kotlin/project/ProjectApi.kt | 29 ++++++----- core/src/main/kotlin/util/Redis.kt | 9 +++- 7 files changed, 115 insertions(+), 21 deletions(-) create mode 100644 core/src/main/kotlin/Metrics.kt diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index d19e7cf..f2a1926 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -35,13 +35,18 @@ class HttpErrorResponseException( class EvaluationProxy( private val projectConfigurations: List, - private val configuration: Configuration = Configuration() + private val configuration: Configuration = Configuration(), + metricsHandler: MetricsHandler? = null ) { companion object { val log by logger() } + init { + Metrics.handler = metricsHandler + } + private val supervisor = SupervisorJob() private val scope = CoroutineScope(supervisor) @@ -189,7 +194,9 @@ class EvaluationProxy( flagKeys: Set? = null ): Map { val projectProxy = getProjectProxy(deploymentKey) - return projectProxy.evaluate(deploymentKey, user, flagKeys) + return Metrics.with({ Evaluation }, { e -> EvaluationFailure(e) }) { + projectProxy.evaluate(deploymentKey, user, flagKeys) + } } suspend fun evaluateV1( @@ -198,7 +205,9 @@ class EvaluationProxy( flagKeys: Set? = null ): Map { val projectProxy = getProjectProxy(deploymentKey) - return projectProxy.evaluateV1(deploymentKey, user, flagKeys) + return Metrics.with({ Evaluation }, { e -> EvaluationFailure(e) }) { + projectProxy.evaluateV1(deploymentKey, user, flagKeys) + } } // Private diff --git a/core/src/main/kotlin/Metrics.kt b/core/src/main/kotlin/Metrics.kt new file mode 100644 index 0000000..f105ae1 --- /dev/null +++ b/core/src/main/kotlin/Metrics.kt @@ -0,0 +1,48 @@ +package com.amplitude + +sealed class Metric +sealed class FailureMetric : Metric() + +interface MetricsHandler { + fun track(metric: Metric) +} + +data object Evaluation: Metric() +data class EvaluationFailure(val exception: Exception): FailureMetric() +data object AssignmentEvent: Metric() +data object AssignmentEventFilter: Metric() +data object AssignmentEventSend: Metric() +data class AssignmentEventSendFailure(val exception: Exception): FailureMetric() +data object DeploymentsFetch: Metric() +data class DeploymentsFetchFailure(val exception: Exception): FailureMetric() +data object FlagsFetch: Metric() +data class FlagsFetchFailure(val exception: Exception): FailureMetric() +data object CohortDescriptionFetch: Metric() +data class CohortDescriptionFetchFailure(val exception: Exception): FailureMetric() +data object CohortDownload: Metric() +data class CohortDownloadFailure(val exception: Exception): FailureMetric() +data object RedisCommand: Metric() +data class RedisCommandFailure(val exception: Exception): FailureMetric() + +internal object Metrics: MetricsHandler { + + internal var handler: MetricsHandler? = null + + override fun track(metric: Metric) { + handler?.track(metric) + } + + internal suspend fun with( + metric: (() -> Metric)?, + failure: ((e: Exception) -> FailureMetric)?, + block: suspend () -> R + ): R { + try { + metric?.invoke() + return block.invoke() + } catch (e: Exception) { + failure?.invoke(e) + throw e + } + } +} diff --git a/core/src/main/kotlin/assignment/AssignmentTracker.kt b/core/src/main/kotlin/assignment/AssignmentTracker.kt index 4f8b177..0bf79c2 100644 --- a/core/src/main/kotlin/assignment/AssignmentTracker.kt +++ b/core/src/main/kotlin/assignment/AssignmentTracker.kt @@ -2,7 +2,12 @@ package com.amplitude.assignment import com.amplitude.Amplitude import com.amplitude.AssignmentConfiguration +import com.amplitude.AssignmentEvent +import com.amplitude.AssignmentEventFilter +import com.amplitude.AssignmentEventSend +import com.amplitude.AssignmentEventSendFailure import com.amplitude.Event +import com.amplitude.Metrics import com.amplitude.util.deviceId import com.amplitude.util.userId import org.json.JSONObject @@ -38,8 +43,13 @@ internal class AmplitudeAssignmentTracker( ) override suspend fun track(assignment: Assignment) { + Metrics.track(AssignmentEvent) if (assignmentFilter.shouldTrack(assignment)) { - amplitude.logEvent(assignment.toAmplitudeEvent()) + Metrics.with({ AssignmentEventSend }, { e -> AssignmentEventSendFailure(e) }) { + amplitude.logEvent(assignment.toAmplitudeEvent()) + } + } else { + Metrics.track(AssignmentEventFilter) } } } diff --git a/core/src/main/kotlin/cohort/CohortLoader.kt b/core/src/main/kotlin/cohort/CohortLoader.kt index 7891218..da5eecc 100644 --- a/core/src/main/kotlin/cohort/CohortLoader.kt +++ b/core/src/main/kotlin/cohort/CohortLoader.kt @@ -1,5 +1,10 @@ package com.amplitude.cohort +import com.amplitude.CohortDescriptionFetch +import com.amplitude.CohortDescriptionFetchFailure +import com.amplitude.CohortDownload +import com.amplitude.CohortDownloadFailure +import com.amplitude.Metrics import com.amplitude.util.logger import kotlinx.coroutines.Job import kotlinx.coroutines.coroutineScope @@ -30,7 +35,12 @@ internal class CohortLoader( private suspend fun loadCohort(cohortId: String) = coroutineScope { log.trace("loadCohort: start - cohortId={}", cohortId) - val networkCohort = cohortApi.getCohortDescription(cohortId) + val networkCohort = Metrics.with( + { CohortDescriptionFetch }, + { e -> CohortDescriptionFetchFailure(e) } + ) { + cohortApi.getCohortDescription(cohortId) + } val storageCohort = cohortStorage.getCohortDescription(cohortId) val shouldDownloadCohort = networkCohort.size <= maxCohortSize && networkCohort.lastComputed > (storageCohort?.lastComputed ?: -1) @@ -39,7 +49,9 @@ internal class CohortLoader( jobs.getOrPut(cohortId) { launch { log.info("Downloading cohort. $networkCohort") - val cohortMembers = cohortApi.getCohortMembers(networkCohort) + val cohortMembers = Metrics.with({ CohortDownload }, { e -> CohortDownloadFailure(e) }) { + cohortApi.getCohortMembers(networkCohort) + } cohortStorage.putCohort(networkCohort, cohortMembers) jobsMutex.withLock { jobs.remove(cohortId) } log.info("Cohort download complete. $networkCohort") diff --git a/core/src/main/kotlin/deployment/DeploymentLoader.kt b/core/src/main/kotlin/deployment/DeploymentLoader.kt index 7b2a9b3..dbebae3 100644 --- a/core/src/main/kotlin/deployment/DeploymentLoader.kt +++ b/core/src/main/kotlin/deployment/DeploymentLoader.kt @@ -1,5 +1,8 @@ package com.amplitude.deployment +import com.amplitude.FlagsFetch +import com.amplitude.FlagsFetchFailure +import com.amplitude.Metrics import com.amplitude.cohort.CohortLoader import com.amplitude.util.getCohortIds import com.amplitude.util.logger @@ -27,7 +30,9 @@ internal class DeploymentLoader( jobsMutex.withLock { jobs.getOrPut(deploymentKey) { launch { - val networkFlags = deploymentApi.getFlagConfigs(deploymentKey) + val networkFlags = Metrics.with({ FlagsFetch }, { e -> FlagsFetchFailure(e) }) { + deploymentApi.getFlagConfigs(deploymentKey) + } for (flag in networkFlags) { val cohortIds = flag.getCohortIds() if (cohortIds.isNotEmpty()) { diff --git a/core/src/main/kotlin/project/ProjectApi.kt b/core/src/main/kotlin/project/ProjectApi.kt index b4d6909..7d24ee7 100644 --- a/core/src/main/kotlin/project/ProjectApi.kt +++ b/core/src/main/kotlin/project/ProjectApi.kt @@ -1,5 +1,8 @@ package com.amplitude.project +import com.amplitude.DeploymentsFetch +import com.amplitude.DeploymentsFetchFailure +import com.amplitude.Metrics import com.amplitude.deployment.Deployment import com.amplitude.util.get import com.amplitude.util.json @@ -48,19 +51,21 @@ internal class ProjectApiV1(private val managementKey: String): ProjectApi { socketTimeoutMillis = 30000 } } - override suspend fun getDeployments(): List { - log.trace("getDeployments: start") - val response = retry(onFailure = { e -> log.error("Get deployments failed: $e") }) { - client.get(MANAGEMENT_SERVER_URL, "/api/1/deployments") { - headers { - set("Authorization", "Bearer $managementKey") - set("Accept", "application/json") + + override suspend fun getDeployments(): List = + Metrics.with({ DeploymentsFetch }, { e -> DeploymentsFetchFailure(e) }) { + log.trace("getDeployments: start") + val response = retry(onFailure = { e -> log.error("Get deployments failed: $e") }) { + client.get(MANAGEMENT_SERVER_URL, "/api/1/deployments") { + headers { + set("Authorization", "Bearer $managementKey") + set("Accept", "application/json") + } } } + json.decodeFromString(response.body()) + .deployments + .mapNotNull { it.toDeployment() } + .also { log.trace("getDeployments: end") } } - return json.decodeFromString(response.body()) - .deployments - .mapNotNull { it.toDeployment() } - .also { log.trace("getDeployments: end") } - } } diff --git a/core/src/main/kotlin/util/Redis.kt b/core/src/main/kotlin/util/Redis.kt index 06bae47..1917217 100644 --- a/core/src/main/kotlin/util/Redis.kt +++ b/core/src/main/kotlin/util/Redis.kt @@ -1,5 +1,8 @@ package com.amplitude.util +import com.amplitude.Metrics +import com.amplitude.RedisCommand +import com.amplitude.RedisCommandFailure import com.amplitude.cohort.CohortDescription import io.lettuce.core.RedisClient import io.lettuce.core.RedisFuture @@ -139,8 +142,10 @@ internal class RedisConnection( } private suspend inline fun Deferred>.run( - action: RedisAsyncCommands.() -> RedisFuture + crossinline action: RedisAsyncCommands.() -> RedisFuture ): R { - return await().async().action().asDeferred().await() + return Metrics.with({ RedisCommand }, { e -> RedisCommandFailure(e) }) { + await().async().action().asDeferred().await() + } } } From 00b4177f2f35faab65dfa2c8b8ff4ba72ac9f87a Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Fri, 3 Nov 2023 12:56:57 -0700 Subject: [PATCH 10/20] fix deployment loader --- README.md | 11 ++++------- core/src/main/kotlin/Config.kt | 2 +- core/src/main/kotlin/deployment/DeploymentLoader.kt | 11 +++++++++++ .../kotlin/util/{FlagConfig.kt => EvaluationFlag.kt} | 0 4 files changed, 16 insertions(+), 8 deletions(-) rename core/src/main/kotlin/util/{FlagConfig.kt => EvaluationFlag.kt} (100%) diff --git a/README.md b/README.md index 3d71dd4..588bda2 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,9 @@ The default location for the configuration yaml file is `/etc/evaluation-proxy-c ```yaml projects: - - id: "YOUR PROJECT ID" - apiKey: "YOUR API KEY" - secretKey: " YOUR SECRET KEY" - deploymentKeys: - - "YOUR DEPLOYMENT KEY 1" - - "YOUR DEPLOYMENT KEY 2" + - apiKey: "YOUR API KEY" + secretKey: "YOUR SECRET KEY" + managementKey: "YOUR MANAGEMENT API KEY" configuration: redis: @@ -43,7 +40,7 @@ Use the evaluation proxy [Helm chart](https://github.com/amplitude/evaluation-pr ### Docker Compose Example -Run the container locally with redis persistence using `docker compose`. You must first update the `compose-config.yaml` file with your project and deployment keys before running the composition. +Run the container locally with redis persistence using `docker compose`. You must first update the `compose-config.yaml` file with your project keys before running the composition. ``` docker compose build diff --git a/core/src/main/kotlin/Config.kt b/core/src/main/kotlin/Config.kt index 04af042..07d6831 100644 --- a/core/src/main/kotlin/Config.kt +++ b/core/src/main/kotlin/Config.kt @@ -49,7 +49,7 @@ data class ConfigurationFile( fun fromFile(path: String): ConfigurationFile { val data = File(path).readText() return if (path.endsWith(".yaml") || path.endsWith(".yml")) { - Yaml.default.decodeFromString(data) + yaml.decodeFromString(data) } else if (path.endsWith(".json")) { json.decodeFromString(data) } else { diff --git a/core/src/main/kotlin/deployment/DeploymentLoader.kt b/core/src/main/kotlin/deployment/DeploymentLoader.kt index dbebae3..49a4265 100644 --- a/core/src/main/kotlin/deployment/DeploymentLoader.kt +++ b/core/src/main/kotlin/deployment/DeploymentLoader.kt @@ -33,6 +33,15 @@ internal class DeploymentLoader( val networkFlags = Metrics.with({ FlagsFetch }, { e -> FlagsFetchFailure(e) }) { deploymentApi.getFlagConfigs(deploymentKey) } + // Remove flags that are no longer deployed. + val networkFlagKeys = networkFlags.map { it.key }.toSet() + val storageFlagKeys = deploymentStorage.getAllFlags(deploymentKey).map { it.key }.toSet() + for (flagToRemove in storageFlagKeys - networkFlagKeys) { + log.debug("Removing flag: $flagToRemove") + deploymentStorage.removeFlag(deploymentKey, flagToRemove) + } + // Load cohorts for each flag independently then put the + // flag into storage. for (flag in networkFlags) { val cohortIds = flag.getCohortIds() if (cohortIds.isNotEmpty()) { @@ -44,6 +53,8 @@ internal class DeploymentLoader( deploymentStorage.putFlag(deploymentKey, flag) } } + // Remove the job + jobsMutex.withLock { jobs.remove(deploymentKey) } } } }.join() diff --git a/core/src/main/kotlin/util/FlagConfig.kt b/core/src/main/kotlin/util/EvaluationFlag.kt similarity index 100% rename from core/src/main/kotlin/util/FlagConfig.kt rename to core/src/main/kotlin/util/EvaluationFlag.kt From 18a5146b4ca9d949e3735859cb1dc2012e82e7a7 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Mon, 6 Nov 2023 13:36:09 -0800 Subject: [PATCH 11/20] format --- core/build.gradle.kts | 2 +- core/src/main/kotlin/Config.kt | 4 +-- core/src/main/kotlin/EvaluationProxy.kt | 2 +- core/src/main/kotlin/Metrics.kt | 34 +++++++++---------- core/src/main/kotlin/assignment/Assignment.kt | 2 -- core/src/main/kotlin/cohort/CohortApi.kt | 6 ++-- core/src/main/kotlin/cohort/CohortStorage.kt | 4 +-- core/src/main/kotlin/deployment/Deployment.kt | 2 +- .../kotlin/deployment/DeploymentStorage.kt | 4 +-- core/src/main/kotlin/project/Project.kt | 2 +- core/src/main/kotlin/project/ProjectApi.kt | 6 ++-- core/src/main/kotlin/project/ProjectProxy.kt | 6 ++-- core/src/main/kotlin/project/ProjectRunner.kt | 1 - .../src/main/kotlin/project/ProjectStorage.kt | 2 +- .../src/main/kotlin/util/EvaluationContext.kt | 1 - core/src/main/kotlin/util/Http.kt | 2 +- core/src/main/kotlin/util/Redis.kt | 2 +- core/src/test/kotlin/Utils.kt | 4 +-- .../assignment/AssignmentServiceTest.kt | 2 +- service/src/main/kotlin/Server.kt | 21 ++++++------ 20 files changed, 52 insertions(+), 57 deletions(-) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index adf4029..54ae2c3 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -52,7 +52,7 @@ publishing { create("core") { groupId = "com.amplitude" artifactId = "evaluation-proxy-core" - version = "0.3.2" + version = "0.4.0" from(components["java"]) pom { name.set("Amplitude Evaluation Proxy") diff --git a/core/src/main/kotlin/Config.kt b/core/src/main/kotlin/Config.kt index 07d6831..8e079e3 100644 --- a/core/src/main/kotlin/Config.kt +++ b/core/src/main/kotlin/Config.kt @@ -6,8 +6,6 @@ import com.amplitude.util.json import com.amplitude.util.longEnv import com.amplitude.util.stringEnv import com.amplitude.util.yaml -import com.charleskorn.kaml.Yaml -import com.charleskorn.kaml.YamlConfiguration import kotlinx.serialization.Serializable import kotlinx.serialization.decodeFromString import java.io.File @@ -92,7 +90,7 @@ data class Configuration( port = intEnv(EnvKey.PORT, Default.PORT)!!, serverUrl = stringEnv(EnvKey.SERVER_URL, Default.SERVER_URL)!!, cohortServerUrl = stringEnv(EnvKey.COHORT_SERVER_URL, Default.COHORT_SERVER_URL)!!, - deploymentSyncIntervalMillis = longEnv( + deploymentSyncIntervalMillis = longEnv( EnvKey.DEPLOYMENT_SYNC_INTERVAL_MILLIS, Default.DEPLOYMENT_SYNC_INTERVAL_MILLIS )!!, diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index f2a1926..feaec96 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -78,7 +78,7 @@ class EvaluationProxy( id = projectId, apiKey = projectConfiguration.apiKey, secretKey = projectConfiguration.secretKey, - managementKey = projectConfiguration.managementKey, + managementKey = projectConfiguration.managementKey ) apiKeysToProject[project.apiKey] = project secretKeysToProject[project.secretKey] = project diff --git a/core/src/main/kotlin/Metrics.kt b/core/src/main/kotlin/Metrics.kt index f105ae1..d187926 100644 --- a/core/src/main/kotlin/Metrics.kt +++ b/core/src/main/kotlin/Metrics.kt @@ -7,24 +7,24 @@ interface MetricsHandler { fun track(metric: Metric) } -data object Evaluation: Metric() -data class EvaluationFailure(val exception: Exception): FailureMetric() -data object AssignmentEvent: Metric() -data object AssignmentEventFilter: Metric() -data object AssignmentEventSend: Metric() -data class AssignmentEventSendFailure(val exception: Exception): FailureMetric() -data object DeploymentsFetch: Metric() -data class DeploymentsFetchFailure(val exception: Exception): FailureMetric() -data object FlagsFetch: Metric() -data class FlagsFetchFailure(val exception: Exception): FailureMetric() -data object CohortDescriptionFetch: Metric() -data class CohortDescriptionFetchFailure(val exception: Exception): FailureMetric() -data object CohortDownload: Metric() -data class CohortDownloadFailure(val exception: Exception): FailureMetric() -data object RedisCommand: Metric() -data class RedisCommandFailure(val exception: Exception): FailureMetric() +data object Evaluation : Metric() +data class EvaluationFailure(val exception: Exception) : FailureMetric() +data object AssignmentEvent : Metric() +data object AssignmentEventFilter : Metric() +data object AssignmentEventSend : Metric() +data class AssignmentEventSendFailure(val exception: Exception) : FailureMetric() +data object DeploymentsFetch : Metric() +data class DeploymentsFetchFailure(val exception: Exception) : FailureMetric() +data object FlagsFetch : Metric() +data class FlagsFetchFailure(val exception: Exception) : FailureMetric() +data object CohortDescriptionFetch : Metric() +data class CohortDescriptionFetchFailure(val exception: Exception) : FailureMetric() +data object CohortDownload : Metric() +data class CohortDownloadFailure(val exception: Exception) : FailureMetric() +data object RedisCommand : Metric() +data class RedisCommandFailure(val exception: Exception) : FailureMetric() -internal object Metrics: MetricsHandler { +internal object Metrics : MetricsHandler { internal var handler: MetricsHandler? = null diff --git a/core/src/main/kotlin/assignment/Assignment.kt b/core/src/main/kotlin/assignment/Assignment.kt index 065ab7d..8b96d3b 100644 --- a/core/src/main/kotlin/assignment/Assignment.kt +++ b/core/src/main/kotlin/assignment/Assignment.kt @@ -21,5 +21,3 @@ internal fun Assignment.canonicalize(): String { } return sb.toString() } - - diff --git a/core/src/main/kotlin/cohort/CohortApi.kt b/core/src/main/kotlin/cohort/CohortApi.kt index 1e06a15..6fc9a45 100644 --- a/core/src/main/kotlin/cohort/CohortApi.kt +++ b/core/src/main/kotlin/cohort/CohortApi.kt @@ -77,7 +77,7 @@ internal class CohortApiV5( override suspend fun getCohortMembers(cohortDescription: CohortDescription): Set { log.debug("getCohortMembers: start - cohortDescription={}", cohortDescription) // Initiate async cohort download - val initialResponse = retry(onFailure = { e -> log.error("Cohort download request failed: $e")}) { + val initialResponse = retry(onFailure = { e -> log.error("Cohort download request failed: $e") }) { client.get(serverUrl, "/api/5/cohorts/request/${cohortDescription.id}") { headers { set("Authorization", "Basic $basicAuth") } parameter("lastComputed", cohortDescription.lastComputed) @@ -87,7 +87,7 @@ internal class CohortApiV5( log.debug("getCohortMembers: poll for status - cohortId=${cohortDescription.id}, requestId=${getCohortResponse.requestId}") // Poll until the cohort is ready for download while (true) { - val statusResponse = retry(onFailure = { e -> log.error("Cohort request status failed: $e")}) { + val statusResponse = retry(onFailure = { e -> log.error("Cohort request status failed: $e") }) { client.get(serverUrl, "/api/5/cohorts/request-status/${getCohortResponse.requestId}") { headers { set("Authorization", "Basic $basicAuth") } } @@ -102,7 +102,7 @@ internal class CohortApiV5( } // Download the cohort log.debug("getCohortMembers: download cohort - cohortId=${cohortDescription.id}, requestId=${getCohortResponse.requestId}") - val downloadResponse = retry(onFailure = { e -> log.error("Cohort file download failed: $e")}) { + val downloadResponse = retry(onFailure = { e -> log.error("Cohort file download failed: $e") }) { client.get(serverUrl, "/api/5/cohorts/request/${getCohortResponse.requestId}/file") { headers { set("Authorization", "Basic $basicAuth") } } diff --git a/core/src/main/kotlin/cohort/CohortStorage.kt b/core/src/main/kotlin/cohort/CohortStorage.kt index 9e6681f..35be2bc 100644 --- a/core/src/main/kotlin/cohort/CohortStorage.kt +++ b/core/src/main/kotlin/cohort/CohortStorage.kt @@ -1,8 +1,6 @@ package com.amplitude.cohort import com.amplitude.RedisConfiguration -import com.amplitude.deployment.InMemoryDeploymentStorage -import com.amplitude.deployment.RedisDeploymentStorage import com.amplitude.util.RedisConnection import com.amplitude.util.RedisKey import com.amplitude.util.json @@ -82,7 +80,7 @@ internal class RedisCohortStorage( private val ttl: Duration, private val prefix: String, private val redis: RedisConnection, - private val readOnlyRedis: RedisConnection, + private val readOnlyRedis: RedisConnection ) : CohortStorage { override suspend fun getCohortDescription(cohortId: String): CohortDescription? { diff --git a/core/src/main/kotlin/deployment/Deployment.kt b/core/src/main/kotlin/deployment/Deployment.kt index 1c4f732..e65d7e2 100644 --- a/core/src/main/kotlin/deployment/Deployment.kt +++ b/core/src/main/kotlin/deployment/Deployment.kt @@ -7,5 +7,5 @@ internal data class Deployment( val id: String, val projectId: String, val label: String, - val key: String, + val key: String ) diff --git a/core/src/main/kotlin/deployment/DeploymentStorage.kt b/core/src/main/kotlin/deployment/DeploymentStorage.kt index 3fcad47..9c09346 100644 --- a/core/src/main/kotlin/deployment/DeploymentStorage.kt +++ b/core/src/main/kotlin/deployment/DeploymentStorage.kt @@ -110,7 +110,7 @@ internal class RedisDeploymentStorage( private val prefix: String, private val projectId: String, private val redis: Redis, - private val readOnlyRedis: Redis, + private val readOnlyRedis: Redis ) : DeploymentStorage { override suspend fun getDeployment(deploymentKey: String): Deployment? { val deploymentJson = redis.hget(RedisKey.Deployments(prefix, projectId), deploymentKey) ?: return null @@ -124,7 +124,7 @@ internal class RedisDeploymentStorage( override suspend fun putDeployment(deployment: Deployment) { val deploymentJson = json.encodeToString(deployment) - redis.hset(RedisKey.Deployments(prefix, projectId), mapOf(deployment.key to deploymentJson)) + redis.hset(RedisKey.Deployments(prefix, projectId), mapOf(deployment.key to deploymentJson)) } override suspend fun removeDeployment(deploymentKey: String) { diff --git a/core/src/main/kotlin/project/Project.kt b/core/src/main/kotlin/project/Project.kt index e6f991c..614cf50 100644 --- a/core/src/main/kotlin/project/Project.kt +++ b/core/src/main/kotlin/project/Project.kt @@ -4,5 +4,5 @@ internal data class Project( val id: String, val apiKey: String, val secretKey: String, - val managementKey: String, + val managementKey: String ) diff --git a/core/src/main/kotlin/project/ProjectApi.kt b/core/src/main/kotlin/project/ProjectApi.kt index 7d24ee7..43f300e 100644 --- a/core/src/main/kotlin/project/ProjectApi.kt +++ b/core/src/main/kotlin/project/ProjectApi.kt @@ -17,18 +17,18 @@ import kotlinx.serialization.Serializable private const val MANAGEMENT_SERVER_URL = "https://experiment.amplitude.com" - @Serializable private data class DeploymentsResponse( val deployments: List ) + @Serializable internal data class SerialDeployment( val id: String, val projectId: String, val label: String, val key: String, - val deleted: Boolean, + val deleted: Boolean ) private fun SerialDeployment.toDeployment(): Deployment? { @@ -40,7 +40,7 @@ internal interface ProjectApi { suspend fun getDeployments(): List } -internal class ProjectApiV1(private val managementKey: String): ProjectApi { +internal class ProjectApiV1(private val managementKey: String) : ProjectApi { companion object { val log by logger() diff --git a/core/src/main/kotlin/project/ProjectProxy.kt b/core/src/main/kotlin/project/ProjectProxy.kt index fcd3f86..6ab6399 100644 --- a/core/src/main/kotlin/project/ProjectProxy.kt +++ b/core/src/main/kotlin/project/ProjectProxy.kt @@ -24,7 +24,7 @@ internal class ProjectProxy( configuration: Configuration, private val assignmentTracker: AssignmentTracker, private val deploymentStorage: DeploymentStorage, - private val cohortStorage: CohortStorage, + private val cohortStorage: CohortStorage ) { companion object { @@ -114,7 +114,9 @@ internal class ProjectProxy( user.toMutableMap().apply { put("cohort_ids", cohortStorage.getCohortMembershipsForUser(userId)) } - } else null + } else { + null + } val evaluationContext = enrichedUser.toEvaluationContext() // Evaluate results log.debug("evaluate - context={}", evaluationContext) diff --git a/core/src/main/kotlin/project/ProjectRunner.kt b/core/src/main/kotlin/project/ProjectRunner.kt index 2b84b4f..3d1e7e9 100644 --- a/core/src/main/kotlin/project/ProjectRunner.kt +++ b/core/src/main/kotlin/project/ProjectRunner.kt @@ -95,7 +95,6 @@ internal class ProjectRunner( log.trace("refresh: end") } - // Must be run within lock private suspend fun addDeploymentInternal(deploymentKey: String) { if (deploymentRunners.contains(deploymentKey)) { diff --git a/core/src/main/kotlin/project/ProjectStorage.kt b/core/src/main/kotlin/project/ProjectStorage.kt index 8e4a73c..6fe1b36 100644 --- a/core/src/main/kotlin/project/ProjectStorage.kt +++ b/core/src/main/kotlin/project/ProjectStorage.kt @@ -52,7 +52,7 @@ internal class InMemoryProjectStorage : ProjectStorage { internal class RedisProjectStorage( private val prefix: String, - private val redis: RedisConnection, + private val redis: RedisConnection ) : ProjectStorage { override val projects = MutableSharedFlow>( diff --git a/core/src/main/kotlin/util/EvaluationContext.kt b/core/src/main/kotlin/util/EvaluationContext.kt index ef7a840..6daf48a 100644 --- a/core/src/main/kotlin/util/EvaluationContext.kt +++ b/core/src/main/kotlin/util/EvaluationContext.kt @@ -38,7 +38,6 @@ internal fun MutableMap?.toEvaluationContext(): EvaluationContext return context } - private fun Map<*, *>.select(vararg selector: Any?): Any? { var map: Map<*, *> = this var result: Any? diff --git a/core/src/main/kotlin/util/Http.kt b/core/src/main/kotlin/util/Http.kt index ff0445d..5d244a5 100644 --- a/core/src/main/kotlin/util/Http.kt +++ b/core/src/main/kotlin/util/Http.kt @@ -12,7 +12,7 @@ import kotlinx.coroutines.delay internal class HttpErrorException( val statusCode: HttpStatusCode, - response: HttpResponse? = null, + response: HttpResponse? = null ) : Exception("HTTP error response: code=$statusCode, message=${statusCode.description}, response=$response") internal data class RetryConfig( diff --git a/core/src/main/kotlin/util/Redis.kt b/core/src/main/kotlin/util/Redis.kt index 1917217..05a853b 100644 --- a/core/src/main/kotlin/util/Redis.kt +++ b/core/src/main/kotlin/util/Redis.kt @@ -59,7 +59,7 @@ internal interface Redis { } internal class RedisConnection( - redisUri: String, + redisUri: String ) : Redis { private val connection: Deferred> diff --git a/core/src/test/kotlin/Utils.kt b/core/src/test/kotlin/Utils.kt index eee8bd9..e038dba 100644 --- a/core/src/test/kotlin/Utils.kt +++ b/core/src/test/kotlin/Utils.kt @@ -3,13 +3,13 @@ fun user( deviceId: String? = null, userProperties: Map? = null, groups: Map>? = null, - groupProperties: Map>>? = null, + groupProperties: Map>>? = null ): MutableMap { return mutableMapOf( "user_id" to userId, "device_id" to deviceId, "user_properties" to userProperties, "groups" to groups, - "group_properties" to groupProperties, + "group_properties" to groupProperties ) } diff --git a/core/src/test/kotlin/assignment/AssignmentServiceTest.kt b/core/src/test/kotlin/assignment/AssignmentServiceTest.kt index 2829105..422c309 100644 --- a/core/src/test/kotlin/assignment/AssignmentServiceTest.kt +++ b/core/src/test/kotlin/assignment/AssignmentServiceTest.kt @@ -27,7 +27,7 @@ class AssignmentServiceTest { metadata = mapOf( "default" to true, "version" to 1, - "segmentName" to "All Other Users", + "segmentName" to "All Other Users" ) ) ) diff --git a/service/src/main/kotlin/Server.kt b/service/src/main/kotlin/Server.kt index e408efd..6d791dd 100644 --- a/service/src/main/kotlin/Server.kt +++ b/service/src/main/kotlin/Server.kt @@ -1,6 +1,5 @@ package com.amplitude -import io.ktor.server.application.Application import com.amplitude.plugins.configureLogging import com.amplitude.plugins.configureMetrics import com.amplitude.util.json @@ -9,6 +8,7 @@ import com.amplitude.util.stringEnv import io.ktor.http.HttpStatusCode import io.ktor.http.parameters import io.ktor.serialization.kotlinx.json.json +import io.ktor.server.application.Application import io.ktor.server.application.ApplicationCall import io.ktor.server.application.call import io.ktor.server.application.createApplicationPlugin @@ -18,7 +18,6 @@ import io.ktor.server.engine.embeddedServer import io.ktor.server.netty.Netty import io.ktor.server.plugins.contentnegotiation.ContentNegotiation import io.ktor.server.request.ApplicationRequest -import io.ktor.server.request.header import io.ktor.server.request.uri import io.ktor.server.response.respond import io.ktor.server.routing.get @@ -83,7 +82,6 @@ fun main() { } fun Application.proxyServer() { - runBlocking { evaluationProxy.start() } @@ -112,7 +110,6 @@ fun Application.proxyServer() { * Configure endpoints. */ routing { - // Local Evaluation get("/sdk/v2/flags") { @@ -290,14 +287,18 @@ private fun ApplicationRequest.getUserFromQuery(): JsonObject { JsonObject(emptyMap()) } if (userId != null) { - user = JsonObject(user.toMutableMap().apply { - put("user_id", JsonPrimitive(userId)) - }) + user = JsonObject( + user.toMutableMap().apply { + put("user_id", JsonPrimitive(userId)) + } + ) } if (deviceId != null) { - user = JsonObject(user.toMutableMap().apply { - put("device_id", JsonPrimitive(userId)) - }) + user = JsonObject( + user.toMutableMap().apply { + put("device_id", JsonPrimitive(userId)) + } + ) } return user } From a77461dbd5d93978e507002a1872490c6ddc725d Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Fri, 26 Jan 2024 16:34:27 -0800 Subject: [PATCH 12/20] feat: add group cohort support --- core/build.gradle.kts | 2 +- core/src/main/kotlin/EvaluationProxy.kt | 8 +++ .../kotlin/assignment/AssignmentTracker.kt | 5 ++ core/src/main/kotlin/cohort/CohortApi.kt | 31 ++++++++--- .../main/kotlin/cohort/CohortDescription.kt | 5 +- core/src/main/kotlin/cohort/CohortStorage.kt | 51 ++++++++++++++++++ .../kotlin/deployment/DeploymentLoader.kt | 4 +- .../kotlin/deployment/DeploymentRunner.kt | 4 +- core/src/main/kotlin/project/ProjectProxy.kt | 52 ++++++++++++++++--- core/src/main/kotlin/project/ProjectRunner.kt | 4 +- .../src/main/kotlin/util/EvaluationContext.kt | 10 +++- core/src/main/kotlin/util/EvaluationFlag.kt | 46 ++++++++++++---- core/src/main/kotlin/util/Redis.kt | 2 +- service/src/main/kotlin/Server.kt | 13 +++++ 14 files changed, 203 insertions(+), 34 deletions(-) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 54ae2c3..dcd7b53 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -52,7 +52,7 @@ publishing { create("core") { groupId = "com.amplitude" artifactId = "evaluation-proxy-core" - version = "0.4.0" + version = "0.4.1" from(components["java"]) pom { name.set("Amplitude Evaluation Proxy") diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index feaec96..ff12871 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -188,6 +188,11 @@ class EvaluationProxy( return projectProxy.getCohortMembershipsForUser(deploymentKey, userId) } + suspend fun getCohortMembershipsForGroup(deploymentKey: String?, groupType: String?, groupName: String?): Set { + val projectProxy = getProjectProxy(deploymentKey) + return projectProxy.getCohortMembershipsForGroup(deploymentKey, groupType, groupName) + } + suspend fun evaluate( deploymentKey: String?, user: Map?, @@ -234,6 +239,9 @@ suspend fun EvaluationProxy.getSerializedFlagConfigs(deploymentKey: String?): St suspend fun EvaluationProxy.getSerializedCohortMembershipsForUser(deploymentKey: String?, userId: String?): String = json.encodeToString(getCohortMembershipsForUser(deploymentKey, userId)) +suspend fun EvaluationProxy.getSerializedCohortMembershipsForGroup(deploymentKey: String?, groupType: String?, groupName: String?): String = + json.encodeToString(getCohortMembershipsForGroup(deploymentKey, groupType, groupName)) + suspend fun EvaluationProxy.serializedEvaluate( deploymentKey: String?, user: Map?, diff --git a/core/src/main/kotlin/assignment/AssignmentTracker.kt b/core/src/main/kotlin/assignment/AssignmentTracker.kt index 0bf79c2..b8c39a9 100644 --- a/core/src/main/kotlin/assignment/AssignmentTracker.kt +++ b/core/src/main/kotlin/assignment/AssignmentTracker.kt @@ -9,6 +9,7 @@ import com.amplitude.AssignmentEventSendFailure import com.amplitude.Event import com.amplitude.Metrics import com.amplitude.util.deviceId +import com.amplitude.util.groups import com.amplitude.util.userId import org.json.JSONObject @@ -60,6 +61,10 @@ internal fun Assignment.toAmplitudeEvent(): Event { this.context.userId(), this.context.deviceId() ) + val groups = this.context.groups() + if (!groups.isNullOrEmpty()) { + event.groups = JSONObject(groups) + } event.eventProperties = JSONObject().apply { for ((flagKey, variant) in this@toAmplitudeEvent.results) { val version = variant.metadata?.get("version") diff --git a/core/src/main/kotlin/cohort/CohortApi.kt b/core/src/main/kotlin/cohort/CohortApi.kt index 6fc9a45..ba53eae 100644 --- a/core/src/main/kotlin/cohort/CohortApi.kt +++ b/core/src/main/kotlin/cohort/CohortApi.kt @@ -19,17 +19,19 @@ import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser +import java.lang.IllegalArgumentException import java.util.Base64 @Serializable -private data class SerialCohortDescription( +private data class SerialCohortInfoResponse( @SerialName("cohort_id") val cohortId: String, @SerialName("app_id") val appId: Int = 0, @SerialName("org_id") val orgId: Int = 0, @SerialName("name") val name: String? = null, @SerialName("size") val size: Int = Int.MAX_VALUE, @SerialName("description") val description: String? = null, - @SerialName("last_computed") val lastComputed: Long = 0 + @SerialName("last_computed") val lastComputed: Long = 0, + @SerialName("group_type") val groupType: String = USER_GROUP_TYPE, ) @Serializable @@ -66,11 +68,12 @@ internal class CohortApiV5( headers { set("Authorization", "Basic $basicAuth") } } } - val serialDescription = json.decodeFromString(response.body()) + val serialDescription = json.decodeFromString(response.body()) return CohortDescription( id = serialDescription.cohortId, lastComputed = serialDescription.lastComputed, - size = serialDescription.size + size = serialDescription.size, + groupType = serialDescription.groupType, ) } @@ -107,8 +110,24 @@ internal class CohortApiV5( headers { set("Authorization", "Basic $basicAuth") } } } + // Parse the csv response val csv = CSVParser.parse(downloadResponse.bodyAsChannel().toInputStream(), Charsets.UTF_8, csvFormat) - return csv.map { it.get("user_id") }.filterNot { it.isNullOrEmpty() }.toSet() - .also { log.debug("getCohortMembers: end - cohortId=${cohortDescription.id}, resultSize=${it.size}") } + return if (cohortDescription.groupType == USER_GROUP_TYPE) { + csv.map { it.get("user_id") }.filterNot { it.isNullOrEmpty() }.toSet() + } else { + csv.map { + try { + // CSV returned from API has all strings prefixed with a tab character + it.get("\tgroup_value") + } catch (e: IllegalArgumentException) { + it.get("group_value") + } + }.filterNot { + it.isNullOrEmpty() + }.map { + // CSV returned from API has all strings prefixed with a tab character + it.removePrefix("\t") + }.toSet() + }.also { log.debug("getCohortMembers: end - resultSize=${it.size}") } } } diff --git a/core/src/main/kotlin/cohort/CohortDescription.kt b/core/src/main/kotlin/cohort/CohortDescription.kt index df27abf..357ca26 100644 --- a/core/src/main/kotlin/cohort/CohortDescription.kt +++ b/core/src/main/kotlin/cohort/CohortDescription.kt @@ -2,9 +2,12 @@ package com.amplitude.cohort import kotlinx.serialization.Serializable +const val USER_GROUP_TYPE = "User" + @Serializable data class CohortDescription( val id: String, val lastComputed: Long, - val size: Int + val size: Int, + val groupType: String = USER_GROUP_TYPE ) diff --git a/core/src/main/kotlin/cohort/CohortStorage.kt b/core/src/main/kotlin/cohort/CohortStorage.kt index 35be2bc..0a9d8be 100644 --- a/core/src/main/kotlin/cohort/CohortStorage.kt +++ b/core/src/main/kotlin/cohort/CohortStorage.kt @@ -14,6 +14,11 @@ internal interface CohortStorage { suspend fun getCohortDescriptions(): Map suspend fun getCohortMembers(cohortDescription: CohortDescription): Set? suspend fun getCohortMembershipsForUser(userId: String, cohortIds: Set? = null): Set + suspend fun getCohortMembershipsForGroup( + groupType: String, + groupName: String, + cohortIds: Set? = null + ): Set suspend fun putCohort(description: CohortDescription, members: Set) suspend fun removeCohort(cohortDescription: CohortDescription) } @@ -73,6 +78,26 @@ internal class InMemoryCohortStorage : CohortStorage { }.toSet() } } + + override suspend fun getCohortMembershipsForGroup( + groupType: String, + groupName: String, + cohortIds: Set? + ): Set { + return lock.withLock { + (cohortIds ?: cohorts.keys).mapNotNull { id -> + val cohort = cohorts[id] + if (cohort?.description?.groupType != groupType) { + null + } else { + when (cohort.members.contains(groupName)) { + true -> id + else -> null + } + } + }.toSet() + } + } } internal class RedisCohortStorage( @@ -101,6 +126,9 @@ internal class RedisCohortStorage( val descriptions = getCohortDescriptions() val memberships = mutableSetOf() for (description in descriptions.values) { + if (cohortIds != null && !cohortIds.contains(description.id)) { + continue + } // High volume, use read connection val isMember = readOnlyRedis.sismember(RedisKey.CohortMembers(prefix, projectId, description), userId) if (isMember) { @@ -110,6 +138,29 @@ internal class RedisCohortStorage( return memberships } + override suspend fun getCohortMembershipsForGroup( + groupType: String, + groupName: String, + cohortIds: Set? + ): Set { + val descriptions = getCohortDescriptions() + val memberships = mutableSetOf() + for (description in descriptions.values) { + if (cohortIds != null && !cohortIds.contains(description.id)) { + continue + } + if (description.groupType != groupType) { + continue + } + // High volume, use read connection + val isMember = readOnlyRedis.sismember(RedisKey.CohortMembers(prefix, projectId, description), groupName) + if (isMember) { + memberships += description.id + } + } + return memberships + } + override suspend fun putCohort(description: CohortDescription, members: Set) { val jsonEncodedDescription = json.encodeToString(description) val existingDescription = getCohortDescription(description.id) diff --git a/core/src/main/kotlin/deployment/DeploymentLoader.kt b/core/src/main/kotlin/deployment/DeploymentLoader.kt index 49a4265..df4348b 100644 --- a/core/src/main/kotlin/deployment/DeploymentLoader.kt +++ b/core/src/main/kotlin/deployment/DeploymentLoader.kt @@ -4,7 +4,7 @@ import com.amplitude.FlagsFetch import com.amplitude.FlagsFetchFailure import com.amplitude.Metrics import com.amplitude.cohort.CohortLoader -import com.amplitude.util.getCohortIds +import com.amplitude.util.getAllCohortIds import com.amplitude.util.logger import kotlinx.coroutines.Job import kotlinx.coroutines.coroutineScope @@ -43,7 +43,7 @@ internal class DeploymentLoader( // Load cohorts for each flag independently then put the // flag into storage. for (flag in networkFlags) { - val cohortIds = flag.getCohortIds() + val cohortIds = flag.getAllCohortIds() if (cohortIds.isNotEmpty()) { launch { cohortLoader.loadCohorts(cohortIds) diff --git a/core/src/main/kotlin/deployment/DeploymentRunner.kt b/core/src/main/kotlin/deployment/DeploymentRunner.kt index b76e9d6..1a5d751 100644 --- a/core/src/main/kotlin/deployment/DeploymentRunner.kt +++ b/core/src/main/kotlin/deployment/DeploymentRunner.kt @@ -2,7 +2,7 @@ package com.amplitude.deployment import com.amplitude.Configuration import com.amplitude.cohort.CohortLoader -import com.amplitude.util.getCohortIds +import com.amplitude.util.getAllCohortIds import com.amplitude.util.logger import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.SupervisorJob @@ -40,7 +40,7 @@ internal class DeploymentRunner( scope.launch { while (true) { delay(configuration.cohortSyncIntervalMillis) - val cohortIds = deploymentStorage.getAllFlags(deploymentKey).values.getCohortIds() + val cohortIds = deploymentStorage.getAllFlags(deploymentKey).values.getAllCohortIds() cohortLoader.loadCohorts(cohortIds) } } diff --git a/core/src/main/kotlin/project/ProjectProxy.kt b/core/src/main/kotlin/project/ProjectProxy.kt index 6ab6399..d4769a1 100644 --- a/core/src/main/kotlin/project/ProjectProxy.kt +++ b/core/src/main/kotlin/project/ProjectProxy.kt @@ -7,13 +7,14 @@ import com.amplitude.assignment.AssignmentTracker import com.amplitude.cohort.CohortApiV5 import com.amplitude.cohort.CohortDescription import com.amplitude.cohort.CohortStorage +import com.amplitude.cohort.USER_GROUP_TYPE import com.amplitude.deployment.DeploymentApiV1 import com.amplitude.deployment.DeploymentStorage import com.amplitude.experiment.evaluation.EvaluationEngineImpl import com.amplitude.experiment.evaluation.EvaluationFlag import com.amplitude.experiment.evaluation.EvaluationVariant import com.amplitude.experiment.evaluation.topologicalSort -import com.amplitude.util.getCohortIds +import com.amplitude.util.getGroupedCohortIds import com.amplitude.util.logger import com.amplitude.util.toEvaluationContext import kotlinx.coroutines.coroutineScope @@ -87,10 +88,31 @@ internal class ProjectProxy( if (userId.isNullOrEmpty()) { throw HttpErrorResponseException(status = 400, message = "Invalid user ID.") } - val cohortIds = deploymentStorage.getAllFlags(deploymentKey).values.getCohortIds() + val cohortIds = deploymentStorage.getAllFlags(deploymentKey).values.getGroupedCohortIds()[USER_GROUP_TYPE] + if (cohortIds.isNullOrEmpty()) { + return setOf() + } return cohortStorage.getCohortMembershipsForUser(userId, cohortIds) } + suspend fun getCohortMembershipsForGroup(deploymentKey: String?, groupType: String?, groupName: String?): Set { + if (deploymentKey.isNullOrEmpty()) { + throw HttpErrorResponseException(status = 401, message = "Invalid deployment.") + } + if (groupType.isNullOrEmpty()) { + throw HttpErrorResponseException(status = 400, message = "Invalid group type.") + } + if (groupName.isNullOrEmpty()) { + throw HttpErrorResponseException(status = 400, message = "Invalid group name.") + } + val cohortIds = deploymentStorage.getAllFlags(deploymentKey).values.getGroupedCohortIds()[groupType] + if (cohortIds.isNullOrEmpty()) { + return setOf() + } + return cohortStorage.getCohortMembershipsForGroup(groupType, groupName, cohortIds) + } + + suspend fun evaluate( deploymentKey: String?, user: Map?, @@ -110,18 +132,32 @@ internal class ProjectProxy( } // Enrich user with cohort IDs and build the evaluation context val userId = user?.get("user_id") as? String - val enrichedUser = if (userId != null) { - user.toMutableMap().apply { - put("cohort_ids", cohortStorage.getCohortMembershipsForUser(userId)) + val enrichedUser = user?.toMutableMap() ?: mutableMapOf() + if (userId != null) { + enrichedUser["cohort_ids"] = cohortStorage.getCohortMembershipsForUser(userId) + } + val groups = enrichedUser["groups"] as? Map<*, *> + if (!groups.isNullOrEmpty()) { + val groupCohortIds = mutableMapOf>>() + for (entry in groups.entries) { + val groupType = entry.key as? String + val groupName = (entry.value as? Collection<*>)?.firstOrNull() as? String + if (groupType != null && groupName != null) { + val cohortIds = cohortStorage.getCohortMembershipsForGroup(groupType, groupName) + if (groupCohortIds.isNotEmpty()) { + groupCohortIds.putIfAbsent(groupType, mutableMapOf(groupName to cohortIds)) + } + } + } + if (groupCohortIds.isNotEmpty()) { + enrichedUser["group_cohort_ids"] = groupCohortIds } - } else { - null } val evaluationContext = enrichedUser.toEvaluationContext() // Evaluate results log.debug("evaluate - context={}", evaluationContext) val result = engine.evaluate(evaluationContext, flags) - if (enrichedUser != null) { + if (enrichedUser.isNotEmpty()) { coroutineScope { launch { assignmentTracker.track(Assignment(evaluationContext, result)) diff --git a/core/src/main/kotlin/project/ProjectRunner.kt b/core/src/main/kotlin/project/ProjectRunner.kt index 3d1e7e9..b4aa0f0 100644 --- a/core/src/main/kotlin/project/ProjectRunner.kt +++ b/core/src/main/kotlin/project/ProjectRunner.kt @@ -8,7 +8,7 @@ import com.amplitude.deployment.DeploymentApi import com.amplitude.deployment.DeploymentRunner import com.amplitude.deployment.DeploymentStorage import com.amplitude.experiment.evaluation.EvaluationFlag -import com.amplitude.util.getCohortIds +import com.amplitude.util.getAllCohortIds import com.amplitude.util.logger import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job @@ -123,7 +123,7 @@ internal class ProjectRunner( for (deploymentKey in deploymentKeys) { allFlagConfigs += deploymentStorage.getAllFlags(deploymentKey).values } - val allTargetedCohortIds = allFlagConfigs.getCohortIds() + val allTargetedCohortIds = allFlagConfigs.getAllCohortIds() val allStoredCohortDescriptions = cohortStorage.getCohortDescriptions().values for (cohortDescription in allStoredCohortDescriptions) { if (!allTargetedCohortIds.contains(cohortDescription.id)) { diff --git a/core/src/main/kotlin/util/EvaluationContext.kt b/core/src/main/kotlin/util/EvaluationContext.kt index 6daf48a..80cc69e 100644 --- a/core/src/main/kotlin/util/EvaluationContext.kt +++ b/core/src/main/kotlin/util/EvaluationContext.kt @@ -9,9 +9,13 @@ internal fun EvaluationContext.deviceId(): String? { return (this["user"] as? Map<*, *>)?.get("device_id") as? String } +internal fun EvaluationContext.groups(): Map<*, *>? { + return this["groups"] as? Map<*, *> +} + internal fun MutableMap?.toEvaluationContext(): EvaluationContext { val context = EvaluationContext() - if (this == null) { + if (this.isNullOrEmpty()) { return context } val groups = mutableMapOf>() @@ -27,6 +31,10 @@ internal fun MutableMap?.toEvaluationContext(): EvaluationContext if (!groupProperties.isNullOrEmpty()) { groupNameMap["group_properties"] = groupProperties } + val groupCohortIds = this.select("group_cohort_ids", groupType, groupName) as? Map<*, *> + if (!groupCohortIds.isNullOrEmpty()) { + groupNameMap["cohort_ids"] = groupCohortIds + } groups[groupType] = groupNameMap } } diff --git a/core/src/main/kotlin/util/EvaluationFlag.kt b/core/src/main/kotlin/util/EvaluationFlag.kt index 6552785..1faf301 100644 --- a/core/src/main/kotlin/util/EvaluationFlag.kt +++ b/core/src/main/kotlin/util/EvaluationFlag.kt @@ -1,35 +1,60 @@ package com.amplitude.util +import com.amplitude.cohort.USER_GROUP_TYPE import com.amplitude.experiment.evaluation.EvaluationCondition import com.amplitude.experiment.evaluation.EvaluationFlag import com.amplitude.experiment.evaluation.EvaluationOperator import com.amplitude.experiment.evaluation.EvaluationSegment -internal fun Collection.getCohortIds(): Set { - val cohortIds = mutableSetOf() +internal fun Collection.getAllCohortIds(): Set { + return getGroupedCohortIds().flatMap { it.value }.toSet() +} + +internal fun Collection.getGroupedCohortIds(): Map> { + val cohortIds = mutableMapOf>() for (flag in this) { - cohortIds += flag.getCohortIds() + for (entry in flag.getGroupedCohortIds()) { + cohortIds.getOrPut(entry.key) { mutableSetOf() } += entry.value + } } return cohortIds } -internal fun EvaluationFlag.getCohortIds(): Set { - val cohortIds = mutableSetOf() +internal fun EvaluationFlag.getAllCohortIds(): Set { + return getGroupedCohortIds().flatMap { it.value }.toSet() +} + +internal fun EvaluationFlag.getGroupedCohortIds(): Map> { + val cohortIds = mutableMapOf>() for (segment in this.segments) { - cohortIds += segment.getCohortConditionIds() + for (entry in segment.getGroupedCohortConditionIds()) { + cohortIds.getOrPut(entry.key) { mutableSetOf() } += entry.value + } } return cohortIds } -private fun EvaluationSegment.getCohortConditionIds(): Set { - val cohortIds = mutableSetOf() +private fun EvaluationSegment.getGroupedCohortConditionIds(): Map> { + val cohortIds = mutableMapOf>() if (conditions == null) { return cohortIds } for (outer in conditions!!) { for (condition in outer) { if (condition.isCohortFilter()) { - cohortIds += condition.values + // User cohort selector is [context, user, cohort_ids] + // Groups cohort selector is [context, groups, {group_type}, cohort_ids] + if (condition.selector.size > 2) { + val contextSubtype = condition.selector[1] + val groupType = if (contextSubtype == "user") { + USER_GROUP_TYPE + } else if (condition.selector.contains("groups")) { + condition.selector[2] + } else { + continue + } + cohortIds.getOrPut(groupType) { mutableSetOf() } += condition.values + } } } } @@ -38,4 +63,5 @@ private fun EvaluationSegment.getCohortConditionIds(): Set { // Only cohort filters use these operators. private fun EvaluationCondition.isCohortFilter(): Boolean = - this.op == EvaluationOperator.SET_CONTAINS_ANY || this.op == EvaluationOperator.SET_DOES_NOT_CONTAIN_ANY + (this.op == EvaluationOperator.SET_CONTAINS_ANY || this.op == EvaluationOperator.SET_DOES_NOT_CONTAIN_ANY) && + this.selector.isNotEmpty() && this.selector.last() == "cohort_ids" diff --git a/core/src/main/kotlin/util/Redis.kt b/core/src/main/kotlin/util/Redis.kt index 05a853b..2cad423 100644 --- a/core/src/main/kotlin/util/Redis.kt +++ b/core/src/main/kotlin/util/Redis.kt @@ -40,7 +40,7 @@ internal sealed class RedisKey(val value: String) { val prefix: String, val projectId: String, val cohortDescription: CohortDescription - ) : RedisKey("$prefix:$STORAGE_PROTOCOL_VERSION:projects:$projectId:cohorts:${cohortDescription.id}:users:${cohortDescription.lastComputed}") + ) : RedisKey("$prefix:$STORAGE_PROTOCOL_VERSION:projects:$projectId:cohorts:${cohortDescription.id}:${cohortDescription.groupType}:${cohortDescription.lastComputed}") } internal interface Redis { diff --git a/service/src/main/kotlin/Server.kt b/service/src/main/kotlin/Server.kt index 6d791dd..b6c6155 100644 --- a/service/src/main/kotlin/Server.kt +++ b/service/src/main/kotlin/Server.kt @@ -159,6 +159,19 @@ fun Application.proxyServer() { call.respond(result) } + get("/sdk/v2/groups/{groupType}/{groupName}/cohorts") { + val deployment = this.call.request.getDeploymentKey() + val groupType = this.call.parameters["groupType"] + val groupName = this.call.parameters["groupName"] + val result = try { + evaluationProxy.getSerializedCohortMembershipsForGroup(deployment, groupType, groupName) + } catch (e: HttpErrorResponseException) { + call.respond(HttpStatusCode.fromValue(e.status), e.message) + return@get + } + call.respond(result) + } + // Remote Evaluation V2 Endpoints get("/sdk/v2/vardata") { From 1840d02214dd7ec3a13184855aaeb0afea50a55a Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Fri, 23 Feb 2024 12:30:29 -0800 Subject: [PATCH 13/20] release: 0.4.2 --- core/build.gradle.kts | 2 +- core/src/main/kotlin/util/Json.kt | 4 ---- core/src/main/kotlin/util/Yaml.kt | 6 ++++++ 3 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 core/src/main/kotlin/util/Yaml.kt diff --git a/core/build.gradle.kts b/core/build.gradle.kts index dcd7b53..eac1ea6 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -52,7 +52,7 @@ publishing { create("core") { groupId = "com.amplitude" artifactId = "evaluation-proxy-core" - version = "0.4.1" + version = "0.4.2" from(components["java"]) pom { name.set("Amplitude Evaluation Proxy") diff --git a/core/src/main/kotlin/util/Json.kt b/core/src/main/kotlin/util/Json.kt index 65cd77f..6bf68f5 100644 --- a/core/src/main/kotlin/util/Json.kt +++ b/core/src/main/kotlin/util/Json.kt @@ -1,11 +1,7 @@ package com.amplitude.util -import com.charleskorn.kaml.Yaml -import com.charleskorn.kaml.YamlConfiguration import kotlinx.serialization.json.Json val json = Json { ignoreUnknownKeys = true } - -val yaml = Yaml(configuration = YamlConfiguration(strictMode = false)) diff --git a/core/src/main/kotlin/util/Yaml.kt b/core/src/main/kotlin/util/Yaml.kt new file mode 100644 index 0000000..c080851 --- /dev/null +++ b/core/src/main/kotlin/util/Yaml.kt @@ -0,0 +1,6 @@ +package com.amplitude.util + +import com.charleskorn.kaml.Yaml +import com.charleskorn.kaml.YamlConfiguration + +val yaml = Yaml(configuration = YamlConfiguration(strictMode = false)) From 2670c7004201b54035bae6410d15c91b71165e6e Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Tue, 27 Feb 2024 10:27:12 -0800 Subject: [PATCH 14/20] release: 0.4.3 --- core/build.gradle.kts | 2 +- core/src/main/kotlin/EvaluationProxy.kt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index eac1ea6..8a8f6a4 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -52,7 +52,7 @@ publishing { create("core") { groupId = "com.amplitude" artifactId = "evaluation-proxy-core" - version = "0.4.2" + version = "0.4.3" from(components["java"]) pom { name.set("Amplitude Evaluation Proxy") diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index ff12871..53240a9 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -147,6 +147,10 @@ class EvaluationProxy( /* * Periodically update the local cache of deployments to project values. */ + for ((project, projectProxy) in projectProxies) { + val deployments = projectProxy.getDeployments().associateWith { project } + mutex.withLock { deploymentKeysToProject.putAll(deployments) } + } scope.launch { while (true) { delay(configuration.deploymentSyncIntervalMillis) From 68ca4c36e55b8bb720cd0e9f93f585bc3bd85636 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 28 Feb 2024 15:02:10 -0800 Subject: [PATCH 15/20] release: 0.4.4 --- core/build.gradle.kts | 2 +- core/src/main/kotlin/EvaluationProxy.kt | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 8a8f6a4..f9c043f 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -52,7 +52,7 @@ publishing { create("core") { groupId = "com.amplitude" artifactId = "evaluation-proxy-core" - version = "0.4.3" + version = "0.4.4" from(components["java"]) pom { name.set("Amplitude Evaluation Proxy") diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index 53240a9..8083705 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -83,6 +83,7 @@ class EvaluationProxy( apiKeysToProject[project.apiKey] = project secretKeysToProject[project.secretKey] = project for (deployment in deployments) { + log.debug("Mapping deployment {} project {}", deployment.key, project.id) deploymentKeysToProject[deployment.key] = project } @@ -147,10 +148,6 @@ class EvaluationProxy( /* * Periodically update the local cache of deployments to project values. */ - for ((project, projectProxy) in projectProxies) { - val deployments = projectProxy.getDeployments().associateWith { project } - mutex.withLock { deploymentKeysToProject.putAll(deployments) } - } scope.launch { while (true) { delay(configuration.deploymentSyncIntervalMillis) @@ -224,7 +221,11 @@ class EvaluationProxy( private suspend fun getProjectProxy(deploymentKey: String?): ProjectProxy { val cachedProject = mutex.withLock { deploymentKeysToProject[deploymentKey] - } ?: throw HttpErrorResponseException(401, "Invalid deployment key.") + } + if (cachedProject == null) { + log.debug("Unable to find project for deployment {}. Current mappings: {}", deploymentKey, deploymentKeysToProject.mapValues { it.value.id }) + throw HttpErrorResponseException(401, "Invalid deployment key.") + } return projectProxies[cachedProject] ?: throw HttpErrorResponseException(404, "Project not found.") } } From 984e01b30eb0c291113230d8200940370e345a64 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 28 Feb 2024 17:16:07 -0800 Subject: [PATCH 16/20] fix: redis key --- core/src/main/kotlin/deployment/DeploymentStorage.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/kotlin/deployment/DeploymentStorage.kt b/core/src/main/kotlin/deployment/DeploymentStorage.kt index 9c09346..6164fcb 100644 --- a/core/src/main/kotlin/deployment/DeploymentStorage.kt +++ b/core/src/main/kotlin/deployment/DeploymentStorage.kt @@ -34,7 +34,7 @@ internal fun getDeploymentStorage(projectId: String, redisConfiguration: RedisCo } else { redis } - RedisDeploymentStorage(projectId, redisConfiguration.prefix, redis, readOnlyRedis) + RedisDeploymentStorage(redisConfiguration.prefix, projectId, redis, readOnlyRedis) } } From d50d6a4690a0ced098b56490d4daeff2a30ebb2f Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 28 Feb 2024 19:24:52 -0800 Subject: [PATCH 17/20] fix: fix assignment tracking and handle exception without failing req --- .../kotlin/assignment/AssignmentTracker.kt | 21 +++++++++++++------ .../src/main/kotlin/util/EvaluationContext.kt | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/core/src/main/kotlin/assignment/AssignmentTracker.kt b/core/src/main/kotlin/assignment/AssignmentTracker.kt index b8c39a9..18a8cee 100644 --- a/core/src/main/kotlin/assignment/AssignmentTracker.kt +++ b/core/src/main/kotlin/assignment/AssignmentTracker.kt @@ -10,6 +10,7 @@ import com.amplitude.Event import com.amplitude.Metrics import com.amplitude.util.deviceId import com.amplitude.util.groups +import com.amplitude.util.logger import com.amplitude.util.userId import org.json.JSONObject @@ -30,6 +31,10 @@ internal class AmplitudeAssignmentTracker( private val assignmentFilter: AssignmentFilter ) : AssignmentTracker { + companion object { + val log by logger() + } + constructor( apiKey: String, config: AssignmentConfiguration @@ -44,13 +49,17 @@ internal class AmplitudeAssignmentTracker( ) override suspend fun track(assignment: Assignment) { - Metrics.track(AssignmentEvent) - if (assignmentFilter.shouldTrack(assignment)) { - Metrics.with({ AssignmentEventSend }, { e -> AssignmentEventSendFailure(e) }) { - amplitude.logEvent(assignment.toAmplitudeEvent()) + try { + Metrics.track(AssignmentEvent) + if (assignmentFilter.shouldTrack(assignment)) { + Metrics.with({ AssignmentEventSend }, { e -> AssignmentEventSendFailure(e) }) { + amplitude.logEvent(assignment.toAmplitudeEvent()) + } + } else { + Metrics.track(AssignmentEventFilter) } - } else { - Metrics.track(AssignmentEventFilter) + } catch (e: Exception) { + log.error("Failed to track assignment event", e) } } } diff --git a/core/src/main/kotlin/util/EvaluationContext.kt b/core/src/main/kotlin/util/EvaluationContext.kt index 80cc69e..e13e88f 100644 --- a/core/src/main/kotlin/util/EvaluationContext.kt +++ b/core/src/main/kotlin/util/EvaluationContext.kt @@ -3,10 +3,10 @@ package com.amplitude.util import com.amplitude.experiment.evaluation.EvaluationContext internal fun EvaluationContext.userId(): String? { - return (this["user"] as? Map<*, *>)?.get("user_id") as? String + return (this["user"] as? Map<*, *>)?.get("user_id")?.toString() } internal fun EvaluationContext.deviceId(): String? { - return (this["user"] as? Map<*, *>)?.get("device_id") as? String + return (this["user"] as? Map<*, *>)?.get("device_id")?.toString() } internal fun EvaluationContext.groups(): Map<*, *>? { From 2ec9452917269c6258636dbad3151dd402760633 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 28 Feb 2024 20:06:25 -0800 Subject: [PATCH 18/20] fix: remote evaluation user parsing --- core/src/main/kotlin/util/Json.kt | 73 +++++++++++++++++++++++++++++++ service/src/main/kotlin/Server.kt | 11 +++-- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/util/Json.kt b/core/src/main/kotlin/util/Json.kt index 6bf68f5..b10159d 100644 --- a/core/src/main/kotlin/util/Json.kt +++ b/core/src/main/kotlin/util/Json.kt @@ -1,7 +1,80 @@ package com.amplitude.util +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.doubleOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.longOrNull val json = Json { ignoreUnknownKeys = true + isLenient = true + coerceInputValues = true + explicitNulls = false } + +object AnySerializer : KSerializer { + private val delegate = JsonElement.serializer() + override val descriptor: SerialDescriptor + get() = SerialDescriptor("Any", delegate.descriptor) + + override fun serialize(encoder: Encoder, value: Any?) { + val jsonElement = value.toJsonElement() + encoder.encodeSerializableValue(delegate, jsonElement) + } + + override fun deserialize(decoder: Decoder): Any? { + val jsonElement = decoder.decodeSerializableValue(delegate) + return jsonElement.toAny() + } +} + +fun Any?.toJsonElement(): JsonElement = when (this) { + null -> JsonNull + is Map<*, *> -> toJsonObject() + is Collection<*> -> toJsonArray() + is Boolean -> JsonPrimitive(this) + is Number -> JsonPrimitive(this) + is String -> JsonPrimitive(this) + else -> JsonPrimitive(toString()) +} + +fun Collection<*>.toJsonArray(): JsonArray = JsonArray(map { it.toJsonElement() }) + +fun Map<*, *>.toJsonObject(): JsonObject = JsonObject( + mapNotNull { + (it.key as? String ?: return@mapNotNull null) to it.value.toJsonElement() + }.toMap(), +) + +fun JsonElement.toAny(): Any? { + return when (this) { + is JsonPrimitive -> toAny() + is JsonArray -> toList() + is JsonObject -> toMap() + } +} + +fun JsonPrimitive.toAny(): Any? { + return if (isString) { + contentOrNull + } else { + booleanOrNull ?: intOrNull ?: longOrNull ?: doubleOrNull + } +} + +fun JsonArray.toList(): List = map { it.toAny() } + +fun JsonObject.toMap(): Map = mapValues { it.value.toAny() } + + diff --git a/service/src/main/kotlin/Server.kt b/service/src/main/kotlin/Server.kt index b6c6155..bfc88a9 100644 --- a/service/src/main/kotlin/Server.kt +++ b/service/src/main/kotlin/Server.kt @@ -1,12 +1,14 @@ +@file:UseSerializers(AnySerializer::class) + package com.amplitude import com.amplitude.plugins.configureLogging import com.amplitude.plugins.configureMetrics +import com.amplitude.util.AnySerializer import com.amplitude.util.json import com.amplitude.util.logger import com.amplitude.util.stringEnv import io.ktor.http.HttpStatusCode -import io.ktor.http.parameters import io.ktor.serialization.kotlinx.json.json import io.ktor.server.application.Application import io.ktor.server.application.ApplicationCall @@ -25,6 +27,9 @@ import io.ktor.server.routing.post import io.ktor.server.routing.routing import io.ktor.util.toByteArray import kotlinx.coroutines.runBlocking +import kotlinx.serialization.Serializable +import kotlinx.serialization.UseSerializers +import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import java.io.FileNotFoundException @@ -276,7 +281,7 @@ private fun ApplicationRequest.getFlagKeys(): Set { private fun ApplicationRequest.getUserFromHeader(): Map { val b64User = this.headers["X-Amp-Exp-User"] val userJson = Base64.getDecoder().decode(b64User).toString(Charsets.UTF_8) - return json.decodeFromString(userJson) + return json.decodeFromString(userJson).toMap() } /** @@ -284,7 +289,7 @@ private fun ApplicationRequest.getUserFromHeader(): Map { */ private suspend fun ApplicationRequest.getUserFromBody(): Map { val userJson = this.receiveChannel().toByteArray().toString(Charsets.UTF_8) - return json.decodeFromString(userJson) + return json.decodeFromString(userJson).toMap() } /** From f29f09c00982a94e16f69822ddb31142e16f0529 Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 28 Feb 2024 20:17:49 -0800 Subject: [PATCH 19/20] update version in code --- core/src/main/kotlin/EvaluationProxy.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/kotlin/EvaluationProxy.kt b/core/src/main/kotlin/EvaluationProxy.kt index 8083705..f1080aa 100644 --- a/core/src/main/kotlin/EvaluationProxy.kt +++ b/core/src/main/kotlin/EvaluationProxy.kt @@ -25,7 +25,7 @@ import kotlinx.serialization.encodeToString import kotlin.time.DurationUnit import kotlin.time.toDuration -const val VERSION = "0.3.2" +const val VERSION = "0.4.4" class HttpErrorResponseException( val status: Int, From 557f04e09c7fedb53b702839f688612252d12d2c Mon Sep 17 00:00:00 2001 From: Brian Giori Date: Wed, 28 Feb 2024 20:21:18 -0800 Subject: [PATCH 20/20] formatting --- core/src/main/kotlin/cohort/CohortApi.kt | 4 ++-- core/src/main/kotlin/project/ProjectProxy.kt | 1 - core/src/main/kotlin/util/Json.kt | 4 +--- service/src/main/kotlin/Server.kt | 2 -- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/core/src/main/kotlin/cohort/CohortApi.kt b/core/src/main/kotlin/cohort/CohortApi.kt index ba53eae..841f9eb 100644 --- a/core/src/main/kotlin/cohort/CohortApi.kt +++ b/core/src/main/kotlin/cohort/CohortApi.kt @@ -31,7 +31,7 @@ private data class SerialCohortInfoResponse( @SerialName("size") val size: Int = Int.MAX_VALUE, @SerialName("description") val description: String? = null, @SerialName("last_computed") val lastComputed: Long = 0, - @SerialName("group_type") val groupType: String = USER_GROUP_TYPE, + @SerialName("group_type") val groupType: String = USER_GROUP_TYPE ) @Serializable @@ -73,7 +73,7 @@ internal class CohortApiV5( id = serialDescription.cohortId, lastComputed = serialDescription.lastComputed, size = serialDescription.size, - groupType = serialDescription.groupType, + groupType = serialDescription.groupType ) } diff --git a/core/src/main/kotlin/project/ProjectProxy.kt b/core/src/main/kotlin/project/ProjectProxy.kt index d4769a1..fa047ce 100644 --- a/core/src/main/kotlin/project/ProjectProxy.kt +++ b/core/src/main/kotlin/project/ProjectProxy.kt @@ -112,7 +112,6 @@ internal class ProjectProxy( return cohortStorage.getCohortMembershipsForGroup(groupType, groupName, cohortIds) } - suspend fun evaluate( deploymentKey: String?, user: Map?, diff --git a/core/src/main/kotlin/util/Json.kt b/core/src/main/kotlin/util/Json.kt index b10159d..c9d9d70 100644 --- a/core/src/main/kotlin/util/Json.kt +++ b/core/src/main/kotlin/util/Json.kt @@ -54,7 +54,7 @@ fun Collection<*>.toJsonArray(): JsonArray = JsonArray(map { it.toJsonElement() fun Map<*, *>.toJsonObject(): JsonObject = JsonObject( mapNotNull { (it.key as? String ?: return@mapNotNull null) to it.value.toJsonElement() - }.toMap(), + }.toMap() ) fun JsonElement.toAny(): Any? { @@ -76,5 +76,3 @@ fun JsonPrimitive.toAny(): Any? { fun JsonArray.toList(): List = map { it.toAny() } fun JsonObject.toMap(): Map = mapValues { it.value.toAny() } - - diff --git a/service/src/main/kotlin/Server.kt b/service/src/main/kotlin/Server.kt index bfc88a9..93ab5e0 100644 --- a/service/src/main/kotlin/Server.kt +++ b/service/src/main/kotlin/Server.kt @@ -27,9 +27,7 @@ import io.ktor.server.routing.post import io.ktor.server.routing.routing import io.ktor.util.toByteArray import kotlinx.coroutines.runBlocking -import kotlinx.serialization.Serializable import kotlinx.serialization.UseSerializers -import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import java.io.FileNotFoundException