diff --git a/gradle.properties b/gradle.properties index 4099085..469ac85 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ group = dev.dialector # Modify this for a custom version for local publishing (will not apply to CI publishing builds) -version = +version = inkt kotlinVersion = 1.8.10 kspVersion = 1.8.10-1.0.9 diff --git a/inkt/README.md b/inkt/README.md new file mode 100644 index 0000000..57dc5c8 --- /dev/null +++ b/inkt/README.md @@ -0,0 +1,13 @@ +# inkt + +An incremental query framework for Kotlin + +Heavily inspired by [Salsa](https://github.com/salsa-rs/salsa) + +## Quick Start + +See [DefinedQueryExample.kt](src/main/kotlin/dev/dialector/inkt/query/example/DefinedQueryExample.kt) for an example of how to define queries and use the database. +A KSP-based code generator will eventually replace manual definition of query database implementations. + + + diff --git a/inkt/build.gradle.kts b/inkt/build.gradle.kts new file mode 100644 index 0000000..ab6a541 --- /dev/null +++ b/inkt/build.gradle.kts @@ -0,0 +1,95 @@ +plugins { + kotlin("jvm") + id("org.jetbrains.kotlinx.kover") + id("maven-publish") + signing +} + +dependencies { + implementation(kotlin("reflect")) +} + +kotlin { + explicitApiWarning() + jvmToolchain { + languageVersion.set(JavaLanguageVersion.of(8)) + } +} + +java { + withJavadocJar() + withSourcesJar() +} + +tasks.withType { + useJUnitPlatform() +} + +kover { + xmlReport { + onCheck.set(true) + } +} + +publishing { + repositories { + maven { + name = "OSSRH" + setUrl("https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/") + credentials { + username = System.getenv("SONATYPE_USERNAME") + password = System.getenv("SONATYPE_PASSWORD") + } + } + maven { + name = "GitHubPackages" + setUrl("https://maven.pkg.github.com/ty1824/dialector") + credentials { + username = System.getenv("GITHUB_ACTOR") + password = System.getenv("GITHUB_TOKEN") + } + } + } + repositories.forEach { println((it as MavenArtifactRepository).url)} + publications { + register("default") { + from(components["java"]) + pom { + name.set("inkt") + description.set("Incremental computation framework for Kotlin") + url.set("http://dialector.dev") + licenses { + license { + name.set("GPL-3.0") + url.set("https://opensource.org/licenses/GPL-3.0") + } + } + issueManagement { + system.set("Github") + url.set("https://github.com/ty1824/dialector/issues") + } + scm { + connection.set("https://github.com/ty1824/dialector.git") + url.set("https://github.com/ty1824/dialector") + } + developers { + developer { + name.set("Tyler Hodgkins") + email.set("ty1824@gmail.com") + } + } + } + } + } +} + +signing { + val gpgPrivateKey = System.getenv("GPG_SIGNING_KEY") + if (!gpgPrivateKey.isNullOrBlank()) { + useInMemoryPgpKeys( + gpgPrivateKey, + System.getenv("GPG_SIGNING_PASSPHRASE") + ) + sign(publishing.publications) + } +} \ No newline at end of file diff --git a/inkt/src/main/kotlin/dev/dialector/inkt/QueryData.kt b/inkt/src/main/kotlin/dev/dialector/inkt/QueryData.kt new file mode 100644 index 0000000..a96ba32 --- /dev/null +++ b/inkt/src/main/kotlin/dev/dialector/inkt/QueryData.kt @@ -0,0 +1,23 @@ +package dev.dialector.inkt + +internal data class QueryKey(val queryDef: DatabaseQuery, val key: K) + +internal sealed interface Value { + var value: V + var changedAt: Int +} + +internal data class InputValue(override var value: V, override var changedAt: Int) : Value + +internal data class DerivedValue( + override var value: V, + val dependencies: MutableList>, + var verifiedAt: Int, + override var changedAt: Int +) : Value + +internal class QueryFrame( + val queryKey: QueryKey, + var maxRevision: Int = 0, + val dependencies: MutableList> = mutableListOf() +) diff --git a/inkt/src/main/kotlin/dev/dialector/inkt/QueryDatabase.kt b/inkt/src/main/kotlin/dev/dialector/inkt/QueryDatabase.kt new file mode 100644 index 0000000..41bf99a --- /dev/null +++ b/inkt/src/main/kotlin/dev/dialector/inkt/QueryDatabase.kt @@ -0,0 +1,122 @@ +package dev.dialector.inkt + +public class QueryDatabase(public val definitions: List>) { + private val storage: Map, MutableMap<*, *>> = definitions.associateWith { it.createMap() } + + private var currentRevision = 0 + + private val currentlyActiveQuery: MutableList> = mutableListOf() + + @Suppress("UNCHECKED_CAST") + private fun getQueryStorage(query: DatabaseQuery): MutableMap> = + storage[query] as MutableMap> + + private fun get(queryKey: QueryKey): Value? = getQueryStorage(queryKey.queryDef)[queryKey.key] + + public fun setInput(inputDef: InputQuery, key: K, value: V) { + val inputStorage = getQueryStorage(inputDef) + val inputValue = inputStorage[key] + if (inputValue == null) { + inputStorage[key] = InputValue(value, ++currentRevision) + } else { + inputValue.value = value + inputValue.changedAt = ++currentRevision + } + } + + public fun inputQuery(queryDef: DatabaseQuery, key: K): V { + val current = QueryKey(queryDef, key) + recordQuery(current) + return get(current)?.let { + trackRevision(it.changedAt) + it.value + } ?: throw RuntimeException("No value when running query $queryDef for input $key") + } + + public fun derivedQuery(queryDef: DatabaseQuery, key: K): V { + val current = QueryKey(queryDef, key) + val derivedStorage = getQueryStorage(queryDef) + if (currentlyActiveQuery.any { it.queryKey == current }) { + throw RuntimeException("Cycle detected: $current already in $currentlyActiveQuery") + } + recordQuery(current) + val existingValue = derivedStorage[key] + return if (existingValue is DerivedValue && verify(existingValue, existingValue.verifiedAt)) { + existingValue.value + } else { + currentlyActiveQuery += QueryFrame(current) + try { + val result = queryDef.get(key) + val frame = currentlyActiveQuery.last() + + derivedStorage[key] = DerivedValue(result, frame.dependencies.toMutableList(), currentRevision, frame.maxRevision) + result + } finally { + val frame = currentlyActiveQuery.removeLast() + recordDependencies(frame.dependencies) + trackRevision(frame.maxRevision) + } + } + } + + private fun verify(value: Value<*>, asOfRevision: Int): Boolean { + return when (value) { + is InputValue<*> -> value.changedAt <= asOfRevision + is DerivedValue<*> -> + if (value.changedAt > asOfRevision) { + // This value has been updated more recently than the expected revision + false + } else if (value.verifiedAt == currentRevision) { + true + } else { + // Recurse through dependencies and verify them + value.dependencies.all { dep -> + get(dep)?.let { verify(it, value.verifiedAt) } ?: true + }.also { + if (it) { + value.verifiedAt = currentRevision + } + } + // TODO: If dependencies are invalid, recompute the query and check if the result is equivalent. + // If so, we can still "verify", preventing dependents from recalculating. + } + } + } + + private fun recordQuery(key: QueryKey<*, *>) { + if (currentlyActiveQuery.isNotEmpty()) { + val deps = currentlyActiveQuery.last().dependencies + if (!deps.contains(key)) deps += key + } + } + + private fun trackRevision(revision: Int) { + if (currentlyActiveQuery.isNotEmpty()) { + val currentFrame = currentlyActiveQuery.last() + if (currentFrame.maxRevision < revision) { + currentFrame.maxRevision = revision + } + } + } + + private fun recordDependencies(dependencies: List>) { + if (currentlyActiveQuery.isNotEmpty()) { + val deps = currentlyActiveQuery.last().dependencies + dependencies.forEach { key -> + if (!deps.contains(key)) deps += key + } + } + } + + public fun print() { + println("=========================") + println("Current revision = $currentRevision") + storage.forEach { (query, store) -> + println("Query store: $query") + store.forEach { (key, value) -> + println(" $key to $value") + } + } + println("=========================") + } +} diff --git a/inkt/src/main/kotlin/dev/dialector/inkt/QueryDsl.kt b/inkt/src/main/kotlin/dev/dialector/inkt/QueryDsl.kt new file mode 100644 index 0000000..747168e --- /dev/null +++ b/inkt/src/main/kotlin/dev/dialector/inkt/QueryDsl.kt @@ -0,0 +1,47 @@ +package dev.dialector.inkt + +import kotlin.reflect.KClass + +public annotation class QueryGroup + +public annotation class Query + +public annotation class Input + +public annotation class Tracked + +public annotation class DatabaseDef(vararg val groups: KClass<*>) + +public class NoInputDefinedException(message: String) : RuntimeException(message) + +public interface DatabaseQuery { + public val name: String + public fun get(key: K): V +} + +/** + * This function is used to create a typesafe storage map for a query. + * The receiver is necessary to properly infer types, even though IntelliJ says otherwise. + */ +internal fun DatabaseQuery.createMap(): MutableMap> = mutableMapOf() + +public data class InputQuery( + override val name: String, + private val query: (K) -> V +) : DatabaseQuery { + override fun get(key: K): V = query(key) +} + +public data class DerivedQuery( + override val name: String, + private val query: (K) -> V +) : DatabaseQuery { + override fun get(key: K): V = query(key) +} + +public fun inputQuery( + name: String, + query: (K) -> V = { throw NoInputDefinedException("No input exists for query $name($it)") } +): InputQuery = InputQuery(name, query) + +public fun derivedQuery(name: String, query: (K) -> V): DerivedQuery = DerivedQuery(name, query) diff --git a/inkt/src/main/kotlin/dev/dialector/inkt/example/DefinedQueryExample.kt b/inkt/src/main/kotlin/dev/dialector/inkt/example/DefinedQueryExample.kt new file mode 100644 index 0000000..8ff3c1e --- /dev/null +++ b/inkt/src/main/kotlin/dev/dialector/inkt/example/DefinedQueryExample.kt @@ -0,0 +1,105 @@ +package dev.dialector.inkt.example + +import dev.dialector.inkt.DerivedQuery +import dev.dialector.inkt.InputQuery +import dev.dialector.inkt.NoInputDefinedException +import dev.dialector.inkt.QueryDatabase +import dev.dialector.inkt.derivedQuery +import dev.dialector.inkt.inputQuery + +/** + * Inkt queries begin as interface definitions. It is easiest to provide default implementations + * as the query database can then invoke the super method, though this approach is not required. + */ +internal interface HelloWorld { + /** + * Input queries are not required to have an implementation - these will generally depend on + * externally-provided inputs. This function corresponds to a [String] -> [String] mapping + * + * A query database will need to provide means to set these input values. + */ + fun inputString(key: String): String? + + /** + * Derived queries are composed from other queries - in this case, [length] returns the length + * of the input string for the given key. + */ + fun length(key: String): Int? { + println("Recomputing length for $key") + return inputString(key)?.length + } + + /** + * This is an example of a derived query that depends on another derived query. + */ + fun longest(keys: Set): String? { + println("recomputing longest") + return keys.maxByOrNull { length(it) ?: -1 }?.let { inputString(it) } + } +} + +/** + * A query database implementation is an implementation of one or more query interfaces, a series of + * [DatabaseQuery] definitions that represent the implemented queries, along with an internal + * [QueryDatabase] to provide incremental behavior. + * + * The [DatabaseQuery] definitions are typesafe handles for the query functionality and must be passed + * to the [QueryDatabase] constructor. This is to allow for internal optimization of query storage. + * + * The [QueryDatabase.inputQuery] and [QueryDatabase.derivedQuery] methods handle fetching different types of data + * from the database and ensuring the queries are incrementalized. + * + * The [QueryDatabase.setInput] method handles assigning input data for input queries. + */ +internal class DefinedQueryExampleDatabase : HelloWorld { + private val inputString: InputQuery = inputQuery("inputString") { + throw NoInputDefinedException("Input not provided for inputString($it)") + } + private val length: DerivedQuery = derivedQuery("length") { super.length(it) } + private val longest: DerivedQuery, String?> = derivedQuery("longest") { super.longest(it) } + private val database = QueryDatabase(listOf(inputString, length, longest)) + + fun setInputString(key: String, value: String?) = database.setInput(inputString, key, value) + + override fun inputString(key: String): String? = database.inputQuery(inputString, key) + + override fun length(key: String): Int? = database.derivedQuery(length, key) + + override fun longest(keys: Set): String? = database.derivedQuery(longest, keys) +} + +internal fun main() { + val db = DefinedQueryExampleDatabase() + db.setInputString("foo", "hello") + + println("foo: Length is ${db.length("foo")}") + println("foo: Length is ${db.length("foo")} shouldn't recompute!") + + db.setInputString("bar", "bye") + + println("foo: Length is ${db.length("foo")} shouldn't recompute!") + println("bar: Length is ${db.length("bar")}") + println("bar: Length is ${db.length("bar")} shouldn't recompute!") + + db.setInputString("foo", "longer") + + println("foo: Length is ${db.length("foo")}") + println("bar: Length is ${db.length("bar")} shouldn't recompute!") + + println("longest {foo, bar} is: ${db.longest(setOf("foo", "bar"))}") + println("longest {foo, bar} is: ${db.longest(setOf("foo", "bar"))}") + + db.setInputString("bar", "even longer") + println("longest {foo, bar} is: ${db.longest(setOf("foo", "bar"))}") + println("longest {foo, bar} is: ${db.longest(setOf("foo", "bar"))}") + + db.setInputString("baz", "definitely the longest") + println("longest {foo, bar, baz} is ${db.longest(setOf("foo", "bar", "baz"))}") + println("longest {foo, bar, baz} is ${db.longest(setOf("foo", "bar", "baz"))}") + + db.setInputString("foo", "long") + db.setInputString("bar", "med") + db.setInputString("baz", "s") + println("longest {foo, bar, baz} is ${db.longest(setOf("foo", "bar", "baz"))}") + println("longest {foo, bar, baz} is ${db.longest(setOf("foo", "bar", "baz"))}") +} diff --git a/inkt/src/main/kotlin/dev/dialector/inkt/example/GeneratedQueryExample.kt b/inkt/src/main/kotlin/dev/dialector/inkt/example/GeneratedQueryExample.kt new file mode 100644 index 0000000..c63d090 --- /dev/null +++ b/inkt/src/main/kotlin/dev/dialector/inkt/example/GeneratedQueryExample.kt @@ -0,0 +1,85 @@ +package dev.dialector.inkt.example + +import dev.dialector.inkt.DatabaseDef +import dev.dialector.inkt.DerivedQuery +import dev.dialector.inkt.Input +import dev.dialector.inkt.InputQuery +import dev.dialector.inkt.NoInputDefinedException +import dev.dialector.inkt.QueryDatabase +import dev.dialector.inkt.QueryGroup +import dev.dialector.inkt.Tracked +import dev.dialector.inkt.derivedQuery +import dev.dialector.inkt.inputQuery + +@QueryGroup +internal interface HelloWorldGen { + @Input + fun inputString(key: String): String? + + @Tracked + fun length(key: String): Int? { + println("Recomputing length for $key") + return inputString(key)?.length + } + + fun longest(keys: Set): String? { + println("recomputing longest") + return keys.maxByOrNull { length(it) ?: -1 }?.let { inputString(it) } + } +} + +@DatabaseDef(HelloWorldGen::class) +internal interface MyDatabase : HelloWorldGen + +internal class GeneratedQueryExample : MyDatabase { + private val inputString: InputQuery = inputQuery("inputString") { throw NoInputDefinedException("Input not provided for inputString($it)") } + private val length: DerivedQuery = derivedQuery("length") { super.length(it) } + private val longest: DerivedQuery, String?> = derivedQuery("longest") { super.longest(it) } + private val database = QueryDatabase(listOf(inputString, length, longest)) + + fun setInputString(key: String, value: String?) = database.setInput(inputString, key, value) + + override fun inputString(key: String): String? = database.inputQuery(inputString, key) + + override fun length(key: String): Int? = database.derivedQuery(length, key) + + override fun longest(keys: Set): String? = database.derivedQuery(longest, keys) +} + +internal fun main() { + val db = GeneratedQueryExample() + db.setInputString("foo", "hello world") + + println("foo: Length is ${db.length("foo")}") + println("foo: Length is ${db.length("foo")} shouldn't recompute!") + + db.setInputString("bar", "bai") + + println("foo: Length is ${db.length("foo")} shouldn't recompute!") + println("bar: Length is ${db.length("bar")}") + println("bar: Length is ${db.length("bar")} shouldn't recompute!") + + db.setInputString("foo", "oh wow this is very long") + + println("foo: Length is ${db.length("foo")}") + println("bar: Length is ${db.length("bar")} shouldn't recompute!") + + println("longest {foo, bar} is: ${db.longest(setOf("foo", "bar"))}") + println("longest {foo, bar} is: ${db.longest(setOf("foo", "bar"))}") +// db.print() + db.setInputString("bar", "super long to verify some stuff hereeeeeeeeee") +// db.print() + println("longest {foo, bar} is: ${db.longest(setOf("foo", "bar"))}") +// db.print() + println("longest {foo, bar} is: ${db.longest(setOf("foo", "bar"))}") + + db.setInputString("baz", "the longest there ever was, because it's criticalllll") + println("longest {foo, bar, baz} is ${db.longest(setOf("foo", "bar", "baz"))}") + println("longest {foo, bar, baz} is ${db.longest(setOf("foo", "bar", "baz"))}") + + db.setInputString("foo", "long") + db.setInputString("bar", "med") + db.setInputString("baz", "s") + println("longest {foo, bar, baz} is ${db.longest(setOf("foo", "bar", "baz"))}") + println("longest {foo, bar, baz} is ${db.longest(setOf("foo", "bar", "baz"))}") +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 06492e1..5db6fdd 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -24,3 +24,4 @@ pluginManagement { include("dialector-kt") include("dialector-lsp") include("dialector-kt-processor") +include("inkt")