eval hybrid pcfg ngram reranker

breandan · Jul 9, 2024 · 2d9ef60 · 2d9ef60
1 parent 09fdc4c
commit 2d9ef60
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/src/commonMain/kotlin/ai/hypergraph/kaliningraph/parsing/SeqValiant.kt b/src/commonMain/kotlin/ai/hypergraph/kaliningraph/parsing/SeqValiant.kt
@@ -223,6 +223,7 @@ class PTree(val root: String = ".ε", val branches: List<Π2A<PTree>> = listOf()
   fun samplePCFG5(pcfgTable: Map<Int, Int>, upUp: Int = 0, upLeft: Int = 0, upRight: Int = 0): Σᐩ {
     if (isLeaf) return epsStr
     val probs = triples.map {
+      /** See [Tree.quintuples] */
       val hash = hash(upUp, upLeft, upRight, it.first, it.second)
       (pcfgTable[hash] ?: 1)
 //      .also { if(Random.nextInt(10000) == 3) if (it == 1) println("$hash Miss"); else println("$hash Hit") }

diff --git a/src/jvmMain/kotlin/ai/hypergraph/markovian/mcmc/MarkovChain.kt b/src/jvmMain/kotlin/ai/hypergraph/markovian/mcmc/MarkovChain.kt
@@ -122,9 +122,10 @@ open class MarkovChain<T>(
 
   // Computes perplexity of a sequence normalized by sequence length (lower is better)
   fun score(seq: List<T?>): Double =
-    if (memory < seq.size) -(scorePrefix + seq + scoreSuffix).windowed(memory)
-      .map { (getAtLeastOne(it) + 1) / (getAtLeastOne(it.dropLast(1) + null) + dictionary.size) }
-      .sumOf { ln(it) } / seq.size
+    if (memory < seq.size)
+      -(scorePrefix + seq + scoreSuffix).windowed(memory).map {
+        (getAtLeastOne(it) + 1) / (getAtLeastOne(it.dropLast(1) + null) + dictionary.size)
+      }.sumOf { ln(it) } / seq.size
     else (seq.sumOf { counter.rawCounts.getEstimate(it) } + 1).toDouble() / counter.total.toDouble()
 
   operator fun get(vararg variables: T?): Double =