Fix continuation-related exploits and bump* stack limit

Common/src/main/java/at/petrak/hexcasting/api/casting/eval/vm/CastingImage.kt

@@ -12,6 +12,7 @@ import net.minecraft.nbt.ListTag
 import net.minecraft.nbt.Tag
 import net.minecraft.server.level.ServerLevel
 import net.minecraft.world.entity.Entity
+import kotlin.math.max
 
 /**
  * The state of a casting VM, containing the stack and all
@@ -35,6 +36,27 @@ data class CastingImage private constructor(
         }
     }
 
+    private val size: Int
+    private val depth: Int
+
+    init {
+        var maxChildDepth = 0
+        var totalSize = 1
+        for (iota in stack) {
+            totalSize += iota.size()
+            maxChildDepth = max(maxChildDepth, iota.depth())
+        }
+        for (iota in parenthesized) {
+            totalSize += iota.iota.size()
+            maxChildDepth = max(maxChildDepth, iota.iota.depth())
+        }
+        depth = maxChildDepth
+        size = totalSize
+    }
+
+    fun size(): Int = size
+    fun depth(): Int = depth
+
     /**
      * Returns an empty list if it's too complicated.
      */

Common/src/main/java/at/petrak/hexcasting/api/casting/eval/vm/CastingVM.kt

@@ -15,8 +15,10 @@ import at.petrak.hexcasting.api.casting.math.HexPattern
 import at.petrak.hexcasting.api.casting.mishaps.*
 import at.petrak.hexcasting.api.utils.*
 import at.petrak.hexcasting.common.lib.hex.HexEvalSounds
+import at.petrak.hexcasting.common.lib.hex.HexIotaTypes
 import net.minecraft.nbt.CompoundTag
 import net.minecraft.server.level.ServerLevel
+import kotlin.math.max
 
 /**
  * The virtual machine! This is the glue that determines the next iteration of a [CastingImage], using a
@@ -52,7 +54,10 @@ class CastingVM(var image: CastingImage, val env: CastingEnvironment) {
             // get caught and folded into CastResult by evaluate.
             val image2 = next.evaluate(continuation.next, world, this).let { result ->
                 // if stack is unable to be serialized, have the result be an error
-                if (result.newData != null && IotaType.isTooLargeToSerialize(result.newData.stack)) {
+                val data = result.newData ?: this.image
+                val size = data.size() + result.continuation.size()
+                val depth = max(data.depth(), result.continuation.depth())
+                if (depth >= HexIotaTypes.MAX_SERIALIZATION_DEPTH || size >= HexIotaTypes.MAX_SERIALIZATION_TOTAL) {
                     result.copy(
                         newData = null,
                         sideEffects = listOf(OperatorSideEffect.DoMishap(MishapStackSize(), Mishap.Context(null, null))),

Common/src/main/java/at/petrak/hexcasting/api/casting/eval/vm/ContinuationFrame.kt

@@ -46,6 +46,8 @@ interface ContinuationFrame {
      * Return the number of iotas contained inside this frame, used for determining whether it is valid to serialise.
      */
     fun size(): Int
+    fun depth(): Int
+    fun subIotas(): Iterable<Iota>?
 
     val type: Type<*>
 

Common/src/main/java/at/petrak/hexcasting/api/casting/eval/vm/FrameEvaluate.kt

@@ -13,6 +13,7 @@ import at.petrak.hexcasting.common.lib.hex.HexIotaTypes
 import net.minecraft.nbt.CompoundTag
 import net.minecraft.nbt.Tag
 import net.minecraft.server.level.ServerLevel
+import kotlin.math.max
 
 /**
  * A list of patterns to be evaluated in sequence.
@@ -53,7 +54,23 @@ data class FrameEvaluate(val list: SpellList, val isMetacasting: Boolean) : Cont
         "isMetacasting" %= isMetacasting
     }
 
-    override fun size() = list.size()
+    private val size: Int
+    private val depth: Int
+
+    init {
+        var maxChildDepth = 0
+        var totalSize = 1
+        for (iota in list) {
+            totalSize += iota.size()
+            maxChildDepth = max(maxChildDepth, iota.depth())
+        }
+        depth = maxChildDepth
+        size = totalSize
+    }
+
+    override fun size() = size
+    override fun depth() = depth
+    override fun subIotas(): Iterable<Iota> = list
 
     override val type: ContinuationFrame.Type<*> = TYPE
 

Common/src/main/java/at/petrak/hexcasting/api/casting/eval/vm/FrameFinishEval.kt

@@ -36,6 +36,8 @@ object FrameFinishEval : ContinuationFrame {
     override fun serializeToNBT() = CompoundTag()
 
     override fun size() = 0
+    override fun depth() = 0
+    override fun subIotas(): Iterable<Iota>? = null
 
     @JvmField
     val TYPE: ContinuationFrame.Type<FrameFinishEval> = object : ContinuationFrame.Type<FrameFinishEval> {

Common/src/main/java/at/petrak/hexcasting/api/casting/eval/vm/FrameForEach.kt

@@ -14,6 +14,7 @@ import at.petrak.hexcasting.common.lib.hex.HexIotaTypes
 import net.minecraft.nbt.CompoundTag
 import net.minecraft.nbt.Tag
 import net.minecraft.server.level.ServerLevel
+import kotlin.math.max
 
 /**
  * A frame representing all the state for a Thoth evaluation.
@@ -89,7 +90,41 @@ data class FrameForEach(
         "accumulator" %= acc.serializeToNBT()
     }
 
-    override fun size() = data.size() + code.size() + acc.size + (baseStack?.size ?: 0)
+    private val size: Int
+    private val depth: Int
+
+    init {
+        var maxChildDepth = 0
+        var totalSize = 1
+        for (iota in data) {
+            totalSize += iota.size()
+            maxChildDepth = max(maxChildDepth, iota.depth())
+        }
+        for (iota in code) {
+            totalSize += iota.size()
+            maxChildDepth = max(maxChildDepth, iota.depth())
+        }
+        for (iota in acc) {
+            totalSize += iota.size()
+            maxChildDepth = max(maxChildDepth, iota.depth())
+        }
+        if (baseStack != null) {
+            for (iota in baseStack) {
+                totalSize += iota.size()
+                maxChildDepth = max(maxChildDepth, iota.depth())
+            }
+        }
+        depth = maxChildDepth
+        size = totalSize
+    }
+
+    override fun size() = size
+    override fun depth() = depth
+    override fun subIotas(): Iterable<Iota> =
+        if (baseStack != null)
+            listOf(data, code, acc, baseStack).flatten()
+        else
+            listOf(data, code, acc).flatten()
 
     override val type: ContinuationFrame.Type<*> = TYPE
 

Common/src/main/java/at/petrak/hexcasting/api/casting/eval/vm/SpellContinuation.kt

@@ -1,21 +1,45 @@
 package at.petrak.hexcasting.api.casting.eval.vm
 
+import at.petrak.hexcasting.api.casting.iota.Iota
 import at.petrak.hexcasting.api.utils.NBTBuilder
 import at.petrak.hexcasting.api.utils.getList
 import net.minecraft.nbt.CompoundTag
 import net.minecraft.nbt.Tag
 import net.minecraft.server.level.ServerLevel
+import kotlin.math.max
 
 /**
  * A continuation during the execution of a spell.
  */
 sealed interface SpellContinuation {
-    object Done : SpellContinuation
+    object Done : SpellContinuation {
+        override fun size(): Int = 0
+        override fun depth(): Int = 0
+        override fun subIotas(): Iterable<Iota>? = null
+    }
 
-    data class NotDone(val frame: ContinuationFrame, val next: SpellContinuation) : SpellContinuation
+    data class NotDone(val frame: ContinuationFrame, val next: SpellContinuation) : SpellContinuation {
+        override fun size(): Int = frame.size() + next.size()
+        override fun depth(): Int = max(frame.depth(), next.depth())
+        override fun subIotas(): Iterable<Iota> {
+            val list: MutableList<Iterable<Iota>> = mutableListOf()
+            var current: SpellContinuation = this
+            while (current is NotDone) {
+                val subIotas = current.frame.subIotas()
+                if (subIotas != null)
+                    list.add(subIotas)
+                current = current.next
+            }
+            return list.flatten()
+        }
+    }
 
     fun pushFrame(frame: ContinuationFrame): SpellContinuation = NotDone(frame, this)
 
+    fun subIotas(): Iterable<Iota>?
+    fun size(): Int
+    fun depth(): Int
+
     fun serializeToNBT() = NBTBuilder {
         TAG_FRAME %= list(getNBTFrames())
     }

Common/src/main/java/at/petrak/hexcasting/api/casting/iota/ContinuationIota.java

@@ -13,6 +13,7 @@
 import net.minecraft.network.chat.Component;
 import net.minecraft.server.level.ServerLevel;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.util.List;
 
@@ -56,17 +57,19 @@ public boolean executable() {
         return true;
     }
 
+    @Override
+    public @Nullable Iterable<Iota> subIotas() {
+        return this.getContinuation().subIotas();
+    }
+
     @Override
     public int size() {
-        var continuation = this.getContinuation();
-        var size = 0;
-        while (continuation instanceof SpellContinuation.NotDone notDone) {
-            size += 1;
-            size += notDone.component1().size();
-            continuation = notDone.component2();
-        }
+        return Math.min(this.getContinuation().size(), 1);
+    }
 
-        return Math.min(size, 1);
+    @Override
+    public int depth() {
+        return this.getContinuation().depth() + 1;
     }
 
     public static IotaType<ContinuationIota> TYPE = new IotaType<>() {

Common/src/main/java/at/petrak/hexcasting/common/lib/hex/HexIotaTypes.java

@@ -23,7 +23,7 @@ public class HexIotaTypes {
         KEY_TYPE = HexAPI.MOD_ID + ":type",
         KEY_DATA = HexAPI.MOD_ID + ":data";
     public static final int MAX_SERIALIZATION_DEPTH = 256;
-    public static final int MAX_SERIALIZATION_TOTAL = 1024;
+    public static final int MAX_SERIALIZATION_TOTAL = 8192;
 
     public static void registerTypes(BiConsumer<IotaType<?>, ResourceLocation> r) {
         for (var e : TYPES.entrySet()) {