serde for undefined

packages/convex-helpers/server/stream.test.ts

@@ -624,4 +624,52 @@
       expect(page1.page.map(stripSystemFields)).toEqual([{ a: 1, b: 3, c: 5 }]);
     });
   });
+  test("undefined cursor serialization roundtrips", async () => {
+    const schema = defineSchema({
+      foo: defineTable({
+        a: v.optional(v.number()),
+        b: v.number(),
+      }).index("ab", ["a", "b"]),
+    });
+  });
+
+  test("undefined cursor serialization roundtrips", async () => {
+    const schema = defineSchema({
+      foo: defineTable({
+        a: v.optional(v.number()),
+        b: v.number(),
+      }).index("ab", ["a", "b"]),
+    });
+    const t = convexTest(schema, modules);
+    await t.run(async (ctx) => {
+      await ctx.db.insert("foo", { a: 1, b: 2 });
+      await ctx.db.insert("foo", { a: undefined, b: 3 });
+      await ctx.db.insert("foo", { a: 2, b: 4 });
+      await ctx.db.insert("foo", { a: undefined, b: 5 });
+      const query = stream(ctx.db, schema).query("foo").withIndex("ab");
+      const result = await query.paginate({ numItems: 1, cursor: null });
+      expect(result.continueCursor).toMatch('["$_",');
+      expect(result.page.map(stripSystemFields)).toEqual([
+        { a: undefined, b: 3 },
+      ]);
+      expect(result.isDone).toBe(false);
+      const page1 = await query.paginate({
+        numItems: 2,
+        cursor: result.continueCursor,
+      });
+      expect(page1.page.map(stripSystemFields)).toEqual([
+        { b: 5 },
+        { a: 1, b: 2 },
+      ]);
+      expect(page1.isDone).toBe(false);
+      const page2 = await query.paginate({
+        numItems: 1,
+        cursor: page1.continueCursor,
+      });
+      expect(page2.page.map(stripSystemFields)).toEqual([
+        { a: undefined, b: 5 },
+      ]);
+      expect(page2.isDone).toBe(true);
+    });
+  });
 });

packages/convex-helpers/server/stream.ts

@@ -1,5 +1,5 @@
 import type { Value } from "convex/values";
-import { convexToJson, jsonToConvex } from "convex/values";
+import { convexToJson, compareValues, jsonToConvex } from "convex/values";
 import type {
   DataModelFromSchemaDefinition,
   DocumentByInfo,
@@ -20,9 +20,8 @@ import type {
   SystemDataModel,
   TableNamesInDataModel,
 } from "convex/server";
-import { compareValues } from "./compare.js";
 
-export type IndexKey = Value[];
+export type IndexKey = (Value | undefined)[];
 
 //
 // Helper functions
@@ -328,7 +327,7 @@ abstract class QueryStream<T extends GenericStreamItem>
     };
     if (opts.cursor !== null) {
       newStartKey = {
-        key: jsonToConvex(JSON.parse(opts.cursor)) as IndexKey,
+        key: deserializeCursor(opts.cursor),
         inclusive: false,
       };
     }
@@ -341,7 +340,7 @@ abstract class QueryStream<T extends GenericStreamItem>
     let maxRows: number | undefined = opts.numItems;
     if (opts.endCursor) {
       newEndKey = {
-        key: jsonToConvex(JSON.parse(opts.endCursor)) as IndexKey,
+        key: deserializeCursor(opts.endCursor),
         inclusive: true,
       };
       // If there's an endCursor, continue until we get there even if it's more
@@ -370,7 +369,7 @@ abstract class QueryStream<T extends GenericStreamItem>
         (maxRowsToRead !== undefined && indexKeys.length >= maxRowsToRead)
       ) {
         hasMore = true;
-        continueCursor = JSON.stringify(convexToJson(indexKey as Value));
+        continueCursor = serializeCursor(indexKey);
         break;
       }
     }
@@ -389,9 +388,7 @@ abstract class QueryStream<T extends GenericStreamItem>
       isDone: !hasMore,
       continueCursor,
       pageStatus,
-      splitCursor: splitCursor
-        ? JSON.stringify(convexToJson(splitCursor as Value))
-        : undefined,
+      splitCursor: splitCursor ? serializeCursor(splitCursor) : undefined,
     };
   }
   async collect() {
@@ -1821,3 +1818,40 @@ function compareKeys(key1: Key, key2: Key): number {
   // of key2.kind is valid...
   throw new Error(`Unexpected key kind: ${key1.kind as any}`);
 }
+
+function serializeCursor(key: IndexKey): string {
+  return JSON.stringify(
+    convexToJson(
+      key.map(
+        (v): Value =>
+          v === undefined
+            ? "$_"
+            : typeof v === "string" && v.endsWith("$_")
+              ? // in the unlikely case their string was "$_" or "$$_" etc.
+                // we need to escape it. Always add a $ so "$$_" becomes "$$$_"
+                "$" + v
+              : v,
+      ),
+    ),
+  );
+}
+
+function deserializeCursor(cursor: string): IndexKey {
+  return (jsonToConvex(JSON.parse(cursor)) as Value[]).map((v) => {
+    if (typeof v === "string") {
+      if (v === "$_") {
+        // This is a special case for the undefined value.
+        // It's not a valid value in the index, but it's a valid value in the
+        // cursor.
+        return undefined;
+      }
+      if (v.endsWith("$_")) {
+        // in the unlikely case their string was "$_" it was changed to "$$_"
+        // in the serialization process. If it was "$$_", it was changed to
+        // "$$$_" and so on.
+        return v.slice(1);
+      }
+    }
+    return v;
+  });
+}