Mikec/upgrade graders

eslint.config.mjs

@@ -3,7 +3,7 @@ import tsparser from "@typescript-eslint/parser";
 
 export default tseslint.config(
   {
-    ignores: ["**/convex/_generated/"],
+    ignores: ["**/convex/_generated/", "scripts/**", "evals/**/grader.test.ts"],
   },
   tseslint.configs.recommendedTypeChecked,
   {

evals/000-fundamentals/000-empty_functions/grader.test.ts

@@ -1,169 +1,89 @@
 import { expect, test } from "vitest";
-import {
-  responseAdminClient,
-  responseClient,
-  compareSchema,
-  compareFunctionSpec,
-} from "../../../grader";
+import { responseAdminClient, responseClient } from "../../../grader";
 import { anyApi } from "convex/server";
 
-test("compare schema", async ({ skip }) => {
-  await compareSchema(skip);
-});
+test("empty public query", async () => {
+  expect(await responseClient.query(anyApi.index.emptyPublicQuery, {})).toBe(
+    null,
+  );
 
-test("compare function spec", async ({ skip }) => {
-  await compareFunctionSpec(skip);
-});
+  await expect(
+    responseClient.query(anyApi.index.emptyPublicQuery, { arg: "test" }),
+  ).rejects.toThrow(/ArgumentValidationError/);
 
-test("empty public query", async () => {
-  const result = await responseClient.query(anyApi.index.emptyPublicQuery, {});
-  expect(result).toBe(null);
+  await expect(
+    responseClient.mutation(anyApi.index.emptyPublicQuery, {}),
+  ).rejects.toBeDefined();
 
-  let error: any = undefined;
-  try {
-    await responseClient.query(anyApi.index.emptyPublicQuery, {
-      arg: "test",
-    });
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
-  expect(error.toString()).toContain("ArgumentValidationError");
-
-  error = undefined;
-  try {
-    await responseClient.mutation(anyApi.index.emptyPublicQuery, {});
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
-
-  error = undefined;
-  try {
-    await responseClient.action(anyApi.index.emptyPublicQuery, {});
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
+  await expect(
+    responseClient.action(anyApi.index.emptyPublicQuery, {}),
+  ).rejects.toBeDefined();
 });
 
 test("empty public mutation", async () => {
-  const result = await responseClient.mutation(
-    anyApi.index.emptyPublicMutation,
-    {},
-  );
-  expect(result).toBe(null);
+  expect(
+    await responseClient.mutation(anyApi.index.emptyPublicMutation, {}),
+  ).toBe(null);
 
-  let error: any = undefined;
-  try {
-    await responseClient.mutation(anyApi.index.emptyPublicMutation, {
+  await expect(
+    responseClient.mutation(anyApi.index.emptyPublicMutation, {
       arg: "test",
-    });
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
-  expect(error.toString()).toContain("ArgumentValidationError");
-
-  error = undefined;
-  try {
-    await responseClient.query(anyApi.index.emptyPublicMutation, {});
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
-
-  error = undefined;
-  try {
-    await responseClient.action(anyApi.index.emptyPublicMutation, {});
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
+    }),
+  ).rejects.toThrow(/ArgumentValidationError/);
+
+  await expect(
+    responseClient.query(anyApi.index.emptyPublicMutation, {}),
+  ).rejects.toBeDefined();
+
+  await expect(
+    responseClient.action(anyApi.index.emptyPublicMutation, {}),
+  ).rejects.toBeDefined();
 });
 
 test("empty public action", async () => {
-  const result = await responseClient.action(
-    anyApi.index.emptyPublicAction,
-    {},
+  expect(await responseClient.action(anyApi.index.emptyPublicAction, {})).toBe(
+    null,
   );
-  expect(result).toBe(null);
 
-  let error: any = undefined;
-  try {
-    await responseClient.action(anyApi.index.emptyPublicAction, {
-      arg: "test",
-    });
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
-  expect(error.toString()).toContain("ArgumentValidationError");
-
-  error = undefined;
-  try {
-    await responseClient.query(anyApi.index.emptyPublicAction, {});
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
-
-  error = undefined;
-  try {
-    await responseClient.mutation(anyApi.index.emptyPublicAction, {});
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
+  await expect(
+    responseClient.action(anyApi.index.emptyPublicAction, { arg: "test" }),
+  ).rejects.toThrow(/ArgumentValidationError/);
+
+  await expect(
+    responseClient.query(anyApi.index.emptyPublicAction, {}),
+  ).rejects.toBeDefined();
+
+  await expect(
+    responseClient.mutation(anyApi.index.emptyPublicAction, {}),
+  ).rejects.toBeDefined();
 });
 
 test("empty private query", async () => {
-  let error: any = undefined;
-  try {
-    await responseClient.query(anyApi.index.emptyPrivateQuery, {});
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
-  expect(error.toString()).toContain("Could not find public function");
-
-  const result = await responseAdminClient.query(
-    anyApi.index.emptyPrivateQuery,
-    {},
-  );
-  expect(result).toBe(null);
+  await expect(
+    responseClient.query(anyApi.index.emptyPrivateQuery, {}),
+  ).rejects.toThrow(/Could not find public function/);
+
+  expect(
+    await responseAdminClient.query(anyApi.index.emptyPrivateQuery, {}),
+  ).toBe(null);
 });
 
 test("empty private mutation", async () => {
-  let error: any = undefined;
-  try {
-    await responseClient.mutation(anyApi.index.emptyPrivateMutation, {});
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
-  expect(error.toString()).toContain("Could not find public function");
-
-  const result = await responseAdminClient.mutation(
-    anyApi.index.emptyPrivateMutation,
-    {},
-  );
-  expect(result).toBe(null);
+  await expect(
+    responseClient.mutation(anyApi.index.emptyPrivateMutation, {}),
+  ).rejects.toThrow(/Could not find public function/);
+
+  expect(
+    await responseAdminClient.mutation(anyApi.index.emptyPrivateMutation, {}),
+  ).toBe(null);
 });
 
 test("empty private action", async () => {
-  let error: any = undefined;
-  try {
-    await responseClient.action(anyApi.index.emptyPrivateAction, {});
-  } catch (e) {
-    error = e;
-  }
-  expect(error).toBeDefined();
-  expect(error.toString()).toContain("Could not find public function");
-
-  const result = await responseAdminClient.action(
-    anyApi.index.emptyPrivateAction,
-    {},
-  );
-  expect(result).toBe(null);
+  await expect(
+    responseClient.action(anyApi.index.emptyPrivateAction, {}),
+  ).rejects.toThrow(/Could not find public function/);
+
+  expect(
+    await responseAdminClient.action(anyApi.index.emptyPrivateAction, {}),
+  ).toBe(null);
 });

evals/000-fundamentals/001-basic_schema/grader.test.ts

@@ -1,16 +1,93 @@
 import { expect, test } from "vitest";
 import {
   responseAdminClient,
-  responseClient,
-  compareSchema,
-  compareFunctionSpec,
+  getSchema,
+  addDocuments,
+  listTable,
+  deleteAllDocuments,
 } from "../../../grader";
-import { api } from "./answer/convex/_generated/api";
 
-test("compare schema", async ({ skip }) => {
-  await compareSchema(skip);
+type SimpleSchema = { tables: Array<{ tableName: string }> };
+
+test("has exactly two tables: users and messages", async () => {
+  const schemaUnknown: unknown = await getSchema(responseAdminClient);
+  expect(schemaUnknown).not.toBeNull();
+  const { tables } = schemaUnknown as SimpleSchema;
+  const tableNames = tables.map((t) => t.tableName);
+  expect(tableNames).toEqual(["messages", "users"]);
 });
 
-test("compare function spec", async ({ skip }) => {
-  await compareFunctionSpec(skip);
+test("users table enforces single string field: name", async () => {
+  await deleteAllDocuments(responseAdminClient, ["users", "messages"]);
+
+  // valid insert
+  await addDocuments(responseAdminClient, "users", [{ name: "Alice" }]);
+  const users = await listTable(responseAdminClient, "users");
+  expect(users.length).toBe(1);
+
+  // extra field should fail
+  await expect(
+    addDocuments(responseAdminClient, "users", [
+      { name: "Bob", extra: "nope" },
+    ]),
+  ).rejects.toBeDefined();
+
+  // wrong type should fail
+  await expect(
+    addDocuments(responseAdminClient, "users", [
+      { name: 123 as unknown as string },
+    ]),
+  ).rejects.toBeDefined();
+
+  // missing required field should fail
+  await expect(
+    addDocuments(responseAdminClient, "users", [
+      {} as unknown as { name: string },
+    ]),
+  ).rejects.toBeDefined();
+});
+
+test("messages table enforces two string fields: text and authorName", async () => {
+  await deleteAllDocuments(responseAdminClient, ["users", "messages"]);
+
+  // valid insert
+  await addDocuments(responseAdminClient, "messages", [
+    { text: "Hello", authorName: "Alice" },
+  ]);
+  const messages = await listTable(responseAdminClient, "messages");
+  expect(messages.length).toBe(1);
+
+  // extra field should fail
+  await expect(
+    addDocuments(responseAdminClient, "messages", [
+      { text: "Hi", authorName: "Bob", extra: "nope" },
+    ]),
+  ).rejects.toBeDefined();
+
+  // wrong types should fail
+  await expect(
+    addDocuments(responseAdminClient, "messages", [
+      { text: 42 as unknown as string, authorName: "Alice" },
+    ]),
+  ).rejects.toBeDefined();
+  await expect(
+    addDocuments(responseAdminClient, "messages", [
+      { text: "Hello", authorName: 99 as unknown as string },
+    ]),
+  ).rejects.toBeDefined();
+
+  // missing required fields should fail
+  await expect(
+    addDocuments(responseAdminClient, "messages", [
+      { text: "Hello" } as unknown as { text: string; authorName: string },
+    ]),
+  ).rejects.toBeDefined();
+  await expect(
+    addDocuments(responseAdminClient, "messages", [
+      { authorName: "Alice" } as unknown as {
+        text: string;
+        authorName: string;
+      },
+    ]),
+  ).rejects.toBeDefined();
 });

evals/000-fundamentals/002-basic_http_endpoint/TASK.txt

@@ -1,3 +1,13 @@
-Create a backend with only two HTTP routes. Serve the text "there" concatenated
-onto the request body for GETs at `/api/hello`. Serve an empty response for POSTs 
-at `/api/messages/*`.
+Create a backend with only two HTTP routes.
+
+1) GET `/api/hello`
+   - Read the request body as text (treat missing/empty body as empty string)
+   - Respond with the body text concatenated with the literal string "there"
+   - Examples:
+     - body: "" -> response: "there"
+     - body: "hello" -> response: "hellothere"
+   - Set `Content-Type: text/plain`
+
+2) POST `/api/messages/*`
+   - Return HTTP 200 with an empty response body
+   - This route should match any path under `/api/messages/` (e.g. `/api/messages/123`). In Convex, use a wildcard route (e.g. `pathPrefix: "/api/messages/"`).

evals/000-fundamentals/002-basic_http_endpoint/grader.test.ts

@@ -1,12 +1,21 @@
 import { expect, test } from "vitest";
-import {
-  responseAdminClient,
-  responseClient,
-  compareSchema,
-  compareFunctionSpec,
-} from "../../../grader";
-import { api } from "./answer/convex/_generated/api";
+import { siteUrl } from "../../../grader";
 
-test("compare function spec", async ({ skip }) => {
-  await compareFunctionSpec(skip);
+test("GET /api/hello returns body + 'there' (empty body => 'there')", async () => {
+  const res = await fetch(`${siteUrl}/api/hello`, { method: "GET" });
+  expect(res.ok).toBe(true);
+  const text = await res.text();
+  expect(text).toBe("there");
+  const contentType = res.headers.get("content-type");
+  expect(contentType && contentType.includes("text/plain")).toBe(true);
+});
+
+test("POST /api/messages/* returns empty response body", async () => {
+  const res = await fetch(`${siteUrl}/api/messages/123`, {
+    method: "POST",
+    body: "ignored",
+  });
+  expect(res.ok).toBe(true);
+  const text = await res.text();
+  expect(text).toBe("");
 });