Fix STACK-BACKEND-5K

.github/workflows/e2e-api-tests.yaml

@@ -125,6 +125,9 @@ jobs:
       - name: Run tests again, to make sure they are stable (attempt 3)
         run: pnpm test
 
+      - name: Verify data integrity
+        run: pnpm run verify-data-integrity
+
       - name: Print Docker Compose logs
         if: always()
         run: docker compose -f dependencies.compose.yaml logs

apps/backend/package.json

@@ -25,7 +25,8 @@
     "watch-docs": "pnpm run with-env tsx watch --clear-screen=false scripts/generate-docs.ts",
     "generate-docs": "pnpm run with-env tsx scripts/generate-docs.ts",
     "generate-keys": "pnpm run with-env tsx scripts/generate-keys.ts",
-    "db-seed-script": "pnpm run with-env tsx prisma/seed.ts"
+    "db-seed-script": "pnpm run with-env tsx prisma/seed.ts",
+    "verify-data-integrity": "pnpm run with-env tsx scripts/verify-data-integrity.ts"
   },
   "prisma": {
     "seed": "pnpm run db-seed-script"

apps/backend/prisma/migrations/20241223225110_fill_empty_project_config_values/migration.sql

@@ -0,0 +1,5 @@
+-- Some older versions allowed the empty string for OAuth provider clientId and clientSecret values.
+-- We fix that.
+
+UPDATE "StandardOAuthProviderConfig" SET "clientId" = 'invalid' WHERE "clientId" = '';
+UPDATE "StandardOAuthProviderConfig" SET "clientSecret" = 'invalid' WHERE "clientSecret" = '';

apps/backend/prisma/migrations/20241223231022_remove_empty_team_profile_images/migration.sql

@@ -0,0 +1,4 @@
+-- Some older versions allowed the empty string as a team profile image.
+-- We fix that.
+
+UPDATE "Team" SET "profileImageUrl" = NULL WHERE "profileImageUrl" = '';

apps/backend/prisma/migrations/20241223231023_onlyhttps_domains/migration.sql

@@ -0,0 +1,4 @@
+-- Some older versions allowed http:// URLs as trusted domains, instead of just https://.
+-- We fix that.
+
+UPDATE "ProjectDomain" SET "domain" = 'https://example.com' WHERE "domain" LIKE 'http://%';

apps/backend/scripts/verify-data-integrity.ts

@@ -0,0 +1,163 @@
+import { PrismaClient } from "@prisma/client";
+import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
+import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
+import { filterUndefined } from "@stackframe/stack-shared/dist/utils/objects";
+import { wait } from "@stackframe/stack-shared/dist/utils/promises";
+import { deindent } from "@stackframe/stack-shared/dist/utils/strings";
+
+const prismaClient = new PrismaClient();
+
+async function main() {
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log("===================================================");
+  console.log("Welcome to verify-data-integrity.ts.");
+  console.log();
+  console.log("This script will ensure that the data in the");
+  console.log("database is not corrupted.");
+  console.log();
+  console.log("It will call the most important endpoints for");
+  console.log("each project and every user, and ensure that");
+  console.log("the status codes are what they should be.");
+  console.log();
+  console.log("It's a good idea to run this script on REPLICAS");
+  console.log("of the production database regularly (not the actual");
+  console.log("prod db!); it should never fail at any point in time.");
+  console.log();
+  console.log("");
+  console.log("\x1b[41mIMPORTANT\x1b[0m: This script may modify");
+  console.log("the database during its execution in all sorts of");
+  console.log("ways, so don't run it on production!");
+  console.log();
+  console.log("===================================================");
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log("Starting in 3 seconds...");
+  await wait(1000);
+  console.log("2...");
+  await wait(1000);
+  console.log("1...");
+  await wait(1000);
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+
+
+  const projects = await prismaClient.project.findMany({
+    select: {
+      id: true,
+      displayName: true,
+    },
+    orderBy: {
+      id: "asc",
+    },
+  });
+  console.log(`Found ${projects.length} projects, iterating over them.`);
+
+  for (let i = 0; i < projects.length; i++) {
+    const projectId = projects[i].id;
+    await recurse(`[project ${i + 1}/${projects.length}] ${projectId} ${projects[i].displayName}`, async (recurse) => {
+      await Promise.all([
+        expectStatusCode(200, `/api/v1/projects/current`, {
+          method: "GET",
+          headers: {
+            "x-stack-project-id": projectId,
+            "x-stack-access-type": "admin",
+            "x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
+          },
+        }),
+        expectStatusCode(200, `/api/v1/users`, {
+          method: "GET",
+          headers: {
+            "x-stack-project-id": projectId,
+            "x-stack-access-type": "admin",
+            "x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"),
+          },
+        }),
+      ]);
+    });
+  }
+
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log();
+  console.log("===================================================");
+  console.log("All good!");
+  console.log();
+  console.log("Goodbye.");
+  console.log("===================================================");
+  console.log();
+  console.log();
+}
+main().catch((...args) => {
+  console.error();
+  console.error();
+  console.error(`\x1b[41mERROR\x1b[0m! Could not verify data integrity. See the error message for more details.`);
+  console.error(...args);
+  process.exit(1);
+});
+
+async function expectStatusCode(expectedStatusCode: number, endpoint: string, request: RequestInit) {
+  const apiUrl = new URL(getEnvVariable("NEXT_PUBLIC_STACK_API_URL"));
+  const response = await fetch(new URL(endpoint, apiUrl), {
+    ...request,
+    headers: {
+      "x-stack-disable-artificial-development-delay": "yes",
+      "x-stack-development-disable-extended-logging": "yes",
+      ...filterUndefined(request.headers ?? {}),
+    },
+  });
+  if (response.status !== expectedStatusCode) {
+    throw new StackAssertionError(deindent`
+      Expected status code ${expectedStatusCode} but got ${response.status} for ${endpoint}:
+
+          ${await response.text()}
+    `, { request, response });
+  }
+  const json = await response.json();
+  return json;
+}
+
+let lastProgress = performance.now() - 9999999999;
+
+type RecurseFunction = (progressPrefix: string, inner: (recurse: RecurseFunction) => Promise<void>) => Promise<void>;
+
+const _recurse = async (progressPrefix: string | ((...args: any[]) => void), inner: Parameters<RecurseFunction>[1]): Promise<void> => {
+  const progressFunc = typeof progressPrefix === "function" ? progressPrefix : (...args: any[]) => {
+    console.log(`${progressPrefix}`, ...args);
+  };
+  if (performance.now() - lastProgress > 1000) {
+    progressFunc();
+    lastProgress = performance.now();
+  }
+  try {
+    return await inner(
+      (progressPrefix, inner) => _recurse(
+        (...args) => progressFunc(progressPrefix, ...args),
+        inner,
+      ),
+    );
+  } catch (error) {
+    progressFunc(`\x1b[41mERROR\x1b[0m!`);
+    throw error;
+  }
+};
+const recurse: RecurseFunction = _recurse;

apps/backend/src/app/api/v1/integrations/neon/oauth-providers/crud.tsx

@@ -14,8 +14,14 @@ import * as yup from "yup";
 const oauthProviderReadSchema = yupObject({
   id: schemaFields.oauthIdSchema.defined(),
   type: schemaFields.oauthTypeSchema.defined(),
-  client_id: schemaFields.yupDefinedWhen(schemaFields.oauthClientIdSchema, 'type', 'standard'),
-  client_secret: schemaFields.yupDefinedWhen(schemaFields.oauthClientSecretSchema, 'type', 'standard'),
+  client_id: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientIdSchema, {
+    when: 'type',
+    is: 'standard',
+  }),
+  client_secret: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientSecretSchema, {
+    when: 'type',
+    is: 'standard',
+  }),
 
   // extra params
   facebook_config_id: schemaFields.oauthFacebookConfigIdSchema.optional(),
@@ -24,8 +30,14 @@ const oauthProviderReadSchema = yupObject({
 
 const oauthProviderUpdateSchema = yupObject({
   type: schemaFields.oauthTypeSchema.optional(),
-  client_id: schemaFields.yupDefinedWhen(schemaFields.oauthClientIdSchema, 'type', 'standard').optional(),
-  client_secret: schemaFields.yupDefinedWhen(schemaFields.oauthClientSecretSchema, 'type', 'standard').optional(),
+  client_id: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientIdSchema, {
+    when: 'type',
+    is: 'standard',
+  }).optional(),
+  client_secret: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientSecretSchema, {
+    when: 'type',
+    is: 'standard',
+  }).optional(),
 
   // extra params
   facebook_config_id: schemaFields.oauthFacebookConfigIdSchema.optional(),

apps/backend/src/route-handlers/crud-handler.tsx

@@ -292,7 +292,7 @@ async function validate<T>(obj: unknown, schema: yup.ISchema<T>, currentUser: Us
           Errors:
             ${error.errors.join("\n")}
         `,
-        { obj: JSON.stringify(obj), schema, cause: error },
+        { obj: obj, schema, cause: error },
       );
     }
     throw error;

apps/backend/src/route-handlers/smart-request.tsx

@@ -9,6 +9,7 @@ import { ProjectsCrud } from "@stackframe/stack-shared/dist/interface/crud/proje
 import { UsersCrud } from "@stackframe/stack-shared/dist/interface/crud/users";
 import { StackAdaptSentinel, yupValidate } from "@stackframe/stack-shared/dist/schema-fields";
 import { groupBy, typedIncludes } from "@stackframe/stack-shared/dist/utils/arrays";
+import { getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env";
 import { StackAssertionError, StatusError, throwErr } from "@stackframe/stack-shared/dist/utils/errors";
 import { ignoreUnhandledRejection } from "@stackframe/stack-shared/dist/utils/promises";
 import { deindent } from "@stackframe/stack-shared/dist/utils/strings";
@@ -145,6 +146,7 @@ async function parseAuth(req: NextRequest): Promise<SmartRequestAuth | null> {
   const adminAccessToken = req.headers.get("x-stack-admin-access-token");
   const accessToken = req.headers.get("x-stack-access-token");
   const refreshToken = req.headers.get("x-stack-refresh-token");
+  const developmentKeyOverride = req.headers.get("x-stack-development-override-key");  // in development, the internal project's API key can optionally be used to access any project
 
   const extractUserFromAccessToken = async (options: { token: string, projectId: string }) => {
     const result = await decodeAccessToken(options.token);
@@ -212,7 +214,13 @@ async function parseAuth(req: NextRequest): Promise<SmartRequestAuth | null> {
   if (!typedIncludes(["client", "server", "admin"] as const, requestType)) throw new KnownErrors.InvalidAccessType(requestType);
   if (!projectId) throw new KnownErrors.AccessTypeWithoutProjectId(requestType);
 
-  if (adminAccessToken) {
+  if (developmentKeyOverride) {
+    if (getNodeEnvironment() !== "development") {
+      throw new StatusError(401, "Development key override is only allowed in development mode");
+    }
+    const result = await checkApiKeySet("internal", { superSecretAdminKey: developmentKeyOverride });
+    if (!result) throw new StatusError(401, "Invalid development key override");
+  } else if (adminAccessToken) {
     if (await queries.internalUser) {
       if (!await queries.project) {
         // this happens if the project is still in the user's managedProjectIds, but has since been deleted

apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/auth-methods/providers.tsx

@@ -3,6 +3,7 @@ import { FormDialog } from "@/components/form-dialog";
 import { InputField, SwitchField } from "@/components/form-fields";
 import { SettingIconButton, SettingSwitch } from "@/components/settings";
 import { AdminProject } from "@stackframe/stack";
+import { yupBoolean, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
 import { sharedProviders } from "@stackframe/stack-shared/dist/utils/oauth";
 import { runAsynchronously } from "@stackframe/stack-shared/dist/utils/promises";
 import { ActionDialog, Badge, InlineCode, Label, SimpleTooltip, Typography } from "@stackframe/stack-ui";
@@ -31,22 +32,22 @@ function toTitle(id: string) {
   }[id];
 }
 
-export const providerFormSchema = yup.object({
-  shared: yup.boolean().defined(),
-  clientId: yup.string()
+export const providerFormSchema = yupObject({
+  shared: yupBoolean().defined(),
+  clientId: yupString()
     .when('shared', {
       is: false,
-      then: (schema) => schema.defined(),
+      then: (schema) => schema.defined().nonEmpty(),
       otherwise: (schema) => schema.optional()
     }),
-  clientSecret: yup.string()
+  clientSecret: yupString()
     .when('shared', {
       is: false,
-      then: (schema) => schema.defined(),
+      then: (schema) => schema.defined().nonEmpty(),
       otherwise: (schema) => schema.optional()
     }),
-  facebookConfigId: yup.string().optional(),
-  microsoftTenantId: yup.string().optional(),
+  facebookConfigId: yupString().optional(),
+  microsoftTenantId: yupString().optional(),
 });
 
 export type ProviderFormValues = yup.InferType<typeof providerFormSchema>
@@ -104,7 +105,8 @@ export function ProviderSettingDialog(props: Props & { open: boolean, onClose: (
               Shared keys are created by the Stack team for development. It helps you get started, but will show a Stack logo and name on the OAuth screen. This should never be enabled in production.
             </Typography> :
             <div className="flex flex-col gap-2">
-              <Label>Redirect URL for the OAuth provider settings
+              <Label>
+                Redirect URL for the OAuth provider settings
               </Label>
               <Typography type="footnote">
                 <InlineCode>{`${process.env.NEXT_PUBLIC_STACK_API_URL}/api/v1/auth/oauth/callback/${props.id}`}</InlineCode>