diff --git a/.github/workflows/e2e-api-tests.yaml b/.github/workflows/e2e-api-tests.yaml index 468b88d74..9e8cc4863 100644 --- a/.github/workflows/e2e-api-tests.yaml +++ b/.github/workflows/e2e-api-tests.yaml @@ -125,6 +125,9 @@ jobs: - name: Run tests again, to make sure they are stable (attempt 3) run: pnpm test + - name: Verify data integrity + run: pnpm run verify-data-integrity + - name: Print Docker Compose logs if: always() run: docker compose -f dependencies.compose.yaml logs diff --git a/apps/backend/package.json b/apps/backend/package.json index 0580e78bb..4d166ee03 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -25,7 +25,8 @@ "watch-docs": "pnpm run with-env tsx watch --clear-screen=false scripts/generate-docs.ts", "generate-docs": "pnpm run with-env tsx scripts/generate-docs.ts", "generate-keys": "pnpm run with-env tsx scripts/generate-keys.ts", - "db-seed-script": "pnpm run with-env tsx prisma/seed.ts" + "db-seed-script": "pnpm run with-env tsx prisma/seed.ts", + "verify-data-integrity": "pnpm run with-env tsx scripts/verify-data-integrity.ts" }, "prisma": { "seed": "pnpm run db-seed-script" diff --git a/apps/backend/prisma/migrations/20241223225110_fill_empty_project_config_values/migration.sql b/apps/backend/prisma/migrations/20241223225110_fill_empty_project_config_values/migration.sql new file mode 100644 index 000000000..51d67f757 --- /dev/null +++ b/apps/backend/prisma/migrations/20241223225110_fill_empty_project_config_values/migration.sql @@ -0,0 +1,5 @@ +-- Some older versions allowed the empty string for OAuth provider clientId and clientSecret values. +-- We fix that. + +UPDATE "StandardOAuthProviderConfig" SET "clientId" = 'invalid' WHERE "clientId" = ''; +UPDATE "StandardOAuthProviderConfig" SET "clientSecret" = 'invalid' WHERE "clientSecret" = ''; diff --git a/apps/backend/prisma/migrations/20241223231022_remove_empty_team_profile_images/migration.sql b/apps/backend/prisma/migrations/20241223231022_remove_empty_team_profile_images/migration.sql new file mode 100644 index 000000000..3cf168d32 --- /dev/null +++ b/apps/backend/prisma/migrations/20241223231022_remove_empty_team_profile_images/migration.sql @@ -0,0 +1,4 @@ +-- Some older versions allowed the empty string as a team profile image. +-- We fix that. + +UPDATE "Team" SET "profileImageUrl" = NULL WHERE "profileImageUrl" = ''; diff --git a/apps/backend/prisma/migrations/20241223231023_onlyhttps_domains/migration.sql b/apps/backend/prisma/migrations/20241223231023_onlyhttps_domains/migration.sql new file mode 100644 index 000000000..785be3f45 --- /dev/null +++ b/apps/backend/prisma/migrations/20241223231023_onlyhttps_domains/migration.sql @@ -0,0 +1,4 @@ +-- Some older versions allowed http:// URLs as trusted domains, instead of just https://. +-- We fix that. + +UPDATE "ProjectDomain" SET "domain" = 'https://example.com' WHERE "domain" LIKE 'http://%'; diff --git a/apps/backend/scripts/verify-data-integrity.ts b/apps/backend/scripts/verify-data-integrity.ts new file mode 100644 index 000000000..6bdeb8628 --- /dev/null +++ b/apps/backend/scripts/verify-data-integrity.ts @@ -0,0 +1,163 @@ +import { PrismaClient } from "@prisma/client"; +import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env"; +import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors"; +import { filterUndefined } from "@stackframe/stack-shared/dist/utils/objects"; +import { wait } from "@stackframe/stack-shared/dist/utils/promises"; +import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; + +const prismaClient = new PrismaClient(); + +async function main() { + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log("==================================================="); + console.log("Welcome to verify-data-integrity.ts."); + console.log(); + console.log("This script will ensure that the data in the"); + console.log("database is not corrupted."); + console.log(); + console.log("It will call the most important endpoints for"); + console.log("each project and every user, and ensure that"); + console.log("the status codes are what they should be."); + console.log(); + console.log("It's a good idea to run this script on REPLICAS"); + console.log("of the production database regularly (not the actual"); + console.log("prod db!); it should never fail at any point in time."); + console.log(); + console.log(""); + console.log("\x1b[41mIMPORTANT\x1b[0m: This script may modify"); + console.log("the database during its execution in all sorts of"); + console.log("ways, so don't run it on production!"); + console.log(); + console.log("==================================================="); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log("Starting in 3 seconds..."); + await wait(1000); + console.log("2..."); + await wait(1000); + console.log("1..."); + await wait(1000); + console.log(); + console.log(); + console.log(); + console.log(); + + + const projects = await prismaClient.project.findMany({ + select: { + id: true, + displayName: true, + }, + orderBy: { + id: "asc", + }, + }); + console.log(`Found ${projects.length} projects, iterating over them.`); + + for (let i = 0; i < projects.length; i++) { + const projectId = projects[i].id; + await recurse(`[project ${i + 1}/${projects.length}] ${projectId} ${projects[i].displayName}`, async (recurse) => { + await Promise.all([ + expectStatusCode(200, `/api/v1/projects/current`, { + method: "GET", + headers: { + "x-stack-project-id": projectId, + "x-stack-access-type": "admin", + "x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"), + }, + }), + expectStatusCode(200, `/api/v1/users`, { + method: "GET", + headers: { + "x-stack-project-id": projectId, + "x-stack-access-type": "admin", + "x-stack-development-override-key": getEnvVariable("STACK_SEED_INTERNAL_PROJECT_SUPER_SECRET_ADMIN_KEY"), + }, + }), + ]); + }); + } + + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log(); + console.log("==================================================="); + console.log("All good!"); + console.log(); + console.log("Goodbye."); + console.log("==================================================="); + console.log(); + console.log(); +} +main().catch((...args) => { + console.error(); + console.error(); + console.error(`\x1b[41mERROR\x1b[0m! Could not verify data integrity. See the error message for more details.`); + console.error(...args); + process.exit(1); +}); + +async function expectStatusCode(expectedStatusCode: number, endpoint: string, request: RequestInit) { + const apiUrl = new URL(getEnvVariable("NEXT_PUBLIC_STACK_API_URL")); + const response = await fetch(new URL(endpoint, apiUrl), { + ...request, + headers: { + "x-stack-disable-artificial-development-delay": "yes", + "x-stack-development-disable-extended-logging": "yes", + ...filterUndefined(request.headers ?? {}), + }, + }); + if (response.status !== expectedStatusCode) { + throw new StackAssertionError(deindent` + Expected status code ${expectedStatusCode} but got ${response.status} for ${endpoint}: + + ${await response.text()} + `, { request, response }); + } + const json = await response.json(); + return json; +} + +let lastProgress = performance.now() - 9999999999; + +type RecurseFunction = (progressPrefix: string, inner: (recurse: RecurseFunction) => Promise) => Promise; + +const _recurse = async (progressPrefix: string | ((...args: any[]) => void), inner: Parameters[1]): Promise => { + const progressFunc = typeof progressPrefix === "function" ? progressPrefix : (...args: any[]) => { + console.log(`${progressPrefix}`, ...args); + }; + if (performance.now() - lastProgress > 1000) { + progressFunc(); + lastProgress = performance.now(); + } + try { + return await inner( + (progressPrefix, inner) => _recurse( + (...args) => progressFunc(progressPrefix, ...args), + inner, + ), + ); + } catch (error) { + progressFunc(`\x1b[41mERROR\x1b[0m!`); + throw error; + } +}; +const recurse: RecurseFunction = _recurse; diff --git a/apps/backend/src/app/api/v1/integrations/neon/oauth-providers/crud.tsx b/apps/backend/src/app/api/v1/integrations/neon/oauth-providers/crud.tsx index 7fb670d5e..6ac180498 100644 --- a/apps/backend/src/app/api/v1/integrations/neon/oauth-providers/crud.tsx +++ b/apps/backend/src/app/api/v1/integrations/neon/oauth-providers/crud.tsx @@ -14,8 +14,14 @@ import * as yup from "yup"; const oauthProviderReadSchema = yupObject({ id: schemaFields.oauthIdSchema.defined(), type: schemaFields.oauthTypeSchema.defined(), - client_id: schemaFields.yupDefinedWhen(schemaFields.oauthClientIdSchema, 'type', 'standard'), - client_secret: schemaFields.yupDefinedWhen(schemaFields.oauthClientSecretSchema, 'type', 'standard'), + client_id: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientIdSchema, { + when: 'type', + is: 'standard', + }), + client_secret: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientSecretSchema, { + when: 'type', + is: 'standard', + }), // extra params facebook_config_id: schemaFields.oauthFacebookConfigIdSchema.optional(), @@ -24,8 +30,14 @@ const oauthProviderReadSchema = yupObject({ const oauthProviderUpdateSchema = yupObject({ type: schemaFields.oauthTypeSchema.optional(), - client_id: schemaFields.yupDefinedWhen(schemaFields.oauthClientIdSchema, 'type', 'standard').optional(), - client_secret: schemaFields.yupDefinedWhen(schemaFields.oauthClientSecretSchema, 'type', 'standard').optional(), + client_id: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientIdSchema, { + when: 'type', + is: 'standard', + }).optional(), + client_secret: schemaFields.yupDefinedAndNonEmptyWhen(schemaFields.oauthClientSecretSchema, { + when: 'type', + is: 'standard', + }).optional(), // extra params facebook_config_id: schemaFields.oauthFacebookConfigIdSchema.optional(), diff --git a/apps/backend/src/route-handlers/crud-handler.tsx b/apps/backend/src/route-handlers/crud-handler.tsx index ff597010b..5ff7302dd 100644 --- a/apps/backend/src/route-handlers/crud-handler.tsx +++ b/apps/backend/src/route-handlers/crud-handler.tsx @@ -292,7 +292,7 @@ async function validate(obj: unknown, schema: yup.ISchema, currentUser: Us Errors: ${error.errors.join("\n")} `, - { obj: JSON.stringify(obj), schema, cause: error }, + { obj: obj, schema, cause: error }, ); } throw error; diff --git a/apps/backend/src/route-handlers/smart-request.tsx b/apps/backend/src/route-handlers/smart-request.tsx index 10777f23e..e7c1c5379 100644 --- a/apps/backend/src/route-handlers/smart-request.tsx +++ b/apps/backend/src/route-handlers/smart-request.tsx @@ -9,6 +9,7 @@ import { ProjectsCrud } from "@stackframe/stack-shared/dist/interface/crud/proje import { UsersCrud } from "@stackframe/stack-shared/dist/interface/crud/users"; import { StackAdaptSentinel, yupValidate } from "@stackframe/stack-shared/dist/schema-fields"; import { groupBy, typedIncludes } from "@stackframe/stack-shared/dist/utils/arrays"; +import { getNodeEnvironment } from "@stackframe/stack-shared/dist/utils/env"; import { StackAssertionError, StatusError, throwErr } from "@stackframe/stack-shared/dist/utils/errors"; import { ignoreUnhandledRejection } from "@stackframe/stack-shared/dist/utils/promises"; import { deindent } from "@stackframe/stack-shared/dist/utils/strings"; @@ -145,6 +146,7 @@ async function parseAuth(req: NextRequest): Promise { const adminAccessToken = req.headers.get("x-stack-admin-access-token"); const accessToken = req.headers.get("x-stack-access-token"); const refreshToken = req.headers.get("x-stack-refresh-token"); + const developmentKeyOverride = req.headers.get("x-stack-development-override-key"); // in development, the internal project's API key can optionally be used to access any project const extractUserFromAccessToken = async (options: { token: string, projectId: string }) => { const result = await decodeAccessToken(options.token); @@ -212,7 +214,13 @@ async function parseAuth(req: NextRequest): Promise { if (!typedIncludes(["client", "server", "admin"] as const, requestType)) throw new KnownErrors.InvalidAccessType(requestType); if (!projectId) throw new KnownErrors.AccessTypeWithoutProjectId(requestType); - if (adminAccessToken) { + if (developmentKeyOverride) { + if (getNodeEnvironment() !== "development") { + throw new StatusError(401, "Development key override is only allowed in development mode"); + } + const result = await checkApiKeySet("internal", { superSecretAdminKey: developmentKeyOverride }); + if (!result) throw new StatusError(401, "Invalid development key override"); + } else if (adminAccessToken) { if (await queries.internalUser) { if (!await queries.project) { // this happens if the project is still in the user's managedProjectIds, but has since been deleted diff --git a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/auth-methods/providers.tsx b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/auth-methods/providers.tsx index fe236b650..7575723a3 100644 --- a/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/auth-methods/providers.tsx +++ b/apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/auth-methods/providers.tsx @@ -3,6 +3,7 @@ import { FormDialog } from "@/components/form-dialog"; import { InputField, SwitchField } from "@/components/form-fields"; import { SettingIconButton, SettingSwitch } from "@/components/settings"; import { AdminProject } from "@stackframe/stack"; +import { yupBoolean, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields"; import { sharedProviders } from "@stackframe/stack-shared/dist/utils/oauth"; import { runAsynchronously } from "@stackframe/stack-shared/dist/utils/promises"; import { ActionDialog, Badge, InlineCode, Label, SimpleTooltip, Typography } from "@stackframe/stack-ui"; @@ -31,22 +32,22 @@ function toTitle(id: string) { }[id]; } -export const providerFormSchema = yup.object({ - shared: yup.boolean().defined(), - clientId: yup.string() +export const providerFormSchema = yupObject({ + shared: yupBoolean().defined(), + clientId: yupString() .when('shared', { is: false, - then: (schema) => schema.defined(), + then: (schema) => schema.defined().nonEmpty(), otherwise: (schema) => schema.optional() }), - clientSecret: yup.string() + clientSecret: yupString() .when('shared', { is: false, - then: (schema) => schema.defined(), + then: (schema) => schema.defined().nonEmpty(), otherwise: (schema) => schema.optional() }), - facebookConfigId: yup.string().optional(), - microsoftTenantId: yup.string().optional(), + facebookConfigId: yupString().optional(), + microsoftTenantId: yupString().optional(), }); export type ProviderFormValues = yup.InferType @@ -104,7 +105,8 @@ export function ProviderSettingDialog(props: Props & { open: boolean, onClose: ( Shared keys are created by the Stack team for development. It helps you get started, but will show a Stack logo and name on the OAuth screen. This should never be enabled in production. :
-