diff --git a/tools/syn2mas/package-lock.json b/tools/syn2mas/package-lock.json index 2e1598da..23423143 100644 --- a/tools/syn2mas/package-lock.json +++ b/tools/syn2mas/package-lock.json @@ -14,6 +14,7 @@ "knex": "^3.0.1", "log4js": "^6.9.1", "pg": "^8.11.3", + "pg-query-stream": "^4.6.0", "sqlite3": "^5.1.6", "ts-command-line-args": "^2.5.1", "yaml": "^2.3.3", @@ -5555,6 +5556,15 @@ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" }, + "node_modules/pg-cursor": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.11.0.tgz", + "integrity": "sha512-TLCOCtu+rqMarzjUi+/Ffc2DV5ZqO/27y5GqnK9Z3w51rWXMwC8FcO96Uf9/ORo5o+qRXEVJxM9Ts3K2K31MLg==", + "license": "MIT", + "peerDependencies": { + "pg": "^8" + } + }, "node_modules/pg-int8": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", @@ -5576,6 +5586,18 @@ "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" }, + "node_modules/pg-query-stream": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/pg-query-stream/-/pg-query-stream-4.6.0.tgz", + "integrity": "sha512-sg2Hewe6ge6osEY07zGu7Z8djrsQBvyiTy5ZjQffoSatEgnNNVsV3EWDm9Px/8R9oaAL1YnfnP8AXPMmfzujZg==", + "license": "MIT", + "dependencies": { + "pg-cursor": "^2.11.0" + }, + "peerDependencies": { + "pg": "^8" + } + }, "node_modules/pg-types": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", diff --git a/tools/syn2mas/package.json b/tools/syn2mas/package.json index c4b953f6..7817d87e 100644 --- a/tools/syn2mas/package.json +++ b/tools/syn2mas/package.json @@ -57,6 +57,7 @@ "knex": "^3.0.1", "log4js": "^6.9.1", "pg": "^8.11.3", + "pg-query-stream": "^4.6.0", "sqlite3": "^5.1.6", "ts-command-line-args": "^2.5.1", "yaml": "^2.3.3", diff --git a/tools/syn2mas/src/migrate.mts b/tools/syn2mas/src/migrate.mts index 9f8a6bcf..fe1beddb 100644 --- a/tools/syn2mas/src/migrate.mts +++ b/tools/syn2mas/src/migrate.mts @@ -192,13 +192,7 @@ export async function migrate(): Promise { ); } - // Get all Synapse users, except appservice owned users who don't need to be migrated - const synapseUsers = await synapse - .select("*") - .from("users") - .whereNull("appservice_id"); - log.info(`Found ${synapseUsers.length} users in Synapse`); - for (const user of synapseUsers) { + async function migrateUser(user: SUser): Promise { const localpart = user.name.split(":")[0].substring(1); log.info(`Processing user ${user.name} as ${localpart}`); @@ -430,10 +424,43 @@ export async function migrate(): Promise { } } } + + // this is a workaround to get the list of columns that we care about from the SUser type + const SUserColumns: Record = { + name: undefined, + password_hash: undefined, + admin: undefined, + is_guest: undefined, + deactivated: undefined, + creation_ts: undefined, + appservice_id: undefined, + }; + + // Get all Synapse users, except appservice owned users who don't need to be migrated + const synapseUserQuery = synapse + .select(Object.keys(SUserColumns) as (keyof SUser)[]) + .from("users") + .whereNull("appservice_id"); + + let synapseUsers = 0; + if (synapseConfig.database.name === "sqlite3") { + // SQLite doesn't support streaming + const synapseUserRows = (await synapseUserQuery) as unknown as SUser[]; + for (const user of synapseUserRows) { + synapseUsers += 1; + await migrateUser(user); + } + } else { + // Stream users from the database + const synapseUserStream = synapseUserQuery.stream(); + for await (const user of synapseUserStream) { + synapseUsers += 1; + await migrateUser(user as unknown as SUser); + } + } + log.info( - `Completed migration ${args.dryRun ? "dry-run " : ""}of ${ - synapseUsers.length - } users with ${fatals} fatals and ${warnings.length} warnings:`, + `Completed migration ${args.dryRun ? "dry-run " : ""}of ${synapseUsers} users with ${fatals} fatals and ${warnings.length} warnings:`, ); warnings.forEach((w) => log.warn(w)); if (fatals > 0) {