diff --git a/package-lock.json b/package-lock.json index 7fed55c4..237b943e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "@tus/s3-store": "https://gitpkg.now.sh/supabase/tus-node-server/packages/s3-store/dist?build", "@tus/server": "https://gitpkg.now.sh/supabase/tus-node-server/packages/server/dist?build", "agentkeepalive": "^4.2.1", + "async-retry": "^1.3.3", "axios": "^0.27.2", "axios-retry": "^3.3.1", "connection-string": "^4.3.6", @@ -4606,6 +4607,14 @@ "node": ">=8" } }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "dependencies": { + "retry": "0.13.1" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -10091,6 +10100,14 @@ "node": ">=4" } }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "engines": { + "node": ">= 4" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -15151,6 +15168,14 @@ "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true }, + "async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "requires": { + "retry": "0.13.1" + } + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -19401,6 +19426,11 @@ "resolved": "https://registry.npmjs.org/ret/-/ret-0.2.2.tgz", "integrity": "sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ==" }, + "retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" + }, "reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", diff --git a/package.json b/package.json index 84935143..b7c610cc 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "@tus/s3-store": "https://gitpkg.now.sh/supabase/tus-node-server/packages/s3-store/dist?build", "@tus/server": "https://gitpkg.now.sh/supabase/tus-node-server/packages/server/dist?build", "agentkeepalive": "^4.2.1", + "async-retry": "^1.3.3", "axios": "^0.27.2", "axios-retry": "^3.3.1", "connection-string": "^4.3.6", diff --git a/src/database/connection.ts b/src/database/connection.ts index 9a447c3c..d829f5a5 100644 --- a/src/database/connection.ts +++ b/src/database/connection.ts @@ -1,8 +1,10 @@ -import pg from 'pg' +import pg, { DatabaseError } from 'pg' import { Knex, knex } from 'knex' import { JwtPayload } from 'jsonwebtoken' +import retry from 'async-retry' import { getConfig } from '../config' import { DbActiveConnection, DbActivePool } from '../monitoring/metrics' +import { StorageBackendError } from '../storage' // https://github.com/knex/knex/issues/387#issuecomment-51554522 pg.types.setTypeParser(20, 'text', parseInt) @@ -115,12 +117,40 @@ export class TenantConnection { } async transaction(instance?: Knex) { - const pool = instance || this.pool - const tnx = await pool.transaction() + const tnx = await retry( + async (bail) => { + try { + const pool = instance || this.pool + return await pool.transaction() + } catch (e) { + if ( + e instanceof DatabaseError && + e.code === '08P01' && + e.message.includes('no more connections allowed') + ) { + throw e + } + + bail(e as Error) + return + } + }, + { + minTimeout: 50, + maxTimeout: 500, + maxRetryTime: 2000, + retries: 10, + } + ) + + if (!tnx) { + throw new StorageBackendError('Could not create transaction', 500, 'transaction_failed') + } if (!instance && this.options.isExternalPool) { await tnx.raw(`SELECT set_config('search_path', ?, true)`, [searchPath.join(', ')]) } + return tnx }