diff --git a/lib/core-manager/index.js b/lib/core-manager/index.js index 646a7e099..b1858beaf 100644 --- a/lib/core-manager/index.js +++ b/lib/core-manager/index.js @@ -63,7 +63,7 @@ export class CoreManager extends TypedEmitter { projectKey, projectSecretKey, encryptionKeys = {}, - storage + storage, }) { super() assert( diff --git a/lib/core-manager/random-access-file-pool.js b/lib/core-manager/random-access-file-pool.js new file mode 100644 index 000000000..512b39407 --- /dev/null +++ b/lib/core-manager/random-access-file-pool.js @@ -0,0 +1,30 @@ +/** + * File descriptor pool for random-access-storage to limit the number of file + * descriptors used. Important particularly for Android where the hard limit for + * the app is 1024. + */ +export class RandomAccessFilePool { + /** @param {number} maxSize max number of file descriptors to use */ + constructor (maxSize) { + this.maxSize = maxSize + /** @type {Set} */ + this.active = new Set() + } + + /** @param {import('random-access-file')} file */ + _onactive (file) { + if (this.active.size >= this.maxSize) { + // suspend least recently inserted this manually iterates in insertion + // order, but only iterates to the first one (least recently inserted) + const toSuspend = this.active[Symbol.iterator]().next().value + toSuspend.suspend() + this.active.delete(toSuspend) + } + this.active.add(file) + } + + /** @param {import('random-access-file')} file */ + _oninactive (file) { + this.active.delete(file) + } +} diff --git a/package-lock.json b/package-lock.json index 69382456b..f57eca137 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "b4a": "^1.6.3", "base32.js": "^0.1.0", "better-sqlite3": "^8.3.0", - "corestore": "^6.5.2", + "corestore": "^6.8.4", "hypercore": "^10.9.0", "hypercore-crypto": "^3.3.1", "hyperdrive": "^11.0.0-alpha.10", @@ -45,8 +45,10 @@ "eslint-config-prettier": "^8.8.0", "nanobench": "^3.0.0", "prettier": "^2.8.8", + "random-access-file": "^4.0.4", "random-access-memory": "^6.2.0", "rimraf": "^5.0.0", + "tempy": "^3.1.0", "ts-proto": "^1.147.1", "type-fest": "^3.10.0", "typedoc": "^0.24.6", @@ -1420,13 +1422,15 @@ "license": "MIT" }, "node_modules/corestore": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/corestore/-/corestore-6.5.2.tgz", - "integrity": "sha512-/pwuChA30v4Uy3rzg3NmdT2PZMRstAMBQmGwPivkETv1rxC8qMt/btjMv9rv4MmanyTo+IHguFFumMEv8yrKjg==", + "version": "6.8.4", + "resolved": "https://registry.npmjs.org/corestore/-/corestore-6.8.4.tgz", + "integrity": "sha512-rJUn1bK2Id18mxZSb64fKGCSsbbBAvPUkSZVzsLB4Nnwhf3pkwxt/JjBvKHtsrvRyPAu9xtxUdUk1cSQ1JnOPw==", "dependencies": { "b4a": "^1.3.1", - "hypercore": "^10.5.3", + "hypercore": "^10.12.0", "hypercore-crypto": "^3.2.1", + "read-write-mutexify": "^2.1.0", + "ready-resource": "^1.0.0", "safety-catch": "^1.0.1", "sodium-universal": "^4.0.0", "xache": "^1.1.0" @@ -1473,6 +1477,33 @@ "node": ">= 8" } }, + "node_modules/crypto-random-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-4.0.0.tgz", + "integrity": "sha512-x8dy3RnvYdlUcPOjkEHqozhiwzKNSq7GcPuXFbnyMOCHxX8V3OgIg/pYuabl2sbUPfIJaeAQB7PMOK8DFIdoRA==", + "dev": true, + "dependencies": { + "type-fest": "^1.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/crypto-random-string/node_modules/type-fest": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-1.4.0.tgz", + "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dataloader": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/dataloader/-/dataloader-1.4.0.tgz", @@ -2258,9 +2289,9 @@ } }, "node_modules/hypercore": { - "version": "10.9.0", - "resolved": "https://registry.npmjs.org/hypercore/-/hypercore-10.9.0.tgz", - "integrity": "sha512-czXbJTqV1UL8A/SIWh6pFWtSkfGignK8oUX/5C4jzwn9pYkxhirjOW88nsmiNzsvnjSSSe2FDhkr8rqw7awtrA==", + "version": "10.17.0", + "resolved": "https://registry.npmjs.org/hypercore/-/hypercore-10.17.0.tgz", + "integrity": "sha512-Yp9lyfUjp81Gy1nPHemNFtYQtngliCGZ6prH+qxvjr2FPVG1QFtNqtOh+ZxFu4hXZ1dAOLYkQOA7Qq2vjFe64A==", "dependencies": { "@hyperswarm/secret-stream": "^6.0.0", "b4a": "^1.1.0", @@ -2272,7 +2303,7 @@ "hypercore-crypto": "^3.2.1", "is-options": "^1.0.1", "protomux": "^3.4.0", - "quickbit-universal": "^2.0.3", + "quickbit-universal": "^2.1.1", "random-access-file": "^4.0.0", "random-array-iterator": "^1.0.0", "safety-catch": "^1.0.1", @@ -2528,6 +2559,18 @@ "node": ">=8" } }, + "node_modules/is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -3582,18 +3625,34 @@ "version": "1.0.1", "license": "MIT" }, + "node_modules/quickbit-native": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/quickbit-native/-/quickbit-native-2.1.5.tgz", + "integrity": "sha512-8fpUjEjqsDoHZezYEbWemf2DCayE6i83GFeCUm6OjSrkole5zDVIgv7lEoE1By+d7wJQnDi1IMNto3xZoWwKmQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "b4a": "^1.6.0", + "napi-macros": "^2.0.0", + "node-gyp-build": "^4.2.3" + } + }, "node_modules/quickbit-universal": { - "version": "2.0.3", - "license": "ISC", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/quickbit-universal/-/quickbit-universal-2.1.2.tgz", + "integrity": "sha512-fPEQ5G9rWm5p0eaBMgTT6+YZ6Ftw2hHgPbZu/olFv/GfeH3lr4LwSL28MFo6L7My572O5fQOIVtngboVtwKU1Q==", "dependencies": { "b4a": "^1.6.0", - "node-gyp-build": "^4.5.0", "simdle-universal": "^1.1.0" + }, + "optionalDependencies": { + "quickbit-native": "^2.1.3" } }, "node_modules/random-access-file": { - "version": "4.0.0", - "license": "MIT", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/random-access-file/-/random-access-file-4.0.4.tgz", + "integrity": "sha512-1W21gZ8ne3RgPyTNpq8INr7feTY0+hPpV4X59yL9Miv5QiZV7U1QpRb/zEG2IuaojW9qVTeWBC19Ty0m0uqFBg==", "dependencies": { "random-access-storage": "^3.0.0" }, @@ -3644,6 +3703,11 @@ "node": ">=0.10.0" } }, + "node_modules/read-write-mutexify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/read-write-mutexify/-/read-write-mutexify-2.1.0.tgz", + "integrity": "sha512-fDw/p5/acI1ytVY1UbxEDma/ej1yJH/n9NcjS9YNzcE6sPBPWdlru3ydRa/UBowUg4zqOvNMD5SOGYJrlQ6MzQ==" + }, "node_modules/readable-stream": { "version": "3.6.0", "license": "MIT", @@ -3667,6 +3731,11 @@ "node": ">=8.10.0" } }, + "node_modules/ready-resource": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ready-resource/-/ready-resource-1.0.0.tgz", + "integrity": "sha512-9/Oj3DXv+QxWinvVcxVVRXn9Jj4b9wssv0PvQh5bO+N/vzqo6kmScUNl+faWWMEu0rYMPa6Tvp50+rP5ujZvqg==" + }, "node_modules/record-cache": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/record-cache/-/record-cache-1.2.0.tgz", @@ -3996,14 +4065,29 @@ "varint": "~5.0.0" } }, - "node_modules/simdle-universal": { - "version": "1.1.0", - "license": "ISC", + "node_modules/simdle-native": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/simdle-native/-/simdle-native-1.1.3.tgz", + "integrity": "sha512-KuD5YD9kyZ7kOEhPWNyQnL+NIGSmYnykjvQ6Yc4RBpJVYcl7bx4z9Jx3bODwVqNhOgbwXbZUOoFXol9/VG74EA==", + "hasInstallScript": true, + "optional": true, "dependencies": { "b4a": "^1.6.0", + "napi-macros": "^2.0.0", "node-gyp-build": "^4.2.3" } }, + "node_modules/simdle-universal": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/simdle-universal/-/simdle-universal-1.1.2.tgz", + "integrity": "sha512-3n3w1bs+uwgHKQjt6arez83EywNlhZzYvNOhvAASTl/8KqNIcqr6aHyGt3JRlfuUC7iB0tomJRPlJ2cRGIpBzA==", + "dependencies": { + "b4a": "^1.6.0" + }, + "optionalDependencies": { + "simdle-native": "^1.1.1" + } + }, "node_modules/simple-concat": { "version": "1.0.1", "funding": [ @@ -4269,6 +4353,45 @@ "node": ">=6" } }, + "node_modules/temp-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-3.0.0.tgz", + "integrity": "sha512-nHc6S/bwIilKHNRgK/3jlhDoIHcp45YgyiwcAk46Tr0LfEqGBVpmiAyuiuxeVE44m3mXnEeVhaipLOEWmH+Njw==", + "dev": true, + "engines": { + "node": ">=14.16" + } + }, + "node_modules/tempy": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tempy/-/tempy-3.1.0.tgz", + "integrity": "sha512-7jDLIdD2Zp0bDe5r3D2qtkd1QOCacylBuL7oa4udvN6v2pqr4+LcCr67C8DR1zkpaZ8XosF5m1yQSabKAW6f2g==", + "dev": true, + "dependencies": { + "is-stream": "^3.0.0", + "temp-dir": "^3.0.0", + "type-fest": "^2.12.2", + "unique-string": "^3.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tempy/node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "dev": true, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "dev": true, @@ -4579,6 +4702,21 @@ "node": ">=0.8.0" } }, + "node_modules/unique-string": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-3.0.0.tgz", + "integrity": "sha512-VGXBUVwxKMBUznyffQweQABPRRW1vHZAbadFZud4pLFAqRGvv/96vafgjWFqzourzr8YonlQiPgH0YCJfawoGQ==", + "dev": true, + "dependencies": { + "crypto-random-string": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/universalify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", diff --git a/package.json b/package.json index b9a1cb2fa..c1f0284d7 100644 --- a/package.json +++ b/package.json @@ -54,8 +54,10 @@ "eslint-config-prettier": "^8.8.0", "nanobench": "^3.0.0", "prettier": "^2.8.8", + "random-access-file": "^4.0.4", "random-access-memory": "^6.2.0", "rimraf": "^5.0.0", + "tempy": "^3.1.0", "ts-proto": "^1.147.1", "type-fest": "^3.10.0", "typedoc": "^0.24.6", @@ -69,7 +71,7 @@ "b4a": "^1.6.3", "base32.js": "^0.1.0", "better-sqlite3": "^8.3.0", - "corestore": "^6.5.2", + "corestore": "^6.8.4", "hypercore": "^10.9.0", "hypercore-crypto": "^3.3.1", "hyperdrive": "^11.0.0-alpha.10", diff --git a/tests/core-manager.js b/tests/core-manager.js index 701f82f6a..9a1dd2c68 100644 --- a/tests/core-manager.js +++ b/tests/core-manager.js @@ -8,6 +8,11 @@ import Sqlite from 'better-sqlite3' import { KeyManager } from '@mapeo/crypto' import { CoreManager } from '../lib/core-manager/index.js' import assert from 'assert' +import { temporaryDirectoryTask } from 'tempy' +import { exec } from 'child_process' +import { RandomAccessFilePool } from '../lib/core-manager/random-access-file-pool.js' +import RandomAccessFile from 'random-access-file' +import path from 'path' async function createCore (...args) { const core = new Hypercore(RAM, ...args) @@ -324,6 +329,63 @@ test('encryption', async function (t) { } }) +test('poolSize limits number of open file descriptors', async function (t) { + const keyManager = new KeyManager(randomBytes(16)) + const { publicKey: projectKey, secretKey: projectSecretKey } = + keyManager.getHypercoreKeypair('auth', randomBytes(32)) + + const CORE_COUNT = 500 + await temporaryDirectoryTask(async tempPath => { + const db = new Sqlite(':memory:') + const storage = name => new RandomAccessFile(path.join(tempPath, name)) + const cm = new CoreManager({ + db, + keyManager, + storage, + projectKey, + projectSecretKey + }) + // -1 because CoreManager creates a writer core already + for (let i = 0; i < CORE_COUNT - 1; i++) { + const coreKey = randomBytes(32) + cm.addCore(coreKey, 'data') + } + const readyPromises = cm.getCores('data').map(({ core }) => core.ready()) + t.is(readyPromises.length, CORE_COUNT) + await Promise.all(readyPromises) + const fdCount = await countOpenFileDescriptors(tempPath) + t.ok(fdCount > CORE_COUNT, 'without pool, at least one fd per core') + }) + + await temporaryDirectoryTask(async tempPath => { + const POOL_SIZE = 100 + const db = new Sqlite(':memory:') + const pool = new RandomAccessFilePool(POOL_SIZE) + const storage = name => + new RandomAccessFile(path.join(tempPath, name), { pool }) + const cm = new CoreManager({ + db, + keyManager, + storage, + projectKey, + projectSecretKey + }) + // -1 because we CoreManager creates a writer core already + for (let i = 0; i < CORE_COUNT - 1; i++) { + const coreKey = randomBytes(32) + cm.addCore(coreKey, 'data') + } + const readyPromises = cm.getCores('data').map(({ core }) => core.ready()) + await Promise.all(readyPromises) + const fdCount = await countOpenFileDescriptors(tempPath) + t.is( + fdCount, + POOL_SIZE, + 'with pool, no more file descriptors than pool size' + ) + }) +}) + async function waitForCores (coreManager, keys) { const allKeys = getAllKeys(coreManager) if (hasKeys(keys, allKeys)) return @@ -388,3 +450,18 @@ function bitfieldEquals (actual, expected, len) { return true } } + +/** + * Count the open file descriptors in a given folder + * + * @param {string} dir folder for counting open file descriptors + * @returns {Promise} + */ +async function countOpenFileDescriptors (dir) { + return new Promise((res, rej) => { + exec(`lsof +D '${dir}' | wc -l`, (error, stdout) => { + if (error) return rej(error) + res(stdout - 1) + }) + }) +} diff --git a/types/corestore.d.ts b/types/corestore.d.ts index b3a572b7f..23c47759b 100644 --- a/types/corestore.d.ts +++ b/types/corestore.d.ts @@ -14,7 +14,7 @@ declare module 'corestore' { class Corestore extends TypedEmitter { constructor( storage: HypercoreStorage, - options?: { primaryKey?: Buffer | Uint8Array } + options?: { primaryKey?: Buffer | Uint8Array, poolSize?: number } ) get(key: Buffer | Uint8Array): Hypercore get(