From 1b6ccaf701042fcde26e7b9e11d1bfbdf6130a78 Mon Sep 17 00:00:00 2001 From: sivanov Date: Tue, 17 Oct 2023 12:21:05 +0300 Subject: [PATCH 1/8] No pause between download --- lib/src/auth/providers.ts | 1 - lib/tdf3/src/client/builders.ts | 2 +- lib/tdf3/src/tdf.ts | 116 ++++++++++++++++++-------------- 3 files changed, 66 insertions(+), 53 deletions(-) diff --git a/lib/src/auth/providers.ts b/lib/src/auth/providers.ts index 2c97c883..3601aef9 100644 --- a/lib/src/auth/providers.ts +++ b/lib/src/auth/providers.ts @@ -134,7 +134,6 @@ export const clientAuthProvider = async (clientConfig: OIDCCredentials): Promise }; export * from './auth.js'; -export * from './OIDCCredentials.js'; export { OIDCClientCredentialsProvider } from './oidc-clientcredentials-provider.js'; export { OIDCExternalJwtProvider } from './oidc-externaljwt-provider.js'; export { OIDCRefreshTokenProvider } from './oidc-refreshtoken-provider.js'; diff --git a/lib/tdf3/src/client/builders.ts b/lib/tdf3/src/client/builders.ts index 94a0d93a..87610453 100644 --- a/lib/tdf3/src/client/builders.ts +++ b/lib/tdf3/src/client/builders.ts @@ -505,7 +505,7 @@ export type DecryptParams = { */ class DecryptParamsBuilder { - private _params: Partial; + _params: Partial; constructor(to_copy: Partial = {}) { this._params = { diff --git a/lib/tdf3/src/tdf.ts b/lib/tdf3/src/tdf.ts index 1faa74f1..00d6f26c 100644 --- a/lib/tdf3/src/tdf.ts +++ b/lib/tdf3/src/tdf.ts @@ -965,60 +965,74 @@ export class TDF extends EventEmitter { zipReader: ZipReader, reconstructedKeyBinary: Binary ) { - const requestsInParallelCount = 100; - let requests = []; - const maxLength = 3; - - for (let i = 0; i < chunkMap.length; i += requestsInParallelCount) { - if (requests.length === maxLength) { - await Promise.all(requests); - requests = []; + const chunksInOneDownload = 500; + + for (let i = 0; i < chunkMap.length; i += chunksInOneDownload) { + try { + const slice = chunkMap.slice(i, i + chunksInOneDownload); + const bufferSize = slice.reduce( + (currentVal, { encryptedSegmentSize }) => + currentVal + (encryptedSegmentSize as number), + 0 + ); + const start = performance.now(); + const buffer: Uint8Array | null = await zipReader.getPayloadSegment( + centralDirectory, + '0.payload', + slice[0].encryptedOffset, + bufferSize + ) + if (buffer) { + this.sliceAndDecrypt({ buffer, reconstructedKeyBinary, slice }) + } + const end = performance.now(); + console.log(`time for one group chunk to download is ${end - start}`) + } catch (e) { + throw new TdfDecryptError( + 'Error decrypting payload. This suggests the key used to decrypt the payload is not correct.', + e + ); } - requests.push( - (async () => { - try { - const slice = chunkMap.slice(i, i + requestsInParallelCount); - const bufferSize = slice.reduce( - (currentVal, { encryptedSegmentSize }) => - currentVal + (encryptedSegmentSize as number), - 0 - ); - let buffer: Uint8Array | null = await zipReader.getPayloadSegment( - centralDirectory, - '0.payload', - slice[0].encryptedOffset, - bufferSize - ); - for (const index in slice) { - const { encryptedOffset, encryptedSegmentSize } = slice[index]; - - const offset = - slice[0].encryptedOffset === 0 - ? encryptedOffset - : encryptedOffset % slice[0].encryptedOffset; - const encryptedChunk = new Uint8Array( - buffer.slice(offset, offset + (encryptedSegmentSize as number)) - ); - - slice[index].decryptedChunk = await this.decryptChunk( - encryptedChunk, - reconstructedKeyBinary, - slice[index]['hash'] - ); - if (slice[index]._resolve) { - (slice[index]._resolve as (value: unknown) => void)(null); - } - } - buffer = null; - } catch (e) { - throw new TdfDecryptError( - 'Error decrypting payload. This suggests the key used to decrypt the payload is not correct.', - e - ); - } - })() + } + } + + async sliceAndDecrypt({ buffer, reconstructedKeyBinary, slice }: { buffer: Uint8Array; reconstructedKeyBinary: Binary; slice: Chunk[] }) { + const start = performance.now(); + + const batchSize = 10; // Start with 10 as a batch size + let promises = []; + + for (const index in slice) { + const { encryptedOffset, encryptedSegmentSize } = slice[index]; + + const offset = + slice[0].encryptedOffset === 0 + ? encryptedOffset + : encryptedOffset % slice[0].encryptedOffset; + const encryptedChunk = new Uint8Array( + buffer.slice(offset, offset + (encryptedSegmentSize as number)) ); + + promises.push( + this.decryptChunk( + encryptedChunk, + reconstructedKeyBinary, + slice[index]['hash'] + ).then((decryptedChunk) => { + if (!decryptedChunk) return; + slice[index].decryptedChunk = decryptedChunk; + if (slice[index]._resolve) { + (slice[index]._resolve as (value: unknown) => void)(null); + } + }) + ) + if (promises.length === batchSize) { + await Promise.all(promises); + promises = []; + } } + const end = performance.now(); + console.log(`time for one group chunk to decrypt is ${end - start}`) } /** From 03e9a73ed60aca2101c323c89cc29f3fb8d5a13f Mon Sep 17 00:00:00 2001 From: sivanov Date: Tue, 17 Oct 2023 14:10:44 +0300 Subject: [PATCH 2/8] working but 20s for decrypt --- lib/package.json | 2 +- lib/tdf3/src/crypto/index.ts | 48 ++++++++++++++++++++++++++++++------ lib/tdf3/types.d.ts | 1 + 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/lib/package.json b/lib/package.json index ca9abc14..7ce6c6c4 100644 --- a/lib/package.json +++ b/lib/package.json @@ -1,6 +1,6 @@ { "name": "@opentdf/client", - "version": "2.0.0", + "version": "2.0.11", "description": "Access and generate tdf protected content", "homepage": "https://github.com/opentdf/client-web", "bugs": { diff --git a/lib/tdf3/src/crypto/index.ts b/lib/tdf3/src/crypto/index.ts index be20fc6f..d0424edc 100644 --- a/lib/tdf3/src/crypto/index.ts +++ b/lib/tdf3/src/crypto/index.ts @@ -29,6 +29,25 @@ export const isSupported = typeof globalThis?.crypto !== 'undefined'; export const method = 'http://www.w3.org/2001/04/xmlenc#aes256-cbc'; export const name = 'BrowserNativeCryptoService'; +window.activeWorkers = new Set(); + +const workerScript = ` +self.onmessage = async (event) => { + const { key, encryptedPayload, algo } = event.data; + + try { + const decryptedData = await crypto.subtle.decrypt( + algo, + key, + encryptedPayload + ); + self.postMessage({ success: true, data: decryptedData }); + } catch (error) { + self.postMessage({ success: false, error: error.message }); + } +}; +`; + /** * Get a DOMString representing the algorithm to use for an * asymmetric key generation. @@ -274,17 +293,30 @@ async function _doDecrypt( const importedKey = await _importKey(key, algoDomString); algoDomString.iv = iv.asArrayBuffer(); - const decrypted = await crypto.subtle - .decrypt(algoDomString, importedKey, payloadBuffer) - // Catching this error so we can specifically check for OperationError - .catch((err) => { - if (err.name === 'OperationError') { - throw new TdfDecryptError(err); + const workerBlob = new Blob([workerScript], { type: 'application/javascript' }); + const worker = new Worker(URL.createObjectURL(workerBlob)); + window.activeWorkers.add(worker); + console.log(`Number of active workers: ${window.activeWorkers.size}`); + + return new Promise((resolve, reject) => { + worker.onmessage = (event) => { + const { success, data, error } = event.data; + worker.terminate(); + window.activeWorkers.delete(worker); + console.log(`Number of active workers: ${window.activeWorkers.size}`); + if (success) { + resolve({ payload: Binary.fromArrayBuffer(data) } as DecryptResult); + } else { + reject(new TdfDecryptError(error)); } + }; - throw err; + worker.postMessage({ + key: importedKey, + encryptedPayload: payloadBuffer, + algo: algoDomString, }); - return { payload: Binary.fromArrayBuffer(decrypted) }; + }); } function _importKey(key: Binary, algorithm: AesCbcParams | AesGcmParams) { diff --git a/lib/tdf3/types.d.ts b/lib/tdf3/types.d.ts index 78d139fe..42d50701 100644 --- a/lib/tdf3/types.d.ts +++ b/lib/tdf3/types.d.ts @@ -2,6 +2,7 @@ declare global { interface Window { TDF: unknown; InstallTrigger: unknown; + activeWorkers: Set; } interface Crypto { From 4334c0992c560c6dff942a58d6424c716f1e657a Mon Sep 17 00:00:00 2001 From: sivanov Date: Fri, 17 Nov 2023 11:55:58 +0200 Subject: [PATCH 3/8] worker optimised --- lib/tdf3/src/crypto/decrypt-worker.ts | 77 +++++++++++++++++++++++++++ lib/tdf3/src/crypto/index.ts | 59 ++++++-------------- 2 files changed, 93 insertions(+), 43 deletions(-) create mode 100644 lib/tdf3/src/crypto/decrypt-worker.ts diff --git a/lib/tdf3/src/crypto/decrypt-worker.ts b/lib/tdf3/src/crypto/decrypt-worker.ts new file mode 100644 index 00000000..467f2503 --- /dev/null +++ b/lib/tdf3/src/crypto/decrypt-worker.ts @@ -0,0 +1,77 @@ +import { TdfDecryptError } from '../errors.js'; + +const maxWorkers = navigator?.hardwareConcurrency || 4; + +interface DecryptData { + key: CryptoKey; + encryptedPayload: ArrayBuffer; + algo: AesCbcParams | AesGcmParams; +} + +const workerScript = async (event: { data: DecryptData }) => { + const { key, encryptedPayload, algo } = event.data; + + try { + const decryptedData = await crypto.subtle.decrypt( + algo, + key, + encryptedPayload + ); + self.postMessage({ success: true, data: decryptedData }); + } catch (error) { + self.postMessage({ success: false, error: error.message }); + } +}; + +const workerBlob = new Blob([`(${workerScript.toString()})()`], { type: 'application/javascript' }); +const workerUrl = URL.createObjectURL(workerBlob); +const workersArray: Worker[] = new Array(maxWorkers).fill(new Worker(workerUrl)); + +interface WorkersQueue { + freeWorkers: Worker[]; + resolvers: ((worker: Worker) => void)[]; + push: (worker: Worker) => void; + pop: () => Promise, +} + +const workersQueue: WorkersQueue = { + freeWorkers: [...workersArray], + resolvers: [], + + push(worker: Worker) { + const resolve = this.resolvers.shift(); + if (typeof resolve === 'function') { + resolve(worker); + } else { + this.freeWorkers.push(worker); + } + }, + + pop(): Promise { + return new Promise((resolve) => { + const worker = this.freeWorkers.shift(); + if (worker instanceof Worker) { + resolve(worker); + } else { + this.resolvers.push(resolve); + } + }); + } +} + +export async function decrypt(data: DecryptData): Promise { + const worker: Worker = await workersQueue.pop(); + return await new Promise((resolve, reject) => { + worker.onmessage = (event) => { + const { success, data, error } = event.data; + workersQueue.push(worker); + if (success) { + resolve((data as ArrayBuffer)); + } else { + reject(new TdfDecryptError(error)); + } + }; + + worker.postMessage(data); + }); +}; \ No newline at end of file diff --git a/lib/tdf3/src/crypto/index.ts b/lib/tdf3/src/crypto/index.ts index d0424edc..72b8033f 100644 --- a/lib/tdf3/src/crypto/index.ts +++ b/lib/tdf3/src/crypto/index.ts @@ -5,6 +5,7 @@ */ import { Algorithms } from '../ciphers/index.js'; +import { decrypt as workerDecrypt } from './decrypt-worker.js'; import { Binary } from '../binary.js'; import { CryptoService, @@ -29,25 +30,6 @@ export const isSupported = typeof globalThis?.crypto !== 'undefined'; export const method = 'http://www.w3.org/2001/04/xmlenc#aes256-cbc'; export const name = 'BrowserNativeCryptoService'; -window.activeWorkers = new Set(); - -const workerScript = ` -self.onmessage = async (event) => { - const { key, encryptedPayload, algo } = event.data; - - try { - const decryptedData = await crypto.subtle.decrypt( - algo, - key, - encryptedPayload - ); - self.postMessage({ success: true, data: decryptedData }); - } catch (error) { - self.postMessage({ success: false, error: error.message }); - } -}; -`; - /** * Get a DOMString representing the algorithm to use for an * asymmetric key generation. @@ -293,30 +275,21 @@ async function _doDecrypt( const importedKey = await _importKey(key, algoDomString); algoDomString.iv = iv.asArrayBuffer(); - const workerBlob = new Blob([workerScript], { type: 'application/javascript' }); - const worker = new Worker(URL.createObjectURL(workerBlob)); - window.activeWorkers.add(worker); - console.log(`Number of active workers: ${window.activeWorkers.size}`); - - return new Promise((resolve, reject) => { - worker.onmessage = (event) => { - const { success, data, error } = event.data; - worker.terminate(); - window.activeWorkers.delete(worker); - console.log(`Number of active workers: ${window.activeWorkers.size}`); - if (success) { - resolve({ payload: Binary.fromArrayBuffer(data) } as DecryptResult); - } else { - reject(new TdfDecryptError(error)); - } - }; - - worker.postMessage({ - key: importedKey, - encryptedPayload: payloadBuffer, - algo: algoDomString, - }); - }); + try { + const decrypted = ( + navigator?.hardwareConcurrency + ? await workerDecrypt({ key: importedKey, encryptedPayload: payloadBuffer, algo: algoDomString }) + : await crypto.subtle.decrypt(algoDomString, importedKey, payloadBuffer) + ) + + return { payload: Binary.fromArrayBuffer(decrypted) }; + } catch (err) { + if (err.name === 'OperationError') { + throw new TdfDecryptError(err); + } + + throw err; + } } function _importKey(key: Binary, algorithm: AesCbcParams | AesGcmParams) { From c9f28a782e3e3c8c9a7f4b49a1d159a4c2b331af Mon Sep 17 00:00:00 2001 From: sivanov Date: Fri, 17 Nov 2023 18:47:19 +0200 Subject: [PATCH 4/8] worker optimised and bug fixed --- lib/package-lock.json | 85 ++++++++++++++++++++------- lib/package.json | 3 +- lib/tdf3/src/crypto/decrypt-worker.ts | 30 +++++----- lib/tdf3/src/tdf.ts | 42 +++++++++---- 4 files changed, 108 insertions(+), 52 deletions(-) diff --git a/lib/package-lock.json b/lib/package-lock.json index aaba0672..54d63118 100644 --- a/lib/package-lock.json +++ b/lib/package-lock.json @@ -1,12 +1,12 @@ { "name": "@opentdf/client", - "version": "2.0.0", + "version": "2.0.19", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@opentdf/client", - "version": "2.0.0", + "version": "2.0.19", "license": "BSD-3-Clause-Clear", "dependencies": { "ajv": "^8.12.0", @@ -19,6 +19,7 @@ "dpop": "^1.2.0", "events": "^3.3.0", "jose": "^4.14.4", + "p-limit": "^5.0.0", "streamsaver": "^2.0.6", "uuid": "~9.0.0" }, @@ -8232,15 +8233,14 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", + "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.0.0" }, "engines": { - "node": ">=10" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -8261,6 +8261,33 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-map": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", @@ -11154,12 +11181,11 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -17135,12 +17161,11 @@ } }, "p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", + "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", "requires": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.0.0" } }, "p-locate": { @@ -17150,6 +17175,23 @@ "dev": true, "requires": { "p-limit": "^3.0.2" + }, + "dependencies": { + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true + } } }, "p-map": { @@ -19217,10 +19259,9 @@ "dev": true }, "yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==" } } } diff --git a/lib/package.json b/lib/package.json index 7652fcfc..d97e16df 100644 --- a/lib/package.json +++ b/lib/package.json @@ -1,6 +1,6 @@ { "name": "@opentdf/client", - "version": "2.0.11", + "version": "2.0.22", "description": "Access and generate tdf protected content", "homepage": "https://github.com/opentdf/client-web", "bugs": { @@ -69,6 +69,7 @@ "dpop": "^1.2.0", "events": "^3.3.0", "jose": "^4.14.4", + "p-limit": "^5.0.0", "streamsaver": "^2.0.6", "uuid": "~9.0.0" }, diff --git a/lib/tdf3/src/crypto/decrypt-worker.ts b/lib/tdf3/src/crypto/decrypt-worker.ts index 467f2503..dfa94efa 100644 --- a/lib/tdf3/src/crypto/decrypt-worker.ts +++ b/lib/tdf3/src/crypto/decrypt-worker.ts @@ -1,4 +1,4 @@ -import { TdfDecryptError } from '../errors.js'; +import { TdfDecryptError } from '../../../src/errors.js'; const maxWorkers = navigator?.hardwareConcurrency || 4; @@ -8,24 +8,22 @@ interface DecryptData { algo: AesCbcParams | AesGcmParams; } -const workerScript = async (event: { data: DecryptData }) => { - const { key, encryptedPayload, algo } = event.data; +const workerScript = ` + self.onmessage = async (event) => { + const { key, encryptedPayload, algo } = event.data; - try { - const decryptedData = await crypto.subtle.decrypt( - algo, - key, - encryptedPayload - ); - self.postMessage({ success: true, data: decryptedData }); - } catch (error) { - self.postMessage({ success: false, error: error.message }); - } -}; + try { + const decryptedData = await crypto.subtle.decrypt(algo, key, encryptedPayload); + self.postMessage({ success: true, data: decryptedData }); + } catch (error) { + self.postMessage({ success: false, error: error.message }); + } + }; +`; -const workerBlob = new Blob([`(${workerScript.toString()})()`], { type: 'application/javascript' }); +const workerBlob = new Blob([workerScript], { type: 'application/javascript' }); const workerUrl = URL.createObjectURL(workerBlob); -const workersArray: Worker[] = new Array(maxWorkers).fill(new Worker(workerUrl)); +const workersArray: Worker[] = Array.from({ length: maxWorkers }, () => new Worker(workerUrl)); interface WorkersQueue { freeWorkers: Worker[]; diff --git a/lib/tdf3/src/tdf.ts b/lib/tdf3/src/tdf.ts index 027ba2c3..699b9622 100644 --- a/lib/tdf3/src/tdf.ts +++ b/lib/tdf3/src/tdf.ts @@ -5,6 +5,7 @@ import { DecoratedReadableStream } from './client/DecoratedReadableStream.js'; import { EntityObject } from '../../src/tdf/EntityObject.js'; import { validateSecureUrl } from '../../src/utils.js'; import { DecryptParams } from './client/builders.js'; +// import pLimit from 'p-limit'; import { AttributeSet, @@ -217,7 +218,7 @@ export async function fetchKasPublicKey( const response: { data: string | KasPublicKeyInfo } = await axios.get(`${kas}/kas_public_key`, { params: { ...params, - v: '2', + // v: '2', }, }); const publicKey = @@ -981,6 +982,10 @@ export async function sliceAndDecrypt({ cryptoService: CryptoService; segmentIntegrityAlgorithm: IntegrityAlgorithm; }) { + // const limit = pLimit(100); + + const promises = []; + for (const index in slice) { const { encryptedOffset, encryptedSegmentSize } = slice[index]; @@ -990,18 +995,28 @@ export async function sliceAndDecrypt({ buffer.slice(offset, offset + (encryptedSegmentSize as number)) ); - slice[index].decryptedChunk = await decryptChunk( - encryptedChunk, - reconstructedKeyBinary, - slice[index]['hash'], - cipher, - segmentIntegrityAlgorithm, - cryptoService - ); - if (slice[index]._resolve) { - (slice[index]._resolve as (value: unknown) => void)(null); - } + promises.push( + // limit( + // () => ( + decryptChunk( + encryptedChunk, + reconstructedKeyBinary, + slice[index]['hash'], + cipher, + segmentIntegrityAlgorithm, + cryptoService + ).then((decryptedChunk) => { + if (!decryptedChunk) return; + slice[index].decryptedChunk = decryptedChunk; + if (slice[index]._resolve) { + (slice[index]._resolve as (value: unknown) => void)(null); + } + }) + // ) + // ) + ) } + await Promise.all(promises); } export async function readStream(cfg: DecryptConfiguration) { @@ -1061,7 +1076,8 @@ export async function readStream(cfg: DecryptConfiguration) { const cipher = new AesGcmCipher(cfg.cryptoService); - await updateChunkQueue( + // no await here + updateChunkQueue( Array.from(chunkMap.values()), centralDirectory, zipReader, From a10b559793b12e20f7fa9933c8c08948c1686740 Mon Sep 17 00:00:00 2001 From: sivanov Date: Mon, 20 Nov 2023 17:28:18 +0200 Subject: [PATCH 5/8] Optimized worker decrypt --- lib/package.json | 2 +- lib/tdf3/src/crypto/decrypt-worker.ts | 2 +- lib/tdf3/src/tdf.ts | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/package.json b/lib/package.json index d97e16df..15d239c8 100644 --- a/lib/package.json +++ b/lib/package.json @@ -1,6 +1,6 @@ { "name": "@opentdf/client", - "version": "2.0.22", + "version": "2.0.0", "description": "Access and generate tdf protected content", "homepage": "https://github.com/opentdf/client-web", "bugs": { diff --git a/lib/tdf3/src/crypto/decrypt-worker.ts b/lib/tdf3/src/crypto/decrypt-worker.ts index dfa94efa..d33665ad 100644 --- a/lib/tdf3/src/crypto/decrypt-worker.ts +++ b/lib/tdf3/src/crypto/decrypt-worker.ts @@ -1,6 +1,6 @@ import { TdfDecryptError } from '../../../src/errors.js'; -const maxWorkers = navigator?.hardwareConcurrency || 4; +const maxWorkers = navigator?.hardwareConcurrency || 4; // save fallback number interface DecryptData { key: CryptoKey; diff --git a/lib/tdf3/src/tdf.ts b/lib/tdf3/src/tdf.ts index 699b9622..88029c2f 100644 --- a/lib/tdf3/src/tdf.ts +++ b/lib/tdf3/src/tdf.ts @@ -5,7 +5,7 @@ import { DecoratedReadableStream } from './client/DecoratedReadableStream.js'; import { EntityObject } from '../../src/tdf/EntityObject.js'; import { validateSecureUrl } from '../../src/utils.js'; import { DecryptParams } from './client/builders.js'; -// import pLimit from 'p-limit'; +import pLimit from 'p-limit'; import { AttributeSet, @@ -923,7 +923,7 @@ async function updateChunkQueue( segmentIntegrityAlgorithm: IntegrityAlgorithm, cryptoService: CryptoService ) { - const chunksInOneDownload = 500; + const chunksInOneDownload = 300; let requests = []; const maxLength = 3; @@ -982,7 +982,7 @@ export async function sliceAndDecrypt({ cryptoService: CryptoService; segmentIntegrityAlgorithm: IntegrityAlgorithm; }) { - // const limit = pLimit(100); + const limit = pLimit(navigator.hardwareConcurrency || 4); // save fallback number const promises = []; @@ -996,8 +996,8 @@ export async function sliceAndDecrypt({ ); promises.push( - // limit( - // () => ( + limit( + () => ( decryptChunk( encryptedChunk, reconstructedKeyBinary, @@ -1012,8 +1012,8 @@ export async function sliceAndDecrypt({ (slice[index]._resolve as (value: unknown) => void)(null); } }) - // ) - // ) + ) + ) ) } await Promise.all(promises); From 223e9a080e26bacc44f4361914d0955632817ca9 Mon Sep 17 00:00:00 2001 From: ivanovSPvirtru Date: Mon, 20 Nov 2023 15:30:00 +0000 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=A4=96=20=F0=9F=8E=A8=20Autoformat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: sivanov --- lib/tdf3/src/crypto/decrypt-worker.ts | 10 ++++---- lib/tdf3/src/crypto/index.ts | 12 ++++++---- lib/tdf3/src/tdf.ts | 34 +++++++++++++-------------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/lib/tdf3/src/crypto/decrypt-worker.ts b/lib/tdf3/src/crypto/decrypt-worker.ts index d33665ad..7f328c0c 100644 --- a/lib/tdf3/src/crypto/decrypt-worker.ts +++ b/lib/tdf3/src/crypto/decrypt-worker.ts @@ -29,7 +29,7 @@ interface WorkersQueue { freeWorkers: Worker[]; resolvers: ((worker: Worker) => void)[]; push: (worker: Worker) => void; - pop: () => Promise, + pop: () => Promise; } const workersQueue: WorkersQueue = { @@ -54,8 +54,8 @@ const workersQueue: WorkersQueue = { this.resolvers.push(resolve); } }); - } -} + }, +}; export async function decrypt(data: DecryptData): Promise { const worker: Worker = await workersQueue.pop(); @@ -64,7 +64,7 @@ export async function decrypt(data: DecryptData): Promise { const { success, data, error } = event.data; workersQueue.push(worker); if (success) { - resolve((data as ArrayBuffer)); + resolve(data as ArrayBuffer); } else { reject(new TdfDecryptError(error)); } @@ -72,4 +72,4 @@ export async function decrypt(data: DecryptData): Promise { worker.postMessage(data); }); -}; \ No newline at end of file +} diff --git a/lib/tdf3/src/crypto/index.ts b/lib/tdf3/src/crypto/index.ts index defa8c48..8f70df92 100644 --- a/lib/tdf3/src/crypto/index.ts +++ b/lib/tdf3/src/crypto/index.ts @@ -276,11 +276,13 @@ async function _doDecrypt( algoDomString.iv = iv.asArrayBuffer(); try { - const decrypted = ( - navigator?.hardwareConcurrency - ? await workerDecrypt({ key: importedKey, encryptedPayload: payloadBuffer, algo: algoDomString }) - : await crypto.subtle.decrypt(algoDomString, importedKey, payloadBuffer) - ) + const decrypted = navigator?.hardwareConcurrency + ? await workerDecrypt({ + key: importedKey, + encryptedPayload: payloadBuffer, + algo: algoDomString, + }) + : await crypto.subtle.decrypt(algoDomString, importedKey, payloadBuffer); return { payload: Binary.fromArrayBuffer(decrypted) }; } catch (err) { diff --git a/lib/tdf3/src/tdf.ts b/lib/tdf3/src/tdf.ts index 88029c2f..befe85d0 100644 --- a/lib/tdf3/src/tdf.ts +++ b/lib/tdf3/src/tdf.ts @@ -996,25 +996,23 @@ export async function sliceAndDecrypt({ ); promises.push( - limit( - () => ( - decryptChunk( - encryptedChunk, - reconstructedKeyBinary, - slice[index]['hash'], - cipher, - segmentIntegrityAlgorithm, - cryptoService - ).then((decryptedChunk) => { - if (!decryptedChunk) return; - slice[index].decryptedChunk = decryptedChunk; - if (slice[index]._resolve) { - (slice[index]._resolve as (value: unknown) => void)(null); - } - }) - ) + limit(() => + decryptChunk( + encryptedChunk, + reconstructedKeyBinary, + slice[index]['hash'], + cipher, + segmentIntegrityAlgorithm, + cryptoService + ).then((decryptedChunk) => { + if (!decryptedChunk) return; + slice[index].decryptedChunk = decryptedChunk; + if (slice[index]._resolve) { + (slice[index]._resolve as (value: unknown) => void)(null); + } + }) ) - ) + ); } await Promise.all(promises); } From 96b88fba97599540ce694bcfa9f950769b956d7e Mon Sep 17 00:00:00 2001 From: sivanov Date: Tue, 21 Nov 2023 18:29:29 +0200 Subject: [PATCH 7/8] test fixed --- lib/package-lock.json | 61 ++++++++++++++++++++++++++++++++++++++-- lib/package.json | 5 ++-- lib/preServer.js | 1 + lib/tdf3/src/tdf.ts | 2 +- lib/tests/mocha/setup.ts | 7 +++++ lib/tests/server.ts | 2 -- 6 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 lib/preServer.js diff --git a/lib/package-lock.json b/lib/package-lock.json index 54d63118..319a9396 100644 --- a/lib/package-lock.json +++ b/lib/package-lock.json @@ -1,12 +1,12 @@ { "name": "@opentdf/client", - "version": "2.0.19", + "version": "2.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@opentdf/client", - "version": "2.0.19", + "version": "2.0.0", "license": "BSD-3-Clause-Clear", "dependencies": { "ajv": "^8.12.0", @@ -49,6 +49,7 @@ "colors": "^1.4.0", "eslint": "^8.46.0", "eslint-config-prettier": "^8.9.0", + "fetch-blob": "^4.0.0", "glob": "^10.3.3", "jsdom": "^22.1.0", "karma": "^6.4.2", @@ -4973,6 +4974,28 @@ "pend": "~1.2.0" } }, + "node_modules/fetch-blob": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-4.0.0.tgz", + "integrity": "sha512-nPmnhRmpNMjYWnp9EBMGs6z5lq9RXed5W1vuZcECrsDVQInM8AMQSooVb3X183Aole60adzjWbH9qlRFWzDDTA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0" + }, + "engines": { + "node": ">=16.7" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "dev": true, @@ -7861,6 +7884,25 @@ "path-to-regexp": "^1.7.0" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-fetch": { "version": "2.6.12", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", @@ -14797,6 +14839,15 @@ "pend": "~1.2.0" } }, + "fetch-blob": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-4.0.0.tgz", + "integrity": "sha512-nPmnhRmpNMjYWnp9EBMGs6z5lq9RXed5W1vuZcECrsDVQInM8AMQSooVb3X183Aole60adzjWbH9qlRFWzDDTA==", + "dev": true, + "requires": { + "node-domexception": "^1.0.0" + } + }, "file-entry-cache": { "version": "6.0.1", "dev": true, @@ -16874,6 +16925,12 @@ "path-to-regexp": "^1.7.0" } }, + "node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true + }, "node-fetch": { "version": "2.6.12", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", diff --git a/lib/package.json b/lib/package.json index 15d239c8..27070359 100644 --- a/lib/package.json +++ b/lib/package.json @@ -53,8 +53,8 @@ "lint": "eslint ./src/**/*.ts ./tdf3/**/*.ts ./tests/**/*.ts", "prepack": "npm run build", "test": "npm run build && npm run test:mocha && npm run test:wtr && npm run test:browser && npm run coverage:merge", - "test:browser": "node dist/web/tests/server.js & trap \"node dist/web/tests/stopServer.js\" EXIT; npx webpack --config webpack.test.config.cjs && npx karma start karma.conf.cjs", - "test:mocha": "node dist/web/tests/server.js & trap \"node dist/web/tests/stopServer.js\" EXIT; c8 --exclude=\"dist/web/tests/\" --exclude=\"dist/web/tdf3/src/utils/aws-lib-storage/\" --exclude=\"dist/web/tests/**/*\" --report-dir=./coverage/mocha mocha 'dist/web/tests/mocha/**/*.spec.js' --file dist/web/tests/mocha/setup.js && npx c8 report --reporter=json --report-dir=./coverage/mocha", + "test:browser": "node --require ./preServer.js dist/web/tests/server.js & trap \"node dist/web/tests/stopServer.js\" EXIT; npx webpack --config webpack.test.config.cjs && npx karma start karma.conf.cjs", + "test:mocha": "node --require ./preServer.js dist/web/tests/server.js & trap \"node dist/web/tests/stopServer.js\" EXIT; c8 --exclude=\"dist/web/tests/\" --exclude=\"dist/web/tdf3/src/utils/aws-lib-storage/\" --exclude=\"dist/web/tests/**/*\" --report-dir=./coverage/mocha mocha 'dist/web/tests/mocha/**/*.spec.js' --file dist/web/tests/mocha/setup.js && npx c8 report --reporter=json --report-dir=./coverage/mocha", "test:wtr": "web-test-runner", "watch": "(trap 'kill 0' SIGINT; npm run build && (npm run build:watch & npm run test -- --watch))" }, @@ -99,6 +99,7 @@ "colors": "^1.4.0", "eslint": "^8.46.0", "eslint-config-prettier": "^8.9.0", + "fetch-blob": "^4.0.0", "glob": "^10.3.3", "jsdom": "^22.1.0", "karma": "^6.4.2", diff --git a/lib/preServer.js b/lib/preServer.js new file mode 100644 index 00000000..232378ac --- /dev/null +++ b/lib/preServer.js @@ -0,0 +1 @@ +import('./dist/web/tests/mocha/setup.js').catch(e => console.error(e)); diff --git a/lib/tdf3/src/tdf.ts b/lib/tdf3/src/tdf.ts index befe85d0..71faf87a 100644 --- a/lib/tdf3/src/tdf.ts +++ b/lib/tdf3/src/tdf.ts @@ -218,7 +218,7 @@ export async function fetchKasPublicKey( const response: { data: string | KasPublicKeyInfo } = await axios.get(`${kas}/kas_public_key`, { params: { ...params, - // v: '2', + v: '2', }, }); const publicKey = diff --git a/lib/tests/mocha/setup.ts b/lib/tests/mocha/setup.ts index 5e9d0041..528b5ad0 100644 --- a/lib/tests/mocha/setup.ts +++ b/lib/tests/mocha/setup.ts @@ -12,6 +12,13 @@ if (typeof globalThis.crypto === 'undefined') { globalThis.crypto = webcrypto; } +if (typeof globalThis.Worker === 'undefined') { + // @ts-expect-error: custom Worker class + globalThis.Worker = class Worker { + constructor() {} + }; +} + const jsdom = new JSDOM(''); const { window } = jsdom; diff --git a/lib/tests/server.ts b/lib/tests/server.ts index 7fec5f85..58874695 100644 --- a/lib/tests/server.ts +++ b/lib/tests/server.ts @@ -1,5 +1,3 @@ -import './mocha/setup.js'; - import * as jose from 'jose'; import { IncomingMessage, RequestListener, createServer } from 'node:http'; From bdc344b40656ffd463ff96272e1ecbe34043803d Mon Sep 17 00:00:00 2001 From: sivanov Date: Mon, 11 Dec 2023 10:51:39 +0200 Subject: [PATCH 8/8] test added --- lib/tdf3/src/crypto/decrypt-worker.ts | 2 +- lib/tests/mocha/unit/decrypt-worker.spec.ts | 25 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 lib/tests/mocha/unit/decrypt-worker.spec.ts diff --git a/lib/tdf3/src/crypto/decrypt-worker.ts b/lib/tdf3/src/crypto/decrypt-worker.ts index 7f328c0c..d27fd53f 100644 --- a/lib/tdf3/src/crypto/decrypt-worker.ts +++ b/lib/tdf3/src/crypto/decrypt-worker.ts @@ -32,7 +32,7 @@ interface WorkersQueue { pop: () => Promise; } -const workersQueue: WorkersQueue = { +export const workersQueue: WorkersQueue = { freeWorkers: [...workersArray], resolvers: [], diff --git a/lib/tests/mocha/unit/decrypt-worker.spec.ts b/lib/tests/mocha/unit/decrypt-worker.spec.ts new file mode 100644 index 00000000..60503bcb --- /dev/null +++ b/lib/tests/mocha/unit/decrypt-worker.spec.ts @@ -0,0 +1,25 @@ +import { decrypt } from '../../../tdf3/src/crypto/decrypt-worker'; +import sinon from 'sinon'; + +const WorkerImplementation = globalThis.Worker; + +describe('', () => { + beforeAll(() => { + globalThis.Worker = class Worker { + private spy: sinon.SinonSpy; + postMessage(data) { + this.spy = sinon.spy(); + this.spy(data); + return this.onmessage(data) + } + onmessage(data) {} + }; + }) + it('', async () => { + + }); + afterAll(() => { + globalThis.Worker = WorkerImplementation; + }) +}); +