diff --git a/.changeset/many-cars-jump.md b/.changeset/many-cars-jump.md new file mode 100644 index 00000000..3d987e15 --- /dev/null +++ b/.changeset/many-cars-jump.md @@ -0,0 +1,5 @@ +--- +'@bnb-chain/reed-solomon': patch +--- + +feat: Add Types diff --git a/README.md b/README.md index f7bfc7ad..16319f3c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ You can find some package documentation below: | Package | Description | Version | | --- | --- | --- | | [@bnb-chain/greenfield-js-sdk](./packages/js-sdk/README.md) | A client library for Greenfield Chain | [![npm](https://img.shields.io/npm/v/%40bnb-chain%2Fgreenfield-js-sdk?color=blue)](https://www.npmjs.com/package/@bnb-chain/greenfield-js-sdk) | -| [@bnb-chain/greenfiled-file-handle](./packages/file-handle/README.md) | WASM module that handle file, such as `checksums` | [![npm](https://img.shields.io/npm/v/%40bnb-chain%2Fgreenfiled-file-handle?color=blue)](https://www.npmjs.com/package/@bnb-chain/greenfiled-file-handle) | +| [@bnb-chain/reed-solomon](./packages/reed-solomon/README.md) | calculate file's `checksums` | [![npm](https://img.shields.io/npm/v/%40bnb-chain%2Freed-solomon?color=blue)](https://www.npmjs.com/package/@bnb-chain/reed-solomon) | | [@bnb-chain/greenfield-zk-crypto](./packages/zk-crypto/README.md) | WASM module about sign crypto | [![npm](https://img.shields.io/npm/v/%40bnb-chain%2Fgreenfield-zk-crypto?color=blue)](https://www.npmjs.com/package/@bnb-chain/greenfield-zk-crypto) | | [@bnb-chain/create-gnfd-app](./packages/create-gnfd-app/README.md) | Create Greenfield App Quickly | [![npm](https://img.shields.io/npm/v/%40bnb-chain%2Fcreate-gnfd-app?color=blue)](https://www.npmjs.com/package/@bnb-chain/create-gnfd-app) | diff --git a/packages/reed-solomon/README.md b/packages/reed-solomon/README.md index 647d9d0a..c62133d1 100644 --- a/packages/reed-solomon/README.md +++ b/packages/reed-solomon/README.md @@ -12,6 +12,17 @@ Compatible with [greenfield-common](https://github.com/bnb-chain/greenfield-comm ## Usage Examples +### ESM + +If you use module bundler such as [Webpack](https://webpack.js.org/), [Rollup](https://rollupjs.org/guide/en/), etc: + +```js +import {ReedSolomon} from '@bnb-chain/reed-solomon' + +const rs = new RS.ReedSolomon(); +const res = rs.encode(new Uint8Array(fileBuffer)) +``` + ### Browser Use directly in the browser via script tag: @@ -22,6 +33,7 @@ Use directly in the browser via script tag: get reed solomon + ``` -### ESM +[Code](./examples/web.html) -If you use module bundler such as [Webpack](https://webpack.js.org/), [Rollup](https://rollupjs.org/guide/en/), etc: +### Browser(WebWorker) -```js -import {ReedSolomon} from '@bnb-chain/reed-solomon' +```html + ``` +[Code](./examples/web-worker.html) + ### Nodejs +```js +const { ReedSolomon } = require('@bnb-chain/reed-solomon') + +const rs = new ReedSolomon(); +const res = await rs.encode(Uint8Array.from(fileBuffer)); +``` + +[Code](./examples/node.js) + +More examples: + +* [calcute several file in a folder](./examples/folder.js) + +### Nodejs(`worker_threads`) + Using in Nodejs: ```js @@ -62,5 +126,8 @@ const rs = new NodeAdapterReedSolomon(); const res = await rs.encodeInWorker(__filename, Uint8Array.from(fileBuffer)) ``` -* [calcute single file](./examples/singlefile.js) -* [calcute several file in a folder](./examples/folder.js) +[Code](./examples/node-worker.js) + +## Benchark + +[benchmark](./benchmark.md) diff --git a/packages/reed-solomon/benchmark.md b/packages/reed-solomon/benchmark.md index be874895..ce1f9180 100644 --- a/packages/reed-solomon/benchmark.md +++ b/packages/reed-solomon/benchmark.md @@ -1,16 +1,9 @@ # benchmarks -## 20M - -* Nodejs: 1s -* Nodejs(worker): 1s - -## 200M - -* Go: 300ms -* Nodejs: 10s -* Nodejs(worker): 2s -* Browser: 15s +| file size | [Nodejs](./examples/node.js) | [Nodejs(worker_threads)](./examples/node-worker.js) | [Browser](./examples/web.html) | [Browser(webworker)](./examples/web-worker.html) | +| - | - | - | - | - | +| 20M | 1s | 1s | 1.6s | 1.3s | +| 200M | 10s | 2s | 15s | 2.2s | ## conclusion diff --git a/packages/reed-solomon/examples/node-worker.js b/packages/reed-solomon/examples/node-worker.js new file mode 100644 index 00000000..712a27d7 --- /dev/null +++ b/packages/reed-solomon/examples/node-worker.js @@ -0,0 +1,16 @@ +/* eslint-disable */ +const fs = require('node:fs'); +const path = require('node:path'); +const { NodeAdapterReedSolomon } = require('../dist/node.adapter'); + +const fileBuffer = fs.readFileSync('./README.md'); + +(async () => { + const rs = new NodeAdapterReedSolomon(); + + console.time('cost worker_threads'); + console.log('file size', sourceData.length / 1024 / 1024, 'm'); + const res = await rs.encodeInWorker(__filename, Uint8Array.from(fileBuffer)); + console.log('res', res); + console.timeEnd('cost worker_threads'); +})(); diff --git a/packages/reed-solomon/examples/node.js b/packages/reed-solomon/examples/node.js new file mode 100644 index 00000000..372739bb --- /dev/null +++ b/packages/reed-solomon/examples/node.js @@ -0,0 +1,15 @@ +/* eslint-disable */ +const fs = require('node:fs'); +const path = require('node:path'); +const { ReedSolomon } = require('../dist/index'); + +const fileBuffer = fs.readFileSync('./README.md'); + +(async () => { + const rs = new ReedSolomon(); + console.log('file size', sourceData.length / 1024 / 1024, 'm'); + console.time('cost'); + const res = await rs.encode(Uint8Array.from(fileBuffer)); + console.log('res', res); + console.timeEnd('cost'); +})(); diff --git a/packages/reed-solomon/examples/web-worker.html b/packages/reed-solomon/examples/web-worker.html new file mode 100644 index 00000000..eadede31 --- /dev/null +++ b/packages/reed-solomon/examples/web-worker.html @@ -0,0 +1,63 @@ + + + + + + RS webworker + + + + + + + + + + + diff --git a/packages/reed-solomon/examples/web.html b/packages/reed-solomon/examples/web.html index 03385091..99c61eb6 100644 --- a/packages/reed-solomon/examples/web.html +++ b/packages/reed-solomon/examples/web.html @@ -3,7 +3,7 @@ - Document + RS @@ -12,13 +12,7 @@ get reed solomon - - - - diff --git a/packages/reed-solomon/package.json b/packages/reed-solomon/package.json index acbe5596..18f1472c 100644 --- a/packages/reed-solomon/package.json +++ b/packages/reed-solomon/package.json @@ -34,15 +34,9 @@ }, "typesVersions": { "*": { - "node.adapter": [ - "./types/node.adapter.d.ts" - ], - "web.adapter": [ - "./types/web.adapter.d.ts" - ], - "utils": [ - "./types/utils.d.ts" - ] + "node.adapter": ["./types/node.adapter.d.ts"], + "web.adapter": ["./types/web.adapter.d.ts"], + "utils": ["./types/utils.d.ts"] } }, "scripts": { diff --git a/packages/reed-solomon/src/web.adapter.js b/packages/reed-solomon/src/web.adapter.js index 9374b1ea..75cca210 100644 --- a/packages/reed-solomon/src/web.adapter.js +++ b/packages/reed-solomon/src/web.adapter.js @@ -2,17 +2,14 @@ import { ReedSolomon } from '.'; import { sha256, getIntegrityUint8Array, toBase64, splitPrice } from './utils'; export class WebAdapterReedSolomon extends ReedSolomon { - async encodeInWorker(workerFn, sourceData, { webAdapterUrl, utilsUrl }) { + async encodeInWorker(workerFn, sourceData) { const chunkList = splitPrice(sourceData, this.segmentSize); const workers = []; for (let i = 0; i < chunkList.length; i++) { // const worker = new Worker('worker.js'); - const worker = createWorker(workerFn, { - webAdapterUrl, - utilsUrl, - }); + const worker = createWorker(workerFn); workers.push(worker); worker.postMessage({ index: i, @@ -60,41 +57,9 @@ export class WebAdapterReedSolomon extends ReedSolomon { } } -function createWorker(f, { webAdapterUrl, utilsUrl }) { - var blob = new Blob([ - '(' + f.toString() + ')(' + `'${webAdapterUrl}'` + ',' + `'${utilsUrl}'` + ')', - ]); +function createWorker(f) { + var blob = new Blob(['(' + f.toString() + ')()']); var url = window.URL.createObjectURL(blob); var worker = new Worker(url); return worker; } - -// inject worker script -export function injectWorker(cdnsUrls) { - importScripts( - cdnsUrls.webAdapterUrl || - 'https://cdn.jsdelivr.net/npm/@bnb-chain/reed-solomon/dist/index.aio.js', - ); - importScripts('https://cdn.jsdelivr.net/npm/@bnb-chain/reed-solomon/dist/utils.aio.js'); - - const rs = new WebAdapter.WebAdapterReedSolomon(); - - onmessage = function (event) { - const { index, chunk } = event.data; - const encodeShards = rs.encodeSegment(chunk); - let encodeDataHash = []; - - for (let i = 0; i < encodeShards.length; i++) { - const priceHash = RSUtils.sha256(encodeShards[i]); - encodeDataHash.push(priceHash); - } - - postMessage({ - index, - segChecksum: RSUtils.sha256(chunk), - encodeDataHash, - }); - - self.close(); - }; -} diff --git a/packages/reed-solomon/types/web.adapter.d.ts b/packages/reed-solomon/types/web.adapter.d.ts index e965106a..ecb48500 100644 --- a/packages/reed-solomon/types/web.adapter.d.ts +++ b/packages/reed-solomon/types/web.adapter.d.ts @@ -1,14 +1,5 @@ declare module '@bnb-chain/reed-solomon/web.adapter' { export class WebAdapterReedSolomon { - encodeInWorker( - workerFn: () => void, - data: Uint8Array, - cdnUrls: { - webAdapterUrl: string; - utilsUrl: string; - }, - ): Promise; + encodeInWorker(workerFn: () => void, data: Uint8Array): Promise; } - - export function injectWorker(cdnsUrls?: { webAdapterUrl: string; utilsUrl: string }): void; }