+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 | + + + + + | import { browserTpl, neutralTpl, nodeTpl } from "@hpcc-js/esbuild-plugins"; + +// config --- +await neutralTpl("src/index.ts", "dist/index"); + + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| esbuild.js | +
+
+ |
+ 0% | +0/4 | +0% | +0/1 | +0% | +0/1 | +0% | +0/4 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | // @ts-expect-error importing from a wasm file is resolved via a custom esbuild plugin +import load, { reset } from "../../../build/packages/base91/src-cpp/base91lib.wasm"; +import { WasmLibrary } from "./wasm-library.ts"; + +// Ref: http://base91.sourceforge.net/#a5 + +let g_base91: Promise<Base91>; + +/** + * Base 91 WASM library, similar to Base 64 but uses more characters resulting in smaller strings. + * + * See [Base91](https://base91.sourceforge.net/) for more details. + * + * ```ts + * import { Base91 } from "@hpcc-js/wasm-base91"; + * + * const base91 = await Base91.load(); + * + * const encoded_data = await base91.encode(data); + * const decoded_data = await base91.decode(encoded_data); + * ``` + */ +export class Base91 extends WasmLibrary { + + private constructor(_module: any) { + super(_module, new _module.CBasE91()); + } + + /** + * Compiles and instantiates the raw wasm. + * + * ::: info + * In general WebAssembly compilation is disallowed on the main thread if the buffer size is larger than 4KB, hence forcing `load` to be asynchronous; + * ::: + * + * @returns A promise to an instance of the Base91 class. + */ + static load(): Promise<Base91> { + if (!g_base91) { + g_base91 = load().then((module: any) => { + return new Base91(module) + }); + } + return g_base91; + } + + /** + * Unloades the compiled wasm instance. + */ + static unload() { + reset(); + } + + /** + * @returns The Base91 c++ version + */ + version(): string { + return this._exports.version(); + } + + /** + * @param data Data to encode. + * @returns string containing the Base 91 encoded data + */ + encode(data: Uint8Array): string { + this._exports.reset(); + + const unencoded = this.uint8_heapu8(data); + const encoded = this.malloc_heapu8(unencoded.size + Math.ceil(unencoded.size / 4)); + + encoded.size = this._exports.encode(unencoded.ptr, unencoded.size, encoded.ptr); + let retVal = this.heapu8_string(encoded); + encoded.size = this._exports.encode_end(encoded.ptr); + retVal += this.heapu8_string(encoded); + + this.free_heapu8(encoded); + this.free_heapu8(unencoded); + return retVal; + } + + /** + * + * @param base91Str encoded string + * @returns origonal data + */ + decode(base91Str: string): Uint8Array { + this._exports.reset(); + + const encoded = this.string_heapu8(base91Str); + const unencoded = this.malloc_heapu8(encoded.size); + + unencoded.size = this._exports.decode(encoded.ptr, encoded.size, unencoded.ptr); + let retVal = this.heapu8_uint8(unencoded); + unencoded.size = this._exports.decode_end(unencoded.ptr); + retVal = new Uint8Array([...retVal, ...this.heapu8_view(unencoded)]); + + this.free_heapu8(unencoded); + this.free_heapu8(encoded); + return retVal; + } +} + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| base91.ts | +
+
+ |
+ 0% | +0/47 | +0% | +0/1 | +0% | +0/1 | +0% | +0/47 | +
| index.ts | +
+
+ |
+ 0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +
| wasm-library.ts | +
+
+ |
+ 0% | +0/41 | +100% | +1/1 | +100% | +1/1 | +0% | +0/41 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "./base91.ts"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | export type PTR = number;
+export interface HeapU8 {
+ ptr: PTR;
+ size: number;
+}
+
+/**
+ * Base class to simplify moving data into and out of Wasm memory.
+ */
+export class WasmLibrary {
+
+ protected _module: any;
+ protected _exports: any;
+
+ protected constructor(_module: any, _export: any) {
+ this._module = _module;
+ this._exports = _export;
+ }
+
+ protected malloc_heapu8(size: number): HeapU8 {
+ const ptr: PTR = this._exports.malloc(size);
+ return {
+ ptr,
+ size
+ };
+ }
+
+ protected free_heapu8(data: HeapU8) {
+ this._exports.free(data.ptr);
+ }
+
+ protected uint8_heapu8(data: Uint8Array): HeapU8 {
+ const retVal = this.malloc_heapu8(data.byteLength);
+ this._module.HEAPU8.set(data, retVal.ptr);
+ return retVal;
+ }
+
+ protected heapu8_view(data: HeapU8): Uint8Array {
+ return this._module.HEAPU8.subarray(data.ptr, data.ptr + data.size);
+ }
+
+ protected heapu8_uint8(data: HeapU8): Uint8Array {
+ return new Uint8Array([...this.heapu8_view(data)]);
+ }
+
+ protected string_heapu8(str: string): HeapU8 {
+ const data = Uint8Array.from(str, x => x.charCodeAt(0));
+ return this.uint8_heapu8(data);
+ }
+
+ protected heapu8_string(data: HeapU8): string {
+ const retVal = Array.from({ length: data.size });
+ const submodule = this._module.HEAPU8.subarray(data.ptr, data.ptr + data.size);
+ submodule.forEach((c: number, i: number) => {
+ retVal[i] = String.fromCharCode(c);
+ });
+ return retVal.join("");
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| base91 | +
+
+ |
+ 0% | +0/4 | +0% | +0/1 | +0% | +0/1 | +0% | +0/4 | +
| base91/src | +
+
+ |
+ 0% | +0/89 | +33.33% | +1/3 | +33.33% | +1/3 | +0% | +0/89 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 | + + + + | import { browserTpl, neutralTpl, nodeTpl } from "@hpcc-js/esbuild-plugins"; + +// config --- +await browserTpl("src/index.ts", "dist/index"); + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| esbuild.js | +
+
+ |
+ 0% | +0/4 | +0% | +0/1 | +0% | +0/1 | +0% | +0/4 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | import load, { reset } from "../build/duckdb-eh.wasm.ts"; +import loadWasmWorker, { reset as resetWasmWorker } from "../build/duckdb-browser-eh.worker.ts"; +import { AsyncDuckDB, ConsoleLogger } from "@duckdb/duckdb-wasm"; + +/** + * DuckDB WASM library, a in-process SQL OLAP Database Management System.. + * + * See [DuckDB](https://github.com/duckdb/duckdb) for more details. + * + * ```ts + * import { DuckDB } from "@hpcc-js/wasm-duckdb"; + * + * let duckdb = await DuckDB.load(); + * const c = await duckdb.db.connect(); + * + * const data = [ + * { "col1": 1, "col2": "foo" }, + * { "col1": 2, "col2": "bar" }, + * ]; + * await duckdb.db.registerFileText("rows.json", JSON.stringify(data)); + * await c.insertJSONFromPath('rows.json', { name: 'rows' }); + * + * const arrowResult = await c.query("SELECT * FROM read_json_auto('rows.json')"); + * const result = arrowResult.toArray().map((row) => row.toJSON()); + * expect(result.length).to.equal(data.length); + * for (let i = 0; i < result.length; i++) { + * expect(result[i].col2).to.equal(data[i].col2); + * } + * + * c.close(); + * ``` + */ +export class DuckDB { + + db: AsyncDuckDB; + + private constructor(db: AsyncDuckDB, protected _version: string) { + this.db = db; + } + + /** + * Compiles and instantiates the raw wasm. + * + * ::: info + * In general WebAssembly compilation is disallowed on the main thread if the buffer size is larger than 4KB, hence forcing `load` to be asynchronous; + * ::: + * + * @returns A promise to an instance of the DuckDB class. + */ + static load(): Promise<DuckDB> { + const workerUrl = URL.createObjectURL( + new Blob([loadWasmWorker()], { type: "text/javascript" }) + ); + const worker = new Worker(workerUrl); + URL.revokeObjectURL(workerUrl); + const logger = new ConsoleLogger(); + const db = new AsyncDuckDB(logger, worker); + const wasmUrl = URL.createObjectURL( + new Blob([load()], { "type": "application/wasm" }) + ); + return db.instantiate(wasmUrl, null).then(async () => { + URL.revokeObjectURL(wasmUrl); + return new DuckDB(db, await db.getVersion()); + }); + } + + /** + * Unloades the compiled wasm instance. + */ + static unload() { + resetWasmWorker(); + reset(); + } + + /** + * @returns The DuckDB version + */ + version(): string { + return this._version; + } +} + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ ++ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "./duckdb.ts"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| sfx-wasm.js | +
+
+ |
+ 0% | +0/7 | +0% | +0/1 | +0% | +0/1 | +0% | +0/7 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 | + + + + + + + | import { wrap } from "@hpcc-js/esbuild-plugins/sfx-wrapper"; + +try { + console.log(await wrap(process.argv[2])); +} catch (e) { + console.error(`Error: ${e?.message} \n`); +} + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| duckdb | +
+
+ |
+ 0% | +0/4 | +0% | +0/1 | +0% | +0/1 | +0% | +0/4 | +
| duckdb/build | +
+
+ |
+ 0% | +0/139 | +0% | +0/3 | +0% | +0/3 | +0% | +0/139 | +
| duckdb/src | +
+
+ |
+ 0% | +0/33 | +0% | +0/2 | +0% | +0/2 | +0% | +0/33 | +
| duckdb/utils | +
+
+ |
+ 0% | +0/7 | +0% | +0/1 | +0% | +0/1 | +0% | +0/7 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 | + + + + + | import { browserTpl, neutralTpl, nodeTpl } from "@hpcc-js/esbuild-plugins"; + +// config --- +await neutralTpl("src/index.ts", "dist/index"); + + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| esbuild.js | +
+
+ |
+ 0% | +0/4 | +0% | +0/1 | +0% | +0/1 | +0% | +0/4 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | // @ts-expect-error importing from a wasm file is resolved via a custom esbuild plugin +import load, { reset } from "../../../build/packages/expat/src-cpp/expatlib/expatlib.wasm"; + +export type Attributes = { [key: string]: string }; +export interface IParser { + startElement(tag: string, attrs: Attributes): void; + endElement(tag: string): void; + characterData(content: string): void; +} + +function parseAttrs(attrs: string): Attributes { + const retVal: Attributes = {}; + const keys = attrs; + const sep = `${String.fromCharCode(1)}`; + const sep2 = `${sep}${sep}`; + keys.split(sep2).filter((key: string) => !!key).forEach((key: string) => { + const parts = key.split(sep); + retVal[parts[0]] = parts[1]; + }); + return retVal; +} + +/** + * Expat XML parser WASM library, provides a simplified wrapper around the Expat XML Parser library. + * + * See [libexpat.github.io](https://libexpat.github.io/) for c++ details. + * + * ```ts + * import { Expat } from "@hpcc-js/wasm-expat"; + * + * const expat = await Expat.load(); + * + * const xml = ` \ + * <root> + * <child xxx="yyy">content</child> + * </root> + * `; + * + * const callback = { + * startElement(tag, attrs) { console.log("start", tag, attrs); }, + * endElement(tag) { console.log("end", tag); }, + * characterData(content) { console.log("characterData", content); } + * }; + * + * expat.parse(xml, callback); + * ``` + + */ +export class Expat { + + private constructor(protected _module: any) { + } + + /** + * Compiles and instantiates the raw wasm. + * + * ::: info + * In general WebAssembly compilation is disallowed on the main thread if the buffer size is larger than 4KB, hence forcing `load` to be asynchronous; + * ::: + * + * @returns A promise to an instance of the Expat class. + */ + static load(): Promise<Expat> { + return load().then((module: any) => { + return new Expat(module); + }); + } + + /** + * Unloades the compiled wasm instance. + */ + static unload() { + reset(); + } + + /** + * + * @returns The Expat c++ version + */ + version(): string { + return this._module.CExpat.prototype.version(); + } + + /** + * Parses the XML with suitable callbacks. + * + * :::tip + * The _IParser.characterData_ callback method can get called several times for a single tag element. + * ::: + * + * @param xml string containing XML + * @param callback Callback interface + * @returns `true`|`false` if the XML parse succeeds. + */ + parse(xml: string, callback: IParser): boolean { + const parser = new this._module.CExpatJS(); + parser.startElement = function () { + callback.startElement(this.tag(), parseAttrs(this.attrs())); + }; + parser.endElement = function () { + callback.endElement(this.tag()); + }; + parser.characterData = function () { + callback.characterData(this.content()); + }; + parser.create(); + const retVal = parser.parse(xml); + parser.destroy(); + this._module.destroy(parser); + return retVal; + } +} + +export class StackElement { + + private _content = ""; + get content(): string { + return this._content; + } + + constructor(readonly tag: string, readonly attrs: Attributes) { + } + + appendContent(content: string) { + this._content += content; + } +} + +export class StackParser implements IParser { + private _stack: StackElement[] = []; + + async parse(xml: string): Promise<boolean> { + const expat = await Expat.load(); + return expat.parse(xml, this); + } + + top(): StackElement { + return this._stack[this._stack.length - 1]; + } + + startElement(tag: string, attrs: Attributes): StackElement { + const retVal = new StackElement(tag, attrs); + this._stack.push(retVal); + return retVal; + } + + endElement(_tag: string): StackElement { + return this._stack.pop()!; + } + + characterData(content: string): void { + this.top().appendContent(content); + } +} + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ ++ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "./expat.ts"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ ++ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 | + + + + | import { nodeTpl } from "@hpcc-js/esbuild-plugins"; + +// config --- +await nodeTpl("src/index.ts", "dist/index"); + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| esbuild.js | +
+
+ |
+ 0% | +0/4 | +0% | +0/1 | +0% | +0/1 | +0% | +0/4 | +
| vitest.workspace copy.ts | +
+
+ |
+ 0% | +0/6 | +0% | +0/1 | +0% | +0/1 | +0% | +0/6 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +140 +141 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +160 +161 +162 +163 +164 +165 +166 +167 +168 +169 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +196 +197 +198 +199 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +217 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +234 +235 +236 +237 +238 +239 +240 +241 +242 +243 +244 +245 +246 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +262 +263 +264 +265 +266 +267 +268 +269 +270 +271 +272 +273 +274 +275 +276 +277 +278 +279 +280 +281 +282 +283 +284 +285 +286 +287 +288 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +303 +304 +305 +306 +307 +308 +309 +310 +311 +312 +313 +314 +315 +316 +317 +318 +319 +320 +321 +322 +323 +324 +325 +326 +327 +328 +329 +330 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +350 +351 +352 +353 +354 +355 +356 +357 +358 +359 +360 +361 +362 +363 +364 +365 +366 +367 +368 +369 +370 +371 +372 +373 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | // @ts-expect-error importing from a wasm file is resolved via a custom esbuild plugin +import load, { reset } from "../../../build/packages/graphviz/src-cpp/graphvizlib.wasm"; + +/** + * Various graphic and data formats for end user, web, documents and other applications. See [Output Formats](https://graphviz.gitlab.io/docs/outputs/) for more information. + */ +export type Format = "svg" | "dot" | "json" | "dot_json" | "xdot_json" | "plain" | "plain-ext" | "canon"; + +/** + * Various algorithms for projecting abstract graphs into a space for visualization. See [Layout Engines](https://graphviz.gitlab.io/docs/layouts/) for more details. + */ +export type Engine = "circo" | "dot" | "fdp" | "sfdp" | "neato" | "osage" | "patchwork" | "twopi" | "nop" | "nop2"; + +/** + * Example: Passing a web hosted Image to GraphViz: + * ```ts + * import { Graphviz } from "@hpcc-js/wasm-graphviz"; + * + * const graphviz = await Graphviz.load(); + * const svg = graphviz.layout('digraph { a[image="https://.../image.png"]; }', "svg", "dot", { + * images: [{ + * path: "https://.../image.png", + * width: "272px", + * height: "92px" + * }] + * }); + * document.getElementById("placeholder").innerHTML = svg; + * ``` + */ +export interface Image { + /** + * Full URL to image + */ + path: string; + width: string; + height: string; +} + +export interface File { + path: string; + data: string; +} + +export interface Options { + images?: Image[]; + files?: File[]; + yInvert?: boolean; + nop?: number; +} + +function imageToFile(image: Image): File { + return { + path: image.path, + data: `<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<svg width="${image.width}" height="${image.height}"></svg>` + }; +} + +function imagesToFiles(images: Image[]) { + return images.map(imageToFile); +} + +function createFiles(graphviz: any, _options?: Options) { + const options = { + images: [], + files: [], + ..._options + }; + [...options.files, ...imagesToFiles(options.images)].forEach(file => graphviz.createFile(file.path, file.data)); +} + +/** + * The Graphviz layout algorithms take descriptions of graphs in a simple text language, and make diagrams in useful formats, such as images and SVG for web pages or display in an interactive graph browser. + * + * Graphviz has many useful features for concrete diagrams, such as options for colors, fonts, tabular node layouts, line styles, hyperlinks, and custom shapes. + * + * See [graphviz.org](https://graphviz.org/) for more details. + * + * ```ts + * import { Graphviz } from "@hpcc-js/wasm/graphviz"; + * + * const graphviz = await Graphviz.load(); + * + * const dot = "digraph G { Hello -> World }"; + * const svg = graphviz.dot(dot); + * ``` + * + * ### Online Demos + * * https://raw.githack.com/hpcc-systems/hpcc-js-wasm/trunk/index.html + * * https://observablehq.com/@gordonsmith/graphviz + */ +export class Graphviz { + + private constructor(protected _module: any) { + } + + /** + * Compiles and instantiates the raw wasm. + * + * ::: info + * In general WebAssembly compilation is disallowed on the main thread if the buffer size is larger than 4KB, hence forcing `load` to be asynchronous; + * ::: + * + * @returns A promise to an instance of the Graphviz class. + */ + static load(): Promise<Graphviz> { + return load().then((module: any) => { + return new Graphviz(module); + }); + } + + /** + * Unloades the compiled wasm instance. + */ + static unload() { + reset(); + } + + /** + * @returns The Graphviz c++ version + */ + version(): string { + return this._module.Graphviz.prototype.version(); + } + + /** + * Performs layout for the supplied _dotSource_, see [The DOT Language](https://graphviz.gitlab.io/doc/info/lang.html) for specification. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param outputFormat The format of the result. + * @param layoutEngine The type of layout to perform. + * @param options Advanced Options for images, files, yInvert and nop. + * @returns A string containing the calculated layout in the format specified by `outputFormat` + */ + layout(dotSource: string, outputFormat: Format = "svg", layoutEngine: Engine = "dot", options?: Options): string { + if (!dotSource) return ""; + const graphViz = new this._module.Graphviz(options?.yInvert ? 1 : 0, options?.nop ? options?.nop : 0); + let retVal = ""; + let errorMsg = ""; + try { + createFiles(graphViz, options); + try { + retVal = graphViz.layout(dotSource, outputFormat, layoutEngine); + } catch (e: any) { + errorMsg = e.message; + }; + errorMsg = graphViz.lastError() || errorMsg; + } finally { + this._module.destroy(graphViz); + } + if (!retVal && errorMsg) { + Graphviz.unload(); + throw new Error(errorMsg); + } + return retVal; + } + + /** + * acyclic is a filter that takes a directed graph as input and outputs a copy of the graph with sufficient edges reversed to make the graph acyclic. The reversed edge inherits all of the attributes of the original edge. The optional file argument specifies where the input graph is stored; by default. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param doWrite Enable output is produced, though the return value will indicate whether the graph is acyclic or not. + * @param verbose Print information about whether the file is acyclic, has a cycle or is undirected. + * @returns `{ acyclic: boolean, num_rev: number, outFile: string }` `acyclic` will be true if a cycle was found, `num_rev` will contain the number of reversed edges and `outFile` will (optionally) contain the output. + */ + acyclic(dotSource: string, doWrite: boolean = false, verbose: boolean = false): { acyclic: boolean, num_rev: number, outFile: string } { + if (!dotSource) return { acyclic: false, num_rev: 0, outFile: "" }; + const graphViz = new this._module.Graphviz(); + let acyclic: boolean = false; + let num_rev: number = 0; + let outFile: string = ""; + let errorMsg = ""; + try { + try { + acyclic = graphViz.acyclic(dotSource, doWrite, verbose); + num_rev = graphViz.acyclic_num_rev; + outFile = graphViz.acyclic_outFile; + } catch (e: any) { + errorMsg = e.message; + }; + errorMsg = graphViz.lastError() || errorMsg; + } finally { + this._module.destroy(graphViz); + } + if (errorMsg) { + Graphviz.unload(); + throw new Error(errorMsg); + } + return { acyclic, num_rev, outFile }; + } + + /** + * tred computes the transitive reduction of directed graphs, and prints the resulting graphs to standard output. This removes edges implied by transitivity. Nodes and subgraphs are not otherwise affected. The ‘‘mean>ing’’ and validity of the reduced graphs is application dependent. tred is particularly useful as a preproces>sor to dot to reduce clutter in dense layouts. Undirected graphs are silently ignored. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param verbose Print additional information. + * @param printRemovedEdges Print information about removed edges. + * @returns `{ out: string, err: string }`. + */ + tred(dotSource: string, verbose: boolean = false, printRemovedEdges: boolean = false): { out: string, err: string } { + if (!dotSource) return { out: "", err: "" }; + const graphViz = new this._module.Graphviz(); + let out: string = ""; + let err: string = ""; + let errorMsg = ""; + try { + try { + graphViz.tred(dotSource, verbose, printRemovedEdges); + out = graphViz.tred_out; + err = graphViz.tred_err; + } catch (e: any) { + errorMsg = e.message; + }; + errorMsg = graphViz.lastError() || errorMsg; + } finally { + this._module.destroy(graphViz); + } + if (!out && errorMsg) { + Graphviz.unload(); + throw new Error(errorMsg); + } + return { out, err }; + } + + /** + * unflatten is a preprocessor to dot that is used to improve the aspect ratio of graphs having many leaves or disconnected nodes. The usual layout for such a graph is generally very wide or tall. unflatten inserts invisible edges or adjusts the minlen on edges to improve layout compaction. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param maxMinlen The minimum length of leaf edges is staggered between 1 and len (a small integer). + * @param do_fans Enables the staggering of the -maxMinlen option to fanout nodes whose indegree and outdegree are both 1. This helps with structures such as a -> \{w x y \} -> b. This option only works if the -maxMinlen flag is set. + * @param chainLimit Form disconnected nodes into chains of up to len nodes. + * @returns A string containing the "unflattened" dotSource. + */ + unflatten(dotSource: string, maxMinlen: number = 0, do_fans: boolean = false, chainLimit: number = 0): string { + if (!dotSource) return ""; + const graphViz = new this._module.Graphviz(); + let retVal = ""; + let errorMsg = ""; + try { + try { + retVal = graphViz.unflatten(dotSource, maxMinlen, do_fans, chainLimit); + } catch (e: any) { + errorMsg = e.message; + }; + errorMsg = graphViz.lastError() || errorMsg; + } finally { + this._module.destroy(graphViz); + } + if (!retVal && errorMsg) { + Graphviz.unload(); + throw new Error(errorMsg); + } + return retVal; + } + + /** + * Convenience function that performs the **circo** layout, is equivalent to `layout(dotSource, outputFormat, "circo");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param outputFormat The format of the result. + * @param options Advanced Options for images, files, yInvert and nop. + * @returns A string containing the calculated layout in the format specified by `outputFormat` + */ + circo(dotSource: string, outputFormat: Format = "svg", options?: Options): string { + return this.layout(dotSource, outputFormat, "circo", options); + } + + /** + * Convenience function that performs the **dot** layout, is equivalent to `layout(dotSource, outputFormat, "dot");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param outputFormat The format of the result. + * @param options Advanced Options for images, files, yInvert and nop. + * @returns A string containing the calculated layout in the format specified by `outputFormat` + */ + dot(dotSource: string, outputFormat: Format = "svg", options?: Options): string { + return this.layout(dotSource, outputFormat, "dot", options); + } + + /** + * Convenience function that performs the **fdp** layout, is equivalent to `layout(dotSource, outputFormat, "fdp");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param outputFormat The format of the result. + * @param options Advanced Options for images, files, yInvert and nop. + * @returns A string containing the calculated layout in the format specified by `outputFormat` + */ + fdp(dotSource: string, outputFormat: Format = "svg", options?: Options): string { + return this.layout(dotSource, outputFormat, "fdp", options); + } + + /** + * Convenience function that performs the **sfdp** layout, is equivalent to `layout(dotSource, outputFormat, "sfdp");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param outputFormat The format of the result. + * @param options Advanced Options for images, files, yInvert and nop. + * @returns A string containing the calculated layout in the format specified by `outputFormat` + */ + sfdp(dotSource: string, outputFormat: Format = "svg", options?: Options): string { + return this.layout(dotSource, outputFormat, "sfdp", options); + } + + /** + * Convenience function that performs the **neato** layout, is equivalent to `layout(dotSource, outputFormat, "neato");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param outputFormat The format of the result. + * @param options Advanced Options for images, files, yInvert and nop. + * @returns A string containing the calculated layout in the format specified by `outputFormat` + */ + neato(dotSource: string, outputFormat: Format = "svg", options?: Options): string { + return this.layout(dotSource, outputFormat, "neato", options); + } + + /** + * Convenience function that performs the **osage** layout, is equivalent to `layout(dotSource, outputFormat, "osage");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param outputFormat The format of the result. + * @param options Advanced Options for images, files, yInvert and nop. + * @returns A string containing the calculated layout in the format specified by `outputFormat` + */ + osage(dotSource: string, outputFormat: Format = "svg", options?: Options): string { + return this.layout(dotSource, outputFormat, "osage", options); + } + + /** + * Convenience function that performs the **patchwork** layout, is equivalent to `layout(dotSource, outputFormat, "patchwork");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param outputFormat The format of the result. + * @param options Advanced Options for images, files, yInvert and nop. + * @returns A string containing the calculated layout in the format specified by `outputFormat` + */ + patchwork(dotSource: string, outputFormat: Format = "svg", options?: Options): string { + return this.layout(dotSource, outputFormat, "patchwork", options); + } + + /** + * Convenience function that performs the **twopi** layout, is equivalent to `layout(dotSource, outputFormat, "twopi");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @param outputFormat The format of the result. + * @param options Advanced Options for images, files, yInvert and nop. + * @returns A string containing the calculated layout in the format specified by `outputFormat` + */ + twopi(dotSource: string, outputFormat: Format = "svg", options?: Options): string { + return this.layout(dotSource, outputFormat, "twopi", options); + } + + /** + * Convenience function that performs the **nop** layout, is equivalent to `layout(dotSource, "dot", "nop");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @returns A string containing the "pretty printed" dotSource. + */ + nop(dotSource: string): string { + return this.layout(dotSource, "dot", "nop"); + } + + /** + * Convenience function that performs the **nop2** layout, is equivalent to `layout(dotSource, "dot", "nop2");`. + * + * @param dotSource Required - graph definition in [DOT](https://graphviz.gitlab.io/doc/info/lang.html) language + * @returns A string containing the "pretty printed" dotSource. + */ + nop2(dotSource: string): string { + return this.layout(dotSource, "dot", "nop2"); + } +} + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| graphviz.ts | +
+
+ |
+ 0% | +0/157 | +0% | +0/1 | +0% | +0/1 | +0% | +0/157 | +
| index.ts | +
+
+ |
+ 0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "./graphviz.ts"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 | + + + + + | import { defineWorkspace } from 'vitest/config'; +import baseWorkspace from '../../vitest.workspace.ts'; + +export default defineWorkspace([ + ...baseWorkspace +]) |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| graphviz | +
+
+ |
+ 0% | +0/10 | +0% | +0/2 | +0% | +0/2 | +0% | +0/10 | +
| graphviz/src | +
+
+ |
+ 0% | +0/158 | +0% | +0/2 | +0% | +0/2 | +0% | +0/158 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ ++ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 | + + + + + | import { browserTpl, neutralTpl, nodeTpl } from "@hpcc-js/esbuild-plugins"; + +// config --- +await neutralTpl("src/index.ts", "dist/index"); + + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| esbuild.js | +
+
+ |
+ 0% | +0/4 | +0% | +0/1 | +0% | +0/1 | +0% | +0/4 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| index.ts | +
+
+ |
+ 0% | +0/2 | +0% | +0/1 | +0% | +0/1 | +0% | +0/2 | +
| llama.ts | +
+
+ |
+ 0% | +0/45 | +0% | +0/1 | +0% | +0/1 | +0% | +0/45 | +
| web-blob.ts | +
+
+ |
+ 0% | +0/74 | +100% | +1/1 | +100% | +1/1 | +0% | +0/74 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 | + | export * from "./llama.ts"; +export * from "./web-blob.ts"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | // @ts-expect-error importing from a wasm file is resolved via a custom esbuild plugin +import load, { reset } from "../../../build/packages/llama/src-cpp/llamalib.wasm"; +import type { MainModule } from "../../../build/packages/llama/src-cpp/llamalib.js"; +import llamaMeta from "../../../vcpkg-overlays/llama/vcpkg.json" with { type: "json" }; + +// Ref: https://github.com/ggerganov/llama.cpp +// Ref: http://facebook.github.io/llama/llama_manual.html +// Ref: https://github.com/facebook/llama + +/** + * The llama WASM library, provides a simplified wrapper around the llama.cpp library. + * + * See [llama.cpp](https://github.com/ggerganov/llama.cpp) for more details. + * + * ```ts + * import { Llama, WebBlob } from "@hpcc-js/wasm-llama"; + * + * let llama = await Llama.load(); + * const model = "https://huggingface.co/CompendiumLabs/bge-base-en-v1.5-gguf/resolve/main/bge-base-en-v1.5-q4_k_m.gguf"; + * const webBlob: Blob = await WebBlob.create(new URL(model)); + * + * const data: ArrayBuffer = await webBlob.arrayBuffer(); + * + * const embeddings = llama.embedding("Hello and Welcome!", new Uint8Array(data)); + * ``` + */ +export class Llama { + + private constructor(protected _module: MainModule) { + } + + /** + * Compiles and instantiates the raw wasm. + * + * ::: info + * In general WebAssembly compilation is disallowed on the main thread if the buffer size is larger than 4KB, hence forcing `load` to be asynchronous; + * ::: + * + * @returns A promise to an instance of the Llama class. + */ + static load(): Promise<Llama> { + return load().then((module: any) => { + return new Llama(module); + }); + } + + /** + * Unloades the compiled wasm instance. + */ + static unload() { + reset(); + } + + /** + * @returns The Llama c++ version + */ + version(): string { + return llamaMeta["version-string"]; + } + + /** + * Calculates the vector representation of the input text. + * + * @param text The input text. + * @param model The model to use for the embedding. + * + * @returns The embedding of the text using the model. + */ + embedding(text: string, model: Uint8Array, format: string = "array"): number[][] { + try { + this._module.FS_createDataFile("/", "embeddingModel.gguf", model, true, false, false); + } catch (e) { + console.error(e); + } + const args = new this._module.VectorString(); + args.push_back("-m"); args.push_back("/embeddingModel.gguf"); + args.push_back("--pooling"); args.push_back("mean"); + args.push_back("--log-disable"); + args.push_back("-p"); args.push_back(text); + args.push_back("--embd-output-format"); args.push_back(format); + const embeddingResult = new this._module.VectorString(); + let retVal: number[][] = []; + try { + this._module.embedding(args, embeddingResult); + const cout = embeddingResult.get(0); + retVal = JSON.parse(cout); + } catch (e) { + console.error(e); + } finally { + embeddingResult.delete(); + args.delete(); + this._module.FS_unlink("/embeddingModel.gguf"); + } + return retVal; + } +} + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | // See: https://github.com/huggingface/huggingface.js/blob/main/packages/hub/src/utils/WebBlob.ts
+
+/**
+ * WebBlob is a Blob implementation for web resources that supports range requests.
+ */
+
+interface WebBlobCreateOptions {
+ /**
+ * @default 1_000_000
+ *
+ * Objects below that size will immediately be fetched and put in RAM, rather
+ * than streamed ad-hoc
+ */
+ cacheBelow?: number;
+ /**
+ * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
+ */
+ fetch?: typeof fetch;
+}
+
+export class WebBlob extends Blob {
+ static async create(url: URL, opts?: WebBlobCreateOptions): Promise<Blob> {
+ const customFetch = opts?.fetch ?? fetch;
+ const response = await customFetch(url, { method: "HEAD" });
+
+ const size = Number(response.headers.get("content-length"));
+ const contentType = response.headers.get("content-type") || "";
+ const supportRange = response.headers.get("accept-ranges") === "bytes";
+
+ if (!supportRange || size < (opts?.cacheBelow ?? 1_000_000)) {
+ return await (await customFetch(url)).blob();
+ }
+
+ return new WebBlob(url, 0, size, contentType, true, customFetch);
+ }
+
+ private url: URL;
+ private start: number;
+ private end: number;
+ private contentType: string;
+ private full: boolean;
+ private fetch: typeof fetch;
+
+ constructor(url: URL, start: number, end: number, contentType: string, full: boolean, customFetch: typeof fetch) {
+ super([]);
+
+ this.url = url;
+ this.start = start;
+ this.end = end;
+ this.contentType = contentType;
+ this.full = full;
+ this.fetch = customFetch;
+ }
+
+ override get size(): number {
+ return this.end - this.start;
+ }
+
+ override get type(): string {
+ return this.contentType;
+ }
+
+ override slice(start = 0, end = this.size): WebBlob {
+ if (start < 0 || end < 0) {
+ new TypeError("Unsupported negative start/end on FileBlob.slice");
+ }
+
+ const slice = new WebBlob(
+ this.url,
+ this.start + start,
+ Math.min(this.start + end, this.end),
+ this.contentType,
+ start === 0 && end === this.size ? this.full : false,
+ this.fetch
+ );
+
+ return slice;
+ }
+
+ override async arrayBuffer(): Promise<ArrayBuffer> {
+ const result = await this.fetchRange();
+
+ return result.arrayBuffer();
+ }
+
+ override async text(): Promise<string> {
+ const result = await this.fetchRange();
+
+ return result.text();
+ }
+
+ override stream(): ReturnType<Blob["stream"]> {
+ const stream = new TransformStream();
+
+ this.fetchRange()
+ .then((response) => response.body?.pipeThrough(stream))
+ .catch((error) => stream.writable.abort(error.message));
+
+ return stream.readable;
+ }
+
+ private fetchRange(): Promise<Response> {
+ const fetch = this.fetch; // to avoid this.fetch() which is bound to the instance instead of globalThis
+ if (this.full) {
+ return fetch(this.url);
+ }
+ return fetch(this.url, {
+ headers: {
+ Range: `bytes=${this.start}-${this.end - 1}`,
+ },
+ });
+ }
+} |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ ++ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 | + + + + + + + + + + + | import { bothTpl } from "@hpcc-js/esbuild-plugins"; + +// config --- +await Promise.all([ + bothTpl("src/base91.ts", "dist/base91", undefined, "@hpcc-js/wasm"), + bothTpl("src/duckdb.ts", "dist/duckdb", undefined, "@hpcc-js/wasm"), + bothTpl("src/graphviz.ts", "dist/graphviz", undefined, "@hpcc-js/wasm"), + bothTpl("src/expat.ts", "dist/expat", undefined, "@hpcc-js/wasm"), + bothTpl("src/zstd.ts", "dist/zstd", undefined, "@hpcc-js/wasm") +]); +await bothTpl("src/index.ts", "dist/index", undefined, "@hpcc-js/wasm", ["./base91.js", "./duckdb.js", "./expat.js", "./graphviz.js", "./zstd.js"]); + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| esbuild.js | +
+
+ |
+ 0% | +0/11 | +0% | +0/1 | +0% | +0/1 | +0% | +0/11 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "@hpcc-js/wasm-base91"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "@hpcc-js/wasm-duckdb"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "@hpcc-js/wasm-expat"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | import { decompress } from "fzstd"; + +// See: https://github.com/Equim-chan/base91 +const table = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!#$%&()*+,./:;<=>?@[]^_`{|}~"'; + +function decode(raw: string): Uint8Array { + const len = raw.length; + const ret: number[] = []; + + let b = 0; + let n = 0; + let v = -1; + + for (let i = 0; i < len; i++) { + const p = table.indexOf(raw[i]); + /* istanbul ignore next */ + if (p === -1) continue; + if (v < 0) { + v = p; + } else { + v += p * 91; + b |= v << n; + n += (v & 8191) > 88 ? 13 : 14; + do { + ret.push(b & 0xff); + b >>= 8; + n -= 8; + } while (n > 7); + v = -1; + } + } + + if (v > -1) { + ret.push((b | v << n) & 0xff); + } + + return new Uint8Array(ret); +} + +export function extract(raw: string): Uint8Array { + const compressed = decode(raw); + return decompress(compressed); +} + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "@hpcc-js/wasm-graphviz"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| base91.ts | +
+
+ |
+ 0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +
| duckdb.ts | +
+
+ |
+ 0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +
| expat.ts | +
+
+ |
+ 0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +
| extract.ts | +
+
+ |
+ 0% | +0/34 | +0% | +0/1 | +0% | +0/1 | +0% | +0/34 | +
| graphviz.ts | +
+
+ |
+ 0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +
| index.ts | +
+
+ |
+ 0% | +0/20 | +0% | +0/1 | +0% | +0/1 | +0% | +0/20 | +
| zstd.ts | +
+
+ |
+ 0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 | + + + + + + + + + + + + + + + + + + + + + + + + + | export namespace Base91 { + export function load() { + return import("./base91.ts").then(mod => mod.Base91.load()); + } +} +export namespace DuckDB { + export function load() { + return import("./duckdb.js").then(mod => mod.DuckDB.load()); + } +} +export namespace Expat { + export function load() { + return import("./expat.ts").then(mod => mod.Expat.load()); + } +} +export namespace Graphviz { + export function load() { + return import("./graphviz.ts").then(mod => mod.Graphviz.load()); + } +} +export namespace Zstd { + export function load() { + return import("./zstd.ts").then(mod => mod.Zstd.load()); + } +} + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "@hpcc-js/wasm-zstd"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ ++ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 | + + + + + | import { browserTpl, neutralTpl, nodeTpl } from "@hpcc-js/esbuild-plugins"; + +// config --- +await neutralTpl("src/index.ts", "dist/index"); + + |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| esbuild.js | +
+
+ |
+ 0% | +0/4 | +0% | +0/1 | +0% | +0/1 | +0% | +0/4 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| index.ts | +
+
+ |
+ 0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +0% | +0/1 | +
| wasm-library.ts | +
+
+ |
+ 0% | +0/41 | +100% | +1/1 | +100% | +1/1 | +0% | +0/41 | +
| zstd.ts | +
+
+ |
+ 0% | +0/61 | +0% | +0/1 | +0% | +0/1 | +0% | +0/61 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 | export * from "./zstd.ts"; |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | export type PTR = number;
+export interface HeapU8 {
+ ptr: PTR;
+ size: number;
+}
+
+/**
+ * Base class to simplify moving data into and out of Wasm memory.
+ */
+export class WasmLibrary {
+
+ protected _module: any;
+ protected _exports: any;
+
+ protected constructor(_module: any, _export: any) {
+ this._module = _module;
+ this._exports = _export;
+ }
+
+ protected malloc_heapu8(size: number): HeapU8 {
+ const ptr: PTR = this._exports.malloc(size);
+ return {
+ ptr,
+ size
+ };
+ }
+
+ protected free_heapu8(data: HeapU8) {
+ this._exports.free(data.ptr);
+ }
+
+ protected uint8_heapu8(data: Uint8Array): HeapU8 {
+ const retVal = this.malloc_heapu8(data.byteLength);
+ this._module.HEAPU8.set(data, retVal.ptr);
+ return retVal;
+ }
+
+ protected heapu8_view(data: HeapU8): Uint8Array {
+ return this._module.HEAPU8.subarray(data.ptr, data.ptr + data.size);
+ }
+
+ protected heapu8_uint8(data: HeapU8): Uint8Array {
+ return new Uint8Array([...this.heapu8_view(data)]);
+ }
+
+ protected string_heapu8(str: string): HeapU8 {
+ const data = Uint8Array.from(str, x => x.charCodeAt(0));
+ return this.uint8_heapu8(data);
+ }
+
+ protected heapu8_string(data: HeapU8): string {
+ const retVal = Array.from({ length: data.size });
+ const submodule = this._module.HEAPU8.subarray(data.ptr, data.ptr + data.size);
+ submodule.forEach((c: number, i: number) => {
+ retVal[i] = String.fromCharCode(c);
+ });
+ return retVal.join("");
+ }
+}
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 +79 +80 +81 +82 +83 +84 +85 +86 +87 +88 +89 +90 +91 +92 +93 +94 +95 +96 +97 +98 +99 +100 +101 +102 +103 +104 +105 +106 +107 +108 +109 +110 +111 +112 +113 +114 +115 +116 +117 +118 +119 +120 +121 +122 +123 +124 +125 +126 +127 +128 +129 +130 +131 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | // @ts-expect-error importing from a wasm file is resolved via a custom esbuild plugin +import load, { reset } from "../../../build/packages/zstd/src-cpp/zstdlib.wasm"; +import { WasmLibrary } from "./wasm-library.ts"; + +// Ref: http://facebook.github.io/zstd/zstd_manual.html +// Ref: https://github.com/facebook/zstd + +let g_zstd: Promise<Zstd>; + +/** + * The Zstandard WASM library, provides a simplified wrapper around the Zstandard c++ library. + * + * See [Zstandard](https://facebook.github.io/zstd/) for more details. + * + * ```ts + * import { Zstd } from "@hpcc-js/wasm-zstd"; + * + * const zstd = await Zstd.load(); + * + * // Generate some "data" + * const data = new Uint8Array(Array.from({ length: 100000 }, (_, i) => i % 256)); + * + * const compressed_data = zstd.compress(data); + * const decompressed_data = zstd.decompress(compressed_data); + * ``` + */ +export class Zstd extends WasmLibrary { + + private constructor(_module: any) { + super(_module, _module.zstd.prototype); + } + + /** + * Compiles and instantiates the raw wasm. + * + * ::: info + * In general WebAssembly compilation is disallowed on the main thread if the buffer size is larger than 4KB, hence forcing `load` to be asynchronous; + * ::: + * + * @returns A promise to an instance of the Zstd class. + */ + static load(): Promise<Zstd> { + if (!g_zstd) { + g_zstd = load().then((module: any) => { + return new Zstd(module) + }); + } + return g_zstd; + } + + /** + * Unloades the compiled wasm instance. + */ + static unload() { + reset(); + } + + /** + * @returns The Zstd c++ version + */ + version(): string { + return this._exports.version(); + } + + /** + * @param data Data to be compressed + * @param compressionLevel Compression v Speed tradeoff, when omitted it will default to `zstd.defaultCLevel()` which is currently 3. + * @returns Compressed data. + * + * :::tip + * A note on compressionLevel: The library supports regular compression levels from 1 up o 22. Levels >= 20, should be used with caution, as they require more memory. The library also offers negative compression levels, which extend the range of speed vs. ratio preferences. The lower the level, the faster the speed (at the cost of compression). + * ::: + */ + compress(data: Uint8Array, compressionLevel: number = this.defaultCLevel()): Uint8Array { + const uncompressed = this.uint8_heapu8(data); + + const compressedSize = this._exports.compressBound(data.length); + const compressed = this.malloc_heapu8(compressedSize); + compressed.size = this._exports.compress(compressed.ptr, compressedSize, uncompressed.ptr, uncompressed.size, compressionLevel); + /* istanbul ignore if */ + if (this._exports.isError(compressed.size)) { + console.error(this._exports.getErrorName(compressed.size)); + } + const retVal = this.heapu8_uint8(compressed); + + this.free_heapu8(compressed); + this.free_heapu8(uncompressed); + return retVal; + } + + /** + * @param compressedData Data to be compressed + * @returns Uncompressed data. + */ + decompress(compressedData: Uint8Array): Uint8Array { + const compressed = this.uint8_heapu8(compressedData); + const uncompressedSize = this._exports.getFrameContentSize(compressed.ptr, compressed.size); + /* istanbul ignore if */ + if (this._exports.isError(uncompressedSize)) { + console.error(this._exports.getErrorName(uncompressedSize)); + } + const uncompressed = this.malloc_heapu8(uncompressedSize); + + uncompressed.size = this._exports.decompress(uncompressed.ptr, uncompressedSize, compressed.ptr, compressed.size); + /* istanbul ignore if */ + if (this._exports.isError(uncompressed.size)) { + console.error(this._exports.getErrorName(uncompressed.size)); + } + const retVal = this.heapu8_uint8(uncompressed); + + this.free_heapu8(uncompressed); + this.free_heapu8(compressed); + return retVal; + } + + /** + * @returns Default compression level (see notes above above). + */ + defaultCLevel(): number { + return this._exports.defaultCLevel(); + } + + minCLevel(): number { + return this._exports.minCLevel(); + } + + maxCLevel(): number { + return this._exports.maxCLevel(); + } +} + |