add nodejs multiprocessing benchmark

.gitignore

@@ -3,4 +3,6 @@
 __pycache__
 *.pth
 *.ort
-*.config
+*.config
+*.data
+*node_modules**

runexamples/node/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "node-rwkv",
+  "version": "1.0.0",
+  "description": "A rwkv node runner",
+  "main": "index.js",
+  "author": "Harrison vanderbyl",
+  "license": "MIT",
+  "dependencies": {
+    "onnxruntime-common": "^1.16.2",
+    "onnxruntime-node": "^1.16.2",
+    "ts-node": "^10.9.1"
+  }
+}

runexamples/node/test.ts

@@ -0,0 +1,264 @@
+// Declare to typescript compiler that we are using onnxruntime-node and to ignore type checking.
+
+// Ignore type checking for onnxruntime-node.
+// @ts-ignore
+import * as ort from 'onnxruntime-node';
+
+import {InferenceSession, Tensor, TypedTensor, InferenceSessionFactory} from 'onnxruntime-common';
+
+
+type WkvStateName = "wkv"|""
+type StateKey = `instate${WkvStateName}${number}`
+
+type State = {
+
+    [key: StateKey]: TypedTensor<"float32">
+}
+
+type TokenJob = {
+    token: number
+    state: State
+    callback: (logits:Tensor,state:State) => void
+}
+
+// function zipState(states:State[]):State {
+//     const result:State = {}
+//     for (const key in states[0]) {
+//         const tensors = states.map(state => state[key])
+//         // result[key] = Tensor.concat(tensors)
+//     }
+//     return result
+// }
+
+
+class RWKV<Initialized extends boolean = false> {
+
+    embed:number = 0
+    layers:number = 0
+    heads:number = 0
+    model: Initialized extends true ? InferenceSession : null = null as Initialized extends true ? InferenceSession : null
+    path : string 
+    jobQueue:TokenJob[] = []
+    currentJobs:TokenJob[] = []
+    stateHolder:State = {}
+
+    constructor(path:string) {
+        this.path = path
+    }
+
+
+    unzipState(state:State):State[] {
+        const result:State[] = []
+
+        const B = state.instate0.dims[0]
+
+        for (let i = 0; i < B; i++) {
+            const newState:State = {}
+            for (const key in state) {
+                const tensor:TypedTensor<"float32"> = state[key as StateKey]
+                const dims = tensor.dims
+                const muldims = dims.slice(1).reduce((a,b) => a*b)
+                const data = tensor.data.slice(i*muldims,(i+1)*muldims)
+                const ten = new Tensor(data,[1,...dims.slice(1)])
+                newState[key as StateKey] = ten
+            }
+            result.push(newState)
+        }
+        return result
+    }
+
+    zipState(states:State[]) {
+        for (const key in states[0]) {
+            const tensors = states.map(state => (state[key as StateKey] as TypedTensor<"float32">))
+            const dims = tensors[0].dims
+            const newdims = [states.length,...dims.slice(1)];
+            const newsize = newdims.reduce((a,b) => a*b)
+
+            if(this.stateHolder[key as StateKey] == undefined ){
+                this.stateHolder[key as StateKey] = new Tensor("float32",new Float32Array(newsize),newdims)
+            }
+
+            if (this.stateHolder[key as StateKey].dims[0] != states.length) {
+                this.stateHolder[key as StateKey] = new Tensor("float32",new Float32Array(newsize),newdims)
+            }
+
+
+            for (let i = 0; i < states.length; i++) {
+                const state = states[i];
+                const tensor = state[key as StateKey]
+                const data = tensor.data
+                this.stateHolder[key as StateKey].data.set(data,i*data.length)
+            }
+
+        // 390 18
+        }
+    }
+
+    async run (){
+        if (this.jobQueue.length > 0) {
+            const jobs = this.jobQueue.splice(0,Math.min(this.jobQueue.length,128))
+
+            this.currentJobs = jobs
+            const states = jobs.map(job => job.state)
+            this.zipState(states)
+            const tokens = new Tensor("int32",jobs.map(job => job.token),[jobs.length])
+            const inputnamesreversed:StateKey[] = this.model!.inputNames as StateKey[]
+            const inputnames = inputnamesreversed.reverse()
+
+            const outputnamesReversed = this.model!.outputNames as string[];
+            const outputnames = outputnamesReversed.reverse()
+            // console.log("outputnames: ", outputnames)
+
+            const currenttime = Date.now()
+
+            const outputs = await this.model!.run({"input0":tokens,...this.stateHolder});
+
+            console.log("Concurrent Jobs: ", jobs.length, " Time: ", Date.now()-currenttime)
+
+            const splits = outputnames.reduce((a,b) => ({
+                "logits": outputs[b].dims[1] == Math.pow(2,16) ? [...a.logits,outputs[b] as TypedTensor<"float32">] : a.logits,
+                "instate": outputs[b].dims.length == 2 && outputs[b].dims[1] != Math.pow(2,16) ?[...a.instate,outputs[b] as TypedTensor<"float32">] : a.instate,
+                "instatewkv": outputs[b].dims.length == 4 ?[...a.instatewkv,outputs[b] as TypedTensor<"float32">] : a.instatewkv
+            }), {
+                "logits": [] as TypedTensor<"float32">[],
+                "instate": [] as TypedTensor<"float32">[],
+                "instatewkv": [] as TypedTensor<"float32">[]
+            })
+
+
+            const logits = splits.logits[0] as Tensor
+
+            // console.log("logits: ", splits.logits.length)
+            // console.log("instate: ", splits.instate.length)
+            // console.log("instatewkv: ", splits.instatewkv.length)
+
+            const nextInputState = {} as State
+
+            for (let i = 0; i < splits.instate.length; i++) {
+                const key = "instate"+i as StateKey;
+                nextInputState[key] = splits.instate[i]
+            }
+
+            for (let i = 0; i < splits.instatewkv.length; i++) {
+                const key = ("instatewkv"+i) as StateKey;
+                nextInputState[key] = splits.instatewkv[i]
+            }
+
+            const newstates = this.unzipState(nextInputState)
+
+            this.stateHolder = {}
+
+            newstates.forEach((state,i) => {
+                // console.log("state: ", Object.keys(state))
+                jobs[i].callback(logits,state)
+            })
+            this.currentJobs = []
+
+        }
+
+    }
+
+
+
+
+    async load():Promise<RWKV<true>> {
+        const sess:InferenceSession = await (ort.InferenceSession as InferenceSessionFactory).create('../../RWKV_32_2560_32_15_QUInt8-pc-norr-ext.onnx', {
+            interOpNumThreads: 8,
+            intraOpNumThreads: 8,
+            executionMode: 'parallel',
+            executionProviders: ["cpu"]
+        });
+
+        // prepare inputs. a tensor need its corresponding TypedArray as data
+        const inputnames = sess.inputNames;
+
+        // console.log("inputnames: ",inputnames)
+
+        // Get the shape of the input
+
+        this.embed = 2560
+        this.layers = (inputnames.length-1)/3
+        this.heads = 40 // 32 if 1b5
+
+        // console.log("embed: ", this.embed)
+        // console.log("layers: ", this.layers)
+
+        this.model = sess as Initialized extends true ? InferenceSession : null
+
+        return this as unknown as RWKV<true>
+    }
+
+    newState():State {
+        const result:State = {}
+        for (let i = 0; i < this.layers*2; i++) {
+            result[`instate${i}` as StateKey] = new Tensor("float32",new Float32Array(this.embed),[1,this.embed])
+        }
+
+        for (let i = 0; i < this.layers; i++) {
+            result[`instatewkv${i}` as StateKey] = new Tensor("float32",new Float32Array(((this.embed * this.embed)/this.heads)),[1,this.heads,this.embed/this.heads, this.embed/this.heads])
+        }
+        return result
+    }
+
+
+}
+
+
+
+// use an async context to call onnxruntime functions.
+async function main() {
+    try {
+        // create a new session and load the specific model.
+        //
+        // the model in this example contains a single MatMul node
+        // it has 2 inputs: 'a'(float32, 3x4) and 'b'(float32, 4x3)
+        // it has 1 output: 'c'(float32, 3x3)
+
+        // const testTensor:Tensor = new Tensor('float32', [0, 0, 0, 0], [1,2,2]);
+
+        // console.log("testTensor: ", testTensor)
+
+        // console.log("testTensor.data: ", testTensor.data.slice(0,2))
+
+        const model = await new RWKV('../../RWKV_32_2560_32_15_QUInt8-pc-norr-ext.onnx').load()
+
+        // run model.run every 100ms
+        setInterval(() => {
+            model.run()
+        }, 100);
+
+        const pushToken = (stuff:number, state:State) => {
+            // console.log("stuff: ", stuff)
+
+            model.jobQueue.push({
+                token: 0,
+                state,
+                callback: (logits,state) => {
+                    pushToken(stuff+1,state)
+                }
+            })
+        }
+
+        // every 3 seconds, push another token
+
+        setInterval(() => {
+            pushToken(0,model.newState())
+        }
+        , 3000);
+
+        // const state = model.newState()
+
+
+        while (true) {
+            await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+
+
+
+
+    } catch (e) {
+        console.error(`failed to inference ONNX model: ${e}.`);
+    }
+}
+
+main();

runexamples/node/yarn-error.log

@@ -0,0 +1,165 @@
+Arguments: 
+  /usr/local/bin/node /usr/local/bin/yarn add @types/onnxruntime-node
+
+PATH: 
+  /home/harrison/.local/bin:/home/harrison/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/snap/bin
+
+Yarn version: 
+  1.22.19
+
+Node version: 
+  18.10.0
+
+Platform: 
+  linux x64
+
+Trace: 
+  Error: https://registry.yarnpkg.com/@types%2fonnxruntime-node: Not found
+      at params.callback [as _callback] (/usr/local/lib/node_modules/yarn/lib/cli.js:66145:18)
+      at self.callback (/usr/local/lib/node_modules/yarn/lib/cli.js:140890:22)
+      at Request.emit (node:events:513:28)
+      at Request.<anonymous> (/usr/local/lib/node_modules/yarn/lib/cli.js:141862:10)
+      at Request.emit (node:events:513:28)
+      at IncomingMessage.<anonymous> (/usr/local/lib/node_modules/yarn/lib/cli.js:141784:12)
+      at Object.onceWrapper (node:events:627:28)
+      at IncomingMessage.emit (node:events:525:35)
+      at endReadableNT (node:internal/streams/readable:1359:12)
+      at process.processTicksAndRejections (node:internal/process/task_queues:82:21)
+
+npm manifest: 
+  {
+    "name": "node-rwkv",
+    "version": "1.0.0",
+    "description": "A rwkv node runner",
+    "main": "index.js",
+    "author": "Harrison vanderbyl",
+    "license": "MIT",
+    "dependencies": {
+      "onnxruntime-node": "^1.16.2",
+      "ts-node": "^10.9.1"
+    }
+  }
+
+yarn manifest: 
+  No manifest
+
+Lockfile: 
+  # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+  # yarn lockfile v1
+
+
+  "@cspotcode/source-map-support@^0.8.0":
+    version "0.8.1"
+    resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
+    integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==
+    dependencies:
+      "@jridgewell/trace-mapping" "0.3.9"
+
+  "@jridgewell/resolve-uri@^3.0.3":
+    version "3.1.1"
+    resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721"
+    integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==
+
+  "@jridgewell/sourcemap-codec@^1.4.10":
+    version "1.4.15"
+    resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32"
+    integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==
+
+  "@jridgewell/trace-mapping@0.3.9":
+    version "0.3.9"
+    resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9"
+    integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==
+    dependencies:
+      "@jridgewell/resolve-uri" "^3.0.3"
+      "@jridgewell/sourcemap-codec" "^1.4.10"
+
+  "@tsconfig/node10@^1.0.7":
+    version "1.0.9"
+    resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2"
+    integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==
+
+  "@tsconfig/node12@^1.0.7":
+    version "1.0.11"
+    resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d"
+    integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==
+
+  "@tsconfig/node14@^1.0.0":
+    version "1.0.3"
+    resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.3.tgz#e4386316284f00b98435bf40f72f75a09dabf6c1"
+    integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==
+
+  "@tsconfig/node16@^1.0.2":
+    version "1.0.4"
+    resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9"
+    integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==
+
+  acorn-walk@^8.1.1:
+    version "8.3.0"
+    resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.3.0.tgz#2097665af50fd0cf7a2dfccd2b9368964e66540f"
+    integrity sha512-FS7hV565M5l1R08MXqo8odwMTB02C2UqzB17RVgu9EyuYFBqJZ3/ZY97sQD5FewVu1UyDFc1yztUDrAwT0EypA==
+
+  acorn@^8.4.1:
+    version "8.11.2"
+    resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.2.tgz#ca0d78b51895be5390a5903c5b3bdcdaf78ae40b"
+    integrity sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w==
+
+  arg@^4.1.0:
+    version "4.1.3"
+    resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
+    integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==
+
+  create-require@^1.1.0:
+    version "1.1.1"
+    resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
+    integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==
+
+  diff@^4.0.1:
+    version "4.0.2"
+    resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
+    integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==
+
+  make-error@^1.1.1:
+    version "1.3.6"
+    resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2"
+    integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==
+
+  onnxruntime-common@~1.16.2:
+    version "1.16.2"
+    resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.16.2.tgz#bfd0d60406e5842185a1be0962e645ebd578ec97"
+    integrity sha512-S2/wPoW2uaq5WzpStS6XKOsy8EvKZOOdpzq++HpIV6XZYl0EIfoWBV/Pi5PWcJPFBcu7Rkopo/oPk9IbD9qRlw==
+
+  onnxruntime-node@^1.16.2:
+    version "1.16.2"
+    resolved "https://registry.yarnpkg.com/onnxruntime-node/-/onnxruntime-node-1.16.2.tgz#54e9aa27f5e85dae9e02b43eb09e356ff586844e"
+    integrity sha512-Qe/Fjx2n5Tlfm++RP/IHAmHrXaXoP75PK9p8XrDf6wI59Jk5ypL5F/bNH7lV3gOT8XIsjJmCBC2sfCVuivz1eQ==
+    dependencies:
+      onnxruntime-common "~1.16.2"
+
+  ts-node@^10.9.1:
+    version "10.9.1"
+    resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b"
+    integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==
+    dependencies:
+      "@cspotcode/source-map-support" "^0.8.0"
+      "@tsconfig/node10" "^1.0.7"
+      "@tsconfig/node12" "^1.0.7"
+      "@tsconfig/node14" "^1.0.0"
+      "@tsconfig/node16" "^1.0.2"
+      acorn "^8.4.1"
+      acorn-walk "^8.1.1"
+      arg "^4.1.0"
+      create-require "^1.1.0"
+      diff "^4.0.1"
+      make-error "^1.1.1"
+      v8-compile-cache-lib "^3.0.1"
+      yn "3.1.1"
+
+  v8-compile-cache-lib@^3.0.1:
+    version "3.0.1"
+    resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
+    integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==
+
+  yn@3.1.1:
+    version "3.1.1"
+    resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"
+    integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==

runexamples/node/yarn.lock

@@ -0,0 +1,119 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@cspotcode/source-map-support@^0.8.0":
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
+  integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==
+  dependencies:
+    "@jridgewell/trace-mapping" "0.3.9"
+
+"@jridgewell/resolve-uri@^3.0.3":
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721"
+  integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==
+
+"@jridgewell/sourcemap-codec@^1.4.10":
+  version "1.4.15"
+  resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32"
+  integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==
+
+"@jridgewell/trace-mapping@0.3.9":
+  version "0.3.9"
+  resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9"
+  integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==
+  dependencies:
+    "@jridgewell/resolve-uri" "^3.0.3"
+    "@jridgewell/sourcemap-codec" "^1.4.10"
+
+"@tsconfig/node10@^1.0.7":
+  version "1.0.9"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2"
+  integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==
+
+"@tsconfig/node12@^1.0.7":
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d"
+  integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==
+
+"@tsconfig/node14@^1.0.0":
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.3.tgz#e4386316284f00b98435bf40f72f75a09dabf6c1"
+  integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==
+
+"@tsconfig/node16@^1.0.2":
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9"
+  integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==
+
+acorn-walk@^8.1.1:
+  version "8.3.0"
+  resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.3.0.tgz#2097665af50fd0cf7a2dfccd2b9368964e66540f"
+  integrity sha512-FS7hV565M5l1R08MXqo8odwMTB02C2UqzB17RVgu9EyuYFBqJZ3/ZY97sQD5FewVu1UyDFc1yztUDrAwT0EypA==
+
+acorn@^8.4.1:
+  version "8.11.2"
+  resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.2.tgz#ca0d78b51895be5390a5903c5b3bdcdaf78ae40b"
+  integrity sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w==
+
+arg@^4.1.0:
+  version "4.1.3"
+  resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
+  integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==
+
+create-require@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333"
+  integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==
+
+diff@^4.0.1:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
+  integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==
+
+make-error@^1.1.1:
+  version "1.3.6"
+  resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2"
+  integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==
+
+onnxruntime-common@^1.16.2, onnxruntime-common@~1.16.2:
+  version "1.16.2"
+  resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.16.2.tgz#bfd0d60406e5842185a1be0962e645ebd578ec97"
+  integrity sha512-S2/wPoW2uaq5WzpStS6XKOsy8EvKZOOdpzq++HpIV6XZYl0EIfoWBV/Pi5PWcJPFBcu7Rkopo/oPk9IbD9qRlw==
+
+onnxruntime-node@^1.16.2:
+  version "1.16.2"
+  resolved "https://registry.yarnpkg.com/onnxruntime-node/-/onnxruntime-node-1.16.2.tgz#54e9aa27f5e85dae9e02b43eb09e356ff586844e"
+  integrity sha512-Qe/Fjx2n5Tlfm++RP/IHAmHrXaXoP75PK9p8XrDf6wI59Jk5ypL5F/bNH7lV3gOT8XIsjJmCBC2sfCVuivz1eQ==
+  dependencies:
+    onnxruntime-common "~1.16.2"
+
+ts-node@^10.9.1:
+  version "10.9.1"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b"
+  integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==
+  dependencies:
+    "@cspotcode/source-map-support" "^0.8.0"
+    "@tsconfig/node10" "^1.0.7"
+    "@tsconfig/node12" "^1.0.7"
+    "@tsconfig/node14" "^1.0.0"
+    "@tsconfig/node16" "^1.0.2"
+    acorn "^8.4.1"
+    acorn-walk "^8.1.1"
+    arg "^4.1.0"
+    create-require "^1.1.0"
+    diff "^4.0.1"
+    make-error "^1.1.1"
+    v8-compile-cache-lib "^3.0.1"
+    yn "3.1.1"
+
+v8-compile-cache-lib@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
+  integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==
+
+yn@3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"
+  integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==

test.py

@@ -133,9 +133,9 @@ def npsample(ozut, temp: float = 1.0, top_p_usual: float = 0.8) -> int:
 
 print("Loaded prompt.")
 
-for i in range(1000):
-    logits, state, state2 = model.forward([prompt[-1],prompt2[-1]],state, state2)
-    prompt = prompt+[npsample(logits[0])]
-    prompt2 = prompt2+[npsample(logits[1])]
-    print(tokenizer.decode(prompt)+":"+tokenizer.decode(prompt2),end="\r", flush=True)
-print(tokenizer.decode(prompt))
+# for i in range(1000):
+#     logits, state, state2 = model.forward([prompt[-1],prompt2[-1]],state, state2)
+#     prompt = prompt+[npsample(logits[0])]
+#     prompt2 = prompt2+[npsample(logits[1])]
+#     print(tokenizer.decode(prompt)+":"+tokenizer.decode(prompt2),end="\r", flush=True)
+# print(tokenizer.decode(prompt))