diff --git a/animated-transformer/package-lock.json b/animated-transformer/package-lock.json index 0028b5a..d7f1c99 100644 --- a/animated-transformer/package-lock.json +++ b/animated-transformer/package-lock.json @@ -30,6 +30,7 @@ "json5": "^2.2.3", "msgpackr": "^1.11.0", "rxjs": "~7.5.0", + "ts-node": "^10.9.2", "tslib": "^2.3.0", "underscore": "^1.13.6", "yargs": "^17.7.2", @@ -2460,6 +2461,28 @@ "node": ">=0.1.90" } }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@discoveryjs/json-ext": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.6.1.tgz", @@ -2816,7 +2839,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, "engines": { "node": ">=6.0.0" } @@ -2843,8 +2865,7 @@ "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", - "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", - "dev": true + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.25", @@ -3856,6 +3877,30 @@ "node": ">=10" } }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "license": "MIT" + }, "node_modules/@tsconfig/node18": { "version": "18.2.4", "resolved": "https://registry.npmjs.org/@tsconfig/node18/-/node18-18.2.4.tgz", @@ -4672,7 +4717,6 @@ "version": "8.12.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz", "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==", - "dev": true, "bin": { "acorn": "bin/acorn" }, @@ -4689,6 +4733,18 @@ "acorn": "^8" } }, + "node_modules/acorn-walk": { + "version": "8.3.3", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.3.tgz", + "integrity": "sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==", + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/adjust-sourcemap-loader": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/adjust-sourcemap-loader/-/adjust-sourcemap-loader-4.0.0.tgz", @@ -4896,6 +4952,12 @@ "node": ">=10" } }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "license": "MIT" + }, "node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -5980,6 +6042,12 @@ } } }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "license": "MIT" + }, "node_modules/crelt": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz", @@ -6794,6 +6862,15 @@ "integrity": "sha512-uJaamHkagcZtHPqCIHZxnFrXlunQXgBOsZSUOWwFw31QJCAbyTBoHMW75YOTur5ZNx8pIeAKgf6GWIgaqqiLhA==", "dev": true }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/dns-packet": { "version": "5.6.1", "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz", @@ -9788,6 +9865,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "license": "ISC" + }, "node_modules/make-fetch-happen": { "version": "13.0.1", "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-13.0.1.tgz", @@ -13088,6 +13171,49 @@ "tree-kill": "cli.js" } }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, "node_modules/tslib": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", @@ -13142,7 +13268,6 @@ "version": "5.5.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", - "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -13349,6 +13474,12 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "license": "MIT" + }, "node_modules/validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -15330,6 +15461,15 @@ "node": ">=8" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/yocto-queue": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz", diff --git a/animated-transformer/package.json b/animated-transformer/package.json index d3b2772..7dc2610 100644 --- a/animated-transformer/package.json +++ b/animated-transformer/package.json @@ -8,7 +8,7 @@ "watch": "ng build --watch --configuration development", "test": "ng test --watch" }, - "node": ">=20.11.1", + "node": ">=22.7.0", "private": true, "dependencies": { "@angular/animations": "^18.0.1", @@ -33,6 +33,7 @@ "json5": "^2.2.3", "msgpackr": "^1.11.0", "rxjs": "~7.5.0", + "ts-node": "^10.9.2", "tslib": "^2.3.0", "underscore": "^1.13.6", "yargs": "^17.7.2", @@ -57,4 +58,4 @@ "karma-jasmine-html-reporter": "~2.0.0", "typescript": "~5.5.0" } -} \ No newline at end of file +} diff --git a/animated-transformer/src/lib/seqtasks/tiny_worlds.run_with_transformer.script.ts b/animated-transformer/src/lib/seqtasks/tiny_worlds.run_with_transformer.script.ts index cd6eb08..38b81b7 100644 --- a/animated-transformer/src/lib/seqtasks/tiny_worlds.run_with_transformer.script.ts +++ b/animated-transformer/src/lib/seqtasks/tiny_worlds.run_with_transformer.script.ts @@ -18,7 +18,7 @@ limitations under the License. Tiny Worlds, run with (gtensor-based) transformers. Run: - ts-node src/lib/seqtasks/tiny_worlds.run_with_transformer.script.ts +npx ts-node src/lib/seqtasks/tiny_worlds.run_with_transformer.script.ts */ @@ -55,7 +55,6 @@ import { prepareBasicTaskTokenRep, BasicTaskTokenRep, } from '../tokens/token_gemb'; -import { GTensorTree, GVariableTree } from '../gtensor/gtensor_tree'; import { layer } from '@tensorflow/tfjs-vis/dist/show/model'; import { example } from 'yargs'; @@ -97,11 +96,7 @@ function getTransformerConfig(): TransformerConfig { return config; } -function* dataGenerator( - task: TinyWorldTask, - batchNum: number, - batchSize: number -) { +function* dataGenerator(task: TinyWorldTask, batchNum: number, batchSize: number) { for (let batchId = 0; batchId < batchNum; batchId += 1) { let batchOriginal = task.exampleIter.takeOutN(batchSize); let batchInput = batchOriginal.map((example) => example.input); @@ -115,7 +110,7 @@ function unbindedLossFn( batchOutput: string[][], tokenRep: BasicTaskTokenRep, transformerConfig: TransformerConfig, - decoderParamsTree: GVariableTree + decoderParamsTree: TransformerParams ): tf.Scalar { let spec = transformerConfig.spec; let computation: TransformerComputation = computeDecoder( @@ -125,18 +120,15 @@ function unbindedLossFn( decoderParamsTree, batchInput ); - let singleNextTokenIdx = singleNextTokenIdxOutputPrepFn( - tokenRep, - batchOutput - ); + let singleNextTokenIdx = singleNextTokenIdxOutputPrepFn(tokenRep, batchOutput); let entropyLoss: tf.Scalar = transformerLastTokenCrossEntropyLoss( computation, - decoderParamsTree.obj.tokenEmbedding, + decoderParamsTree.tokenEmbedding, singleNextTokenIdx ); let accuracy: tf.Scalar = transformerAccuracy( computation, - decoderParamsTree.obj.tokenEmbedding, + decoderParamsTree.tokenEmbedding, singleNextTokenIdx ); @@ -171,13 +163,7 @@ function unbindedLossFn( let [batchInput, batchOutput] = batch; let bindedLossFn = () => - unbindedLossFn( - batchInput, - batchOutput, - tokenRep, - transformerConfig, - decoderParamsTree - ); + unbindedLossFn(batchInput, batchOutput, tokenRep, transformerConfig, decoderParamsTree); optimizer.minimize(bindedLossFn); } @@ -200,16 +186,9 @@ function unbindedLossFn( decoderParamsTree, batchInput ); - let singleNextTokenIdx = singleNextTokenIdxOutputPrepFn( - tokenRep, - batchOutput - ); - let singleNextTokenIdxArrayData = - singleNextTokenIdx.tensor.arraySync() as number[]; - let logits = transformerLastTokenLogits( - computation, - decoderParamsTree.obj.tokenEmbedding - ); + let singleNextTokenIdx = singleNextTokenIdxOutputPrepFn(tokenRep, batchOutput); + let singleNextTokenIdxArrayData = singleNextTokenIdx.tensor.arraySync() as number[]; + let logits = transformerLastTokenLogits(computation, decoderParamsTree.tokenEmbedding); let probs = logits.softmax('tokenId'); let probsArrayData = probs.tensor.arraySync() as number[][]; @@ -237,8 +216,6 @@ function unbindedLossFn( batchInput = batchInput.map((subArray, batchIndex) => subArray.slice(1).concat(batchOutput[batchIndex]) ); - batchOutput = batchOutputAll.map((subArray) => - subArray.slice(inferStep, inferStep + 1) - ); + batchOutput = batchOutputAll.map((subArray) => subArray.slice(inferStep, inferStep + 1)); } }