Auto generate model folder

- Also rename model docs writing script
thekevinscott · Nov 15, 2023 · a7d5d84 · a7d5d84
1 parent 14aac57
commit a7d5d84
Show file tree

Hide file tree

Showing 10 changed files with 1,007 additions and 585 deletions.
diff --git a/internals/scripts/package.json b/internals/scripts/package.json
@@ -18,10 +18,13 @@
     "write:docs:guides": "wireit",
     "write:docs:models": "wireit",
     "write:docs:api": "wireit",
+    "write:model:docs": "wireit",
+    "write:model:folder": "wireit",
     "start:guide": "wireit",
     "validate": "wireit",
     "update:npm-dependencies": "wireit",
     "build": "wireit",
+    "copy-files": "wireit",
     "test:run": "wireit",
     "test": "wireit"
   },
@@ -58,6 +61,18 @@
         "build"
       ]
     },
+    "write:model:docs": {
+      "command": "node ./dist/bin/write/model/docs/index.js",
+      "dependencies": [
+        "build"
+      ]
+    },
+    "write:model:folder": {
+      "command": "node ./dist/bin/write/model/folder/index.js",
+      "dependencies": [
+        "build"
+      ]
+    },
     "start:guide": {
       "command": "node ./dist/bin/start/guide.js",
       "dependencies": [
@@ -71,15 +86,14 @@
       ]
     },
     "build": {
-      "command": "tsc -p ./tsconfig.json",
+      "command": "pnpm copy-files && tsc -p ./tsconfig.json",
       "dependencies": [
         "../common:build"
       ],
       "files": [
         "src/**/*.ts",
-        "src/**/*.mts",
         "!src/**/*.test.ts",
-        "!src/**/*.test.mts",
+        "**/_templates/**/*.ejs",
         "package.json",
         "vite.config.ts",
         "tsconfig.json"
@@ -88,6 +102,9 @@
         "dist/**"
       ]
     },
+    "copy-files": {
+      "command": "rsync -avmq --include='*.ejs' -f 'hide,! */' ./src/ ./dist"
+    },
     "test:run": {
       "command": "vitest run --config ./vite.config.ts",
       "dependencies": [

diff --git a/...rnals/scripts/src/bin/write/model-docs.ts → ...scripts/src/bin/write/model/docs/index.ts b/...rnals/scripts/src/bin/write/model-docs.ts → ...scripts/src/bin/write/model/docs/index.ts
@@ -1,9 +1,9 @@
 import path from 'path';
-import { parseArgs } from "node:util";
-import { exists, readFile, readdir, stat, writeFile } from '@internals/common/fs';
+import { exists, readFile, readdir, writeFile } from '@internals/common/fs';
 import { MODELS_DIR, SHARED_DIR } from '@internals/common/constants';
-import { error, info, warn } from '@internals/common/logger';
+import { error, info } from '@internals/common/logger';
 import { JSONSchema, getPackageJSON } from '@internals/common/package-json';
+import { getModels } from '../shared/getModels.js';
 
 const getModelFamily = (packageJSON: JSONSchema) => packageJSON['@upscalerjs']?.['modelFamily'];
 
@@ -100,63 +100,17 @@ const writeModelDocs = async (
   }));
 }
 
-const isDirectory = async (path: string) => (await stat(path)).isDirectory();
-
-const isValidModel = async (modelDirectoryName: string) => {
-  const modelDirectoryPath = path.resolve(MODELS_DIR, modelDirectoryName);
-  return await exists(modelDirectoryPath) && await isDirectory(modelDirectoryPath);
-};
-
-const getModelDirectories = async () => {
-  const modelDirectories: string[] = [];
-  for (const modelDirectoryName of await readdir(MODELS_DIR)) {
-    if (await isDirectory(path.resolve(MODELS_DIR, modelDirectoryName))) {
-      modelDirectories.push(modelDirectoryName);
-    }
-  };
-  return modelDirectories;
-}
-
-const expandModel = async (model: string): Promise<string[]> => {
-  if (model.includes('*')) {
-    const modelNameMatch = model.split('*')[0];
-    const models: string[] = [];
-    for (const modelDirectoryName of await getModelDirectories()) {
-      if (modelDirectoryName.startsWith(modelNameMatch)) {
-        models.push(modelDirectoryName);
-      }
-    }
-    return models;
-  }
-  return [model];
-};
-
 const main = async () => {
-  const {
-    positionals: models,
-  } = parseArgs({
-    allowPositionals: true,
-  });
-
-  const validModels = new Set<string>();
-  for (const modelName of models) {
-    for (const model of await expandModel(modelName)) {
-      if (await isValidModel(model)) {
-        validModels.add(model);
-      } else {
-        warn(`Invalid model: ${model}`);
-      }
-    }
-  };
+  const validModels = await getModels();
 
-  if (models.length === 0) {
+  if (validModels.length === 0) {
     error('No models selected, nothing to do.')
     return;
   }
 
 
-  info(`Writing model docs for models:\n${Array.from(validModels).map(m => `- ${m}`).join('\n')}`);
-  await writeModelDocs(Array.from(validModels));
+  info(`Writing model docs for models:\n${validModels.map(m => `- ${m}`).join('\n')}`);
+  await writeModelDocs(validModels);
 };
 
 main();
diff --git a/internals/scripts/src/bin/write/model/folder/_templates/package.json.ejs b/internals/scripts/src/bin/write/model/folder/_templates/package.json.ejs
@@ -0,0 +1,82 @@
+{
+  "name": "<%- name %>",
+  "version": "<%- version %>",
+  "description": "<%- description %>",
+  "keywords": <%- JSON.stringify(keywords, null, 2) %>,
+  "author": "Kevin Scott",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/thekevinscott/upscaler.git"
+  },
+  "exports": <%- JSON.stringify(exports, null, 2) %>,
+  "module": "<%- module %>",
+  "types": "<%- types %>",
+  "umd:main": "<%- umdMain %>",
+  "scripts": <%- JSON.stringify(Object.keys(scripts).reduce((obj, scriptName) => ({
+    ...obj,
+    [scriptName]: 'wireit',
+  }), {
+    "lint": "wireit",
+    "prepublishOnly": "wireit",
+    "validate:build": "wireit",
+    "scaffold": "wireit",
+    "build:umd:tsc": "wireit",
+    "build:umd:index": "wireit",
+    "build:umd:rollup:index": "wireit",
+    "build:umd:uglify:index": "wireit",
+  }), null, 2) %>,
+  "files": [
+    "assets/**/*",
+    "license",
+    "models/**/*",
+    "dist/**/*"
+  ],
+  "peerDependencies": {
+    "@tensorflow/tfjs": "~4.11.0"
+  },
+  "devDependencies": {
+    "@tensorflow/tfjs": "~4.11.0",
+    "@tensorflow/tfjs-core": "~4.11.0",
+    "@tensorflow/tfjs-layers": "~4.11.0",
+    "@tensorflow/tfjs-node": "~4.11.0",
+    "@tensorflow/tfjs-node-gpu": "~4.11.0",
+    "seedrandom": "3.0.5",
+    "wireit": "^0.14.0"
+  },
+  "@upscalerjs": {
+    "title": "<%- title %>",
+    <% if (!!locals.models) { %>"models": <%- JSON.stringify(models, null, 2) %>,<% } %>
+    "modelFamily": "<%- modelFamily %>"
+  },
+  "wireit": <%- JSON.stringify(Object.entries(scripts).reduce((obj, [name, value]) => ({
+    ...obj,
+    [name]: value,
+  }), {
+    "lint": {
+      "command": "eslint -c ../.eslintrc.js src --ext .ts",
+      "dependencies": [
+        "scaffold"
+      ]
+    },
+    "prepublishOnly": {
+      "command": "pnpm lint && pnpm build && pnpm validate:build"
+    },
+    "scaffold": {
+      "command": "node -e 'const fs = require(\"fs\"); const {name, version} = require(\"./package.json\"); fs.writeFileSync(\"./src/constants.generated.ts\", `export const NAME = \"${name}\";\\nexport const VERSION=\"${version}\"`, \"utf-8\");'",
+      "files": [
+        "package.json"
+      ],
+      "output": [
+        "./src/constants.generated.ts"
+      ]
+    },
+    "build:umd:tsc": {
+      "command": "tsc -p ./tsconfig.umd.json --outDir ./dist/umd-tmp"
+    },
+    "build:umd:index": {
+      "command": "pnpm build:umd:rollup:index && pnpm build:umd:uglify:index"
+    },
+  }), null, 2) %>
+}
+