Skip to content

Commit

Permalink
Auto generate model folder
Browse files Browse the repository at this point in the history
- Also rename model docs writing script
  • Loading branch information
thekevinscott committed Nov 15, 2023
1 parent 14aac57 commit a7d5d84
Show file tree
Hide file tree
Showing 10 changed files with 1,007 additions and 585 deletions.
23 changes: 20 additions & 3 deletions internals/scripts/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,13 @@
"write:docs:guides": "wireit",
"write:docs:models": "wireit",
"write:docs:api": "wireit",
"write:model:docs": "wireit",
"write:model:folder": "wireit",
"start:guide": "wireit",
"validate": "wireit",
"update:npm-dependencies": "wireit",
"build": "wireit",
"copy-files": "wireit",
"test:run": "wireit",
"test": "wireit"
},
Expand Down Expand Up @@ -58,6 +61,18 @@
"build"
]
},
"write:model:docs": {
"command": "node ./dist/bin/write/model/docs/index.js",
"dependencies": [
"build"
]
},
"write:model:folder": {
"command": "node ./dist/bin/write/model/folder/index.js",
"dependencies": [
"build"
]
},
"start:guide": {
"command": "node ./dist/bin/start/guide.js",
"dependencies": [
Expand All @@ -71,15 +86,14 @@
]
},
"build": {
"command": "tsc -p ./tsconfig.json",
"command": "pnpm copy-files && tsc -p ./tsconfig.json",
"dependencies": [
"../common:build"
],
"files": [
"src/**/*.ts",
"src/**/*.mts",
"!src/**/*.test.ts",
"!src/**/*.test.mts",
"**/_templates/**/*.ejs",
"package.json",
"vite.config.ts",
"tsconfig.json"
Expand All @@ -88,6 +102,9 @@
"dist/**"
]
},
"copy-files": {
"command": "rsync -avmq --include='*.ejs' -f 'hide,! */' ./src/ ./dist"
},
"test:run": {
"command": "vitest run --config ./vite.config.ts",
"dependencies": [
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import path from 'path';
import { parseArgs } from "node:util";
import { exists, readFile, readdir, stat, writeFile } from '@internals/common/fs';
import { exists, readFile, readdir, writeFile } from '@internals/common/fs';
import { MODELS_DIR, SHARED_DIR } from '@internals/common/constants';
import { error, info, warn } from '@internals/common/logger';
import { error, info } from '@internals/common/logger';
import { JSONSchema, getPackageJSON } from '@internals/common/package-json';
import { getModels } from '../shared/getModels.js';

const getModelFamily = (packageJSON: JSONSchema) => packageJSON['@upscalerjs']?.['modelFamily'];

Expand Down Expand Up @@ -100,63 +100,17 @@ const writeModelDocs = async (
}));
}

const isDirectory = async (path: string) => (await stat(path)).isDirectory();

const isValidModel = async (modelDirectoryName: string) => {
const modelDirectoryPath = path.resolve(MODELS_DIR, modelDirectoryName);
return await exists(modelDirectoryPath) && await isDirectory(modelDirectoryPath);
};

const getModelDirectories = async () => {
const modelDirectories: string[] = [];
for (const modelDirectoryName of await readdir(MODELS_DIR)) {
if (await isDirectory(path.resolve(MODELS_DIR, modelDirectoryName))) {
modelDirectories.push(modelDirectoryName);
}
};
return modelDirectories;
}

const expandModel = async (model: string): Promise<string[]> => {
if (model.includes('*')) {
const modelNameMatch = model.split('*')[0];
const models: string[] = [];
for (const modelDirectoryName of await getModelDirectories()) {
if (modelDirectoryName.startsWith(modelNameMatch)) {
models.push(modelDirectoryName);
}
}
return models;
}
return [model];
};

const main = async () => {
const {
positionals: models,
} = parseArgs({
allowPositionals: true,
});

const validModels = new Set<string>();
for (const modelName of models) {
for (const model of await expandModel(modelName)) {
if (await isValidModel(model)) {
validModels.add(model);
} else {
warn(`Invalid model: ${model}`);
}
}
};
const validModels = await getModels();

if (models.length === 0) {
if (validModels.length === 0) {
error('No models selected, nothing to do.')
return;
}


info(`Writing model docs for models:\n${Array.from(validModels).map(m => `- ${m}`).join('\n')}`);
await writeModelDocs(Array.from(validModels));
info(`Writing model docs for models:\n${validModels.map(m => `- ${m}`).join('\n')}`);
await writeModelDocs(validModels);
};

main();
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
{
"name": "<%- name %>",
"version": "<%- version %>",
"description": "<%- description %>",
"keywords": <%- JSON.stringify(keywords, null, 2) %>,
"author": "Kevin Scott",
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/thekevinscott/upscaler.git"
},
"exports": <%- JSON.stringify(exports, null, 2) %>,
"module": "<%- module %>",
"types": "<%- types %>",
"umd:main": "<%- umdMain %>",
"scripts": <%- JSON.stringify(Object.keys(scripts).reduce((obj, scriptName) => ({
...obj,
[scriptName]: 'wireit',
}), {
"lint": "wireit",
"prepublishOnly": "wireit",
"validate:build": "wireit",
"scaffold": "wireit",
"build:umd:tsc": "wireit",
"build:umd:index": "wireit",
"build:umd:rollup:index": "wireit",
"build:umd:uglify:index": "wireit",
}), null, 2) %>,
"files": [
"assets/**/*",
"license",
"models/**/*",
"dist/**/*"
],
"peerDependencies": {
"@tensorflow/tfjs": "~4.11.0"
},
"devDependencies": {
"@tensorflow/tfjs": "~4.11.0",
"@tensorflow/tfjs-core": "~4.11.0",
"@tensorflow/tfjs-layers": "~4.11.0",
"@tensorflow/tfjs-node": "~4.11.0",
"@tensorflow/tfjs-node-gpu": "~4.11.0",
"seedrandom": "3.0.5",
"wireit": "^0.14.0"
},
"@upscalerjs": {
"title": "<%- title %>",
<% if (!!locals.models) { %>"models": <%- JSON.stringify(models, null, 2) %>,<% } %>
"modelFamily": "<%- modelFamily %>"
},
"wireit": <%- JSON.stringify(Object.entries(scripts).reduce((obj, [name, value]) => ({
...obj,
[name]: value,
}), {
"lint": {
"command": "eslint -c ../.eslintrc.js src --ext .ts",
"dependencies": [
"scaffold"
]
},
"prepublishOnly": {
"command": "pnpm lint && pnpm build && pnpm validate:build"
},
"scaffold": {
"command": "node -e 'const fs = require(\"fs\"); const {name, version} = require(\"./package.json\"); fs.writeFileSync(\"./src/constants.generated.ts\", `export const NAME = \"${name}\";\\nexport const VERSION=\"${version}\"`, \"utf-8\");'",
"files": [
"package.json"
],
"output": [
"./src/constants.generated.ts"
]
},
"build:umd:tsc": {
"command": "tsc -p ./tsconfig.umd.json --outDir ./dist/umd-tmp"
},
"build:umd:index": {
"command": "pnpm build:umd:rollup:index && pnpm build:umd:uglify:index"
},
}), null, 2) %>
}

Loading

0 comments on commit a7d5d84

Please sign in to comment.