Add new Maxim models:

- Deblurring
- Denoising
- Deraining
- Dehazing
  - Indoor
  - Outdoor
- Low Light Enhancement
- Retouching

dev/browser/package.json

@@ -1,11 +1,13 @@
 {
-  "name": "@upscalerjs/dev",
+  "name": "@upscalerjs/dev-browser",
   "private": true,
   "scripts": {
-    "dev": "vite --config ./vite.config.ts"
+    "dev": "pnpm run dev:only",
+    "dev:only": "vite --config ./vite.config.ts"
   },
   "devDependencies": {
     "upscaler": "workspace:*",
+    "@upscalerjs/default-model": "workspace:*",
     "@upscalerjs/esrgan-thick": "workspace:*",
     "@upscalerjs/esrgan-slim": "workspace:*",
     "@upscalerjs/esrgan-medium": "workspace:*",

dev/browser/public/.gitignore

@@ -0,0 +1 @@
+models

dev/browser/specific-model/image.ts

@@ -1,21 +1,58 @@
-export const makeImg = (path: string, label: string) => {
+export const getCanvas = (img: HTMLImageElement) => {
+  const canvas = document.createElement('canvas');
+  canvas.width = img.width;
+  canvas.height = img.height;
+  canvas.getContext('2d')?.drawImage(img, 0, 0, canvas.width, canvas.height);
+  return canvas;
+};
+
+const scaleCanvas = (canvas: HTMLCanvasElement, scale: number) => {
+  const scaledCanvas = document.createElement('canvas');
+  scaledCanvas.width = canvas.width * scale;
+  scaledCanvas.height = canvas.height * scale;
+
+  scaledCanvas
+    .getContext('2d')
+    ?.drawImage(canvas, 0, 0, scaledCanvas.width, scaledCanvas.height);
+
+  return scaledCanvas;
+};
+
+export const resizeImage = (img: HTMLImageElement, scale: number) => {
+  const canvas = getCanvas(img);
+  const scaledCanvas = scaleCanvas(canvas, scale);
+  return scaledCanvas;
+};
+
+const loadImage = (path: string) => new Promise<HTMLImageElement>((resolve, reject) => {
   const img = new Image();
   img.src = path;
   img.onload = () => {
-    const divEl = document.createElement('div');
-    const imgEl = document.createElement('img');
-    const labelEl = document.createElement('label');
-    labelEl.innerText = label;
-    imgEl.src = path;
-    imgEl.width = img.width;
-    imgEl.height = img.height;
-    imgEl.appendChild(img);
-
-    divEl.appendChild(labelEl);
-    divEl.appendChild(imgEl);
-    divEl.appendChild(document.createElement('hr'));
-
-    document.body.appendChild(divEl);
-    return imgEl;
+    resolve(img);
+  };
+  img.onerror = reject;
+});
+
+export const makeImg = async (path: string, label: string, scale?: number): Promise<HTMLImageElement | HTMLCanvasElement> => {
+  let img: HTMLImageElement | HTMLCanvasElement = await loadImage(path);
+
+  if (scale) {
+    img = resizeImage(img, scale);
   }
+
+  const divEl = document.createElement('div');
+  const imgEl = document.createElement('img');
+  const labelEl = document.createElement('label');
+  labelEl.innerText = label;
+  imgEl.src = path;
+  imgEl.width = img.width;
+  imgEl.height = img.height;
+  imgEl.appendChild(img);
+
+  divEl.appendChild(labelEl);
+  divEl.appendChild(imgEl);
+  divEl.appendChild(document.createElement('hr'));
+
+  document.body.appendChild(divEl);
+  return imgEl;
 }

dev/browser/specific-model/index.html

@@ -12,7 +12,7 @@
       <hr />
     </div>
     <div>
-      <p id="status"></p>
+      <p id="status">Waiting for status...</p>
     </div>
   </div>
   <script type="module" src="./index.ts"></script>

dev/browser/specific-model/index.ts

@@ -1,9 +1,9 @@
-import Upscaler from '../../packages/upscalerjs/src/index';
-import model from '../../models/esrgan-legacy/src/gans';
-import flower from '../../models/esrgan-legacy/test/__fixtures__/fixture.png';
+import Upscaler from '../../../packages/upscalerjs/src/index';
+import model from '../../../models/maxim-denoising/src/small';
+import fixture from '../../../models/maxim-denoising/test/__fixtures__/fixture.png';
 import * as tf from '@tensorflow/tfjs';
 import { makeImg } from './image';
-const MODEL = '/models/esrgan-legacy/models/gans/model.json';
+const MODEL = '/models/maxim-denoising/models/small/model.json';
 
 const status = document.getElementById('status')!;
 
@@ -16,14 +16,14 @@ const getModel = (path: string) => {
 }
 
 (async () => {
-  makeImg(flower, 'Original');
+  const img = await makeImg(fixture, 'Original', 1.5);
   const model = getModel(MODEL);
   status.innerHTML = 'Starting';
   const upscaler = new Upscaler({
     model,
   });
   status.innerHTML = 'Upscaling...';
-  const upscaledImg = await upscaler.upscale(flower);
+  const upscaledImg = await upscaler.upscale(img);
   status.innerHTML = 'Image upscaled';
   makeImg(upscaledImg, 'Upscaled');
   status.innerHTML = 'Image printed';

dev/browser/vite.config.ts

@@ -1,8 +1,6 @@
 import { defineConfig, } from 'vite';
 import path from 'path';
 
-const ROOT = path.resolve(__dirname, '../');
-
 export default defineConfig({
-  root: path.resolve(ROOT, './dev'),
+  root: './',
 });

dev/node/.gitignore

@@ -0,0 +1 @@
+output

dev/node/index.ts

@@ -0,0 +1,53 @@
+const fs = require('fs');
+const tf = require('@tensorflow/tfjs-node-gpu');
+const Upscaler = require('../../packages/upscalerjs/dist/node-gpu/cjs/index.js').default;
+const { mkdirpSync } = require("fs-extra");
+
+const getModel = (model: (tf: any) => any) => {
+  const { packageInformation, ...rest } = model(tf);
+  return {
+    ...rest,
+    scale: 1,
+    path: tf.io.fileSystem(`../../models/${packageInformation?.name.split('/').pop()}/${rest.path}`),
+  };
+}
+
+const upscaleImage = async (modelPath: string, imagePath: string, outputPath: string) => {
+  const model = require(modelPath).default;
+  const upscaler = new Upscaler({
+    model: getModel(model),
+  });
+
+  const imageBuffer = fs.readFileSync(imagePath);
+  const tensor = tf.node.decodeImage(imageBuffer).slice([0, 0, 0], [-1, -1, 3]);
+  const start = performance.now();
+  const upscaledTensor = await upscaler.upscale(tensor);
+  console.log(`Duration for ${outputPath}: ${((performance.now() - start) / 1000).toFixed(2)}s`);
+  tensor.dispose();
+  const upscaledPng = await tf.node.encodePng(upscaledTensor);
+  fs.writeFileSync(outputPath, upscaledPng);
+  upscaledTensor.dispose();
+}
+
+(async () => {
+  const models = [
+    // 'deblurring',
+    // 'denoising',
+    'dehazing-indoor',
+    // 'dehazing-outdoor',
+    // 'deraining',
+    // 'enhancement',
+    // 'retouching',
+  ];
+
+  for (const model of models) {
+    for (const size of ['large', 'small']) {
+      console.log('Running', size, model);
+      const modelPath = `../../models/maxim-${model}/src/${size}`;
+      const imagePath = `../../models/maxim-${model}/test/__fixtures__/fixture.png`;
+      const outputPath = `../../models/maxim-${model}/test/__fixtures__/${size}/result.png`;
+      mkdirpSync('output');
+      await upscaleImage(modelPath, imagePath, outputPath);
+    }
+  }
+})();

dev/node/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@upscalerjs/dev-node",
+  "private": true,
+  "scripts": {
+    "dev": "ts-node index.ts"
+  },
+  "dependencies": {
+    "upscaler": "workspace:*",
+    "@tensorflow/tfjs-node-gpu": "^4.1.0",
+    "seedrandom": "3.0.5",
+    "ts-node": "^10.9.1"
+  },
+  "version": "1.0.0-beta.10"
+}

docs/docs/documentation/troubleshooting.md

@@ -82,28 +82,6 @@ const upscaler = new Upscaler({
 
 Ensure you pass a valid `path` argument in the `model` payload. [See the guide on models for more information](/documentation/guides/browser/models).
 
-## Missing Model Scale
-
-If you see an error like:
-
-```
-Error: You must provide a "scale" when providing a model definition
-```
-
-You've passed a `null` or `undefined` scale argument in the `model` argument to UpscalerJS:
-
-```javascript
-const upscaler = new Upscaler({
-  model: {
-    scale: null,
-  },
-})
-```
-
-Every model must have an explicit `scale` defined.
-
-Ensure you pass a valid `scale` argument in the `model` payload. [See the guide on models for more information](/documentation/guides/browser/models).
-
 ## Invalid Warmup Value
 
 If you see an error like:
@@ -201,3 +179,11 @@ This error implies that the given model does not have an input layer compatible
 UpscalerJS only supports models whose input layers are set up to accept rank 4 tensors.
 
 If you believe this is in error, or you have a particular use case you think would be appropriate for UpscalerJS, [please open a Github issue](https://github.com/thekevinscott/UpscalerJS/issues/new/choose).
+
+## Input size and patch size
+
+If a model is defined with an `inputSize` and you provide an explicit `patchSize`, `patchSize` will be ignored.
+
+This is because the model expects the input to be a specific size, and cannot accept dynamically sized patches.
+
+UpscalerJS will automatically handle padding or slicing the image to fit the expected `inputSize` of the provided model.

models/maxim-deblurring/.gitignore

@@ -0,0 +1,3 @@
+dist
+node_modules
+*.generated.ts

models/maxim-deblurring/.npmignore

@@ -0,0 +1,4 @@
+src
+yarn-error.log
+node_modules
+test

models/maxim-deblurring/LICENSE

@@ -0,0 +1,23 @@
+
+MIT License
+
+Copyright (c) 2023 Kevin Scott
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

models/maxim-deblurring/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@upscalerjs/maxim-deblurring",
+  "version": "0.1.0",
+  "description": "Maxim: Deblurring Model",
+  "exports": {
+    "./small": {
+      "require": "./dist/cjs/models/maxim-deblurring/src/small.js",
+      "import": "./dist/esm/models/maxim-deblurring/src/small.js"
+    },
+    "./large": {
+      "require": "./dist/cjs/models/maxim-deblurring/src/large.js",
+      "import": "./dist/esm/models/maxim-deblurring/src/large.js"
+    },
+    ".": {
+      "require": "./dist/cjs/models/maxim-deblurring/src/index.js",
+      "import": "./dist/esm/models/maxim-deblurring/src/index.js"
+    }
+  },
+  "scripts": {
+    "scaffold:dependencies": "ts-node ../../scripts/package-scripts/scaffold-dependencies.ts --src models/maxim-deblurring --config models/scaffolder.ts",
+    "lint:fix": "pnpm lint --fix",
+    "lint": "pnpm scaffold:dependencies && eslint -c ../.eslintrc.js src --ext .ts",
+    "prepublishOnly": "pnpm lint && pnpm build && pnpm validate:build",
+    "validate:build": "ts-node ../../scripts/package-scripts/validate-build.ts models/maxim-deblurring",
+    "build": "ts-node ../../scripts/package-scripts/build-model.ts maxim-deblurring -o cjs -o esm -o umd"
+  },
+  "keywords": [
+    "image super resolution",
+    "image upscaling",
+    "image enhancement",
+    "tensorflow.js",
+    "pretrained models",
+    "esrgan"
+  ],
+  "files": [
+    "license",
+    "src/**/*",
+    "models/**/*",
+    "dist/**/*"
+  ],
+  "peerDependencies": {
+    "@tensorflow/tfjs": "^4.1.0"
+  },
+  "dependencies": {
+    "@upscalerjs/core": "workspace:*"
+  },
+  "devDependencies": {
+    "@tensorflow/tfjs-core": "^4.1.0",
+    "@tensorflow/tfjs-layers": "^4.1.0",
+    "@tensorflow/tfjs": "^4.1.0",
+    "@tensorflow/tfjs-node": "^4.1.0",
+    "@tensorflow/tfjs-node-gpu": "^4.1.0",
+    "seedrandom": "3.0.5"
+  },
+  "author": "Kevin Scott",
+  "license": "MIT"
+}

models/maxim-deblurring/src/index.ts

@@ -0,0 +1,2 @@
+export { default as large, } from './large';
+export { default as small, } from './small';

models/maxim-deblurring/src/large.ts

@@ -0,0 +1,22 @@
+import { ModelDefinitionFn, } from '@upscalerjs/core';
+import { NAME, VERSION, } from './constants.generated';
+import { registerKernels, modelDefinition as sharedModelDefinition, } from '../../../packages/shared/src/maxim';
+
+const modelDefinition: ModelDefinitionFn = tf => {
+  registerKernels(tf);
+
+  return {
+    ...sharedModelDefinition,
+    path: 'models/large/model.json',
+    packageInformation: {
+      name: NAME,
+      version: VERSION,
+    },
+    meta: {
+      dataset: 'GoPro',
+      quantization: null,
+    },
+  };
+};
+
+export default modelDefinition;

models/maxim-deblurring/src/small.ts

@@ -0,0 +1,22 @@
+import { ModelDefinitionFn, } from '@upscalerjs/core';
+import { NAME, VERSION, } from './constants.generated';
+import { registerOps, modelDefinition as sharedModelDefinition, } from '../../../packages/shared/src/maxim';
+
+const modelDefinition: ModelDefinitionFn = tf => {
+  registerOps(tf);
+
+  return {
+    ...sharedModelDefinition,
+    path: 'models/small/model.json',
+    packageInformation: {
+      name: NAME,
+      version: VERSION,
+    },
+    meta: {
+      dataset: 'GoPro',
+      quantization: 'float16',
+    },
+  };
+};
+
+export default modelDefinition;