Testing out plugin versions from local

.eslintrc.json

@@ -22,6 +22,7 @@
     // examples, and because it is not a recommended rule, you should either
     // disable it, or understand what it enforces.
     // https://typescript-eslint.io/rules/explicit-function-return-type/
-    "@typescript-eslint/explicit-function-return-type": "warn"
+    "@typescript-eslint/explicit-function-return-type": "warn",
+    "@typescript-eslint/no-explicit-any": "warn"
   }
 }

.gitignore

@@ -11,6 +11,7 @@ coverage
 
 # Transpiled files
 build/
+out/
 
 # VS Code
 .vscode

package.json

@@ -40,7 +40,10 @@
     "node": ">= 20.9 < 21"
   },
   "dependencies": {
-    "js-yaml": "^4.1.0"
+    "@studiorack/core": "^1.2.6",
+    "graphql-request": "^6.1.0",
+    "js-yaml": "^4.1.0",
+    "semver": "^7.6.0"
   },
   "devDependencies": {
     "@types/jest": "~29.5",

src/main.ts

@@ -1,33 +1,61 @@
-import fs from 'fs';
-import yaml from 'js-yaml';
-import { PluginInterface, PluginRegistry } from './types/Plugin.js';
+import { dirCreate, fileJsonCreate } from '@studiorack/core';
+import {
+  PluginEntry,
+  PluginInterface,
+  PluginRegistry,
+} from './types/Plugin.js';
+import { githubGetPack } from './sources/github.js';
+import { localGetPack } from './sources/local.js';
 
-function registryLoad(repo: string, version: string, id: string) {
-  const file: string = fs.readFileSync(`./src/plugins/${repo}/${version}/${id}.yaml`, 'utf8');
-  return yaml.load(file) as PluginInterface;
-}
+const DIST_PATH: string = './out';
+const REGISTRY_OUT: string = 'index.json';
+const REGISTRY_OUT_EFFECTS: string = 'effects.json';
+const REGISTRY_OUT_INSTRUMENTS: string = 'instruments.json';
+const REGISTRY_OUT_SFZ: string = 'sfz.json';
 
-function registryNew() {
+export function registryNew(type: string): PluginRegistry {
   return {
-    name: 'StudioRack Registry',
-    url: 'https://studiorack.github.io/studiorack-registry',
+    name: `StudioRack Registry - ${type}`,
+    url: `https://studiorack.github.io/studiorack-registry/${type}.json`,
     version: '2.0.0',
-    objects: [],
+    objects: {},
   };
 }
 
-function run() {
-  const registry: PluginRegistry = registryNew();
-  const pluginRepo: string = 'surge-synthesizer/releases-xt';
-  const pluginVersion: string = '1.3.1';
-  const pluginId: string = 'surge';
-  const plugin: PluginInterface = registryLoad(pluginRepo, pluginVersion, pluginId);
-  // registry.objects[`${pluginRepo}/${pluginId}`] = {
-  //   id: pluginId,
-  //   version: pluginVersion,
-  //   versions[pluginVersion]: plugin;,
-  // }
-  console.log(plugin);
+async function registrySave(path: string, file: PluginRegistry) {
+  dirCreate(DIST_PATH);
+  fileJsonCreate(path, file);
+}
+
+async function run() {
+  const registry: PluginRegistry = registryNew('registry');
+  registry.objects = Object.assign(registry.objects, await githubGetPack());
+  registry.objects = Object.assign(registry.objects, localGetPack());
+
+  // Create separate registries for Effects and Instruments
+  const effects: PluginRegistry = registryNew('effects');
+  const index: PluginRegistry = registryNew('index');
+  const instruments: PluginRegistry = registryNew('instruments');
+  const sfz: PluginRegistry = registryNew('sfz');
+  for (const pluginId in registry.objects) {
+    const pluginEntry: PluginEntry = registry.objects[pluginId];
+    const plugin: PluginInterface = pluginEntry.versions[pluginEntry.version];
+    // Check if tags include Effect/Fx
+    if (plugin.tags.includes('Effect') || plugin.tags.includes('Fx')) {
+      effects.objects[pluginId] = pluginEntry;
+    }
+    if (plugin.tags.includes('Instrument')) {
+      instruments.objects[pluginId] = pluginEntry;
+    }
+    if (plugin.tags.includes('sfz')) {
+      sfz.objects[pluginId] = pluginEntry;
+    }
+    index.objects[pluginId] = pluginEntry;
+  }
+  registrySave(`${DIST_PATH}/${REGISTRY_OUT}`, index);
+  registrySave(`${DIST_PATH}/${REGISTRY_OUT_EFFECTS}`, effects);
+  registrySave(`${DIST_PATH}/${REGISTRY_OUT_INSTRUMENTS}`, instruments);
+  registrySave(`${DIST_PATH}/${REGISTRY_OUT_SFZ}`, sfz);
 }
 
 run();

src/plugins/sfztools/sfizz/1.2.3.yaml

@@ -0,0 +1,27 @@
+---
+name: Sfizz
+author: SFZTools
+homepage: https://github.com/sfztools/sfizz
+description: SFZ parser and synth c++ library, providing AU / LV2 / VST3 plugins and JACK standalone client.
+date: 2024-01-14T00:00:00.000Z
+license: bsd-2-clause
+tags:
+  - Instrument
+  - Sampler
+  - Synth
+files:
+  audio:
+    url: https://studiorack.github.io/studiorack-registry/plugins/sfztools/sfizz/sfizz.flac
+    size: 47910
+  image:
+    url: https://studiorack.github.io/studiorack-registry/plugins/sfztools/sfizz/sfizz.jpg
+    size: 33976
+  linux:
+    url: https://github.com/sfztools/sfizz/releases/download/1.2.3/sfizz-1.2.3.tar.gz
+    size: 19102967
+  mac:
+    url: https://github.com/sfztools/sfizz/releases/download/1.2.3/sfizz-1.2.3-macos.tar.gz
+    size: 1748833
+  win:
+    url: https://github.com/sfztools/sfizz/releases/download/1.2.3/sfizz-1.2.3-win64.zip
+    size: 8286178

src/plugins/surge-synthesizer/surge/1.3.0.yaml

@@ -0,0 +1,27 @@
+---
+name: Surge XT
+author: Surge Synth Team
+homepage: https://github.com/surge-synthesizer/surge
+description: Hybrid synthesizer featuring many synthesis techniques, a great selection of filters, a flexible modulation engine, a smorgasbord of effects, and modern features like MPE and microtuning.
+date: 2023-12-08T00:00:00.000Z
+license: gpl-3.0
+tags:
+  - Instrument
+  - Synth
+  - Modulation
+files:
+  audio:
+    url: https://studiorack.github.io/studiorack-registry/plugins/surge-synthesizer/surge/surge.flac
+    size: 141339
+  image:
+    url: https://studiorack.github.io/studiorack-registry/plugins/surge-synthesizer/surge/surge.jpg
+    size: 159518
+  linux:
+    url: https://github.com/surge-synthesizer/releases-xt/releases/download/1.3.0/surge-xt-linux-1.3.0-pluginsonly.tar.gz
+    size: 94448096
+  mac:
+    url: https://github.com/surge-synthesizer/releases-xt/releases/download/1.3.0/surge-xt-macos-1.3.0-pluginsonly.zip
+    size: 180726292
+  win:
+    url: https://github.com/surge-synthesizer/releases-xt/releases/download/1.3.0/surge-xt-win64-1.3.0-pluginsonly.zip
+    size: 48165645

src/plugins/surge-synthesizer/surge/1.3.1.yaml

@@ -0,0 +1,27 @@
+---
+name: Surge XT
+author: Surge Synth Team
+homepage: https://github.com/surge-synthesizer/surge
+description: Hybrid synthesizer featuring many synthesis techniques, a great selection of filters, a flexible modulation engine, a smorgasbord of effects, and modern features like MPE and microtuning.
+date: 2024-02-06T00:00:00.000Z
+license: gpl-3.0
+tags:
+  - Instrument
+  - Synth
+  - Modulation
+files:
+  audio:
+    url: https://studiorack.github.io/studiorack-registry/plugins/surge-synthesizer/surge/surge.flac
+    size: 141339
+  image:
+    url: https://studiorack.github.io/studiorack-registry/plugins/surge-synthesizer/surge/surge.jpg
+    size: 159518
+  linux:
+    url: https://github.com/surge-synthesizer/releases-xt/releases/download/1.3.1/surge-xt-linux-1.3.1-pluginsonly.tar.gz
+    size: 94448096
+  mac:
+    url: https://github.com/surge-synthesizer/releases-xt/releases/download/1.3.1/surge-xt-macos-1.3.1-pluginsonly.zip
+    size: 180726292
+  win:
+    url: https://github.com/surge-synthesizer/releases-xt/releases/download/1.3.1/surge-xt-win64-1.3.1-pluginsonly.zip
+    size: 48165645

src/sources/github.ts

@@ -0,0 +1,130 @@
+import * as semver from 'semver';
+import { PluginLocal, pluginValidateSchema, safeSlug } from '@studiorack/core';
+import fetch from 'node-fetch';
+import { gql, GraphQLClient, RequestDocument } from 'graphql-request';
+import { PluginPack, PluginRelease } from '../types/Plugin.js';
+
+// Plugins need to have a topic `studiorack-plugin` to appear in the results
+// https://github.com/topics/studiorack-plugin
+const GITHUB_API: string = 'https://api.github.com/graphql';
+const GITHUB_TOPIC: string = 'studiorack-plugin';
+const GITHUB_REPO_PAGINATION: number = 100;
+const GITHUB_RELEASES_PAGINATION: number = 100;
+
+interface GitHubRelease {
+  tagName: string;
+}
+
+interface GitHubRepository {
+  nameWithOwner: string;
+  licenseInfo: {
+    key: string;
+  };
+  releases: {
+    nodes: GitHubRelease[];
+  };
+}
+
+interface GitHubSearch {
+  search: {
+    nodes: GitHubRepository[];
+  };
+}
+
+async function githubGetPack(): Promise<PluginPack> {
+  const pluginPack: PluginPack = {};
+  const results: GitHubSearch = await githubSearchRepos(GITHUB_API);
+  for (const repo of results.search.nodes) {
+    for (const release of repo.releases.nodes) {
+      await githubGetRelease(pluginPack, repo, release);
+    }
+  }
+  console.log(pluginPack);
+  return pluginPack;
+}
+
+async function githubSearchRepos(url: string): Promise<GitHubSearch> {
+  const headers: any = {};
+  if (process.env.GITHUB_TOKEN)
+    headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`;
+  const graphQLClient = new GraphQLClient(url, { headers });
+  const query: RequestDocument = gql`
+    {
+      search(query: "topic:${GITHUB_TOPIC} fork:true", type: REPOSITORY, first: ${GITHUB_REPO_PAGINATION}) {
+        nodes {
+          ... on Repository {
+            nameWithOwner
+            licenseInfo {
+              key
+            }
+            releases(first: ${GITHUB_RELEASES_PAGINATION}) {
+              nodes {
+                tagName
+              }
+            }
+          }
+        }
+      }
+    }
+  `;
+  return graphQLClient.request(query);
+}
+
+async function githubGetRelease(
+  pluginPack: PluginPack,
+  repo: GitHubRepository,
+  release: GitHubRelease,
+) {
+  const pluginsJsonList = await githubGetPlugins(
+    `https://github.com/${repo.nameWithOwner}/releases/download/${release.tagName}/plugins.json`,
+  );
+  pluginsJsonList.plugins.forEach((plugin: PluginRelease) => {
+    // For each plugin sanitize the id and add to registry
+    const pluginId = safeSlug(`${repo.nameWithOwner}/${safeSlug(plugin.id)}`);
+    const pluginVersion = semver.coerce(plugin.version)?.version || '0.0.0';
+    console.log('github', pluginId, pluginVersion);
+    if (!pluginPack[pluginId]) {
+      pluginPack[pluginId] = {
+        version: pluginVersion,
+        versions: {},
+      };
+    }
+    // TODO update plugins.json to not need these fields
+    delete plugin.id;
+    delete plugin.version;
+    pluginPack[pluginId].versions[pluginVersion] = plugin;
+    // If plugin version is greater than the current, set as latest version
+    if (semver.gt(pluginVersion, pluginPack[pluginId].version)) {
+      pluginPack[pluginId].version = pluginVersion;
+    }
+  });
+  return pluginPack;
+}
+
+async function githubGetPlugins(url: string) {
+  const pluginsValid: PluginRelease[] = [];
+  const pluginsJson = await getJSONSafe(url);
+  pluginsJson.plugins.forEach((plugin: PluginRelease) => {
+    const error = pluginValidateSchema(plugin as unknown as PluginLocal);
+    if (error === false) {
+      pluginsValid.push(plugin);
+    } else {
+      console.log(error, plugin);
+    }
+  });
+  return { plugins: pluginsValid };
+}
+
+async function getJSONSafe(url: string): Promise<any> {
+  console.log('⤓', url);
+  try {
+    const response = await fetch(url);
+    const json = await response.json();
+    return json;
+  } catch (error) {
+    // console.log(error);
+    return { plugins: [] };
+  }
+}
+
+export { githubGetPack };