Create NPM package @ethdebug/format with schemas

- Export list of schema IDs

- Export `describeSchema` function that takes a schema ID (or a schema
  ID and pointer) and returns information about that schema (including
  raw schema object and source YAML)

- Use `prepare` step for generating a TS file with the full contents of
  all the schemas, so that nothing requires `fs` or any of that garbage

- Update web package to use new @ethdebug/format's `describeSchema`
  function (and get rid of the ever-growing list of yaml imports)

- Get rid of custom webpack shenanigans inside docusaurus config, since
  we won't need to import raw yaml anymore. x fingers crossed x

- Update tests package to remove all the `fs.readFileSync` nonsense

- Oh yeah, also get rid of all explicit listing of files... switch
  schema reading to do recursive `fs.readdirSync` for great less typing

- Generally clean up unused dependencies/imports (probably incompletely)

- Add a `yarn start` command in the repo root (and a bunch of other
  watchers), to accommodate the chain of dependent rebuilds this setup
  now effects.

package.json

@@ -4,9 +4,11 @@
     "packages/*"
   ],
   "scripts": {
-    "test": "node --experimental-vm-modules $(yarn bin jest)"
+    "test": "node --experimental-vm-modules ./node_modules/.bin/jest",
+    "start": "concurrently --names=format,web,jest \"cd ./packages/format && yarn watch\" \"cd ./packages/web && yarn start\" \"yarn test --watch\""
   },
   "devDependencies": {
+    "jest": "^29.7.0",
     "lerna": "^8.0.2"
   }
 }

packages/format/.gitignore

@@ -0,0 +1,2 @@
+dist
+yamls.ts

packages/format/bin/generate-schema-yamls.js

@@ -0,0 +1,40 @@
+const fs = require("fs");
+const path = require("path");
+const YAML = require("yaml");
+
+const filename = __dirname;
+const repositoryRoot = path.resolve(filename, "../../../");
+const schemasRoot = path.resolve(repositoryRoot, "./schemas");
+
+const readSchemaYamls = (directory) => {
+  const schemaYamls = {};
+  const entries = fs.readdirSync(directory, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = path.join(directory, entry.name);
+    if (entry.isDirectory()) {
+      const subdirectorySchemaYamls = readSchemaYamls(fullPath);
+      for (const id in subdirectorySchemaYamls) {
+        schemaYamls[id] = subdirectorySchemaYamls[id];
+      }
+    } else if (entry.isFile() && entry.name.endsWith(".schema.yaml")) {
+      const contents = fs.readFileSync(fullPath, "utf8");
+      const { $id: id } = YAML.parse(contents);
+
+      schemaYamls[id] = contents;
+    }
+  }
+
+  return schemaYamls;
+}
+
+const schemaYamls = readSchemaYamls(schemasRoot);
+
+console.log(`export type SchemaYamlsById = {
+  [id: string]: string;
+};
+
+export const schemaYamls: SchemaYamlsById = ${
+  JSON.stringify(schemaYamls, undefined, 2)
+};
+`);

packages/format/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@ethdebug/format",
+  "version": "0.1.0-0",
+  "description": "ethdebug/format schemas distributed as NPM package",
+  "main": "dist/src/index.js",
+  "repository": "https://github.com/ethdebug/format",
+  "license": "MIT",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "prepare:yamls": "node ./bin/generate-schema-yamls.js > yamls.ts",
+    "prepare": "yarn prepare:yamls && tsc",
+    "clean": "rm -rf dist && rm yamls.ts",
+    "watch:typescript": "tsc --watch",
+    "watch:schemas": "nodemon --watch ../../schemas -e 'yaml' --exec 'yarn prepare:yamls'",
+    "watch": "concurrently --names=tsc,schemas \"yarn watch:typescript\" \"yarn watch:schemas\""
+  },
+  "dependencies": {
+    "json-schema-typed": "^8.0.1",
+    "yaml": "^2.3.4"
+  },
+  "devDependencies": {
+    "concurrently": "^8.2.2",
+    "nodemon": "^3.0.2"
+  }
+}

packages/format/src/describe.ts

@@ -0,0 +1,200 @@
+import * as YAML from "yaml";
+
+import { schemaYamls } from "../yamls";
+
+import type { JSONSchema as JSONSchemaTyped } from "json-schema-typed/draft-2020-12"
+
+export type JSONSchema = Exclude<JSONSchemaTyped, boolean>;
+
+export interface DescribeSchemaOptions<
+  S extends SchemaReference = SchemaReference
+> {
+  schema: S;
+  pointer?: SchemaPointer;
+};
+
+export interface SchemaInfo {
+  id?: string; // root ID only
+  pointer?: SchemaPointer; // normalized from root ID
+  yaml: string;
+  schema: JSONSchema;
+  rootSchema: JSONSchema;
+}
+
+export function describeSchema({
+  schema,
+  pointer
+}: DescribeSchemaOptions): SchemaInfo {
+  if (typeof pointer === "string" && !pointer.startsWith("#")) {
+    throw new Error("`pointer` option must start with '#'");
+  }
+
+  return referencesId(schema)
+    ? describeSchemaById({ schema, pointer })
+    : referencesYaml(schema)
+      ? describeSchemaByYaml({ schema, pointer })
+      : describeSchemaByObject({ schema, pointer });
+}
+
+function describeSchemaById({
+  schema: { id: referencedId },
+  pointer: relativePointer
+}: DescribeSchemaOptions<SchemaById>): SchemaInfo {
+  // we need to handle the case where the schema is referenced by an ID
+  // with a pointer specified, possibly with a separate `pointer` field too
+  const [id, rawReferencedPointer] = referencedId.split("#");
+
+  const pointer = rawReferencedPointer
+    ? joinSchemaPointers([`#${rawReferencedPointer}`, relativePointer])
+    : relativePointer;
+
+  const rootYaml = schemaYamls[id]
+  if (!rootYaml) {
+    throw new Error(`Unknown schema with $id "${id}"`);
+  }
+
+  const yaml = pointToYaml(rootYaml, pointer);
+
+  const schema = YAML.parse(yaml);
+  const rootSchema = YAML.parse(rootYaml);
+
+  return {
+    id,
+    pointer,
+    yaml,
+    schema,
+    rootSchema
+  }
+}
+
+function describeSchemaByYaml({
+  schema: { yaml: referencedYaml },
+  pointer
+}: DescribeSchemaOptions<SchemaByYaml>): SchemaInfo {
+  const yaml = pointToYaml(referencedYaml, pointer);
+
+  const schema = YAML.parse(yaml);
+  const rootSchema = YAML.parse(referencedYaml);
+
+  const id = schema.$id;
+
+  if (id) {
+    return {
+      id,
+      pointer,
+      yaml,
+      schema,
+      rootSchema
+    }
+  } else {
+    return {
+      pointer,
+      yaml,
+      schema,
+      rootSchema
+    }
+  }
+}
+
+function describeSchemaByObject({
+  schema: rootSchema,
+  pointer
+}: DescribeSchemaOptions<object>): SchemaInfo {
+  const rootYaml = YAML.stringify(rootSchema);
+
+  const yaml = pointToYaml(rootYaml, pointer);
+
+  const schema = YAML.parse(yaml);
+
+  const id = schema.$id;
+
+  if (id) {
+    return {
+      id,
+      pointer,
+      yaml,
+      schema,
+      rootSchema
+    }
+  } else {
+    return {
+      pointer,
+      yaml,
+      schema,
+      rootSchema
+    }
+  }
+}
+
+function joinSchemaPointers(
+  pointers: (SchemaPointer | undefined)[]
+): SchemaPointer | undefined {
+  const joined = pointers
+    .filter((pointer): pointer is SchemaPointer => typeof pointer === "string")
+    .map(pointer => pointer.slice(1))
+    .join("");
+
+  if (joined.length === 0) {
+    return;
+  }
+
+  return `#${joined}`;
+}
+
+function pointToYaml(
+  yaml: string,
+  pointer?: SchemaPointer
+): string {
+  if (!pointer) {
+    return yaml;
+  }
+
+  let doc = YAML.parseDocument(yaml);
+
+  // slice(2) because we want to remove leading #/
+  for (const step of pointer.slice(2).split("/")) {
+    // @ts-ignore
+    doc = doc.get(step, true);
+
+    if (!doc) {
+      throw new Error(`Pointer ${pointer} not found in schema`);
+    }
+  }
+
+  return YAML.stringify(doc);
+}
+
+type Impossible<K extends keyof any> = {
+  [P in K]: never;
+};
+
+type NoExtraProperties<T, U extends T = T> =
+  & U
+  & Impossible<Exclude<keyof U, keyof T>>;
+
+export type SchemaPointer = `#${string}`;
+
+export type SchemaReference =
+  | SchemaById
+  | SchemaByYaml
+  | object /* JSONSchema object itself */;
+
+export type SchemaById = NoExtraProperties<{
+  id: string;
+}>;
+
+export type SchemaByYaml = NoExtraProperties<{
+  yaml: string;
+}>;
+
+export function referencesId(
+  schema: SchemaReference
+): schema is SchemaById {
+  return Object.keys(schema).length === 1 && "id" in schema;
+}
+
+export function referencesYaml(
+  schema: SchemaReference
+): schema is SchemaByYaml {
+  return Object.keys(schema).length === 1 && "yaml" in schema;
+}

packages/format/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./describe";
+export { schemas, schemaIds } from "./schemas";

packages/format/src/schemas.ts

@@ -0,0 +1,9 @@
+import { schemaYamls } from "../yamls";
+import { describeSchema } from "./describe";
+
+export const schemaIds: string[] = Object.keys(schemaYamls);
+export const schemas = schemaIds
+  .map(id => ({
+    [id]: describeSchema({ schema: { id } }).schema
+  }))
+  .reduce((a, b) => ({ ...a, ...b }), {});