wip: support client-side search

package.json

@@ -51,5 +51,10 @@
     "npm": ">=7.0.0",
     "node": ">=14.0.0"
   },
-  "packageManager": "[email protected]"
+  "packageManager": "[email protected]",
+  "dependencies": {
+    "@myst-theme/search": "^0.0.0",
+    "minisearch": "^7.1.0",
+    "string.prototype.matchall": "^4.0.11"
+  }
 }

packages/common/package.json

@@ -19,6 +19,7 @@
     "build": "npm-run-all -l clean -p build:esm"
   },
   "dependencies": {
+    "@myst-theme/search": "^0.0.0",
     "myst-common": "^1.6.0",
     "myst-config": "^1.5.0",
     "myst-spec-ext": "^1.6.0",

packages/common/src/types.ts

@@ -2,6 +2,7 @@ import type { Dependency, SourceFileKind } from 'myst-spec-ext';
 import type { GenericParent, References } from 'myst-common';
 import type { SiteAction, SiteExport, SiteManifest } from 'myst-config';
 import type { PageFrontmatter } from 'myst-frontmatter';
+import type { MystSearchIndex } from '@myst-theme/search';
 
 export enum Theme {
   light = 'light',
@@ -25,6 +26,7 @@ export type Heading = {
 export type SiteLoader = {
   theme?: Theme;
   config?: SiteManifest;
+  searchIndex?: MystSearchIndex;
   CONTENT_CDN_PORT?: string | number;
   MODE?: 'app' | 'static';
   BASE_URL?: string;

packages/providers/src/index.tsx

@@ -5,6 +5,7 @@ export * from './references.js';
 export * from './baseurl.js';
 export * from './ui.js';
 export * from './site.js';
+export * from './search.js';
 export * from './tabs.js';
 export * from './xref.js';
 export * from './renderers.js';

packages/providers/src/search.tsx

@@ -0,0 +1,19 @@
+import React, { useContext } from 'react';
+import type { ISearch } from '@myst-theme/search';
+
+const SearchContext = React.createContext<ISearch | undefined>(undefined);
+
+export function SearchProvider({
+  search,
+  children,
+}: {
+  search?: ISearch;
+  children: React.ReactNode;
+}) {
+  return <SearchContext.Provider value={search}>{children}</SearchContext.Provider>;
+}
+
+export function useSearch() {
+  const config = useContext(SearchContext);
+  return config;
+}

packages/search-minisearch/.eslintrc.cjs

@@ -0,0 +1,5 @@
+module.exports = {
+  root: true,
+  extends: ['curvenote'],
+  ignorePatterns: ['src/**/*.spec.ts'],
+};

packages/search-minisearch/README.md

@@ -0,0 +1,3 @@
+# @myst-theme/search-minisearch
+
+A minisearch implementation for client-side searching in MyST sites.

packages/search-minisearch/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@myst-theme/search-minisearch",
+  "version": "0.0.0",
+  "type": "module",
+  "exports": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "license": "MIT",
+  "sideEffects": false,
+  "scripts": {
+    "clean": "rimraf dist",
+    "lint": "eslint \"src/**/*.ts*\" -c ./.eslintrc.cjs",
+    "lint:format": "prettier --check \"src/**/*.{ts,tsx,md}\"",
+    "test": "vitest run",
+    "test:watch": "vitest watch",
+    "build:esm": "tsc --project ./tsconfig.json --module Node16 --outDir dist --declaration",
+    "build": "npm-run-all -l clean -p build:esm"
+  },
+  "dependencies": {
+    "@myst-theme/search": "^0.0.0"
+  }
+}

packages/search-minisearch/src/index.ts

@@ -0,0 +1,80 @@
+import MiniSearch, { type Options, type SearchResult as MiniSearchResult } from 'minisearch';
+import type { SearchRecord, SearchResult, ISearch } from '@myst-theme/search';
+
+export type ExtendedOptions = Options & Required<Pick<Options, 'tokenize' | 'processTerm'>>;
+
+export function prepareOptions(options: Options): ExtendedOptions {
+  return {
+    ...options,
+    tokenize: MiniSearch.getDefault('tokenize'),
+    processTerm: MiniSearch.getDefault('processTerm'),
+  };
+}
+
+export type RawSearchResult = SearchRecord & MiniSearchResult;
+
+export function combineResults(results: Map<string, Map<string, RawSearchResult>>) {
+  const [firstEntry, ...restEntries] = results.entries();
+
+  const firstRawResults = firstEntry[1];
+  const initialValue = new Map<string, SearchResult>(
+    Array.from(firstRawResults.entries(), ([id, rawResult]) => {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const { id: _, score, terms, queryTerms, match, ...rest } = rawResult;
+      return [
+        id,
+        {
+          id,
+          queries: [
+            {
+              term: queryTerms[0],
+              matches: match,
+            },
+          ],
+          ...rest,
+        },
+      ];
+    }),
+  );
+  const mergedResults = restEntries.reduce(
+    (accumulator: Map<string, SearchResult>, value: [string, Map<string, RawSearchResult>]) => {
+      const nextAccumulator = new Map<string, SearchResult>();
+
+      const rawResults = value[1];
+      rawResults.forEach((rawResult, docID) => {
+        const existing = accumulator.get(docID);
+        if (existing == null) {
+          return;
+        }
+        const { queryTerms, match } = rawResult;
+        existing.queries.push({
+          term: queryTerms[0],
+          matches: match,
+        });
+        nextAccumulator.set(docID, existing);
+      });
+      return nextAccumulator;
+    },
+    initialValue,
+  );
+  return Array.from(mergedResults.values());
+}
+
+export function createSearch(documents: SearchRecord[], options: Options): ISearch {
+  const extendedOptions = prepareOptions(options);
+  const search = new MiniSearch(extendedOptions);
+  search.addAll(documents.map((doc, index) => ({ ...doc, id: index })));
+  return (query: string) => {
+    // Implement executeQuery whilst retaining distinction between terms
+    // TODO: should we check for unique terms?
+    const terms = extendedOptions.tokenize(query);
+    const termResults = new Map(
+      terms.map((term) => [
+        term,
+        new Map(search.search(term).map((doc) => [doc.id, doc as RawSearchResult])),
+      ]),
+    );
+
+    return combineResults(termResults);
+  };
+}

packages/search-minisearch/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "../tsconfig/esm.json",
+  "include": ["."],
+  "exclude": ["dist", "node_modules", "src/**/*.spec.ts", "tests"]
+}

packages/search/.eslintrc.cjs

@@ -0,0 +1,5 @@
+module.exports = {
+  root: true,
+  extends: ['curvenote'],
+  ignorePatterns: ['src/**/*.spec.ts'],
+};

packages/search/README.md

@@ -0,0 +1,3 @@
+# @myst-theme/search
+
+An implementation and spec for client-side searching in MyST sites.

packages/search/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@myst-theme/search",
+  "version": "0.0.0",
+  "type": "module",
+  "exports": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "license": "MIT",
+  "sideEffects": false,
+  "scripts": {
+    "clean": "rimraf dist",
+    "lint": "eslint \"src/**/*.ts*\" -c ./.eslintrc.cjs",
+    "lint:format": "prettier --check \"src/**/*.{ts,tsx,md}\"",
+    "test": "vitest run",
+    "test:watch": "vitest watch",
+    "build:esm": "tsc --project ./tsconfig.json --module Node16 --outDir dist --declaration",
+    "build": "npm-run-all -l clean -p build:esm"
+  },
+  "dependencies": {}
+}

packages/search/src/index.ts

@@ -0,0 +1,3 @@
+export * from './types.js';
+export * from './rank.js';
+export * from './search.js';