From 37d59d7d66ef867de9d99fad3d68437f28641fd9 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Fri, 13 Sep 2024 13:14:06 +0100 Subject: [PATCH] wip: support client-side search --- package-lock.json | 119 ++++--- package.json | 7 +- packages/common/package.json | 1 + packages/common/src/types.ts | 2 + packages/providers/src/index.tsx | 1 + packages/providers/src/search.tsx | 19 ++ packages/search-minisearch/.eslintrc.cjs | 5 + packages/search-minisearch/README.md | 3 + packages/search-minisearch/package.json | 24 ++ packages/search-minisearch/src/index.ts | 80 +++++ packages/search-minisearch/tsconfig.json | 5 + packages/search/.eslintrc.cjs | 5 + packages/search/README.md | 3 + packages/search/package.json | 22 ++ packages/search/src/index.ts | 3 + packages/search/src/rank.ts | 301 ++++++++++++++++++ packages/search/src/search.ts | 7 + packages/search/src/types.ts | 76 +++++ packages/search/tsconfig.json | 5 + packages/site/package.json | 3 + .../site/src/components/Navigation/Search.tsx | 144 +++++++++ .../src/components/Navigation/ThemeButton.tsx | 2 +- .../site/src/components/Navigation/TopNav.tsx | 4 + themes/book/app/root.tsx | 68 ++-- .../app/routes/($project)_.$slug[.json].tsx | 2 +- themes/book/app/utils/loaders.server.ts | 3 +- themes/book/package.json | 2 + 27 files changed, 846 insertions(+), 70 deletions(-) create mode 100644 packages/providers/src/search.tsx create mode 100644 packages/search-minisearch/.eslintrc.cjs create mode 100644 packages/search-minisearch/README.md create mode 100644 packages/search-minisearch/package.json create mode 100644 packages/search-minisearch/src/index.ts create mode 100644 packages/search-minisearch/tsconfig.json create mode 100644 packages/search/.eslintrc.cjs create mode 100644 packages/search/README.md create mode 100644 packages/search/package.json create mode 100644 packages/search/src/index.ts create mode 100644 packages/search/src/rank.ts create mode 100644 packages/search/src/search.ts create mode 100644 packages/search/src/types.ts create mode 100644 packages/search/tsconfig.json create mode 100644 packages/site/src/components/Navigation/Search.tsx diff --git a/package-lock.json b/package-lock.json index 9b41a8e89..edd1b25a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,11 @@ "styles", "docs" ], + "dependencies": { + "@myst-theme/search": "^0.0.0", + "minisearch": "^7.1.0", + "string.prototype.matchall": "^4.0.11" + }, "devDependencies": { "@changesets/cli": "^2.26.1", "@remix-run/node": "~1.17.0", @@ -7707,6 +7712,14 @@ "resolved": "packages/providers", "link": true }, + "node_modules/@myst-theme/search": { + "resolved": "packages/search", + "link": true + }, + "node_modules/@myst-theme/search-minisearch": { + "resolved": "packages/search-minisearch", + "link": true + }, "node_modules/@myst-theme/site": { "resolved": "packages/site", "link": true @@ -8232,6 +8245,41 @@ } } }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.1.tgz", + "integrity": "sha512-zysS+iU4YP3STKNS6USvFVqI4qqx8EpiwmT5TuCApVEBca+eRCbONi4EgzfNSuVnOXvC5UPHHMjs8RXO6DH9Bg==", + "dependencies": { + "@radix-ui/primitive": "1.1.0", + "@radix-ui/react-compose-refs": "1.1.0", + "@radix-ui/react-context": "1.1.0", + "@radix-ui/react-dismissable-layer": "1.1.0", + "@radix-ui/react-focus-guards": "1.1.0", + "@radix-ui/react-focus-scope": "1.1.0", + "@radix-ui/react-id": "1.1.0", + "@radix-ui/react-portal": "1.1.1", + "@radix-ui/react-presence": "1.1.0", + "@radix-ui/react-primitive": "2.0.0", + "@radix-ui/react-slot": "1.1.0", + "@radix-ui/react-use-controllable-state": "1.1.0", + "aria-hidden": "^1.1.1", + "react-remove-scroll": "2.5.7" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-direction": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.0.1.tgz", @@ -16335,7 +16383,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz", "integrity": "sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.5", @@ -16527,7 +16574,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/arraybuffer.prototype.slice/-/arraybuffer.prototype.slice-1.0.3.tgz", "integrity": "sha512-bMxMKAjg13EBSVscxTaYA4mRc5t1UAXa2kXiGTNfZ079HIWXEkKmkgFrh/nJqamaLSrXO5H4WFFkPEaLJWbs3A==", - "dev": true, "license": "MIT", "dependencies": { "array-buffer-byte-length": "^1.0.1", @@ -19687,7 +19733,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.1.tgz", "integrity": "sha512-0lht7OugA5x3iJLOWFhWK/5ehONdprk0ISXqVFn/NFrDu+cuc8iADFrGQz5BnRK7LLU3JmkbXSxaqX+/mXYtUA==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.6", @@ -19705,7 +19750,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/data-view-byte-length/-/data-view-byte-length-1.0.1.tgz", "integrity": "sha512-4J7wRJD3ABAzr8wP+OcIcqq2dlUKp4DVflx++hs5h5ZKydWMI6/D/fAot+yh6g2tHh8fLFTvNOaVN357NvSrOQ==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7", @@ -19723,7 +19767,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/data-view-byte-offset/-/data-view-byte-offset-1.0.0.tgz", "integrity": "sha512-t/Ygsytq+R995EJ5PZlD4Cu56sWa8InXySaViRzw9apusqsOO2bQP+SbYzAhR0pFKoB+43lYy8rWban9JSuXnA==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.6", @@ -20017,7 +20060,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "dev": true, "license": "MIT", "dependencies": { "define-data-property": "^1.0.1", @@ -20872,7 +20914,6 @@ "version": "1.23.3", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.23.3.tgz", "integrity": "sha512-e+HfNH61Bj1X9/jLc5v1owaLYuHdeHHSQlkhCBiTK8rBvKaULl/beGMxwrMXjpYrv4pz22BlY570vVePA2ho4A==", - "dev": true, "license": "MIT", "dependencies": { "array-buffer-byte-length": "^1.0.1", @@ -21008,7 +21049,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz", "integrity": "sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0" @@ -21021,7 +21061,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.3.tgz", "integrity": "sha512-3T8uNMC3OQTHkFUsFq8r/BwAXLHvU/9O9mE0fBc/MY5iq/8H7ncvO947LmYA6ldWw9Uh8Yhf25zu6n7nML5QWQ==", - "dev": true, "license": "MIT", "dependencies": { "get-intrinsic": "^1.2.4", @@ -21046,7 +21085,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", - "dev": true, "license": "MIT", "dependencies": { "is-callable": "^1.1.4", @@ -24172,7 +24210,6 @@ "version": "1.1.6", "resolved": "https://registry.npmjs.org/function.prototype.name/-/function.prototype.name-1.1.6.tgz", "integrity": "sha512-Z5kx79swU5P27WEayXM1tBi5Ze/lbIyiNgU3qyXUOf9b2rgXYyF9Dy9Cx+IQv/Lc8WCG6L82zwUPpSS9hGehIg==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.2", @@ -24191,7 +24228,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.3.tgz", "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -24339,7 +24375,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.0.2.tgz", "integrity": "sha512-g0QYk1dZBxGwk+Ngc+ltRH2IBp2f7zBkBMBJZCDerh6EhlhSR6+9irMCuT/09zD6qkarHUSn529sK/yL4S27mg==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.5", @@ -24648,7 +24683,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", - "dev": true, "license": "MIT", "dependencies": { "define-properties": "^1.2.1", @@ -25000,7 +25034,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -26146,7 +26179,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.7.tgz", "integrity": "sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -26349,7 +26381,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.4.tgz", "integrity": "sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.2", @@ -26388,7 +26419,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", - "dev": true, "license": "MIT", "dependencies": { "has-bigints": "^1.0.1" @@ -26414,7 +26444,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.2", @@ -26517,7 +26546,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-data-view/-/is-data-view-1.0.1.tgz", "integrity": "sha512-AHkaJrsUVW6wq6JS8y3JnM/GJF/9cf+k20+iDzlSaJrinEo5+7vRiteOSwBhHRiAyQATN1AmY4hwzxJKPmYf+w==", - "dev": true, "license": "MIT", "dependencies": { "is-typed-array": "^1.1.13" @@ -26533,7 +26561,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.5.tgz", "integrity": "sha512-9YQaSxsAiSwcvS33MBk3wTCVnWK+HhF8VZR2jRxehM16QcVOdHqPn4VPHmRK4lSr38n9JriurInLcP90xsYNfQ==", - "dev": true, "license": "MIT", "dependencies": { "has-tostringtag": "^1.0.0" @@ -26756,7 +26783,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.3.tgz", "integrity": "sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -26778,7 +26804,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", - "dev": true, "license": "MIT", "dependencies": { "has-tostringtag": "^1.0.0" @@ -26857,7 +26882,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", "integrity": "sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.2", @@ -26887,7 +26911,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.3.tgz", "integrity": "sha512-nA2hv5XIhLR3uVzDDfCIknerhx8XUKnstuOERPNNIinXG7v9u+ohXF67vxm4TPTEPU6lm61ZkwP3c9PCB97rhg==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7" @@ -26915,7 +26938,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.7.tgz", "integrity": "sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==", - "dev": true, "license": "MIT", "dependencies": { "has-tostringtag": "^1.0.0" @@ -26956,7 +26978,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==", - "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.2" @@ -27013,7 +27034,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", "integrity": "sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.2" @@ -30944,6 +30964,11 @@ "dev": true, "license": "ISC" }, + "node_modules/minisearch": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.1.0.tgz", + "integrity": "sha512-tv7c/uefWdEhcu6hvrfTihflgeEi2tN6VV7HJnCjK6VxM75QQJh4t9FwJCsA2EsRS8LCnu3W87CuGPWMocOLCA==" + }, "node_modules/minizlib": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", @@ -32397,7 +32422,6 @@ "version": "1.13.2", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -32427,7 +32451,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -32437,7 +32460,6 @@ "version": "4.1.5", "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.5.tgz", "integrity": "sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.5", @@ -35044,7 +35066,6 @@ "version": "1.5.2", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz", "integrity": "sha512-NcDiDkTLuPR+++OCKB0nWafEmhg/Da8aUPLPMQbK+bxKKCm1/S5he+AqYa4PlMCVBalb4/yxIRub6qkEx5yJbw==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.6", @@ -35986,7 +36007,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz", "integrity": "sha512-vj6RsCsWBCf19jIeHEfkRMw8DPiBb+DMXklQ/1SGDHOMlHdPUkZXFQ2YdplS23zESTijAcurb1aSgJA3AgMu1Q==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7", @@ -36011,7 +36031,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.3.tgz", "integrity": "sha512-CdASjNJPvRa7roO6Ra/gLYBTzYzzPyyBXxIMdGW3USQLyjWEls2RgW5UBTXaQVp+OrpeCK3bLem8smtmheoRuw==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.6", @@ -36236,7 +36255,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.2.tgz", "integrity": "sha512-7PGFlmtwsEADb0WYyvCMa1t+yke6daIG4Wirafur5kcf+MhUnPms1UeR0CKQdTZD81yESwMHbtn+TR+dMviakQ==", - "dev": true, "license": "MIT", "dependencies": { "define-data-property": "^1.1.4", @@ -36342,7 +36360,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7", @@ -37065,8 +37082,6 @@ "version": "4.0.11", "resolved": "https://registry.npmjs.org/string.prototype.matchall/-/string.prototype.matchall-4.0.11.tgz", "integrity": "sha512-NUdh0aDavY2og7IbBPenWqR9exH+E26Sv8e0/eTe1tltDGZL+GtBkDAnnyBtmekfK6/Dq3MkcGtzXFEd1LQrtg==", - "dev": true, - "license": "MIT", "dependencies": { "call-bind": "^1.0.7", "define-properties": "^1.2.1", @@ -37122,7 +37137,6 @@ "version": "1.2.9", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.9.tgz", "integrity": "sha512-klHuCNxiMZ8MlsOihJhJEBJAiMVqU3Z2nEXWfWnIqjN0gEFS9J9+IxKozWWtQGcgoa1WUZzLjKPTr4ZHNFTFxw==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7", @@ -37141,7 +37155,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.8.tgz", "integrity": "sha512-p73uL5VCHCO2BZZ6krwwQE3kCzM7NKmis8S//xEC6fQonchbum4eP6kR4DLEjQFO3Wnj3Fuo8NM0kOSjVdHjZQ==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7", @@ -37156,7 +37169,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.8.tgz", "integrity": "sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7", @@ -38755,7 +38767,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.2.tgz", "integrity": "sha512-gEymJYKZtKXzzBzM4jqa9w6Q1Jjm7x2d+sh19AdsD4wqnMPDYyvwpsIc2Q/835kHuo3BEQ7CjelGhfTsoBb2MQ==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7", @@ -38770,7 +38781,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/typed-array-byte-length/-/typed-array-byte-length-1.0.1.tgz", "integrity": "sha512-3iMJ9q0ao7WE9tWcaYKIptkNBuOIcZCCT0d4MRvuuH88fEoEH62IuQe0OtraD3ebQEoTRk8XCBoknUNc1Y67pw==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7", @@ -38790,7 +38800,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/typed-array-byte-offset/-/typed-array-byte-offset-1.0.2.tgz", "integrity": "sha512-Ous0vodHa56FviZucS2E63zkgtgrACj7omjwd/8lTEMEPFFyjfixMZ1ZXenpgCFBBt4EC1J2XsyVS2gkG0eTFA==", - "dev": true, "license": "MIT", "dependencies": { "available-typed-arrays": "^1.0.7", @@ -38811,7 +38820,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.6.tgz", "integrity": "sha512-/OxDN6OtAk5KBpGb28T+HZc2M+ADtvRxXrKKbUwtsLgdoxgX13hyy7ek6bFRl5+aBs2yZzB0c4CnQfAtVypW/g==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.7", @@ -38920,7 +38928,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", - "dev": true, "license": "MIT", "dependencies": { "call-bind": "^1.0.2", @@ -40432,7 +40439,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==", - "dev": true, "license": "MIT", "dependencies": { "is-bigint": "^1.0.1", @@ -40967,6 +40973,7 @@ "version": "0.12.0", "license": "MIT", "dependencies": { + "@myst-theme/search": "^0.0.0", "myst-common": "^1.6.0", "myst-config": "^1.5.0", "myst-spec-ext": "^1.6.0", @@ -41255,6 +41262,19 @@ "react-dom": "^16.8 || ^17.0 || ^18.0" } }, + "packages/search": { + "name": "@myst-theme/search", + "version": "0.0.0", + "license": "MIT" + }, + "packages/search-minisearch": { + "name": "@myst-theme/search-minisearch", + "version": "0.0.0", + "license": "MIT", + "dependencies": { + "@myst-theme/search": "^0.0.0" + } + }, "packages/site": { "name": "@myst-theme/site", "version": "0.12.0", @@ -41267,7 +41287,9 @@ "@myst-theme/frontmatter": "^0.12.0", "@myst-theme/jupyter": "^0.12.0", "@myst-theme/providers": "^0.12.0", + "@myst-theme/search": "^0.0.0", "@radix-ui/react-collapsible": "^1.0.3", + "@radix-ui/react-dialog": "^1.0.3", "classnames": "^2.3.2", "lodash.throttle": "^4.1.1", "myst-common": "^1.6.0", @@ -41278,6 +41300,7 @@ "nbtx": "^0.2.3", "node-cache": "^5.1.2", "node-fetch": "^2.6.11", + "string.prototype.matchall": "^4.0.11", "thebe-react": "0.4.10", "unist-util-select": "^4.0.1" }, @@ -41360,6 +41383,8 @@ "@myst-theme/icons": "^0.12.0", "@myst-theme/jupyter": "^0.12.0", "@myst-theme/providers": "^0.12.0", + "@myst-theme/search": "^0.0.0", + "@myst-theme/search-minisearch": "^0.0.0", "@myst-theme/site": "^0.12.0", "@myst-theme/styles": "^0.12.0", "@remix-run/node": "~1.17.0", diff --git a/package.json b/package.json index b94d33f6c..20b732027 100644 --- a/package.json +++ b/package.json @@ -51,5 +51,10 @@ "npm": ">=7.0.0", "node": ">=14.0.0" }, - "packageManager": "npm@8.10.0" + "packageManager": "npm@8.10.0", + "dependencies": { + "@myst-theme/search": "^0.0.0", + "minisearch": "^7.1.0", + "string.prototype.matchall": "^4.0.11" + } } diff --git a/packages/common/package.json b/packages/common/package.json index b1d48871c..52eed93e9 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -19,6 +19,7 @@ "build": "npm-run-all -l clean -p build:esm" }, "dependencies": { + "@myst-theme/search": "^0.0.0", "myst-common": "^1.6.0", "myst-config": "^1.5.0", "myst-spec-ext": "^1.6.0", diff --git a/packages/common/src/types.ts b/packages/common/src/types.ts index db1ce01b5..02e9a936a 100644 --- a/packages/common/src/types.ts +++ b/packages/common/src/types.ts @@ -2,6 +2,7 @@ import type { Dependency, SourceFileKind } from 'myst-spec-ext'; import type { GenericParent, References } from 'myst-common'; import type { SiteAction, SiteExport, SiteManifest } from 'myst-config'; import type { PageFrontmatter } from 'myst-frontmatter'; +import type { MystSearchIndex } from '@myst-theme/search'; export enum Theme { light = 'light', @@ -25,6 +26,7 @@ export type Heading = { export type SiteLoader = { theme?: Theme; config?: SiteManifest; + searchIndex?: MystSearchIndex; CONTENT_CDN_PORT?: string | number; MODE?: 'app' | 'static'; BASE_URL?: string; diff --git a/packages/providers/src/index.tsx b/packages/providers/src/index.tsx index 8c68d21e6..2b5564e54 100644 --- a/packages/providers/src/index.tsx +++ b/packages/providers/src/index.tsx @@ -5,6 +5,7 @@ export * from './references.js'; export * from './baseurl.js'; export * from './ui.js'; export * from './site.js'; +export * from './search.js'; export * from './tabs.js'; export * from './xref.js'; export * from './renderers.js'; diff --git a/packages/providers/src/search.tsx b/packages/providers/src/search.tsx new file mode 100644 index 000000000..c6d8499fd --- /dev/null +++ b/packages/providers/src/search.tsx @@ -0,0 +1,19 @@ +import React, { useContext } from 'react'; +import type { ISearch } from '@myst-theme/search'; + +const SearchContext = React.createContext(undefined); + +export function SearchProvider({ + search, + children, +}: { + search?: ISearch; + children: React.ReactNode; +}) { + return {children}; +} + +export function useSearch() { + const config = useContext(SearchContext); + return config; +} diff --git a/packages/search-minisearch/.eslintrc.cjs b/packages/search-minisearch/.eslintrc.cjs new file mode 100644 index 000000000..f9a3f7ae7 --- /dev/null +++ b/packages/search-minisearch/.eslintrc.cjs @@ -0,0 +1,5 @@ +module.exports = { + root: true, + extends: ['curvenote'], + ignorePatterns: ['src/**/*.spec.ts'], +}; diff --git a/packages/search-minisearch/README.md b/packages/search-minisearch/README.md new file mode 100644 index 000000000..54edb14c9 --- /dev/null +++ b/packages/search-minisearch/README.md @@ -0,0 +1,3 @@ +# @myst-theme/search-minisearch + +A minisearch implementation for client-side searching in MyST sites. diff --git a/packages/search-minisearch/package.json b/packages/search-minisearch/package.json new file mode 100644 index 000000000..9ad2fcd84 --- /dev/null +++ b/packages/search-minisearch/package.json @@ -0,0 +1,24 @@ +{ + "name": "@myst-theme/search-minisearch", + "version": "0.0.0", + "type": "module", + "exports": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist" + ], + "license": "MIT", + "sideEffects": false, + "scripts": { + "clean": "rimraf dist", + "lint": "eslint \"src/**/*.ts*\" -c ./.eslintrc.cjs", + "lint:format": "prettier --check \"src/**/*.{ts,tsx,md}\"", + "test": "vitest run", + "test:watch": "vitest watch", + "build:esm": "tsc --project ./tsconfig.json --module Node16 --outDir dist --declaration", + "build": "npm-run-all -l clean -p build:esm" + }, + "dependencies": { + "@myst-theme/search": "^0.0.0" + } +} diff --git a/packages/search-minisearch/src/index.ts b/packages/search-minisearch/src/index.ts new file mode 100644 index 000000000..c57dcca07 --- /dev/null +++ b/packages/search-minisearch/src/index.ts @@ -0,0 +1,80 @@ +import MiniSearch, { type Options, type SearchResult as MiniSearchResult } from 'minisearch'; +import type { SearchRecord, SearchResult, ISearch } from '@myst-theme/search'; + +export type ExtendedOptions = Options & Required>; + +export function prepareOptions(options: Options): ExtendedOptions { + return { + ...options, + tokenize: MiniSearch.getDefault('tokenize'), + processTerm: MiniSearch.getDefault('processTerm'), + }; +} + +export type RawSearchResult = SearchRecord & MiniSearchResult; + +export function combineResults(results: Map>) { + const [firstEntry, ...restEntries] = results.entries(); + + const firstRawResults = firstEntry[1]; + const initialValue = new Map( + Array.from(firstRawResults.entries(), ([id, rawResult]) => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { id: _, score, terms, queryTerms, match, ...rest } = rawResult; + return [ + id, + { + id, + queries: [ + { + term: queryTerms[0], + matches: match, + }, + ], + ...rest, + }, + ]; + }), + ); + const mergedResults = restEntries.reduce( + (accumulator: Map, value: [string, Map]) => { + const nextAccumulator = new Map(); + + const rawResults = value[1]; + rawResults.forEach((rawResult, docID) => { + const existing = accumulator.get(docID); + if (existing == null) { + return; + } + const { queryTerms, match } = rawResult; + existing.queries.push({ + term: queryTerms[0], + matches: match, + }); + nextAccumulator.set(docID, existing); + }); + return nextAccumulator; + }, + initialValue, + ); + return Array.from(mergedResults.values()); +} + +export function createSearch(documents: SearchRecord[], options: Options): ISearch { + const extendedOptions = prepareOptions(options); + const search = new MiniSearch(extendedOptions); + search.addAll(documents.map((doc, index) => ({ ...doc, id: index }))); + return (query: string) => { + // Implement executeQuery whilst retaining distinction between terms + // TODO: should we check for unique terms? + const terms = extendedOptions.tokenize(query); + const termResults = new Map( + terms.map((term) => [ + term, + new Map(search.search(term).map((doc) => [doc.id, doc as RawSearchResult])), + ]), + ); + + return combineResults(termResults); + }; +} diff --git a/packages/search-minisearch/tsconfig.json b/packages/search-minisearch/tsconfig.json new file mode 100644 index 000000000..ffd179e41 --- /dev/null +++ b/packages/search-minisearch/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../tsconfig/esm.json", + "include": ["."], + "exclude": ["dist", "node_modules", "src/**/*.spec.ts", "tests"] +} diff --git a/packages/search/.eslintrc.cjs b/packages/search/.eslintrc.cjs new file mode 100644 index 000000000..f9a3f7ae7 --- /dev/null +++ b/packages/search/.eslintrc.cjs @@ -0,0 +1,5 @@ +module.exports = { + root: true, + extends: ['curvenote'], + ignorePatterns: ['src/**/*.spec.ts'], +}; diff --git a/packages/search/README.md b/packages/search/README.md new file mode 100644 index 000000000..e28bb3fb9 --- /dev/null +++ b/packages/search/README.md @@ -0,0 +1,3 @@ +# @myst-theme/search + +An implementation and spec for client-side searching in MyST sites. diff --git a/packages/search/package.json b/packages/search/package.json new file mode 100644 index 000000000..20f7ccf48 --- /dev/null +++ b/packages/search/package.json @@ -0,0 +1,22 @@ +{ + "name": "@myst-theme/search", + "version": "0.0.0", + "type": "module", + "exports": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist" + ], + "license": "MIT", + "sideEffects": false, + "scripts": { + "clean": "rimraf dist", + "lint": "eslint \"src/**/*.ts*\" -c ./.eslintrc.cjs", + "lint:format": "prettier --check \"src/**/*.{ts,tsx,md}\"", + "test": "vitest run", + "test:watch": "vitest watch", + "build:esm": "tsc --project ./tsconfig.json --module Node16 --outDir dist --declaration", + "build": "npm-run-all -l clean -p build:esm" + }, + "dependencies": {} +} diff --git a/packages/search/src/index.ts b/packages/search/src/index.ts new file mode 100644 index 000000000..773f6599d --- /dev/null +++ b/packages/search/src/index.ts @@ -0,0 +1,3 @@ +export * from './types.js'; +export * from './rank.js'; +export * from './search.js'; diff --git a/packages/search/src/rank.ts b/packages/search/src/rank.ts new file mode 100644 index 000000000..67ed2ebb5 --- /dev/null +++ b/packages/search/src/rank.ts @@ -0,0 +1,301 @@ +import type { Query, SearchResult, RankedSearchResult, AttributeType } from './types.js'; +import { SEARCH_ATTRIBUTES_ORDERED } from './types.js'; +import { extractField, SPACE_OR_PUNCTUATION } from './search.js'; + +export const POSITIONAL_SEARCH_ATTRIBUTES: AttributeType[] = ['content'] as const; + +// Weights that prioritise headings over content +const TYPE_WEIGHTS = new Map([ + ['lvl1', 90], + ['lvl2', 80], + ['lvl3', 70], + ['lvl4', 60], + ['lvl5', 50], + ['lvl6', 40], + ['content', 0], +]); + +/* + * Generic `cmp` helper function + * + * @param left - left value + * @param right - right value + */ +function cmp(left: number, right: number): number { + if (left < right) { + return -1; + } else if (left > right) { + return +1; + } else { + return 0; + } +} + +/** + * Build a RegExp that matches a single TOKEN bounded by SPACE_OR_PUNCTUATION, or string boundaries + * + * @param text - text to match, e.g. ` foo `, ` foo bar `, `foo bar` + */ +function buildRegExpToken(token: string): RegExp { + return new RegExp( + `(?:(?:${SPACE_OR_PUNCTUATION.source})|^)${token}(?:(?:${SPACE_OR_PUNCTUATION.source})|$)`, + `${SPACE_OR_PUNCTUATION.flags}i`, + ); +} + +/** + * Compute the proximity between two queries, bounded by a limit + * + * @param record - parent search record + * @param left - first query + * @param right - second query + * @param bound - upper limit on computed proximity + */ +function queryPairProximity( + record: SearchResult, + left: Query, + right: Query, + bound: number, +): number { + // TODO: this is highly-nested, and probably slow + // it should be re-written for performance + let bestProximity = bound; + + // For each term in the left query + for (const [leftTerm, leftFields] of Object.entries(left.matches)) { + const leftPattern = buildRegExpToken(leftTerm); + + // For each field matched with this left term + for (const leftField of leftFields) { + // Pull out the (left) field content + const content = extractField(record, leftField); + + // For each term in the right query + for (const [rightTerm, rightFields] of Object.entries(right.matches)) { + const rightPattern = buildRegExpToken(rightTerm); + // For each field matched with this right term + for (const rightField of rightFields) { + // Terms matching different fields can never be better than the bound + if (leftField !== rightField) { + continue; + } + + // Find all of the matches in the content for each pattern + const leftMatches = content.matchAll(leftPattern); + const rightMatches = content.matchAll(rightPattern); + + // Iterate over match pairs + for (const leftMatch of leftMatches) { + for (const rightMatch of rightMatches) { + // Find the ordered (start, stop) pairs for these two matches + const [start, stop] = + leftMatch.index < rightMatch.index + ? [leftMatch.index, rightMatch.index] + : [rightMatch.index, leftMatch.index]; + + // Identify how many token separators there are in this range + const numSeparators = Array.from( + content.slice(start, stop).matchAll(SPACE_OR_PUNCTUATION), + ).length; + + // Fast-path, can never beat 1! + if (numSeparators === 1) { + return 1; + } + + // Does this result improve our current proximity? + if (numSeparators < bestProximity) { + bestProximity = numSeparators; + } + } + } + } + } + } + } + return bestProximity; +} + +/** + * Compute the associative pair-wise proximity of a search result + * + * @param result - search result + * @param bound - upper bound on final proximity + */ +function wordsProximity(result: SearchResult, bound: number) { + const { queries } = result; + let proximity = 0; + for (let i = 0; i < queries.length - 1; i++) { + const left = queries[i]; + const right = queries[i + 1]; + + proximity += queryPairProximity(result, left, right, bound); + } + return Math.min(proximity, bound); +} + +/** + * Identify the best-matched attribute and the match position + * + * @param result - search result + */ +function matchedAttributePosition(result: SearchResult): { + attribute: AttributeType; + position: number | undefined; +} { + // Build mapping from fields to terms matching that field + // i.e. invert and flatten `result.queries[...].matches` + const fieldToTerms = new Map(); + result.queries.forEach((query) => { + Object.entries(query.matches).forEach(([term, fields]) => { + fields.forEach((field) => { + let terms = fieldToTerms.get(field); + if (!terms) { + terms = []; + fieldToTerms.set(field, terms); + } + terms.push(term); + }); + }); + }); + + // Find first field that we matched + const attribute = SEARCH_ATTRIBUTES_ORDERED.find((field) => fieldToTerms.has(field))!; + + let position; + // If this field is positional, find the start of the text match + if (POSITIONAL_SEARCH_ATTRIBUTES.includes(attribute)) { + // Find the terms that this field matches + const attributeTerms = fieldToTerms.get(attribute)!; + // Extract the field value + const value = extractField(result, attribute); + // Match each term against the field value, and extract the match position + const matchPositions = attributeTerms + .flatMap( + (term) => + Array.from(value.matchAll(buildRegExpToken(term))) as { + index: number; + }[], + ) + .map((match) => match.index); + // Find the smallest (earliest) match position + position = Math.min(...matchPositions); + } + // Otherwise, we don't care about the position + else { + position = undefined; + } + + return { attribute, position }; +} +/** + * Determine how many terms matched the corpus exactly + * + * @param result - search result + */ +function matchedExactWords(result: SearchResult) { + const allMatches = result.queries.flatMap( + // For each query (foo bar baz -> foo, then bar, then baz) + (query) => + Object.entries(query.matches) + .flatMap( + // For each (match, matched fields) pair in the query matches + ([match, fields]) => { + const pattern = buildRegExpToken(match); + return fields.flatMap( + // For each matched field + (field) => { + // Retrieve corpus and test for pattern + const value = extractField(result, field); + return Array.from(value.matchAll(pattern)).map((m) => (m ? query.term : undefined)); + }, + ); + }, + ) + .filter((item) => item), + ); + const uniqueMatches = new Set(allMatches); + return uniqueMatches.size; +} + +/** + * Determine the number of fuzzy matches in a search result + * + * @param result - search result + */ +function numberOfTypos(result: SearchResult): number { + return result.queries + .map((query) => { + const typoTerms = Object.keys(query.matches).filter((match) => match !== query.term); + return typoTerms.length; + }) + .reduce((sum, value) => sum + value); +} + +/** + * Rank a search result using Algolia-derived metrics + * + * @param result - search result + */ +function rankSearchResult(result: SearchResult): RankedSearchResult { + return { + ...result, + ranking: { + typos: numberOfTypos(result), + ...matchedAttributePosition(result), + proximity: wordsProximity(result, 8), // TODO + exact: matchedExactWords(result), + level: TYPE_WEIGHTS.get(result.type)!, + appearance: result.position, + }, + }; +} + +/** + * Compare ranked search results to prioritise higher rankings + * + * @param left - ranked search result + * @param right - ranked search result + */ +function cmpRankedSearchResults(left: RankedSearchResult, right: RankedSearchResult) { + const leftRank = left.ranking; + const rightRank = right.ranking; + + if (leftRank.typos !== rightRank.typos) { + return cmp(leftRank.typos, rightRank.typos); + } + if (leftRank.attribute !== rightRank.attribute) { + const i = SEARCH_ATTRIBUTES_ORDERED.findIndex((item) => item === leftRank.attribute); + const j = SEARCH_ATTRIBUTES_ORDERED.findIndex((item) => item === rightRank.attribute); + + return cmp(i, j); + } + if ( + leftRank.position != null && + rightRank.position != null && + leftRank.position !== rightRank.position + ) { + return cmp(leftRank.position, rightRank.position); + } + if (leftRank.proximity !== rightRank.proximity) { + return cmp(leftRank.proximity, rightRank.proximity); + } + if (leftRank.exact !== rightRank.exact) { + return cmp(rightRank.exact, leftRank.exact); + } + if (leftRank.level !== rightRank.level) { + return cmp(rightRank.level, leftRank.level); + } + if (leftRank.appearance !== rightRank.appearance) { + return cmp(leftRank.appearance, rightRank.appearance); + } + + return 0; +} + +/** + * Rank and then filter raw search results + */ +export function rankAndFilterResults(results: SearchResult[]): RankedSearchResult[] { + return results.map(rankSearchResult).sort(cmpRankedSearchResults); +} diff --git a/packages/search/src/search.ts b/packages/search/src/search.ts new file mode 100644 index 000000000..36a322344 --- /dev/null +++ b/packages/search/src/search.ts @@ -0,0 +1,7 @@ +export const SPACE_OR_PUNCTUATION = /[\n\r\p{Z}\p{P}]+/gu; +export function extractField(document: Record, fieldName: string) { + // Access nested fields + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + return fieldName.split('.').reduce((doc, key) => doc && doc[key], document); +} diff --git a/packages/search/src/types.ts b/packages/search/src/types.ts new file mode 100644 index 000000000..cac166914 --- /dev/null +++ b/packages/search/src/types.ts @@ -0,0 +1,76 @@ +// myst-spec-ext types +export type DocumentHierarchy = { + lvl1?: string; + lvl2?: string; + lvl3?: string; + lvl4?: string; + lvl5?: string; + lvl6?: string; +}; + +export type HeadingLevel = keyof DocumentHierarchy; + +export type SearchRecordBase = { + hierarchy: DocumentHierarchy; + url: string; + + position: number; +}; +export type HeadingRecord = SearchRecordBase & { + type: HeadingLevel; +}; +export type ContentRecord = SearchRecordBase & { + type: 'content'; + content: string; +}; + +export type SearchRecord = ContentRecord | HeadingRecord; + +export type MystSearchIndex = { + version: '1'; + records: SearchRecord[]; +}; +/// + +export type Query = { + term: string; // Raw search query term + matches: Record; // Match results (match token -> fields[]) +}; + +export type SearchResult = SearchRecord & { + id: string | number; + queries: Query[]; +}; + +export interface ISearch { + (query: string): SearchResult[]; +} + +/// Search ranking +export const SEARCH_ATTRIBUTES_ORDERED = [ + 'hierarchy.lvl1', + 'hierarchy.lvl2', + 'hierarchy.lvl3', + 'hierarchy.lvl4', + 'hierarchy.lvl5', + 'hierarchy.lvl6', + 'content', +] as const; + +export type AttributeType = (typeof SEARCH_ATTRIBUTES_ORDERED)[number]; + +/** + * Type describing a seach result that has ranking + */ +export type RankedSearchResult = SearchResult & { + ranking: { + // words: number; (Aloglia supports dropping words, we don't) + typos: number; + attribute: AttributeType; + position?: number; + proximity: number; + exact: number; + level: number; + appearance: number; + }; +}; diff --git a/packages/search/tsconfig.json b/packages/search/tsconfig.json new file mode 100644 index 000000000..ffd179e41 --- /dev/null +++ b/packages/search/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../tsconfig/esm.json", + "include": ["."], + "exclude": ["dist", "node_modules", "src/**/*.spec.ts", "tests"] +} diff --git a/packages/site/package.json b/packages/site/package.json index 2154ae1c6..b126ce870 100644 --- a/packages/site/package.json +++ b/packages/site/package.json @@ -26,7 +26,9 @@ "@myst-theme/frontmatter": "^0.12.0", "@myst-theme/jupyter": "^0.12.0", "@myst-theme/providers": "^0.12.0", + "@myst-theme/search": "^0.0.0", "@radix-ui/react-collapsible": "^1.0.3", + "@radix-ui/react-dialog": "^1.0.3", "classnames": "^2.3.2", "lodash.throttle": "^4.1.1", "myst-common": "^1.6.0", @@ -37,6 +39,7 @@ "nbtx": "^0.2.3", "node-cache": "^5.1.2", "node-fetch": "^2.6.11", + "string.prototype.matchall": "^4.0.11", "thebe-react": "0.4.10", "unist-util-select": "^4.0.1" }, diff --git a/packages/site/src/components/Navigation/Search.tsx b/packages/site/src/components/Navigation/Search.tsx new file mode 100644 index 000000000..b541e3263 --- /dev/null +++ b/packages/site/src/components/Navigation/Search.tsx @@ -0,0 +1,144 @@ +import { useEffect, useState } from 'react'; +import { + MagnifyingGlassIcon, + DocumentTextIcon, + HashtagIcon, + PencilIcon, +} from '@heroicons/react/24/solid'; +import classNames from 'classnames'; +import * as Dialog from '@radix-ui/react-dialog'; +import type { ISearch, RankedSearchResult, HeadingLevel } from '@myst-theme/search'; +import { SPACE_OR_PUNCTUATION, rankAndFilterResults } from '@myst-theme/search'; +import { withBaseurl, useBaseurl } from '@myst-theme/providers'; + +function matchAll(text: string, pattern: RegExp) { + const matches = []; + let match; + while ((match = pattern.exec(text))) { + matches.push(match); + } + return matches; +} + +function highlightTitle(text: string, result: RankedSearchResult): string { + const allTerms = result.queries.flatMap((query) => Object.keys(query.matches)).join('|'); + const pattern = new RegExp(`\\b(${allTerms})\\b`, 'gi'); + const allMatches = Array.from(matchAll(text, pattern)).map((m) => m); + + const { index: start } = allMatches[0] ?? { index: 0 }; + + const tokens = [ + ...matchAll(text.slice(start), SPACE_OR_PUNCTUATION), + { index: text.length - start }, + ]; + + const limitedTokens = tokens.slice(0, 16); + const { index: offset } = limitedTokens[limitedTokens.length - 1]; + + let title = text.slice(start, start + offset).replace(pattern, '$&'); + if (start !== 0) { + title = `... ${title}`; + } + if (offset !== text.length) { + title = `${title} ...`; + } + + return title; +} + +function SearchItem({ result }: { result: RankedSearchResult }) { + const { hierarchy, type, url } = result; + const baseURL = useBaseurl(); + + // Generic "this document matched" + const kind = type === 'content' ? 'text' : type === 'lvl1' ? 'file' : 'heading'; + const title = highlightTitle( + result.type === 'content' ? result['content'] : hierarchy[type as HeadingLevel]!, + result, + ); + + const icon = + kind === 'file' ? ( + + ) : kind === 'heading' ? ( + + ) : ( + + ); + return ( + +
+ {icon} + +
+ + ); +} + +export function Search({ className, doSearch }: { className?: string; doSearch: ISearch }) { + const [searchResults, setSearchResults] = useState([]); + const [query, setQuery] = useState(); + useEffect(() => { + const timeoutId = setTimeout(() => { + if (query != undefined) { + const rankedResults = rankAndFilterResults(doSearch(query)); + setSearchResults(rankedResults); + } + }, 500); + return () => clearTimeout(timeoutId); + }, [doSearch, query]); + + const handleSearchChange = (event: React.ChangeEvent) => { + setQuery(event.target.value); + }; + return ( + + +
+ + +
+ + + + +
+ + +
+ +
    + {searchResults.map((result) => ( +
  • + {' '} + {' '} +
    • + ))} +
+ + + + + + + + ); +} diff --git a/packages/site/src/components/Navigation/ThemeButton.tsx b/packages/site/src/components/Navigation/ThemeButton.tsx index 2d97e3f70..1aea2ece7 100644 --- a/packages/site/src/components/Navigation/ThemeButton.tsx +++ b/packages/site/src/components/Navigation/ThemeButton.tsx @@ -8,7 +8,7 @@ export function ThemeButton({ className = 'w-8 h-8 mx-3' }: { className?: string return (