Support description shards (#285)

* Load desc shards * Add searchState compatible js * Bump std search index * Feat DescShardManager * Integrate DocSearchV2 * Store desc shards into storage * Remove legacy DocSearch * Fix nightly search index * Add std desc shards * Split desc shards setter * Compatible with libdocrust DocSearch for searchState * Polish desc shards index format * Load search.js to get descShards * Store searchIndex as json array format * Support source mode for std docs search * Support add new crate's searchIndex * Move loadScript to script/lib.js * Check crate version compatibility in docs.rs
huhu · Jun 23, 2024 · 3597c31 · 3597c31
1 parent 46afbd0
commit 3597c31
Show file tree

Hide file tree

Showing 19 changed files with 3,605 additions and 808 deletions.
diff --git a/content-script-bundle.js b/content-script-bundle.js
@@ -8,5 +8,5 @@ export {
     storage,
     settings,
     IndexSetter,
-    CrateDocManager
+    CrateDocManager,
 }
diff --git a/extension/content-script-bundle.js b/extension/content-script-bundle.js
diff --git a/extension/crate-manager.js b/extension/crate-manager.js
@@ -1,4 +1,5 @@
 import storage from "./core/storage.js";
+import IndexSetter from "./index-setter.js";
 
 export default class CrateDocManager {
     static async getCrates() {
@@ -30,7 +31,7 @@ export default class CrateDocManager {
             let crate = Object.entries(crates).find(([_, { crateName }]) => crateName == name);
             if (crate) {
                 let libName = crate[0];
-                return await storage.getItem(`@${libName}`);
+                return new Map(await storage.getItem(`@${libName}`));
             } else {
                 return null;
             }
@@ -43,26 +44,26 @@ export default class CrateDocManager {
     // 
     // Here is the rule: https://docs.rs/{crateName}/{crateVersion}/{libName}
     //
-    // Ensure `searchIndex` is a Object, not a Map.
-    static async addCrate({ libName, crateVersion, searchIndex, crateName }) {
-        if (searchIndex && libName in searchIndex) {
-            await storage.setItem(`@${libName}`, searchIndex);
-            let doc = searchIndex[libName]["doc"];
-            let crates = await CrateDocManager.getCrates();
-            if (libName in crates) {
-                // Don't override the time if the crate exists
-                crates[libName] = { version: crateVersion, doc, time: crates[libName].time, crateName };
-            } else {
-                crates[libName] = { version: crateVersion, doc, time: Date.now(), crateName };
-            }
-            await storage.setItem("crates", crates);
+    // The caller should ensure `searchIndex` is a Map, not a Object.
+    static async addCrate({ libName, crateVersion, crateTitle, searchIndex, crateName, descShards }) {
+        await storage.setItem(`@${libName}`, searchIndex);
+        let doc = crateTitle;
+        let crates = await CrateDocManager.getCrates();
+        if (libName in crates) {
+            // Don't override the time if the crate exists
+            crates[libName] = { version: crateVersion, doc, time: crates[libName].time, crateName };
+        } else {
+            crates[libName] = { version: crateVersion, doc, time: Date.now(), crateName };
         }
+        await storage.setItem("crates", crates);
+        IndexSetter.setDescShards(libName, descShards);
     }
 
     static async removeCrate(name) {
         let crates = await CrateDocManager.getCrates();
         delete crates[name];
         await storage.setItem("crates", crates);
         await storage.removeItem(`@${name}`);
+        await storage.removeItem(`desc-shards-${name}`);
     }
 };
diff --git a/extension/index-manager.js b/extension/index-manager.js
@@ -7,6 +7,7 @@ import rfcsIndex from "./index/rfcs.js";
 import rustcIndex from "./index/rustc.js";
 import targetsIndex from "./index/targets.js";
 import searchIndex from "./index/std-docs.js";
+import stdDescShards from "./index/desc-shards/std.js";
 import { mapping, crateIndex } from "./index/crates.js";
 import storage from "./core/storage.js";
 import IndexSetter from "./index-setter.js";
@@ -18,13 +19,32 @@ import IndexSetter from "./index-setter.js";
 
 export default class IndexManager extends IndexSetter {
     static async getStdStableIndex() {
-        // Convert default map searchIndex to Object since rust 1.76.0
-        return await storage.getItem('index-std-stable') || Object.fromEntries(searchIndex);
+        let index = await storage.getItem('index-std-stable');
+        if (index?.length > 0) {
+            return new Map(index);
+        } else {
+            return searchIndex;
+        }
     }
 
     static async getStdNightlyIndex() {
-        // Convert default map searchIndex to Object since rust 1.76.0
-        return await storage.getItem('index-std-nightly') || Object.fromEntries(searchIndex);
+        let index = await storage.getItem('index-std-nightly');
+        if (index?.length > 0) {
+            return new Map(index);
+        } else {
+            // Structure clone search index is required
+            return structuredClone(searchIndex);
+        }
+
+    }
+
+    static async getDescShards(crate) {
+        let descShards = await storage.getItem(`desc-shards-${crate}`);
+        if (descShards) {
+            return new Map(descShards);
+        } else {
+            return stdDescShards;
+        }
     }
 
     static async getBookIndex() {

diff --git a/extension/index-setter.js b/extension/index-setter.js
@@ -9,6 +9,12 @@ export default class IndexSetter {
         storage.setItem('index-std-nightly', index);
     }
 
+    static setDescShards(crate, shards) {
+        if (shards) {
+            storage.setItem(`desc-shards-${crate}`, shards);
+        }
+    }
+
     static setBookIndex(index) {
         storage.setItem('index-book', index);
     }

diff --git a/extension/index/desc-shards/std.js b/extension/index/desc-shards/std.js
diff --git a/extension/index/std-docs.js b/extension/index/std-docs.js
diff --git a/extension/lib.js b/extension/lib.js
@@ -21,23 +21,13 @@ export default class RustSearchOmnibox {
         lintSearcher,
         commandManager,
     }) {
-        // All dynamic setting items. Those items will been updated
-        // in chrome.storage.onchange listener callback.
-        let isOfflineMode = await settings.isOfflineMode;
-
         function formatDoc(index, doc) {
             let content = doc.href;
             let description = doc.displayPath + `<match>${doc.name}</match>`;
             if (doc.desc) {
                 description += ` - <dim>${Compat.escape(Compat.eliminateTags(doc.desc))}</dim>`;
             }
 
-            if (doc.queryType === "s" || doc.queryType === "src") {
-                let url = new URL(doc.href);
-                url.search = "?mode=src";
-                content = url.toString();
-                description = `[Source code] ${description}`;
-            }
             return { content, description };
         }
 
@@ -51,21 +41,18 @@ export default class RustSearchOmnibox {
             ];
         }
 
-        const docsSearchMixins = {
-            onSearch: (query) => {
-                return stdSearcher.search(query);
+        omnibox.bootstrap({
+            onSearch: async (query) => {
+                const result = await stdSearcher.search(query);
+                return result.others || [];
             },
             onFormat: formatDoc,
             onAppend: async (query) => {
                 return [{
-                    content: stdSearcher.getSearchUrl(query),
-                    description: `Search Rust docs <match>${query}</match> on ${await settings.isOfflineMode ? "offline mode" : stdSearcher.getRootPath()}`,
+                    content: await stdSearcher.getSearchUrl(query),
+                    description: `Search Rust docs <match>${query}</match> on ${await settings.isOfflineMode ? "offline mode" : await stdSearcher.rootPath}`,
                 }];
             },
-        };
-
-        omnibox.bootstrap({
-            ...docsSearchMixins,
             onEmptyNavigate: (content, disposition) => {
                 commandManager.handleCommandEnterEvent(content, disposition);
             },
@@ -97,25 +84,44 @@ export default class RustSearchOmnibox {
 
         omnibox.addRegexQueryEvent(/^s(?:rc)?:/i, {
             name: "Source code",
-            ...docsSearchMixins,
+            onSearch: async (query) => {
+                query = query.replace(/^s(?:rc)?:/i, "");
+                const result = await stdSearcher.search(query);
+                return result.others || [];
+            },
+            onFormat: (index, doc) => {
+                let { content, description } = formatDoc(index, doc);
+                let url = new URL(doc.href);
+                url.search = "?mode=src";
+                content = url.toString();
+                description = `[Source code] ${description}`;
+                return { content, description };
+            },
+            onAppend: async (query) => {
+                return [{
+                    content: await stdSearcher.getSearchUrl(query),
+                    description: `Search Rust docs <match>${query}</match> on ${await settings.isOfflineMode ? "offline mode" : await stdSearcher.rootPath}`,
+                }];
+            },
         });
 
         // Nightly std docs search
         omnibox.addPrefixQueryEvent("/", {
             name: "Nightly docs",
-            onSearch: (query) => {
+            onSearch: async (query) => {
                 query = query.replaceAll("/", "").trim();
-                return nightlySearcher.search(query);
+                const result = await nightlySearcher.search(query);
+                return result.others || [];
             },
             onFormat: (index, doc) => {
                 let { content, description } = formatDoc(index, doc);
                 return { content, description: '[Nightly] ' + description };
             },
-            onAppend: (query) => {
+            onAppend: async (query) => {
                 query = query.replaceAll("/", "").trim();
                 return [{
-                    content: nightlySearcher.getSearchUrl(query),
-                    description: `Search nightly Rust docs <match>${query}</match> on ${nightlySearcher.getRootPath()}`,
+                    content: await nightlySearcher.getSearchUrl(query),
+                    description: `Search nightly Rust docs <match>${query}</match> on ${nightlySearcher.rootPath}`,
                 }];
             },
         });

diff --git a/extension/main.js b/extension/main.js
@@ -82,13 +82,8 @@ async function start(omnibox) {
         }),
     );
 
-    let stdSearcher = new DocSearch("std", await IndexManager.getStdStableIndex(), () => {
-        return isOfflineMode ? offlineDocPath : "https://doc.rust-lang.org/";
-    });
-    let nightlySearcher = new DocSearch("std", await IndexManager.getStdNightlyIndex(), () => {
-        // Nightly docs doesn't support offline mode yet.
-        return "https://doc.rust-lang.org/nightly/";
-    });
+    let nightlySearcher = new DocSearch("std", await IndexManager.getStdNightlyIndex(), "https://doc.rust-lang.org/nightly/");
+    let stdSearcher = new DocSearch("std", await IndexManager.getStdStableIndex(), isOfflineMode ? offlineDocPath : "https://doc.rust-lang.org/");
 
     RustSearchOmnibox.run({
         omnibox,