Support for search in odoc

CHANGES.md

@@ -13,6 +13,7 @@ Tags:
 
 ### Added
 - Display 'private' keyword for private type extensions (@gpetiot, #1019)
+- Add support for search (@panglesd, @EmileTrotignon, #972)
 
 ### Fixed
 

doc/driver.mld

@@ -260,12 +260,26 @@ let link ?(ignore_output = false) file =
   if not ignore_output then
     add_prefixed_output cmd link_output (Fpath.to_string file) lines
 
-let html_generate ?(ignore_output = false) file source =
+let html_generate ?(ignore_output = false) ?(assets = []) ?(search_uris = []) file source =
   let open Cmd in
-  let source = match source with None -> empty | Some source -> v "--source" % p source in
+  let source =
+    match source with None -> empty | Some source -> v "--source" % p source
+  in
+  let assets =
+    List.fold_left
+      (fun acc filename -> acc % "--asset" % filename)
+      empty
+      assets
+  in
+  let search_uris =
+    List.fold_left
+      (fun acc filename -> acc % "--search-uri" % p filename)
+      empty
+      search_uris
+  in
   let cmd =
-    odoc % "html-generate" %% source % p file % "-o" % "html" % "--theme-uri" % "odoc"
-    % "--support-uri" % "odoc"
+    odoc % "html-generate" %% source % p file %% assets %% search_uris % "-o" % "html"
+    % "--theme-uri" % "odoc" % "--support-uri" % "odoc"
   in
   let lines = run cmd in
   if not ignore_output then
@@ -277,6 +291,7 @@ let support_files () =
   run cmd
 ]}
 
+
 We'll now make some library lists. We have not only external dependency
 libraries, but [odoc] itself is also separated into libraries. These two
 sets of libraries will be documented in different sections, so we'll keep them
@@ -318,7 +333,8 @@ let dep_libraries =
 let odoc_libraries = [
     "odoc_xref_test"; "odoc_xref2"; "odoc_odoc"; "odoc_html_support_files";
     "odoc_model_desc"; "odoc_model"; "odoc_manpage"; "odoc_loader";
-    "odoc_latex"; "odoc_html"; "odoc_document"; "odoc_examples"; "odoc_parser"; "ocamlary" ];;
+    "odoc_latex"; "odoc_html"; "odoc_document"; "odoc_examples"; "odoc_parser";
+    "ocamlary"; "odoc_search" ; "odoc_html_frontend" ; "odoc_json_index" ];;
 
 let all_libraries = dep_libraries @ odoc_libraries;;
 
@@ -429,6 +445,20 @@ let compile_deps f =
   | _ -> Error (`Msg "odd")
 ]}
 
+For each compiled odoc file, we'll need to remember some options given at
+[odoc compile]-time. An example of this is the source code rendering: when we
+enable the feature at compile time, we need to provide the source file at html
+generation.
+
+{[
+type unit = {
+  file : Fpath.t;
+  ignore_output : bool;
+  source : Fpath.t option;
+  assets : string list;
+}
+]}
+
 For [odoc] libraries, we infer the implementation and interface source file path
 from the library name. We list them in a file, passed to [odoc source-tree], to
 generate [src-source.odoc]. This file contains the source hierarchy, and will be
@@ -490,7 +520,7 @@ let compile_source_tree units =
   let source_map = Fpath.v "source.map" in
   let () = Bos.OS.File.write_lines source_map sources |> get_ok in
   let () = source_tree ~parent:"odoc" ~output:odoc_source_tree source_map in
-  (odoc_source_tree, false, None)
+  { file = odoc_source_tree ; ignore_output = false ; source = None ; assets = [] }
 
 ]}
 
@@ -533,6 +563,8 @@ let all_units =
 Now we'll compile all of the parent [.mld] files. To ensure that the parents are compiled before the children, we start with [odoc.mld], then [deps.mld], and so on. The result of this file is a list of the resulting [odoc] files.
 
 {[
+let search_file = "index.js"
+
 let compile_mlds () =
   let mkpage x = "page-\"" ^ x ^ "\"" in
   let mkmod x = "module-" ^ String.capitalize_ascii x in
@@ -563,9 +595,10 @@ let compile_mlds () =
         "page-" ^ library ^ ".odoc")
       all_libraries
   in
+  { file = Fpath.v "page-odoc.odoc" ; ignore_output = false ; source = None ; assets = [] } ::
   List.map
-    (fun f -> (Fpath.v f, false, None))
-    ("page-odoc.odoc" :: "page-deps.odoc" :: odocs @ extra_odocs)
+    (fun f -> { file = Fpath.v f ; ignore_output = false ; source = None; assets = [] })
+    ( "page-deps.odoc" :: odocs @ extra_odocs)
 ]}
 
 Now we get to the compilation phase. For each unit, we query its dependencies, then recursively call to compile these dependencies. Once this is done we compile the unit itself. If the unit has already been compiled we don't do anything. Note that we aren't checking the hashes of the dependencies which a build system should do to ensure that the module being compiled is the correct one. Again we benefit from the fact that we're creating the docs for one leaf package and that there must be no module name clashes in its dependencies. The result of this function is a list of the resulting [odoc] files.
@@ -600,42 +633,122 @@ let compile_all () =
       let ignore_output = parent = "deps" in
       let source_args = source_args impl in
       compile file ~parent:lib ?source_args ~ignore_output [];
-      (output, ignore_output, impl) :: files
+      { file = output ; ignore_output ; source = impl; assets = [] } :: files
   in
   source_tree
   :: List.fold_left
     (fun acc (parent, lib, dep, impl) ->
       acc @ rec_compile ?impl parent lib (best_file dep))
-    [] all_units
-  @ mld_odocs
+      [] all_units
+   @ mld_odocs
 ]}
 
 Linking is now straightforward. We link all [odoc] files.
 
 {[
 let link_all odoc_files =
   List.map
-    (fun (odoc_file, ignore_output, source) ->
+    (fun ({ file = odoc_file ; ignore_output ; _ } as unit) ->
       ignore (link ~ignore_output odoc_file);
-      Fpath.set_ext "odocl" odoc_file, source)
+      { unit with file = Fpath.set_ext "odocl" odoc_file })
     odoc_files
 ]}
 
 Now we simply run [odoc html-generate] over all of the resulting [odocl] files.
 This will generate sources, as well as documentation for non-hidden units.
+We notify the generator that the javascript file to use for search is [index.js].
 
 {[
 let generate_all odocl_files =
   let relativize_opt = function None -> None | Some file -> Some (relativize file) in
-  List.iter (fun (f, source) -> ignore(html_generate f (relativize_opt source))) odocl_files;
+  let search_uris = [Fpath.v "minisearch.js"; Fpath.v "index.js"] in
+  List.iter
+     (fun ({file = f ; ignore_output = _ ; source ; assets}) ->
+     ignore(html_generate ~assets ~search_uris f (relativize_opt source)))
+     odocl_files;
   support_files ()
 ]}
 
+
+Finally, we generate an index of all values, types, ... This index is meant to be consumed by search engines, to create their own index. It consists of a JSON array, containing entries with the name, full name, associated comment, link and anchor, and kind. Generating the index is done via [odoc compile-index], which create a json file.
+
+Search engines written in OCaml can also call the [Odoc_model.Fold.unit] and  [Odoc_model.Fold.page] function, in conjunction with [Odoc_search.Entry.entry_of_item] in order to get an OCaml value of each element to be indexed.
+
+{[
+let index_generate ?(ignore_output = false) () =
+  let open Cmd in
+  let files =
+    OS.Dir.contents (Fpath.v ".")
+    |> get_ok
+    |> List.filter (Fpath.has_ext "odocl")
+    |> List.filter (fun p -> not (String.equal "src-source.odocl" (Fpath.filename p)))
+    |> List.filter (fun p -> not (is_hidden p))
+    |> List.map Fpath.to_string
+  in
+  let index_map = Fpath.v "index.map" in
+  let () = Bos.OS.File.write_lines index_map files |> get_ok in
+  let cmd =
+    odoc % "compile-index" % "-o" % "html/index.json" % "--file-list"
+    % p index_map
+  in
+  let lines = run cmd in
+  if not ignore_output then
+    add_prefixed_output cmd generate_output "index compilation" lines
+]}
+
+We turn the JSON index into a javascript file. In order to never block the UI, this file will be used as a web worker by [odoc], to perform searches:
+
+- The search query will be sent as a plain string to the web worker, using the standard mechanism of message passing
+- The web worker has to sent back the result as a message to the main thread, containing the list of result. Each entry of this list must have the same form as it had in the original JSON file.
+- The file must be given to the [odoc-support] URI.
+
+In this driver, we use the minisearch javascript library. For more involved application, we could use [index.js] to call a server-side search engine via an API call.
+
+{[
+  let js_index () =
+    let index = Bos.OS.File.read Fpath.(v "html" / "index.json") |> get_ok in
+ Bos.OS.File.writef (Fpath.v search_file) {|
+ let documents =
+   %s
+ ;
+
+ let miniSearch = new MiniSearch({
+  fields: ['id', 'doc', 'entry_id'], // fields to index for full-text search
+   storeFields: ['display'], // fields to return with search results
+   idField: 'entry_id',
+   extractField: (document, fieldName) => {
+     if (fieldName === 'id') {
+       return document.id.map(e => e.kind + "-" + e.name).join('.')
+     }
+     return document[fieldName]
+   }
+ })
+
+
+ // Use a unique id since some entries' id are not unique (type extension or
+ // standalone doc comments for instance)
+ documents.forEach((entry,i) => entry.entry_id = i)
+ miniSearch.addAll(documents);
+
+ onmessage = (m) => {
+   let query = m.data;
+   let result = miniSearch.search(query);
+   postMessage(result.slice(0,200).map(a => a.display));
+ }
+ |} index |> get_ok ;
+    Bos.OS.Cmd.run Bos.Cmd.(v "cp" % search_file % "html/") |> get_ok;
+    Bos.OS.Cmd.run Bos.Cmd.(v "cp" % "minisearch.js" % "html/") |> get_ok;
+]}
+
+
+
 The following code executes all of the above, and we're done!
 
 {[
 let compiled = compile_all () in
 let linked = link_all compiled in
+let () = index_generate () in
+let _ = js_index () in
 generate_all linked
 ]}
 
@@ -647,9 +760,9 @@ Let's see if there was any output from the [odoc] invocations:
 # !link_output;;
 - : string list =
 ["";
- "'../src/odoc/bin/main.exe' 'link' 'odoc_odoc.odoc' '-o' 'odoc_odoc.odocl' '-I' '.'";
- "odoc_odoc.odoc: File \"src/odoc/fs.mli\", line 37, characters 6-73:";
- "odoc_odoc.odoc: Warning: Failed to resolve reference unresolvedroot(Invalid_arg) Couldn't find \"Invalid_arg\"";
+ "'../src/odoc/bin/main.exe' 'link' 'odoc_html_frontend.odoc' '-o' 'odoc_html_frontend.odocl' '-I' '.'";
+ "odoc_html_frontend.odoc: File \"src/search/odoc_html_frontend.mli\", line 3, characters 15-25:";
+ "odoc_html_frontend.odoc: Warning: Failed to resolve reference unresolvedroot(Entry).t Couldn't find \"Entry\"";
  "'../src/odoc/bin/main.exe' 'link' 'page-deps.odoc' '-o' 'page-deps.odocl' '-I' '.'";
  "page-deps.odoc: File \"src/fmt.mli\", line 6, characters 4-13:";
  "page-deps.odoc: Warning: Failed to resolve reference unresolvedroot(Format) Couldn't find \"Format\"";
@@ -674,17 +787,11 @@ Let's see if there was any output from the [odoc] invocations:
  "page-deps.odoc: File \"src/fpath.mli\", line 7, characters 59-71:";
  "page-deps.odoc: Warning: Failed to resolve reference unresolvedroot(Set) Couldn't find \"Set\"";
  "page-deps.odoc: File \"src/fpath.mli\", line 7, characters 28-52:";
- "page-deps.odoc: Warning: Failed to resolve reference unresolvedroot(file_exts) Couldn't find \"file_exts\"";
- "'../src/odoc/bin/main.exe' 'link' 'page-odoc_for_authors.odoc' '-o' 'page-odoc_for_authors.odocl' '-I' '.'";
- "page-odoc_for_authors.odoc: File \"odoc_for_authors.mld\", line 496, character 59 to line 497, character 18:";
- "page-odoc_for_authors.odoc: Warning: Failed to resolve reference unresolvedroot(Features).canonical Couldn't find page \"Features\""]
+ "page-deps.odoc: Warning: Failed to resolve reference unresolvedroot(file_exts) Couldn't find \"file_exts\""]
 # !source_tree_output;;
 - : string list = [""]
 # !generate_output;;
-- : string list =
-["";
- "'../src/odoc/bin/main.exe' 'html-generate' 'odoc_examples.odocl' '-o' 'html' '--theme-uri' 'odoc' '--support-uri' 'odoc'";
- "odoc_examples.odocl: Warning, resolved hidden path: Odoc_examples__.Unexposed.t"]
+- : string list = [""]
 ]}
 
 We can have a look at the produced hierarchy of files, which matches the desired output. Note that source files with a [.ml.html] extension are generated for modules compiled with the [--source] option.
@@ -707,13 +814,18 @@ odoc_document
 odoc_examples
 odoc_for_authors.html
 odoc_html
+odoc_html_frontend
 odoc_html_support_files
+odoc_json_index
 odoc_latex
 odoc_loader
 odoc_manpage
 odoc_model
 odoc_model_desc
 odoc_odoc
+odoc_parser
+odoc_search
+odoc_search.js
 odoc_xref2
 odoc_xref_test
 parent_child_spec.html
@@ -756,13 +868,17 @@ html/odoc/odoc_html/index.html
 html/odoc/odoc_html/Odoc_html
 html/odoc/odoc_html/Odoc_html/Config
 html/odoc/odoc_html/Odoc_html/Config/index.html
+html/odoc/odoc_html/Odoc_html_frontend
+html/odoc/odoc_html/Odoc_html_frontend/index.html
 html/odoc/odoc_html/Odoc_html/Generator
 html/odoc/odoc_html/Odoc_html/Generator/index.html
 html/odoc/odoc_html/Odoc_html/Html_fragment_json
 html/odoc/odoc_html/Odoc_html/Html_fragment_json/index.html
 html/odoc/odoc_html/Odoc_html/Html_page
 html/odoc/odoc_html/Odoc_html/Html_page/index.html
 html/odoc/odoc_html/Odoc_html/index.html
+html/odoc/odoc_html/Odoc_html/Json
+html/odoc/odoc_html/Odoc_html/Json/index.html
 html/odoc/odoc_html/Odoc_html/Link
 html/odoc/odoc_html/Odoc_html/Link/index.html
 html/odoc/odoc_html/Odoc_html/Link/Path

doc/dune

@@ -33,6 +33,7 @@
   (package odoc)
   (universe) ; Benchmark depends on the running time of odoc commands
   (glob_files *.mld)
+  (glob_files *.js)
   (glob_files library_mlds/*)
   (glob_files examples/*.ml*))
  (action

doc/library_mlds/odoc_html_frontend.mld

@@ -0,0 +1,3 @@
+{0 odoc_html_frontend}
+
+{!childmodule-Odoc_html_frontend}

doc/library_mlds/odoc_json_index.mld

@@ -0,0 +1,3 @@
+{0 odoc_json_index}
+
+{!childmodule-Odoc_json_index}

doc/library_mlds/odoc_search.mld

@@ -0,0 +1,3 @@
+{0 odoc_search}
+
+{!childmodule-Odoc_search}

doc/odoc_for_authors.mld

@@ -494,7 +494,7 @@ examples below. They are:
 - [@since string] - declares from which version the element has been available.
 - [@version string] - declares the version of the element itself.
 - [@canonical string] - declares the path to the canonical instance of this element.
-Can be applied to modules, module types and types. See the {{!page-Features.canonical}
+Can be applied to modules, module types and types. See the {{!page-features.canonical}
 Language Features} page for more details.
 
 The content of the tag is then the rest of the line, and it is uninterpreted,

src/document/ML.mli

@@ -30,3 +30,7 @@ val source_page :
   Lang.Source_info.infos ->
   string ->
   Types.Document.t
+
+val type_expr : ?needs_parentheses:bool -> Lang.TypeExpr.t -> Codefmt.t
+
+val record : Lang.TypeDecl.Field.t list -> Types.DocumentedSrc.one list

src/document/generator.ml

@@ -470,6 +470,8 @@ module Make (Syntax : SYNTAX) = struct
 
     val extension : Lang.Extension.t -> Item.t
 
+    val record : Lang.TypeDecl.Field.t list -> DocumentedSrc.one list
+
     val exn : Lang.Exception.t -> Item.t
 
     val format_params :
@@ -1888,6 +1890,10 @@ module Make (Syntax : SYNTAX) = struct
 
   include Page
 
+  let type_expr = type_expr
+
+  let record = record
+
   let source_page id syntax_info infos source_code =
     Document.Source_page (Source_page.source id syntax_info infos source_code)
 end

src/document/generator_signatures.ml

@@ -114,4 +114,8 @@ module type GENERATOR = sig
     Lang.Source_info.infos ->
     string ->
     Document.t
+
+  val type_expr : ?needs_parentheses:bool -> Lang.TypeExpr.t -> text
+
+  val record : Lang.TypeDecl.Field.t list -> DocumentedSrc.one list
 end