From b5f01f0723abab077f649339bdf4ca20d660525a Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Fri, 1 Aug 2025 14:07:05 +0200 Subject: [PATCH 01/20] Add markdown alias and dune rules --- src/dune_rules/alias0.ml | 1 + src/dune_rules/alias0.mli | 1 + src/dune_rules/odoc.ml | 232 ++++++++++++++---- .../test-cases/odoc/doc-markdown.t | 71 ++++++ .../test-cases/odoc/markdown-with-mld.t | 69 ++++++ 5 files changed, 326 insertions(+), 48 deletions(-) create mode 100644 test/blackbox-tests/test-cases/odoc/doc-markdown.t create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-with-mld.t diff --git a/src/dune_rules/alias0.ml b/src/dune_rules/alias0.ml index 3fc892ca0c7..dba8d5f92f2 100644 --- a/src/dune_rules/alias0.ml +++ b/src/dune_rules/alias0.ml @@ -20,6 +20,7 @@ let lint = standard "lint" let private_doc = standard "doc-private" let doc = standard "doc" let doc_json = standard "doc-json" +let doc_markdown = standard "doc-markdown" let doc_new = standard "doc-new" let check = standard "check" let install = standard "install" diff --git a/src/dune_rules/alias0.mli b/src/dune_rules/alias0.mli index c5f86390e33..2b37a713684 100644 --- a/src/dune_rules/alias0.mli +++ b/src/dune_rules/alias0.mli @@ -7,6 +7,7 @@ module Name := Dune_engine.Alias.Name val fmt : Name.t val doc : Name.t val doc_json : Name.t +val doc_markdown : Name.t val lint : Name.t val private_doc : Name.t val doc_new : Name.t diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index d52f92ea8bd..079d6214574 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -75,6 +75,7 @@ type odoc_artefact = ; odocl_file : Path.Build.t ; html_file : Path.Build.t ; json_file : Path.Build.t + ; markdown_file : Path.Build.t } let add_rule sctx = @@ -94,6 +95,7 @@ module Paths = struct ;; let html_root ctx = root ctx ++ "_html" + let markdown_root ctx = root ctx ++ "_markdown" let odocl_root ctx = root ctx ++ "_odocls" let add_pkg_lnu base m = @@ -105,40 +107,47 @@ module Paths = struct ;; let html ctx m = add_pkg_lnu (html_root ctx) m + let markdown ctx m = add_pkg_lnu (markdown_root ctx) m let odocl ctx m = add_pkg_lnu (odocl_root ctx) m let gen_mld_dir ctx pkg = root ctx ++ "_mlds" ++ Package.Name.to_string pkg let odoc_support ctx = html_root ctx ++ odoc_support_dirname let toplevel_index ctx = html_root ctx ++ "index.html" + let markdown_index ctx = markdown_root ctx ++ "index.md" end module Output_format = struct type t = | Html | Json + | Markdown - let all = [ Html; Json ] + let all = [ Html; Json; Markdown ] let iter ~f = Memo.parallel_iter all ~f let extension = function | Html -> ".html" | Json -> ".html.json" + | Markdown -> ".md" ;; let args = function | Html -> Command.Args.empty | Json -> A "--as-json" + | Markdown -> Command.Args.empty ;; let target t odoc_file = match t with | Html -> odoc_file.html_file | Json -> odoc_file.json_file + | Markdown -> odoc_file.markdown_file ;; let alias t ~dir = match t with | Html -> Alias.make Alias0.doc ~dir | Json -> Alias.make Alias0.doc_json ~dir + | Markdown -> Alias.make Alias0.doc_markdown ~dir ;; let toplevel_index_path format ctx = @@ -146,6 +155,7 @@ module Output_format = struct match format with | Html -> base | Json -> Path.Build.extend_basename base ~suffix:".json" + | Markdown -> Paths.markdown_index ctx ;; end @@ -167,7 +177,15 @@ module Dep : sig These dependencies may be used using the [deps] function *) val setup_deps : Context.t -> target -> Path.Set.t -> unit Memo.t end = struct - let format_alias f ctx m = Output_format.alias f ~dir:(Paths.html ctx m) + let format_alias f ctx m = + let dir = + match (f : Output_format.t) with + | Html | Json -> Paths.html ctx m + | Markdown -> Paths.markdown ctx m + in + Output_format.alias f ~dir + ;; + let alias = Alias.make (Alias.Name.of_string ".odoc-all") let deps ctx pkg requires = @@ -454,33 +472,52 @@ let setup_library_odoc_rules cctx (local_lib : Lib.Local.t) = let setup_generate sctx ~search_db odoc_file out = let ctx = Super_context.context sctx in let odoc_support_path = Paths.odoc_support ctx in - let search_args = - Sherlodoc.odoc_args sctx ~search_db ~dir_sherlodoc_dot_js:(Paths.html_root ctx) + let command, output_dir, args = + match out with + | Output_format.Markdown -> + ( "markdown-generate" + , Paths.markdown_root ctx + , [ Command.Args.A "-o" + ; Command.Args.Path (Path.build (Paths.markdown_root ctx)) + ; Command.Args.Dep (Path.build odoc_file.odocl_file) + ; Command.Args.Hidden_targets [ Output_format.target out odoc_file ] + ] ) + | Html | Json -> + let search_args = + match search_db with + | None -> Command.Args.empty + | Some search_db -> + Sherlodoc.odoc_args sctx ~search_db ~dir_sherlodoc_dot_js:(Paths.html_root ctx) + in + ( "html-generate" + , Paths.html_root ctx + , [ search_args + ; Command.Args.A "-o" + ; Command.Args.Path (Path.build (Paths.html_root ctx)) + ; Command.Args.A "--support-uri" + ; Command.Args.Path (Path.build odoc_support_path) + ; Command.Args.A "--theme-uri" + ; Command.Args.Path (Path.build odoc_support_path) + ; Command.Args.Dep (Path.build odoc_file.odocl_file) + ; Output_format.args out + ; Command.Args.Hidden_targets [ Output_format.target out odoc_file ] + ] ) in let run_odoc = - run_odoc - sctx - ~dir:(Path.build (Paths.html_root ctx)) - "html-generate" - ~quiet:false - ~flags_for:None - [ search_args - ; A "-o" - ; Path (Path.build (Paths.html_root ctx)) - ; A "--support-uri" - ; Path (Path.build odoc_support_path) - ; A "--theme-uri" - ; Path (Path.build odoc_support_path) - ; Dep (Path.build odoc_file.odocl_file) - ; Output_format.args out - ; Hidden_targets [ Output_format.target out odoc_file ] - ] + run_odoc sctx ~dir:(Path.build output_dir) command ~quiet:false ~flags_for:None args in add_rule sctx run_odoc ;; -let setup_generate_all sctx ~search_db odoc_file = - Output_format.iter ~f:(setup_generate sctx ~search_db odoc_file) +(* let setup_generate_all sctx ~search_db odoc_file = + Output_format.iter ~f:(setup_generate sctx ~search_db:(Some search_db) odoc_file) *) +let setup_generate_html_and_json sctx ~search_db odoc_file = + let* () = setup_generate sctx ~search_db:(Some search_db) odoc_file Html in + setup_generate sctx ~search_db:(Some search_db) odoc_file Json +;; + +let setup_generate_markdown sctx odoc_file = + setup_generate sctx ~search_db:None odoc_file Markdown ;; let setup_css_rule sctx = @@ -510,10 +547,15 @@ module Toplevel_index = struct ; link : string } - let of_packages packages = + let of_packages packages output_format = Package.Name.Map.to_list_map packages ~f:(fun name package -> let name = Package.Name.to_string name in - { name; version = Package.version package; link = sp "%s/index.html" name }) + let extension = + match (output_format : Output_format.t) with + | Markdown -> "md" + | Html | Json -> "html" + in + { name; version = Package.version package; link = sp "%s/index.%s" name extension }) ;; let html_list_items t = @@ -576,26 +618,35 @@ module Toplevel_index = struct let json t = Dune_stats.Json.to_string (to_json t) + let markdown t = + let b = Buffer.create 256 in + Buffer.add_string b "# OCaml Package Documentation\n\n"; + List.iter t ~f:(fun { name; version; link } -> + Buffer.add_string b (sp "- [%s](%s)" name link); + (match version with + | None -> () + | Some v -> Buffer.add_string b (sp " (version %s)" (Package_version.to_string v))); + Buffer.add_char b '\n'); + Buffer.contents b + ;; + let content (output : Output_format.t) t = match output with | Html -> html t | Json -> json t + | Markdown -> markdown t ;; end let setup_toplevel_index_rule sctx output = let* packages = Dune_load.packages () in - let index = Toplevel_index.of_packages packages in + let index = Toplevel_index.of_packages packages output in let content = Toplevel_index.content output index in let ctx = Super_context.context sctx in let path = Output_format.toplevel_index_path output ctx in add_rule sctx (Action_builder.write_file path content) ;; -let setup_toplevel_index_rules sctx = - Output_format.iter ~f:(setup_toplevel_index_rule sctx) -;; - let libs_of_pkg ctx ~pkg = let+ { Scope.DB.Lib_entry.Set.libraries; _ } = Scope.DB.lib_entries_of_package ctx pkg @@ -631,6 +682,7 @@ let entry_modules sctx ~pkg = let create_odoc ctx ~target odoc_file = let html_base = Paths.html ctx target in + let markdown_base = Paths.markdown ctx target in let odocl_base = Paths.odocl ctx target in let basename = Path.Build.basename odoc_file |> Filename.remove_extension in let odocl_file = odocl_base ++ (basename ^ ".odocl") in @@ -638,16 +690,36 @@ let create_odoc ctx ~target odoc_file = | Lib _ -> let html_dir = html_base ++ Stdune.String.capitalize basename in let file output = - html_dir ++ "index" - |> Path.Build.extend_basename ~suffix:(Output_format.extension output) + match output with + | Output_format.Html | Json -> + html_dir ++ "index" + |> Path.Build.extend_basename ~suffix:(Output_format.extension output) + | Markdown -> + markdown_base ++ basename + |> Path.Build.extend_basename ~suffix:(Output_format.extension output) in - { odoc_file; odocl_file; html_file = file Html; json_file = file Json } + { odoc_file + ; odocl_file + ; html_file = file Html + ; json_file = file Json + ; markdown_file = file Markdown + } | Pkg _ -> let file output = - html_base ++ (basename |> String.drop_prefix ~prefix:"page-" |> Option.value_exn) + let base = + match (output : Output_format.t) with + | Markdown -> markdown_base + | Html | Json -> html_base + in + base ++ (basename |> String.drop_prefix ~prefix:"page-" |> Option.value_exn) |> Path.Build.extend_basename ~suffix:(Output_format.extension output) in - { odoc_file; odocl_file; html_file = file Html; json_file = file Json } + { odoc_file + ; odocl_file + ; html_file = file Html + ; json_file = file Json + ; markdown_file = file Markdown + } ;; let check_mlds_no_dupes ~pkg ~mlds = @@ -774,6 +846,7 @@ let out_file (output : Output_format.t) odoc = match output with | Html -> odoc.html_file | Json -> odoc.json_file + | Markdown -> odoc.markdown_file ;; let out_files ctx (output : Output_format.t) odocs = @@ -781,6 +854,7 @@ let out_files ctx (output : Output_format.t) odocs = match output with | Html -> [ Path.build (Paths.odoc_support ctx) ] | Json -> [] + | Markdown -> [] in Path.build (Output_format.toplevel_index_path output ctx) :: List.rev_append @@ -788,6 +862,13 @@ let out_files ctx (output : Output_format.t) odocs = (List.map odocs ~f:(fun odoc -> Path.build (out_file output odoc))) ;; +let add_format_alias_deps ctx format target odocs = + let paths = out_files ctx format odocs in + Rules.Produce.Alias.add_deps + (Dep.format_alias format ctx target) + (Action_builder.paths paths) +;; + let setup_lib_html_rules_def = let module Input = struct module Super_context = Super_context.As_memo_key @@ -803,11 +884,8 @@ let setup_lib_html_rules_def = let ctx = Super_context.context sctx in let target = Lib lib in let* odocs = odoc_artefacts sctx target in - Output_format.iter ~f:(fun output -> - let paths = out_files ctx output odocs in - Rules.Produce.Alias.add_deps - (Dep.format_alias output ctx target) - (Action_builder.paths paths)) + let* () = add_format_alias_deps ctx Html target odocs in + add_format_alias_deps ctx Json target odocs in Memo.With_implicit_output.create "setup-library-html-rules" @@ -829,7 +907,8 @@ let setup_lib_html_rules sctx ~search_db lib = let target = Lib lib in let* odocs = odoc_artefacts sctx target in let* () = - Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_all sctx ~search_db odoc) + Memo.parallel_iter odocs ~f:(fun odoc -> + setup_generate_html_and_json sctx ~search_db odoc) in Memo.With_implicit_output.exec setup_lib_html_rules_def (sctx, lib) ;; @@ -849,12 +928,11 @@ let setup_pkg_html_rules_def = Sherlodoc.search_db sctx ~dir ~external_odocls:[] odocls in let* () = Memo.parallel_iter libs ~f:(setup_lib_html_rules sctx ~search_db) in - let* () = Memo.parallel_iter pkg_odocs ~f:(setup_generate_all ~search_db sctx) in - Output_format.iter ~f:(fun output -> - let paths = out_files ctx output all_odocs in - Rules.Produce.Alias.add_deps - (Dep.format_alias output ctx (Pkg pkg)) - (Action_builder.paths paths)) + let* () = + Memo.parallel_iter pkg_odocs ~f:(setup_generate_html_and_json ~search_db sctx) + in + let* () = add_format_alias_deps ctx Html (Pkg pkg) all_odocs in + add_format_alias_deps ctx Json (Pkg pkg) all_odocs in setup_pkg_rules_def "setup-package-html-rules" f ;; @@ -863,6 +941,34 @@ let setup_pkg_html_rules sctx ~pkg : unit Memo.t = Memo.With_implicit_output.exec setup_pkg_html_rules_def (sctx, pkg) ;; +let setup_lib_markdown_rules sctx lib = + let ctx = Super_context.context sctx in + let target = Lib lib in + let* odocs = odoc_artefacts sctx target in + let* () = Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) in + add_format_alias_deps ctx Markdown target odocs +;; + +let setup_pkg_markdown_rules_def = + let f (sctx, pkg) = + let ctx = Super_context.context sctx in + let* libs = Context.name ctx |> libs_of_pkg ~pkg in + let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in + let* lib_odocs = + Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) + in + let all_odocs = pkg_odocs @ lib_odocs in + let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in + let* () = Memo.parallel_iter pkg_odocs ~f:(setup_generate_markdown sctx) in + add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs + in + setup_pkg_rules_def "setup-package-markdown-rules" f +;; + +let setup_pkg_markdown_rules sctx ~pkg : unit Memo.t = + Memo.With_implicit_output.exec setup_pkg_markdown_rules_def (sctx, pkg) +;; + let setup_package_aliases_format sctx (pkg : Package.t) (output : Output_format.t) = let ctx = Super_context.context sctx in let name = Package.name pkg in @@ -1011,7 +1117,37 @@ let gen_rules sctx ~dir rest = ~directory_targets (Sherlodoc.sherlodoc_dot_js sctx ~dir:(Paths.html_root ctx) >>> setup_css_rule sctx - >>> setup_toplevel_index_rules sctx) + >>> setup_toplevel_index_rule sctx Html + >>> setup_toplevel_index_rule sctx Json) + | [ "_markdown" ] -> has_rules (setup_toplevel_index_rule sctx Markdown) + | [ "_markdown"; lib_unique_name_or_pkg ] -> + has_rules + (let ctx = Super_context.context sctx in + let* lib, lib_db = Scope_key.of_string (Context.name ctx) lib_unique_name_or_pkg in + let* lib = + let+ lib = Lib.DB.find lib_db lib in + Option.bind ~f:Lib.Local.of_lib lib + in + let+ () = + match lib with + | None -> Memo.return () + | Some lib -> + (match Lib_info.package (Lib.Local.info lib) with + | None -> + (* lib with no package above it *) + setup_lib_markdown_rules sctx lib + | Some pkg -> setup_pkg_markdown_rules sctx ~pkg) + and+ () = + let* packages = Dune_load.packages () in + match + Package.Name.Map.find packages (Package.Name.of_string lib_unique_name_or_pkg) + with + | None -> Memo.return () + | Some pkg -> + let name = Package.name pkg in + setup_pkg_markdown_rules sctx ~pkg:name + in + ()) | [ "_mlds"; pkg ] -> with_package pkg ~f:(fun pkg -> let pkg = Package.name pkg in diff --git a/test/blackbox-tests/test-cases/odoc/doc-markdown.t b/test/blackbox-tests/test-cases/odoc/doc-markdown.t new file mode 100644 index 00000000000..4c58cdc7bb4 --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/doc-markdown.t @@ -0,0 +1,71 @@ + $ cat > dune-project << EOF + > (lang dune 3.10) + > + > (package + > (name mylib)) + > EOF + + $ cat > dune << EOF + > (library + > (public_name mylib)) + > EOF + + $ cat > mylib.ml << EOF + > (** This is the main module for mylib *) + > + > (** A simple type definition *) + > type t = int + > + > (** A function that adds one *) + > val add_one : int -> int + > let add_one x = x + 1 + > + > module SubModule = struct + > (** A nested module *) + > type nested = string + > end + > EOF + + $ cat > mylib.mli << EOF + > (** This is the main module for mylib *) + > + > (** A simple type definition *) + > type t = int + > + > (** A function that adds one *) + > val add_one : int -> int + > + > module SubModule : sig + > (** A nested module *) + > type nested = string + > end + > EOF + + $ list_markdown_docs () { + > find _build/default/_doc/_markdown -name '*.md' | sort + > } + +Build markdown documentation: + + $ dune build @doc-markdown + $ list_markdown_docs + _build/default/_doc/_markdown/index.md + _build/default/_doc/_markdown/mylib/Mylib-SubModule.md + _build/default/_doc/_markdown/mylib/Mylib.md + _build/default/_doc/_markdown/mylib/index.md + +Check the top-level index contains markdown: + + $ cat _build/default/_doc/_markdown/index.md + # OCaml Package Documentation + + - [mylib](mylib/index.md) + +Test that @doc still generates HTML as usual: + + $ dune build @doc + $ find _build/default/_doc/_html -name '*.html' | sort + _build/default/_doc/_html/index.html + _build/default/_doc/_html/mylib/Mylib/SubModule/index.html + _build/default/_doc/_html/mylib/Mylib/index.html + _build/default/_doc/_html/mylib/index.html diff --git a/test/blackbox-tests/test-cases/odoc/markdown-with-mld.t b/test/blackbox-tests/test-cases/odoc/markdown-with-mld.t new file mode 100644 index 00000000000..9aff16f889d --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-with-mld.t @@ -0,0 +1,69 @@ +Test markdown generation with package documentation (.mld files) + + $ cat > dune-project << EOF + > (lang dune 3.10) + > (package + > (name example)) + > EOF + + $ cat > dune << EOF + > (library + > (public_name example)) + > EOF + + $ cat > example.ml << EOF + > (** Example library module *) + > + > let greet name = Printf.sprintf "Hello, %s!" name + > EOF + +Create a package documentation file: + + $ cat > index.mld << EOF + > {0 Example Package} + > + > This is the documentation for the example package. + > + > {1 Overview} + > + > This package provides a simple greeting function. + > + > {2 Usage} + > + > {[ + > let message = Example.greet "World" + > ]} + > + > See {!Example} for the API documentation. + > EOF + +Build the markdown documentation: + + $ dune build @doc-markdown + +Check that markdown files are generated: + + $ find _build/default/_doc/_markdown -name '*.md' | sort + _build/default/_doc/_markdown/example/Example.md + _build/default/_doc/_markdown/example/index.md + _build/default/_doc/_markdown/index.md + +The package-level documentation should be present: + + $ ls _build/default/_doc/_markdown/example/ + Example.md + index.md + +Test building documentation from current directory (where the package is defined): + + $ dune build @doc-markdown + +Verify that both HTML and markdown can coexist: + + $ dune build @doc @doc-markdown + $ find _build/default/_doc -name 'index.*' | grep -E '(html|md)$' | sort + _build/default/_doc/_html/example/Example/index.html + _build/default/_doc/_html/example/index.html + _build/default/_doc/_html/index.html + _build/default/_doc/_markdown/example/index.md + _build/default/_doc/_markdown/index.md From 4998400084541cc185ca48ab4571fc12ad1b7729 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Fri, 5 Sep 2025 12:09:12 +0200 Subject: [PATCH 02/20] Add more tests --- .../test-cases/odoc/markdown-submodules.t | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-submodules.t diff --git a/test/blackbox-tests/test-cases/odoc/markdown-submodules.t b/test/blackbox-tests/test-cases/odoc/markdown-submodules.t new file mode 100644 index 00000000000..5048e2d2088 --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-submodules.t @@ -0,0 +1,75 @@ +Test that markdown generation includes all modules following naming conventions. + + $ cat > dune-project << EOF + > (lang dune 3.0) + > (package (name mylib)) + > EOF + + $ cat > dune << EOF + > (library + > (public_name mylib) + > (modules main main_sub main_nested main_nested_deep)) + > EOF + +Create separate module files following a naming convention: + + $ cat > main.mli << EOF + > (** Main module *) + > val x : int + > EOF + + $ cat > main.ml << EOF + > let x = 42 + > EOF + + $ cat > main_sub.mli << EOF + > (** Main sub module *) + > val y : string + > EOF + + $ cat > main_sub.ml << EOF + > let y = "hello" + > EOF + + $ cat > main_nested.mli << EOF + > (** Main nested module *) + > val z : bool + > EOF + + $ cat > main_nested.ml << EOF + > let z = true + > EOF + + $ cat > main_nested_deep.mli << EOF + > (** Main nested deep module *) + > val w : float + > EOF + + $ cat > main_nested_deep.ml << EOF + > let w = 3.14 + > EOF + +Build the library and generate docs: + + $ dune build @doc-markdown + +Check what markdown files were generated: + + $ find _build/default/_doc/_markdown -name "*.md" | sort + _build/default/_doc/_markdown/index.md + _build/default/_doc/_markdown/mylib/Mylib-Main.md + _build/default/_doc/_markdown/mylib/Mylib-Main_nested.md + _build/default/_doc/_markdown/mylib/Mylib-Main_nested_deep.md + _build/default/_doc/_markdown/mylib/Mylib-Main_sub.md + _build/default/_doc/_markdown/mylib/Mylib.md + _build/default/_doc/_markdown/mylib/index.md + +Great! All modules are being generated as separate files. +Let's verify that the content is correct: + + $ head -5 _build/default/_doc/_markdown/mylib/Mylib-Main_sub.md + + # Module `Mylib.Main_sub` + + ``` + val y : string From ccbb6906ffe7becbeabc7244b519be6efb558fe3 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Fri, 5 Sep 2025 14:37:40 +0200 Subject: [PATCH 03/20] Have files already there, do not create them --- .../odoc/markdown-modules.t/main.ml | 4 ++ .../odoc/markdown-modules.t/main.mli | 6 ++ .../odoc/markdown-modules.t/main_nested.ml | 3 + .../odoc/markdown-modules.t/main_nested.mli | 4 ++ .../markdown-modules.t/main_nested_deep.ml | 1 + .../markdown-modules.t/main_nested_deep.mli | 2 + .../odoc/markdown-modules.t/main_sub.ml | 1 + .../odoc/markdown-modules.t/main_sub.mli | 2 + .../run.t} | 57 +++++-------------- 9 files changed, 37 insertions(+), 43 deletions(-) create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.ml create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.mli create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested.ml create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested.mli create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested_deep.ml create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested_deep.mli create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_sub.ml create mode 100644 test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_sub.mli rename test/blackbox-tests/test-cases/odoc/{markdown-submodules.t => markdown-modules.t/run.t} (50%) diff --git a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.ml b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.ml new file mode 100644 index 00000000000..5990dbe2725 --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.ml @@ -0,0 +1,4 @@ +let x = 42 + +module Sub = Main_sub +module Nested = Main_nested \ No newline at end of file diff --git a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.mli b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.mli new file mode 100644 index 00000000000..3ccc36eac8f --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.mli @@ -0,0 +1,6 @@ +(** Main module *) +val x : int + +module Sub = Main_sub +module Nested = Main_nested +module Nested_deep = Main_nested_deep \ No newline at end of file diff --git a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested.ml b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested.ml new file mode 100644 index 00000000000..59cb6e62a7a --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested.ml @@ -0,0 +1,3 @@ +let z = true + +module Nested_deep = Main_nested_deep \ No newline at end of file diff --git a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested.mli b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested.mli new file mode 100644 index 00000000000..df000074ed5 --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested.mli @@ -0,0 +1,4 @@ +(** Nested module *) +val z : bool + +module Nested_deep = Main_nested_deep \ No newline at end of file diff --git a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested_deep.ml b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested_deep.ml new file mode 100644 index 00000000000..562bc739f7a --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested_deep.ml @@ -0,0 +1 @@ +let w = 3.14 diff --git a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested_deep.mli b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested_deep.mli new file mode 100644 index 00000000000..3952fdb9542 --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_nested_deep.mli @@ -0,0 +1,2 @@ +(** Main nested deep module *) +val w : float diff --git a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_sub.ml b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_sub.ml new file mode 100644 index 00000000000..3d6a356065d --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_sub.ml @@ -0,0 +1 @@ +let y = "hello" diff --git a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_sub.mli b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_sub.mli new file mode 100644 index 00000000000..faa6f6aa0b8 --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main_sub.mli @@ -0,0 +1,2 @@ +(** Sub module *) +val y : string diff --git a/test/blackbox-tests/test-cases/odoc/markdown-submodules.t b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/run.t similarity index 50% rename from test/blackbox-tests/test-cases/odoc/markdown-submodules.t rename to test/blackbox-tests/test-cases/odoc/markdown-modules.t/run.t index 5048e2d2088..fab438e20c3 100644 --- a/test/blackbox-tests/test-cases/odoc/markdown-submodules.t +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/run.t @@ -7,46 +7,7 @@ Test that markdown generation includes all modules following naming conventions. $ cat > dune << EOF > (library - > (public_name mylib) - > (modules main main_sub main_nested main_nested_deep)) - > EOF - -Create separate module files following a naming convention: - - $ cat > main.mli << EOF - > (** Main module *) - > val x : int - > EOF - - $ cat > main.ml << EOF - > let x = 42 - > EOF - - $ cat > main_sub.mli << EOF - > (** Main sub module *) - > val y : string - > EOF - - $ cat > main_sub.ml << EOF - > let y = "hello" - > EOF - - $ cat > main_nested.mli << EOF - > (** Main nested module *) - > val z : bool - > EOF - - $ cat > main_nested.ml << EOF - > let z = true - > EOF - - $ cat > main_nested_deep.mli << EOF - > (** Main nested deep module *) - > val w : float - > EOF - - $ cat > main_nested_deep.ml << EOF - > let w = 3.14 + > (public_name mylib)) > EOF Build the library and generate docs: @@ -67,9 +28,19 @@ Check what markdown files were generated: Great! All modules are being generated as separate files. Let's verify that the content is correct: - $ head -5 _build/default/_doc/_markdown/mylib/Mylib-Main_sub.md + $ cat _build/default/_doc/_markdown/mylib/Mylib.md - # Module `Mylib.Main_sub` + # Module `Mylib` ``` - val y : string + module Main : sig ... end + ``` + ``` + module Main_nested : sig ... end + ``` + ``` + module Main_nested_deep : sig ... end + ``` + ``` + module Main_sub : sig ... end + ``` From 623425f6e3f92158ca5c5b5cfea1e5242c73f1e6 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Fri, 5 Sep 2025 22:16:21 +0200 Subject: [PATCH 04/20] Add debuging --- src/dune_rules/odoc.ml | 28 +++++++++++++++++++ .../odoc/markdown-modules.t/main.mli | 4 ++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index a4dadfee5dd..e6fdc49b15c 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -986,11 +986,36 @@ let setup_pkg_markdown_rules_def = let f (sctx, pkg) = let ctx = Super_context.context sctx in let* libs = Context.name ctx |> libs_of_pkg ~pkg in + (* Debug: Print to stderr for immediate visibility *) + Printf.eprintf "[DEBUG] Processing package: %s\n%!" (Package.Name.to_string pkg); + (* Debug: Log libraries found for the package *) + Printf.eprintf + "[DEBUG] Libraries found: %s\n%!" + (libs + |> List.map ~f:(fun lib -> Lib.Local.to_lib lib |> Lib.name |> Lib_name.to_string) + |> String.concat ~sep:", "); let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in + (* Debug: Log pkg_odocs *) + Printf.eprintf "[DEBUG] Package odocs (%d items):\n%!" (List.length pkg_odocs); + List.iter pkg_odocs ~f:(fun odoc -> + Printf.eprintf "[DEBUG] - odoc: %s\n%!" (Path.Build.to_string odoc.odoc_file); + Printf.eprintf "[DEBUG] html: %s\n%!" (Path.Build.to_string odoc.html_file); + Printf.eprintf + "[DEBUG] markdown: %s\n%!" + (Path.Build.to_string odoc.markdown_file)); let* lib_odocs = Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) in + (* Debug: Log lib_odocs *) + Printf.eprintf "[DEBUG] Library odocs (%d items):\n%!" (List.length lib_odocs); + List.iter lib_odocs ~f:(fun odoc -> + Printf.eprintf "[DEBUG] - odoc: %s\n%!" (Path.Build.to_string odoc.odoc_file); + Printf.eprintf "[DEBUG] html: %s\n%!" (Path.Build.to_string odoc.html_file); + Printf.eprintf + "[DEBUG] markdown: %s\n%!" + (Path.Build.to_string odoc.markdown_file)); let all_odocs = pkg_odocs @ lib_odocs in + Printf.eprintf "[DEBUG] Total odocs: %d\n%!" (List.length all_odocs); let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in let* () = Memo.parallel_iter pkg_odocs ~f:(setup_generate_markdown sctx) in add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs @@ -999,6 +1024,9 @@ let setup_pkg_markdown_rules_def = ;; let setup_pkg_markdown_rules sctx ~pkg : unit Memo.t = + Printf.eprintf + "[DEBUG-ENTRY] setup_pkg_markdown_rules called for package: %s\n%!" + (Package.Name.to_string pkg); Memo.With_implicit_output.exec setup_pkg_markdown_rules_def (sctx, pkg) ;; diff --git a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.mli b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.mli index 3ccc36eac8f..ab95e4c0f20 100644 --- a/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.mli +++ b/test/blackbox-tests/test-cases/odoc/markdown-modules.t/main.mli @@ -1,6 +1,8 @@ (** Main module *) val x : int +(** Sub module *) module Sub = Main_sub + +(** Nested module *) module Nested = Main_nested -module Nested_deep = Main_nested_deep \ No newline at end of file From 90b7cc998650d93d6a067c53afd636615f0ec092 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Fri, 19 Sep 2025 09:36:29 +0200 Subject: [PATCH 05/20] Revert logging 623425f6e3f92158ca5c5b5cfea1e5242c73f1e6 --- src/dune_rules/odoc.ml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index e6fdc49b15c..a4dadfee5dd 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -986,36 +986,11 @@ let setup_pkg_markdown_rules_def = let f (sctx, pkg) = let ctx = Super_context.context sctx in let* libs = Context.name ctx |> libs_of_pkg ~pkg in - (* Debug: Print to stderr for immediate visibility *) - Printf.eprintf "[DEBUG] Processing package: %s\n%!" (Package.Name.to_string pkg); - (* Debug: Log libraries found for the package *) - Printf.eprintf - "[DEBUG] Libraries found: %s\n%!" - (libs - |> List.map ~f:(fun lib -> Lib.Local.to_lib lib |> Lib.name |> Lib_name.to_string) - |> String.concat ~sep:", "); let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in - (* Debug: Log pkg_odocs *) - Printf.eprintf "[DEBUG] Package odocs (%d items):\n%!" (List.length pkg_odocs); - List.iter pkg_odocs ~f:(fun odoc -> - Printf.eprintf "[DEBUG] - odoc: %s\n%!" (Path.Build.to_string odoc.odoc_file); - Printf.eprintf "[DEBUG] html: %s\n%!" (Path.Build.to_string odoc.html_file); - Printf.eprintf - "[DEBUG] markdown: %s\n%!" - (Path.Build.to_string odoc.markdown_file)); let* lib_odocs = Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) in - (* Debug: Log lib_odocs *) - Printf.eprintf "[DEBUG] Library odocs (%d items):\n%!" (List.length lib_odocs); - List.iter lib_odocs ~f:(fun odoc -> - Printf.eprintf "[DEBUG] - odoc: %s\n%!" (Path.Build.to_string odoc.odoc_file); - Printf.eprintf "[DEBUG] html: %s\n%!" (Path.Build.to_string odoc.html_file); - Printf.eprintf - "[DEBUG] markdown: %s\n%!" - (Path.Build.to_string odoc.markdown_file)); let all_odocs = pkg_odocs @ lib_odocs in - Printf.eprintf "[DEBUG] Total odocs: %d\n%!" (List.length all_odocs); let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in let* () = Memo.parallel_iter pkg_odocs ~f:(setup_generate_markdown sctx) in add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs @@ -1024,9 +999,6 @@ let setup_pkg_markdown_rules_def = ;; let setup_pkg_markdown_rules sctx ~pkg : unit Memo.t = - Printf.eprintf - "[DEBUG-ENTRY] setup_pkg_markdown_rules called for package: %s\n%!" - (Package.Name.to_string pkg); Memo.With_implicit_output.exec setup_pkg_markdown_rules_def (sctx, pkg) ;; From 463b7b877995e169dd6ee246a3cbf240d66f4b66 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Fri, 19 Sep 2025 09:58:43 +0200 Subject: [PATCH 06/20] Capitaliza basename on markdown --- src/dune_rules/odoc.ml | 2 +- test/blackbox-tests/test-cases/odoc/doc-markdown.t | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index a4dadfee5dd..33b4c1e31c6 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -702,7 +702,7 @@ let create_odoc ctx ~target odoc_file = html_dir ++ "index" |> Path.Build.extend_basename ~suffix:(Output_format.extension output) | Markdown -> - markdown_base ++ basename + markdown_base ++ Stdune.String.capitalize basename |> Path.Build.extend_basename ~suffix:(Output_format.extension output) in { odoc_file diff --git a/test/blackbox-tests/test-cases/odoc/doc-markdown.t b/test/blackbox-tests/test-cases/odoc/doc-markdown.t index 4c58cdc7bb4..a0dc408e08a 100644 --- a/test/blackbox-tests/test-cases/odoc/doc-markdown.t +++ b/test/blackbox-tests/test-cases/odoc/doc-markdown.t @@ -60,12 +60,3 @@ Check the top-level index contains markdown: # OCaml Package Documentation - [mylib](mylib/index.md) - -Test that @doc still generates HTML as usual: - - $ dune build @doc - $ find _build/default/_doc/_html -name '*.html' | sort - _build/default/_doc/_html/index.html - _build/default/_doc/_html/mylib/Mylib/SubModule/index.html - _build/default/_doc/_html/mylib/Mylib/index.html - _build/default/_doc/_html/mylib/index.html From 9e0eb9bab9efe0511395301db2254e6007732596 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Fri, 19 Sep 2025 14:01:48 +0200 Subject: [PATCH 07/20] setup lib markdown rules definition --- src/dune_rules/odoc.ml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index 33b4c1e31c6..e29e09e3fca 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -902,6 +902,30 @@ let add_format_alias_deps ctx format target odocs = (Action_builder.paths paths) ;; +let setup_lib_markdown_rules_def = + let module Input = struct + module Super_context = Super_context.As_memo_key + + type t = Super_context.t * Lib.Local.t + + let equal (sc1, l1) (sc2, l2) = Super_context.equal sc1 sc2 && Lib.Local.equal l1 l2 + let hash = Tuple.T2.hash Super_context.hash Lib.Local.hash + let to_dyn _ = Dyn.Opaque + end + in + let f (sctx, lib) = + let ctx = Super_context.context sctx in + let target = Lib lib in + let* odocs = odoc_artefacts sctx target in + add_format_alias_deps ctx Markdown target odocs; + in + Memo.With_implicit_output.create + "setup-library-markdown-rules" + ~implicit_output:Rules.implicit_output + ~input:(module Input) + f +;; + let setup_lib_html_rules_def = let module Input = struct module Super_context = Super_context.As_memo_key @@ -979,7 +1003,8 @@ let setup_lib_markdown_rules sctx lib = let target = Lib lib in let* odocs = odoc_artefacts sctx target in let* () = Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) in - add_format_alias_deps ctx Markdown target odocs + let* () = add_format_alias_deps ctx Markdown target odocs in + Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib) ;; let setup_pkg_markdown_rules_def = From 1450c5e06b71365af92793278727e2e7ed1a7a93 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Fri, 19 Sep 2025 14:21:35 +0200 Subject: [PATCH 08/20] Remove commented generate_all --- src/dune_rules/odoc.ml | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index e29e09e3fca..f9ceb1e7026 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -518,8 +518,6 @@ let setup_generate sctx ~search_db odoc_file out = add_rule sctx run_odoc ;; -(* let setup_generate_all sctx ~search_db odoc_file = - Output_format.iter ~f:(setup_generate sctx ~search_db:(Some search_db) odoc_file) *) let setup_generate_html_and_json sctx ~search_db odoc_file = let* () = setup_generate sctx ~search_db:(Some search_db) odoc_file Html in setup_generate sctx ~search_db:(Some search_db) odoc_file Json From c4b68396148d157e411f27bf54e85d53f9a4e6cd Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Tue, 14 Oct 2025 14:01:08 +0800 Subject: [PATCH 09/20] Consider directory targets for markdown --- src/dune_rules/odoc.ml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index 5c21575185d..8be23c3eab9 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -1174,9 +1174,11 @@ let gen_rules sctx ~dir rest = >>> setup_toplevel_index_rule sctx Json) | [ "_markdown" ] -> has_rules (setup_toplevel_index_rule sctx Markdown) | [ "_markdown"; lib_unique_name_or_pkg ] -> + let ctx = Super_context.context sctx in + let directory_targets = Path.Build.Map.singleton dir Loc.none in has_rules - (let ctx = Super_context.context sctx in - let* lib, lib_db = Scope_key.of_string (Context.name ctx) lib_unique_name_or_pkg in + ~directory_targets + (let* lib, lib_db = Scope_key.of_string (Context.name ctx) lib_unique_name_or_pkg in let* lib = let+ lib = Lib.DB.find lib_db lib in Option.bind ~f:Lib.Local.of_lib lib From 4c83d4edb61234d78c10050d1f917e3da645bc21 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Tue, 14 Oct 2025 20:27:38 +0800 Subject: [PATCH 10/20] Link with the markdown path --- src/dune_rules/odoc.ml | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index 8be23c3eab9..d4141999323 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -417,10 +417,11 @@ let odoc_include_flags ctx pkg requires = let link_odoc_rules sctx (odoc_file : odoc_artefact) ~pkg ~requires = let ctx = Super_context.context sctx in let deps = Dep.deps ctx pkg requires in + let dir = Path.build (Path.Build.parent_exn odoc_file.odocl_file) in let run_odoc = run_odoc sctx - ~dir:(Path.build (Paths.html_root ctx)) + ~dir "link" ~quiet:false ~flags_for:(Some odoc_file.odoc_file) @@ -896,7 +897,7 @@ let add_format_alias_deps ctx format target odocs = (Action_builder.paths paths) ;; -let setup_lib_markdown_rules_def = +let setup_lib_html_rules_def = let module Input = struct module Super_context = Super_context.As_memo_key @@ -911,16 +912,17 @@ let setup_lib_markdown_rules_def = let ctx = Super_context.context sctx in let target = Lib lib in let* odocs = odoc_artefacts sctx target in - add_format_alias_deps ctx Markdown target odocs; + let* () = add_format_alias_deps ctx Html target odocs in + add_format_alias_deps ctx Json target odocs in Memo.With_implicit_output.create - "setup-library-markdown-rules" + "setup-library-html-rules" ~implicit_output:Rules.implicit_output ~input:(module Input) f ;; -let setup_lib_html_rules_def = +let setup_lib_markdown_rules_def = let module Input = struct module Super_context = Super_context.As_memo_key @@ -935,11 +937,10 @@ let setup_lib_html_rules_def = let ctx = Super_context.context sctx in let target = Lib lib in let* odocs = odoc_artefacts sctx target in - let* () = add_format_alias_deps ctx Html target odocs in - add_format_alias_deps ctx Json target odocs + add_format_alias_deps ctx Markdown target odocs in Memo.With_implicit_output.create - "setup-library-html-rules" + "setup-library-markdown-rules" ~implicit_output:Rules.implicit_output ~input:(module Input) f @@ -993,11 +994,9 @@ let setup_pkg_html_rules sctx ~pkg : unit Memo.t = ;; let setup_lib_markdown_rules sctx lib = - let ctx = Super_context.context sctx in let target = Lib lib in let* odocs = odoc_artefacts sctx target in let* () = Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) in - let* () = add_format_alias_deps ctx Markdown target odocs in Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib) ;; @@ -1174,11 +1173,10 @@ let gen_rules sctx ~dir rest = >>> setup_toplevel_index_rule sctx Json) | [ "_markdown" ] -> has_rules (setup_toplevel_index_rule sctx Markdown) | [ "_markdown"; lib_unique_name_or_pkg ] -> - let ctx = Super_context.context sctx in - let directory_targets = Path.Build.Map.singleton dir Loc.none in has_rules - ~directory_targets - (let* lib, lib_db = Scope_key.of_string (Context.name ctx) lib_unique_name_or_pkg in + ( + let ctx = Super_context.context sctx in + let* lib, lib_db = Scope_key.of_string (Context.name ctx) lib_unique_name_or_pkg in let* lib = let+ lib = Lib.DB.find lib_db lib in Option.bind ~f:Lib.Local.of_lib lib From 53a22e802ed3733e6c978d6e32e482824dfb7c2f Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Wed, 15 Oct 2025 09:22:18 +0800 Subject: [PATCH 11/20] Add logging into gen_project_rules --- src/dune_rules/odoc.ml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index d4141999323..c5f8fb88dfe 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -1116,10 +1116,24 @@ let setup_package_odoc_rules sctx ~pkg = ;; let gen_project_rules sctx project = + let () = + User_message.print + (User_message.make + [ Pp.textf "Odoc.gen_project_rules called for project: %s" + (Dune_project.name project |> Dune_project_name.to_string_hum) + ]) + in Dune_project.packages project |> Dune_lang.Package_name.Map.to_seq |> Memo.parallel_iter_seq ~f:(fun (_, (pkg : Package.t)) -> (* setup @doc to build the correct html for the package *) + let () = + User_message.print + (User_message.make + [ Pp.textf "Setting up package aliases for: %s" + (Package.name pkg |> Package.Name.to_string) + ]) + in setup_package_aliases sctx pkg) ;; From 0cc967972ab016635e8d339c581d65edbf56e16c Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Wed, 15 Oct 2025 10:02:15 +0800 Subject: [PATCH 12/20] Include subdirs --- src/dune_rules/odoc.ml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index c5f8fb88dfe..c2f93f71b0c 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -784,7 +784,21 @@ let odoc_artefacts sctx target = | Lib lib -> let info = Lib.Local.info lib in let obj_dir = Lib_info.obj_dir info in - let+ modules = entry_modules_by_lib sctx lib in + let+ modules = + let+ modules = Dir_contents.modules_of_local_lib sctx lib in + Modules.fold modules ~init:[] ~f:(fun m acc -> + (* Only include modules that: + 1. Have public visibility + 2. Are not wrapped library implementation modules (those with __ in the name) *) + if Module.visibility m = Public + && (Module.obj_name m + |> Module_name.Unique.to_name ~loc:Loc.none + |> Module_name.to_string + |> String.contains_double_underscore + |> not) + then m :: acc + else acc) + in List.map modules ~f:(fun m -> let odoc_file = Obj_dir.Module.odoc obj_dir m in create_odoc ctx ~target odoc_file) @@ -1116,24 +1130,10 @@ let setup_package_odoc_rules sctx ~pkg = ;; let gen_project_rules sctx project = - let () = - User_message.print - (User_message.make - [ Pp.textf "Odoc.gen_project_rules called for project: %s" - (Dune_project.name project |> Dune_project_name.to_string_hum) - ]) - in Dune_project.packages project |> Dune_lang.Package_name.Map.to_seq |> Memo.parallel_iter_seq ~f:(fun (_, (pkg : Package.t)) -> (* setup @doc to build the correct html for the package *) - let () = - User_message.print - (User_message.make - [ Pp.textf "Setting up package aliases for: %s" - (Package.name pkg |> Package.Name.to_string) - ]) - in setup_package_aliases sctx pkg) ;; From 47fb0b505939c872b4ba80104625688b003c4159 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Wed, 15 Oct 2025 13:46:50 +0800 Subject: [PATCH 13/20] Create bash to unify directory targets unique --- src/dune_rules/odoc.ml | 150 ++++++++++++++++++++++++++--------------- 1 file changed, 95 insertions(+), 55 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index c2f93f71b0c..16f50514b2b 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -784,21 +784,7 @@ let odoc_artefacts sctx target = | Lib lib -> let info = Lib.Local.info lib in let obj_dir = Lib_info.obj_dir info in - let+ modules = - let+ modules = Dir_contents.modules_of_local_lib sctx lib in - Modules.fold modules ~init:[] ~f:(fun m acc -> - (* Only include modules that: - 1. Have public visibility - 2. Are not wrapped library implementation modules (those with __ in the name) *) - if Module.visibility m = Public - && (Module.obj_name m - |> Module_name.Unique.to_name ~loc:Loc.none - |> Module_name.to_string - |> String.contains_double_underscore - |> not) - then m :: acc - else acc) - in + let+ modules = entry_modules_by_lib sctx lib in List.map modules ~f:(fun m -> let odoc_file = Obj_dir.Module.odoc obj_dir m in create_odoc ctx ~target odoc_file) @@ -905,10 +891,15 @@ let out_files ctx (output : Output_format.t) odocs = ;; let add_format_alias_deps ctx format target odocs = - let paths = out_files ctx format odocs in - Rules.Produce.Alias.add_deps - (Dep.format_alias format ctx target) - (Action_builder.paths paths) + match (format : Output_format.t) with + | Markdown -> + (* skip intermediate aliases since package directories are directory targets *) + Memo.return () + | Html | Json -> + let paths = out_files ctx format odocs in + Rules.Produce.Alias.add_deps + (Dep.format_alias format ctx target) + (Action_builder.paths paths) ;; let setup_lib_html_rules_def = @@ -1010,7 +1001,15 @@ let setup_pkg_html_rules sctx ~pkg : unit Memo.t = let setup_lib_markdown_rules sctx lib = let target = Lib lib in let* odocs = odoc_artefacts sctx target in - let* () = Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) in + (* For libraries WITH a package, generation happens in the package-level rule. + Only generate for libraries WITHOUT a package. *) + let* () = + match Lib_info.package (Lib.Local.info lib) with + | Some _ -> Memo.return () (* Package-level rule handles it *) + | None -> + (* No package, so we need individual rules *) + Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) + in Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib) ;; @@ -1023,8 +1022,45 @@ let setup_pkg_markdown_rules_def = Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) in let all_odocs = pkg_odocs @ lib_odocs in + (* Generate ALL markdown for this package in ONE rule with directory target. + Since odoc generates unpredictable files (Belt.md, Belt-Array.md, Belt-List.md, etc.) + from nested submodules, we must use a directory target and batch all odoc commands. *) + let* () = + if List.is_empty all_odocs + then Memo.return () + else + let pkg_markdown_dir = Paths.markdown ctx (Pkg pkg) in + let markdown_root = Paths.markdown_root ctx in + let rule = + let bash_cmd_args = + let open Action_builder.O in + let* odoc_prog = odoc_program sctx (Context.build_dir ctx) in + let odoc_path = Action.Prog.ok_exn odoc_prog |> Path.to_string in + let bash_cmd = + List.map all_odocs ~f:(fun odoc -> + let odocl_rel = Path.reach (Path.build odoc.odocl_file) ~from:(Path.build markdown_root) in + Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel) + |> String.concat ~sep:" && " + in + let* () = + List.map all_odocs ~f:(fun odoc -> Action_builder.path (Path.build odoc.odocl_file)) + |> Action_builder.all + >>| ignore + in + Action_builder.return (Command.Args.S [ A "-c"; A bash_cmd ]) + in + let deps = Action_builder.env_var "ODOC_SYNTAX" in + let open Action_builder.With_targets.O in + Action_builder.with_no_targets deps + >>> Command.run + ~dir:(Path.build markdown_root) + (Ok (Path.of_string "/bin/bash")) + [ Dyn bash_cmd_args ] + |> Action_builder.With_targets.add_directories ~directory_targets:[ pkg_markdown_dir ] + in + add_rule sctx rule + in let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in - let* () = Memo.parallel_iter pkg_odocs ~f:(setup_generate_markdown sctx) in add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs in setup_pkg_rules_def "setup-package-markdown-rules" f @@ -1045,11 +1081,22 @@ let setup_package_aliases_format sctx (pkg : Package.t) (output : Output_format. let* libs = Context.name ctx |> libs_of_pkg ~pkg:name >>| List.map ~f:(fun lib -> Lib lib) in - Pkg name :: libs - |> List.map ~f:(Dep.format_alias output ctx) - |> Dune_engine.Dep.Set.of_list_map ~f:(fun f -> Dune_engine.Dep.alias f) - |> Action_builder.deps - |> Rules.Produce.Alias.add_deps alias + let deps = + match (output : Output_format.t) with + | Markdown -> + let directory_target = Paths.markdown ctx (Pkg name) in + let toplevel_index = Paths.markdown_index ctx in + let open Action_builder.O in + let+ () = Action_builder.path (Path.build directory_target) + and+ () = Action_builder.path (Path.build toplevel_index) in + () + | Html | Json -> + Pkg name :: libs + |> List.map ~f:(Dep.format_alias output ctx) + |> Dune_engine.Dep.Set.of_list_map ~f:(fun f -> Dune_engine.Dep.alias f) + |> Action_builder.deps + in + Rules.Produce.Alias.add_deps alias deps ;; let setup_package_aliases sctx (pkg : Package.t) = @@ -1185,36 +1232,29 @@ let gen_rules sctx ~dir rest = >>> setup_css_rule sctx >>> setup_toplevel_index_rule sctx Html >>> setup_toplevel_index_rule sctx Json) - | [ "_markdown" ] -> has_rules (setup_toplevel_index_rule sctx Markdown) - | [ "_markdown"; lib_unique_name_or_pkg ] -> + | [ "_markdown" ] -> + let* packages = Dune_load.packages () in + let ctx = Super_context.context sctx in + let all_package_dirs = + Package.Name.Map.to_list packages + |> List.map ~f:(fun (_, (pkg : Package.t)) -> + let pkg_name = Package.name pkg in + Paths.markdown ctx (Pkg pkg_name)) + in + let directory_targets = + List.fold_left all_package_dirs ~init:Path.Build.Map.empty ~f:(fun acc dir -> + Path.Build.Map.set acc dir Loc.none) + in has_rules - ( - let ctx = Super_context.context sctx in - let* lib, lib_db = Scope_key.of_string (Context.name ctx) lib_unique_name_or_pkg in - let* lib = - let+ lib = Lib.DB.find lib_db lib in - Option.bind ~f:Lib.Local.of_lib lib - in - let+ () = - match lib with - | None -> Memo.return () - | Some lib -> - (match Lib_info.package (Lib.Local.info lib) with - | None -> - (* lib with no package above it *) - setup_lib_markdown_rules sctx lib - | Some pkg -> setup_pkg_markdown_rules sctx ~pkg) - and+ () = - let* packages = Dune_load.packages () in - match - Package.Name.Map.find packages (Package.Name.of_string lib_unique_name_or_pkg) - with - | None -> Memo.return () - | Some pkg -> - let name = Package.name pkg in - setup_pkg_markdown_rules sctx ~pkg:name - in - ()) + ~directory_targets + (let* () = setup_toplevel_index_rule sctx Markdown in + Package.Name.Map.to_seq packages + |> Memo.parallel_iter_seq ~f:(fun (_, (pkg : Package.t)) -> + let pkg_name = Package.name pkg in + setup_pkg_markdown_rules sctx ~pkg:pkg_name)) + | [ "_markdown"; _lib_unique_name_or_pkg ] -> + (* package directories are directory targets *) + Memo.return Gen_rules.no_rules | [ "_mlds"; pkg ] -> with_package pkg ~f:(fun pkg -> let pkg = Package.name pkg in From 0ae18bdc8ddfdb4065ad6456b853e97225327f0f Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Wed, 15 Oct 2025 15:40:26 +0800 Subject: [PATCH 14/20] Improve documentation for the single action for packages --- src/dune_rules/odoc.ml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index 16f50514b2b..3868910cfaf 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -1001,13 +1001,12 @@ let setup_pkg_html_rules sctx ~pkg : unit Memo.t = let setup_lib_markdown_rules sctx lib = let target = Lib lib in let* odocs = odoc_artefacts sctx target in - (* For libraries WITH a package, generation happens in the package-level rule. - Only generate for libraries WITHOUT a package. *) let* () = + (* because libraries with a package are handled in the package-level rule with the bash script for all directory target, we skip packages *) match Lib_info.package (Lib.Local.info lib) with - | Some _ -> Memo.return () (* Package-level rule handles it *) + | Some _ -> Memo.return () | None -> - (* No package, so we need individual rules *) + (* when there's no package, we still need have rules for each odoc file *) Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) in Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib) @@ -1022,9 +1021,7 @@ let setup_pkg_markdown_rules_def = Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) in let all_odocs = pkg_odocs @ lib_odocs in - (* Generate ALL markdown for this package in ONE rule with directory target. - Since odoc generates unpredictable files (Belt.md, Belt-Array.md, Belt-List.md, etc.) - from nested submodules, we must use a directory target and batch all odoc commands. *) + (* odoc generates all markdown files on the same level for the package so we use one rule with directory target and batch all odoc commands. *) let* () = if List.is_empty all_odocs then Memo.return () From 5431ccddbedcdb6908895f1954ddb3a95fb027c9 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Wed, 15 Oct 2025 16:27:13 +0800 Subject: [PATCH 15/20] Enable version check for odoc --- src/dune_rules/odoc.ml | 220 ++++++++++++++++++++++++++++++++--------- 1 file changed, 171 insertions(+), 49 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index 3868910cfaf..31e70b74a7f 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -261,6 +261,49 @@ module Flags = struct ;; end +module Version = struct + type t = int * int * int (* major * minor * patch *) + + let of_string s : t option = + (* strip any suffix *) + let s = + match + String.findi s ~f:(function + | '+' | '-' | '~' -> true + | _ -> false) + with + | None -> s + | Some i -> String.take s i + in + try + match String.split s ~on:'.' with + | [ major; minor; patch ] -> + Some (int_of_string major, int_of_string minor, int_of_string patch) + | [ major; minor ] -> Some (int_of_string major, int_of_string minor, 0) + | _ -> None + with + | _ -> None + ;; + + let compare (ma1, mi1, pa1) (ma2, mi2, pa2) = + match Int.compare ma1 ma2 with + | Ordering.Eq -> + (match Int.compare mi1 mi2 with + | Ordering.Eq -> Int.compare pa1 pa2 + | n -> n) + | n -> n + ;; + + let supports_doc_markdown version = + match version with + | None -> false + | Some v -> + (match compare v (3, 1, 0) with + | Ordering.Lt | Ordering.Eq -> false + | Ordering.Gt -> true) + ;; +end + let odoc_base_flags quiet build_dir = let open Action_builder.O in let+ conf = Flags.get ~dir:build_dir in @@ -272,6 +315,40 @@ let odoc_base_flags quiet build_dir = | Nonfatal -> S [] ;; +let get_odoc_version_impl bin = + let* _ = Build_system.build_file bin in + Memo.of_reproducible_fiber + @@ + let open Fiber.O in + let+ output, exit_code = + Process.run_capture_lines + ~display:Quiet + ~stderr_to: + (Process.Io.make_stderr + ~output_on_success:Swallow + ~output_limit:Execution_parameters.Action_output_limit.default) + Return + bin + [ "--version" ] + in + output, exit_code +;; + +let odoc_version_memo = + Memo.create "odoc-version" ~input:(module Path) get_odoc_version_impl +;; + +let get_odoc_version odoc_path = + let open Memo.O in + let+ output, exit_code = Memo.exec odoc_version_memo odoc_path in + if exit_code <> 0 + then None + else ( + match output with + | [ version_line ] -> Version.of_string version_line + | _ -> None) +;; + let odoc_dev_tool_exe_path_building_if_necessary () = let open Action_builder.O in let path = Path.build (Pkg_dev_tool.exe_path Odoc) in @@ -999,66 +1076,111 @@ let setup_pkg_html_rules sctx ~pkg : unit Memo.t = ;; let setup_lib_markdown_rules sctx lib = - let target = Lib lib in - let* odocs = odoc_artefacts sctx target in - let* () = - (* because libraries with a package are handled in the package-level rule with the bash script for all directory target, we skip packages *) - match Lib_info.package (Lib.Local.info lib) with - | Some _ -> Memo.return () - | None -> - (* when there's no package, we still need have rules for each odoc file *) - Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) + let ctx = Super_context.context sctx in + let* odoc_prog = + Super_context.resolve_program_memo + sctx + ~dir:(Context.build_dir ctx) + ~where:Original_path + "odoc" + ~loc:None in - Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib) + match odoc_prog with + | Error _ -> + (* odoc not found, skip markdown generation *) + Memo.return () + | Ok odoc_path -> + let* version = get_odoc_version odoc_path in + if not (Version.supports_doc_markdown version) + then Memo.return () + else ( + let target = Lib lib in + let* odocs = odoc_artefacts sctx target in + let* () = + (* because libraries with a package are handled in the package-level rule with the system shell script for all directory target, we skip packages *) + match Lib_info.package (Lib.Local.info lib) with + | Some _ -> Memo.return () + | None -> + (* when there's no package, we still need have rules for each odoc file *) + Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) + in + Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib)) ;; let setup_pkg_markdown_rules_def = let f (sctx, pkg) = let ctx = Super_context.context sctx in - let* libs = Context.name ctx |> libs_of_pkg ~pkg in - let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in - let* lib_odocs = - Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) + (* Check if odoc version supports markdown generation *) + let* odoc_prog = + Super_context.resolve_program_memo + sctx + ~dir:(Context.build_dir ctx) + ~where:Original_path + "odoc" + ~loc:None in - let all_odocs = pkg_odocs @ lib_odocs in - (* odoc generates all markdown files on the same level for the package so we use one rule with directory target and batch all odoc commands. *) - let* () = - if List.is_empty all_odocs + match odoc_prog with + | Error _ -> + (* odoc not found, skip markdown generation *) + Memo.return () + | Ok odoc_path -> + let* version = get_odoc_version odoc_path in + if not (Version.supports_doc_markdown version) then Memo.return () else - let pkg_markdown_dir = Paths.markdown ctx (Pkg pkg) in - let markdown_root = Paths.markdown_root ctx in - let rule = - let bash_cmd_args = - let open Action_builder.O in - let* odoc_prog = odoc_program sctx (Context.build_dir ctx) in - let odoc_path = Action.Prog.ok_exn odoc_prog |> Path.to_string in - let bash_cmd = - List.map all_odocs ~f:(fun odoc -> - let odocl_rel = Path.reach (Path.build odoc.odocl_file) ~from:(Path.build markdown_root) in - Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel) - |> String.concat ~sep:" && " - in - let* () = - List.map all_odocs ~f:(fun odoc -> Action_builder.path (Path.build odoc.odocl_file)) - |> Action_builder.all - >>| ignore + let* libs = Context.name ctx |> libs_of_pkg ~pkg in + let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in + let* lib_odocs = + Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) + in + let all_odocs = pkg_odocs @ lib_odocs in + (* odoc generates all markdown files on the same level for the package so we use one rule with directory target and batch all odoc commands. *) + let* () = + if List.is_empty all_odocs + then Memo.return () + else ( + let pkg_markdown_dir = Paths.markdown ctx (Pkg pkg) in + let markdown_root = Paths.markdown_root ctx in + let rule = + let prog, shell_arg = + Env_path.system_shell_exn ~needed_to:"generate markdown documentation" + in + let system_shell_cmd_args = + let open Action_builder.O in + let* odoc_prog = odoc_program sctx (Context.build_dir ctx) in + let odoc_path = Action.Prog.ok_exn odoc_prog |> Path.to_string in + let shell_cmd = + List.map all_odocs ~f:(fun odoc -> + let odocl_rel = + Path.reach + (Path.build odoc.odocl_file) + ~from:(Path.build markdown_root) + in + Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel) + |> String.concat ~sep:" && " + in + let* () = + List.map all_odocs ~f:(fun odoc -> + Action_builder.path (Path.build odoc.odocl_file)) + |> Action_builder.all + >>| ignore + in + Action_builder.return (Command.Args.S [ A shell_arg; A shell_cmd ]) + in + let deps = Action_builder.env_var "ODOC_SYNTAX" in + let open Action_builder.With_targets.O in + Action_builder.with_no_targets deps + >>> Command.run + ~dir:(Path.build markdown_root) + (Ok prog) + [ Dyn system_shell_cmd_args ] + |> Action_builder.With_targets.add_directories + ~directory_targets:[ pkg_markdown_dir ] in - Action_builder.return (Command.Args.S [ A "-c"; A bash_cmd ]) - in - let deps = Action_builder.env_var "ODOC_SYNTAX" in - let open Action_builder.With_targets.O in - Action_builder.with_no_targets deps - >>> Command.run - ~dir:(Path.build markdown_root) - (Ok (Path.of_string "/bin/bash")) - [ Dyn bash_cmd_args ] - |> Action_builder.With_targets.add_directories ~directory_targets:[ pkg_markdown_dir ] + add_rule sctx rule) in - add_rule sctx rule - in - let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in - add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs + let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in + add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs in setup_pkg_rules_def "setup-package-markdown-rules" f ;; From b2fe8d310ffc7088a6328920d53a7cc708b8fa24 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Wed, 15 Oct 2025 17:32:28 +0800 Subject: [PATCH 16/20] Refactor and add extensive test --- src/dune_rules/odoc.ml | 193 +++++++++------- .../odoc/odoc-markdown-version-check.t | 214 ++++++++++++++++++ 2 files changed, 320 insertions(+), 87 deletions(-) create mode 100644 test/blackbox-tests/test-cases/odoc/odoc-markdown-version-check.t diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index 31e70b74a7f..cf41c976a97 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -294,13 +294,13 @@ module Version = struct | n -> n ;; - let supports_doc_markdown version = + let higher_than_310 version = match version with | None -> false | Some v -> (match compare v (3, 1, 0) with - | Ordering.Lt | Ordering.Eq -> false - | Ordering.Gt -> true) + | Ordering.Lt -> false + | Ordering.Eq | Ordering.Gt -> true) ;; end @@ -349,6 +349,35 @@ let get_odoc_version odoc_path = | _ -> None) ;; +let odoc_path_memo sctx = + let odoc_dev_tool_lock_dir_exists = + match Config.get Compile_time.lock_dev_tools with + | `Enabled -> true + | `Disabled -> false + in + match odoc_dev_tool_lock_dir_exists with + | true -> + let path = Path.build (Pkg_dev_tool.exe_path Odoc) in + Memo.return (Ok path) + | false -> + let ctx = Super_context.context sctx in + Super_context.resolve_program_memo + sctx + ~dir:(Context.build_dir ctx) + ~where:Original_path + "odoc" + ~loc:None +;; + +let supports_doc_markdown sctx = + let* odoc_prog = odoc_path_memo sctx in + match odoc_prog with + | Error _ -> Memo.return false + | Ok odoc_path -> + let* version = get_odoc_version odoc_path in + Memo.return (Version.higher_than_310 version) +;; + let odoc_dev_tool_exe_path_building_if_necessary () = let open Action_builder.O in let path = Path.build (Pkg_dev_tool.exe_path Odoc) in @@ -1076,59 +1105,31 @@ let setup_pkg_html_rules sctx ~pkg : unit Memo.t = ;; let setup_lib_markdown_rules sctx lib = - let ctx = Super_context.context sctx in - let* odoc_prog = - Super_context.resolve_program_memo - sctx - ~dir:(Context.build_dir ctx) - ~where:Original_path - "odoc" - ~loc:None - in - match odoc_prog with - | Error _ -> - (* odoc not found, skip markdown generation *) - Memo.return () - | Ok odoc_path -> - let* version = get_odoc_version odoc_path in - if not (Version.supports_doc_markdown version) - then Memo.return () - else ( - let target = Lib lib in - let* odocs = odoc_artefacts sctx target in - let* () = - (* because libraries with a package are handled in the package-level rule with the system shell script for all directory target, we skip packages *) - match Lib_info.package (Lib.Local.info lib) with - | Some _ -> Memo.return () - | None -> - (* when there's no package, we still need have rules for each odoc file *) - Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) - in - Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib)) + let* markdown_supported = supports_doc_markdown sctx in + if not markdown_supported + then Memo.return () + else ( + let target = Lib lib in + let* odocs = odoc_artefacts sctx target in + let* () = + (* because libraries with a package are handled in the package-level rule with the system shell script for all directory target, we skip packages *) + match Lib_info.package (Lib.Local.info lib) with + | Some _ -> Memo.return () + | None -> + (* when there's no package, we still need have rules for each odoc file *) + Memo.parallel_iter odocs ~f:(fun odoc -> setup_generate_markdown sctx odoc) + in + Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib)) ;; let setup_pkg_markdown_rules_def = let f (sctx, pkg) = let ctx = Super_context.context sctx in - (* Check if odoc version supports markdown generation *) - let* odoc_prog = - Super_context.resolve_program_memo - sctx - ~dir:(Context.build_dir ctx) - ~where:Original_path - "odoc" - ~loc:None - in - match odoc_prog with - | Error _ -> - (* odoc not found, skip markdown generation *) - Memo.return () - | Ok odoc_path -> - let* version = get_odoc_version odoc_path in - if not (Version.supports_doc_markdown version) - then Memo.return () - else - let* libs = Context.name ctx |> libs_of_pkg ~pkg in + let* markdown_supported = supports_doc_markdown sctx in + if not markdown_supported + then Memo.return () + else + let* libs = Context.name ctx |> libs_of_pkg ~pkg in let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in let* lib_odocs = Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) @@ -1191,31 +1192,44 @@ let setup_pkg_markdown_rules sctx ~pkg : unit Memo.t = let setup_package_aliases_format sctx (pkg : Package.t) (output : Output_format.t) = let ctx = Super_context.context sctx in - let name = Package.name pkg in - let alias = - let pkg_dir = Package.dir pkg in - let dir = Path.Build.append_source (Context.build_dir ctx) pkg_dir in - Output_format.alias output ~dir - in - let* libs = - Context.name ctx |> libs_of_pkg ~pkg:name >>| List.map ~f:(fun lib -> Lib lib) - in - let deps = - match (output : Output_format.t) with - | Markdown -> + match (output : Output_format.t) with + | Markdown -> + let* is_markdown_supported = supports_doc_markdown sctx in + if not is_markdown_supported + then Memo.return () + else ( + let name = Package.name pkg in + let alias = + let pkg_dir = Package.dir pkg in + let dir = Path.Build.append_source (Context.build_dir ctx) pkg_dir in + Output_format.alias output ~dir + in let directory_target = Paths.markdown ctx (Pkg name) in let toplevel_index = Paths.markdown_index ctx in - let open Action_builder.O in - let+ () = Action_builder.path (Path.build directory_target) - and+ () = Action_builder.path (Path.build toplevel_index) in - () - | Html | Json -> + let deps = + let open Action_builder.O in + let+ () = Action_builder.path (Path.build directory_target) + and+ () = Action_builder.path (Path.build toplevel_index) in + () + in + Rules.Produce.Alias.add_deps alias deps) + | Html | Json -> + let name = Package.name pkg in + let alias = + let pkg_dir = Package.dir pkg in + let dir = Path.Build.append_source (Context.build_dir ctx) pkg_dir in + Output_format.alias output ~dir + in + let* libs = + Context.name ctx |> libs_of_pkg ~pkg:name >>| List.map ~f:(fun lib -> Lib lib) + in + let deps = Pkg name :: libs |> List.map ~f:(Dep.format_alias output ctx) |> Dune_engine.Dep.Set.of_list_map ~f:(fun f -> Dune_engine.Dep.alias f) |> Action_builder.deps - in - Rules.Produce.Alias.add_deps alias deps + in + Rules.Produce.Alias.add_deps alias deps ;; let setup_package_aliases sctx (pkg : Package.t) = @@ -1354,23 +1368,28 @@ let gen_rules sctx ~dir rest = | [ "_markdown" ] -> let* packages = Dune_load.packages () in let ctx = Super_context.context sctx in - let all_package_dirs = - Package.Name.Map.to_list packages - |> List.map ~f:(fun (_, (pkg : Package.t)) -> - let pkg_name = Package.name pkg in - Paths.markdown ctx (Pkg pkg_name)) - in - let directory_targets = - List.fold_left all_package_dirs ~init:Path.Build.Map.empty ~f:(fun acc dir -> - Path.Build.Map.set acc dir Loc.none) - in - has_rules - ~directory_targets - (let* () = setup_toplevel_index_rule sctx Markdown in - Package.Name.Map.to_seq packages - |> Memo.parallel_iter_seq ~f:(fun (_, (pkg : Package.t)) -> - let pkg_name = Package.name pkg in - setup_pkg_markdown_rules sctx ~pkg:pkg_name)) + let* is_markdown_supported = supports_doc_markdown sctx in + if not is_markdown_supported + then + Memo.return Gen_rules.no_rules + else + let all_package_dirs = + Package.Name.Map.to_list packages + |> List.map ~f:(fun (_, (pkg : Package.t)) -> + let pkg_name = Package.name pkg in + Paths.markdown ctx (Pkg pkg_name)) + in + let directory_targets = + List.fold_left all_package_dirs ~init:Path.Build.Map.empty ~f:(fun acc dir -> + Path.Build.Map.set acc dir Loc.none) + in + has_rules + ~directory_targets + (let* () = setup_toplevel_index_rule sctx Markdown in + Package.Name.Map.to_seq packages + |> Memo.parallel_iter_seq ~f:(fun (_, (pkg : Package.t)) -> + let pkg_name = Package.name pkg in + setup_pkg_markdown_rules sctx ~pkg:pkg_name)) | [ "_markdown"; _lib_unique_name_or_pkg ] -> (* package directories are directory targets *) Memo.return Gen_rules.no_rules diff --git a/test/blackbox-tests/test-cases/odoc/odoc-markdown-version-check.t b/test/blackbox-tests/test-cases/odoc/odoc-markdown-version-check.t new file mode 100644 index 00000000000..d133763e0c1 --- /dev/null +++ b/test/blackbox-tests/test-cases/odoc/odoc-markdown-version-check.t @@ -0,0 +1,214 @@ +Testing that odoc markdown generation is only enabled with odoc >= 3.1.0 + +Create a simple project with odoc documentation: + $ cat > dune-project < (lang dune 3.0) + > (package (name foo)) + > EOF + + $ cat > dune < (library + > (public_name foo) + > (name foo)) + > EOF + + $ cat > foo.ml < (** This is a test module *) + > let x = 42 + > EOF + + $ cat > foo.mli < (** This is the interface for the test module *) + > val x : int + > (** The answer to everything *) + > EOF + + $ cat > odoc << 'EOF' + > #!/bin/bash + > case "$1" in + > --version) + > echo "2.0.0" + > ;; + > compile | compile-index) + > # Find the -o flag and create the output file + > output="" + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > output="$2" + > shift 2 + > else + > shift + > fi + > done + > if [[ -n "$output" ]]; then + > mkdir -p $(dirname "$output") + > touch "$output" + > fi + > exit 0;; + > link) + > # Find the -o flag and create the output .odocl file + > output="" + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > output="$2" + > shift 2 + > else + > shift + > fi + > done + > if [[ -n "$output" ]]; then + > mkdir -p $(dirname "$output") + > touch "$output" + > fi + > exit 0;; + > html-generate) + > # Find the -o flag and create output directory + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > mkdir -p "$2" + > break + > fi + > shift + > done + > exit 0;; + > support-files) exit 0;; + > *) exit 0;; + > esac + > EOF + $ chmod +x odoc + $ PATH=.:$PATH dune build @doc-markdown + $ ls _build/default/_doc/_markdown 2>/dev/null || echo "No markdown directory created (expected for odoc 2.0.0)" + No markdown directory created (expected for odoc 2.0.0) + + $ cat > odoc << 'EOF' + > #!/bin/bash + > case "$1" in + > --version) + > echo "3.1.0" + > ;; + > compile | compile-index) + > # Find the -o flag and create the output file + > output="" + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > output="$2" + > shift 2 + > else + > shift + > fi + > done + > if [[ -n "$output" ]]; then + > mkdir -p $(dirname "$output") + > touch "$output" + > fi + > exit 0;; + > link) + > # Find the -o flag and create the output .odocl file + > output="" + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > output="$2" + > shift 2 + > else + > shift + > fi + > done + > if [[ -n "$output" ]]; then + > mkdir -p $(dirname "$output") + > touch "$output" + > fi + > exit 0;; + > html-generate) + > # Find the -o flag and create output directory + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > mkdir -p "$2" + > break + > fi + > shift + > done + > exit 0;; + > markdown-generate) + > # Find the -o flag and create markdown files + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > mkdir -p "$2" + > # Create some dummy markdown files to satisfy the build + > touch "$2/index.md" + > mkdir -p "$2/foo" + > touch "$2/foo/index.md" + > break + > fi + > shift + > done + > exit 0;; + > support-files) exit 0;; + > *) exit 0;; + > esac + > EOF + $ chmod +x odoc + + $ PATH=.:$PATH dune build @doc-markdown + $ ls _build/default/_doc/_markdown/foo/index.md 2>/dev/null && echo "Markdown files created (expected for odoc 3.1.0)" + _build/default/_doc/_markdown/foo/index.md + Markdown files created (expected for odoc 3.1.0) + + $ rm -rf _build + $ cat > odoc << 'EOF' + > #!/bin/bash + > case "$1" in + > --version) + > echo "not a valid version" + > ;; + > compile | compile-index) + > # Find the -o flag and create the output file + > output="" + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > output="$2" + > shift 2 + > else + > shift + > fi + > done + > if [[ -n "$output" ]]; then + > mkdir -p $(dirname "$output") + > touch "$output" + > fi + > exit 0;; + > link) + > # Find the -o flag and create the output .odocl file + > output="" + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > output="$2" + > shift 2 + > else + > shift + > fi + > done + > if [[ -n "$output" ]]; then + > mkdir -p $(dirname "$output") + > touch "$output" + > fi + > exit 0;; + > html-generate) + > # Find the -o flag and create output directory + > while [[ $# -gt 0 ]]; do + > if [[ "$1" == "-o" && -n "$2" ]]; then + > mkdir -p "$2" + > break + > fi + > shift + > done + > exit 0;; + > support-files) exit 0;; + > *) exit 0;; + > esac + > EOF + $ chmod +x odoc + + $ PATH=.:$PATH dune build @doc-markdown + $ ls _build/default/_doc/_markdown + ls: _build/default/_doc/_markdown: No such file or directory + [1] From 556330db4f0bd1212b71977261bc0208f95b1fe6 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Wed, 15 Oct 2025 21:57:01 +0800 Subject: [PATCH 17/20] Format dune_rules/odoc --- src/dune_rules/odoc.ml | 105 ++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 53 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index cf41c976a97..1dd471fccfe 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -1130,58 +1130,58 @@ let setup_pkg_markdown_rules_def = then Memo.return () else let* libs = Context.name ctx |> libs_of_pkg ~pkg in - let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in - let* lib_odocs = - Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) - in - let all_odocs = pkg_odocs @ lib_odocs in - (* odoc generates all markdown files on the same level for the package so we use one rule with directory target and batch all odoc commands. *) - let* () = - if List.is_empty all_odocs - then Memo.return () - else ( - let pkg_markdown_dir = Paths.markdown ctx (Pkg pkg) in - let markdown_root = Paths.markdown_root ctx in - let rule = - let prog, shell_arg = - Env_path.system_shell_exn ~needed_to:"generate markdown documentation" + let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in + let* lib_odocs = + Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) + in + let all_odocs = pkg_odocs @ lib_odocs in + (* odoc generates all markdown files on the same level for the package so we use one rule with directory target and batch all odoc commands. *) + let* () = + if List.is_empty all_odocs + then Memo.return () + else ( + let pkg_markdown_dir = Paths.markdown ctx (Pkg pkg) in + let markdown_root = Paths.markdown_root ctx in + let rule = + let prog, shell_arg = + Env_path.system_shell_exn ~needed_to:"generate markdown documentation" + in + let system_shell_cmd_args = + let open Action_builder.O in + let* odoc_prog = odoc_program sctx (Context.build_dir ctx) in + let odoc_path = Action.Prog.ok_exn odoc_prog |> Path.to_string in + let shell_cmd = + List.map all_odocs ~f:(fun odoc -> + let odocl_rel = + Path.reach + (Path.build odoc.odocl_file) + ~from:(Path.build markdown_root) + in + Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel) + |> String.concat ~sep:" && " in - let system_shell_cmd_args = - let open Action_builder.O in - let* odoc_prog = odoc_program sctx (Context.build_dir ctx) in - let odoc_path = Action.Prog.ok_exn odoc_prog |> Path.to_string in - let shell_cmd = - List.map all_odocs ~f:(fun odoc -> - let odocl_rel = - Path.reach - (Path.build odoc.odocl_file) - ~from:(Path.build markdown_root) - in - Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel) - |> String.concat ~sep:" && " - in - let* () = - List.map all_odocs ~f:(fun odoc -> - Action_builder.path (Path.build odoc.odocl_file)) - |> Action_builder.all - >>| ignore - in - Action_builder.return (Command.Args.S [ A shell_arg; A shell_cmd ]) + let* () = + List.map all_odocs ~f:(fun odoc -> + Action_builder.path (Path.build odoc.odocl_file)) + |> Action_builder.all + >>| ignore in - let deps = Action_builder.env_var "ODOC_SYNTAX" in - let open Action_builder.With_targets.O in - Action_builder.with_no_targets deps - >>> Command.run - ~dir:(Path.build markdown_root) - (Ok prog) - [ Dyn system_shell_cmd_args ] - |> Action_builder.With_targets.add_directories - ~directory_targets:[ pkg_markdown_dir ] + Action_builder.return (Command.Args.S [ A shell_arg; A shell_cmd ]) in - add_rule sctx rule) - in - let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in - add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs + let deps = Action_builder.env_var "ODOC_SYNTAX" in + let open Action_builder.With_targets.O in + Action_builder.with_no_targets deps + >>> Command.run + ~dir:(Path.build markdown_root) + (Ok prog) + [ Dyn system_shell_cmd_args ] + |> Action_builder.With_targets.add_directories + ~directory_targets:[ pkg_markdown_dir ] + in + add_rule sctx rule) + in + let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in + add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs in setup_pkg_rules_def "setup-package-markdown-rules" f ;; @@ -1370,9 +1370,8 @@ let gen_rules sctx ~dir rest = let ctx = Super_context.context sctx in let* is_markdown_supported = supports_doc_markdown sctx in if not is_markdown_supported - then - Memo.return Gen_rules.no_rules - else + then Memo.return Gen_rules.no_rules + else ( let all_package_dirs = Package.Name.Map.to_list packages |> List.map ~f:(fun (_, (pkg : Package.t)) -> @@ -1389,7 +1388,7 @@ let gen_rules sctx ~dir rest = Package.Name.Map.to_seq packages |> Memo.parallel_iter_seq ~f:(fun (_, (pkg : Package.t)) -> let pkg_name = Package.name pkg in - setup_pkg_markdown_rules sctx ~pkg:pkg_name)) + setup_pkg_markdown_rules sctx ~pkg:pkg_name))) | [ "_markdown"; _lib_unique_name_or_pkg ] -> (* package directories are directory targets *) Memo.return Gen_rules.no_rules From ea3c270f9aa0238fcbb387f71bbfe0e5fd027de5 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Wed, 15 Oct 2025 22:37:42 +0800 Subject: [PATCH 18/20] Add debuging --- src/dune_rules/odoc.ml | 147 +++++++++++++++++++++++++++++------------ 1 file changed, 106 insertions(+), 41 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index 1dd471fccfe..b9c4305b400 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -372,10 +372,20 @@ let odoc_path_memo sctx = let supports_doc_markdown sctx = let* odoc_prog = odoc_path_memo sctx in match odoc_prog with - | Error _ -> Memo.return false + | Error _ -> + let () = Console.print_user_message (User_message.make [ Pp.text "[DEBUG] odoc not found, markdown not supported" ]) in + Memo.return false | Ok odoc_path -> let* version = get_odoc_version odoc_path in - Memo.return (Version.higher_than_310 version) + let supported = Version.higher_than_310 version in + let version_str = match version with + | None -> "unknown" + | Some (major, minor, patch) -> Printf.sprintf "%d.%d.%d" major minor patch + in + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] odoc version: %s, markdown supported: %b" version_str supported) + ]) in + Memo.return supported ;; let odoc_dev_tool_exe_path_building_if_necessary () = @@ -1125,63 +1135,96 @@ let setup_lib_markdown_rules sctx lib = let setup_pkg_markdown_rules_def = let f (sctx, pkg) = let ctx = Super_context.context sctx in + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] setup_pkg_markdown_rules called for package: %s" (Package.Name.to_string pkg)) + ]) in let* markdown_supported = supports_doc_markdown sctx in if not markdown_supported - then Memo.return () - else + then ( + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] Markdown not supported, skipping rules for package: %s" (Package.Name.to_string pkg)) + ]) in + Memo.return () + ) + else ( + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] Markdown supported, generating rules for package: %s" (Package.Name.to_string pkg)) + ]) in let* libs = Context.name ctx |> libs_of_pkg ~pkg in let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in let* lib_odocs = Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) in - let all_odocs = pkg_odocs @ lib_odocs in - (* odoc generates all markdown files on the same level for the package so we use one rule with directory target and batch all odoc commands. *) - let* () = - if List.is_empty all_odocs - then Memo.return () - else ( - let pkg_markdown_dir = Paths.markdown ctx (Pkg pkg) in - let markdown_root = Paths.markdown_root ctx in - let rule = + let all_odocs = pkg_odocs @ lib_odocs in + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] Package %s has %d odoc files to generate markdown from" + (Package.Name.to_string pkg) (List.length all_odocs)) + ]) in + (* odoc generates all markdown files on the same level for the package so we use one rule with directory target and batch all odoc commands. *) + let* () = + if List.is_empty all_odocs + then Memo.return () + else ( + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] Creating batched markdown generation rule for package: %s" (Package.Name.to_string pkg)) + ]) in + let pkg_markdown_dir = Paths.markdown ctx (Pkg pkg) in + let markdown_root = Paths.markdown_root ctx in + let rule = let prog, shell_arg = Env_path.system_shell_exn ~needed_to:"generate markdown documentation" in - let system_shell_cmd_args = - let open Action_builder.O in - let* odoc_prog = odoc_program sctx (Context.build_dir ctx) in - let odoc_path = Action.Prog.ok_exn odoc_prog |> Path.to_string in - let shell_cmd = - List.map all_odocs ~f:(fun odoc -> - let odocl_rel = - Path.reach - (Path.build odoc.odocl_file) - ~from:(Path.build markdown_root) - in - Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel) - |> String.concat ~sep:" && " - in + let system_shell_cmd_args = + let open Action_builder.O in + let* odoc_prog = odoc_program sctx (Context.build_dir ctx) in + let odoc_path = Action.Prog.ok_exn odoc_prog |> Path.to_string in + let shell_cmd = + List.map all_odocs ~f:(fun odoc -> + let odocl_rel = + Path.reach + (Path.build odoc.odocl_file) + ~from:(Path.build markdown_root) + in + let expected_output = Path.Build.to_string odoc.markdown_file in + Printf.sprintf + "(echo '[DEBUG] Generating markdown for %s -> %s' >&2 && %s markdown-generate -o . %s && echo '[DEBUG] Success: %s' >&2) || (echo '[DEBUG] FAILED to generate markdown for %s' >&2 && exit 1)" + odocl_rel expected_output odoc_path odocl_rel odocl_rel odocl_rel) + |> String.concat ~sep:" && " + in let* () = List.map all_odocs ~f:(fun odoc -> Action_builder.path (Path.build odoc.odocl_file)) |> Action_builder.all >>| ignore + in + let* () = + Action_builder.return () + >>| fun () -> + Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] About to run markdown generation shell script for package: %s" (Package.Name.to_string pkg)) + ]) + in + Action_builder.return (Command.Args.S [ A shell_arg; A shell_cmd ]) in - Action_builder.return (Command.Args.S [ A shell_arg; A shell_cmd ]) + let deps = Action_builder.env_var "ODOC_SYNTAX" in + let open Action_builder.With_targets.O in + Action_builder.with_no_targets deps + >>> Command.run + ~dir:(Path.build markdown_root) + (Ok prog) + [ Dyn system_shell_cmd_args ] + |> Action_builder.With_targets.add_directories + ~directory_targets:[ pkg_markdown_dir ] in - let deps = Action_builder.env_var "ODOC_SYNTAX" in - let open Action_builder.With_targets.O in - Action_builder.with_no_targets deps - >>> Command.run - ~dir:(Path.build markdown_root) - (Ok prog) - [ Dyn system_shell_cmd_args ] - |> Action_builder.With_targets.add_directories - ~directory_targets:[ pkg_markdown_dir ] - in - add_rule sctx rule) + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] Adding markdown generation rule for package: %s, target dir: %s" + (Package.Name.to_string pkg) (Path.Build.to_string pkg_markdown_dir)) + ]) in + add_rule sctx rule) in let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs + ) in setup_pkg_rules_def "setup-package-markdown-rules" f ;; @@ -1194,10 +1237,21 @@ let setup_package_aliases_format sctx (pkg : Package.t) (output : Output_format. let ctx = Super_context.context sctx in match (output : Output_format.t) with | Markdown -> + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] setup_package_aliases_format called for Markdown, package: %s" (Package.Name.to_string (Package.name pkg))) + ]) in let* is_markdown_supported = supports_doc_markdown sctx in if not is_markdown_supported - then Memo.return () + then ( + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] Markdown not supported, skipping alias for package: %s" (Package.Name.to_string (Package.name pkg))) + ]) in + Memo.return () + ) else ( + let () = Console.print_user_message (User_message.make [ + Pp.text (Printf.sprintf "[DEBUG] Markdown supported, creating alias for package: %s" (Package.Name.to_string (Package.name pkg))) + ]) in let name = Package.name pkg in let alias = let pkg_dir = Package.dir pkg in @@ -1366,12 +1420,23 @@ let gen_rules sctx ~dir rest = >>> setup_toplevel_index_rule sctx Html >>> setup_toplevel_index_rule sctx Json) | [ "_markdown" ] -> + let () = Console.print_user_message (User_message.make [ + Pp.text "[DEBUG] gen_rules called for _markdown directory" + ]) in let* packages = Dune_load.packages () in let ctx = Super_context.context sctx in let* is_markdown_supported = supports_doc_markdown sctx in if not is_markdown_supported - then Memo.return Gen_rules.no_rules + then ( + let () = Console.print_user_message (User_message.make [ + Pp.text "[DEBUG] Markdown not supported, returning no_rules for _markdown" + ]) in + Memo.return Gen_rules.no_rules + ) else ( + let () = Console.print_user_message (User_message.make [ + Pp.text "[DEBUG] Markdown supported, setting up markdown rules for all packages" + ]) in let all_package_dirs = Package.Name.Map.to_list packages |> List.map ~f:(fun (_, (pkg : Package.t)) -> From dad4ff24e4fa0a397ab2f57b58dedd801e42d305 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Thu, 16 Oct 2025 00:20:43 +0800 Subject: [PATCH 19/20] Remove logging --- src/dune_rules/odoc.ml | 87 ++++++------------------------------------ 1 file changed, 11 insertions(+), 76 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index b9c4305b400..8644d7fd41f 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -372,20 +372,10 @@ let odoc_path_memo sctx = let supports_doc_markdown sctx = let* odoc_prog = odoc_path_memo sctx in match odoc_prog with - | Error _ -> - let () = Console.print_user_message (User_message.make [ Pp.text "[DEBUG] odoc not found, markdown not supported" ]) in - Memo.return false + | Error _ -> Memo.return false | Ok odoc_path -> let* version = get_odoc_version odoc_path in - let supported = Version.higher_than_310 version in - let version_str = match version with - | None -> "unknown" - | Some (major, minor, patch) -> Printf.sprintf "%d.%d.%d" major minor patch - in - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] odoc version: %s, markdown supported: %b" version_str supported) - ]) in - Memo.return supported + Memo.return (Version.higher_than_310 version) ;; let odoc_dev_tool_exe_path_building_if_necessary () = @@ -1135,39 +1125,21 @@ let setup_lib_markdown_rules sctx lib = let setup_pkg_markdown_rules_def = let f (sctx, pkg) = let ctx = Super_context.context sctx in - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] setup_pkg_markdown_rules called for package: %s" (Package.Name.to_string pkg)) - ]) in let* markdown_supported = supports_doc_markdown sctx in if not markdown_supported - then ( - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] Markdown not supported, skipping rules for package: %s" (Package.Name.to_string pkg)) - ]) in - Memo.return () - ) - else ( - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] Markdown supported, generating rules for package: %s" (Package.Name.to_string pkg)) - ]) in + then Memo.return () + else let* libs = Context.name ctx |> libs_of_pkg ~pkg in let* pkg_odocs = odoc_artefacts sctx (Pkg pkg) in let* lib_odocs = Memo.List.concat_map libs ~f:(fun lib -> odoc_artefacts sctx (Lib lib)) in let all_odocs = pkg_odocs @ lib_odocs in - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] Package %s has %d odoc files to generate markdown from" - (Package.Name.to_string pkg) (List.length all_odocs)) - ]) in (* odoc generates all markdown files on the same level for the package so we use one rule with directory target and batch all odoc commands. *) let* () = if List.is_empty all_odocs then Memo.return () else ( - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] Creating batched markdown generation rule for package: %s" (Package.Name.to_string pkg)) - ]) in let pkg_markdown_dir = Paths.markdown ctx (Pkg pkg) in let markdown_root = Paths.markdown_root ctx in let rule = @@ -1185,24 +1157,14 @@ let setup_pkg_markdown_rules_def = (Path.build odoc.odocl_file) ~from:(Path.build markdown_root) in - let expected_output = Path.Build.to_string odoc.markdown_file in - Printf.sprintf - "(echo '[DEBUG] Generating markdown for %s -> %s' >&2 && %s markdown-generate -o . %s && echo '[DEBUG] Success: %s' >&2) || (echo '[DEBUG] FAILED to generate markdown for %s' >&2 && exit 1)" - odocl_rel expected_output odoc_path odocl_rel odocl_rel odocl_rel) + Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel) |> String.concat ~sep:" && " in - let* () = - List.map all_odocs ~f:(fun odoc -> - Action_builder.path (Path.build odoc.odocl_file)) - |> Action_builder.all - >>| ignore - in let* () = - Action_builder.return () - >>| fun () -> - Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] About to run markdown generation shell script for package: %s" (Package.Name.to_string pkg)) - ]) + List.map all_odocs ~f:(fun odoc -> + Action_builder.path (Path.build odoc.odocl_file)) + |> Action_builder.all + >>| ignore in Action_builder.return (Command.Args.S [ A shell_arg; A shell_cmd ]) in @@ -1216,15 +1178,10 @@ let setup_pkg_markdown_rules_def = |> Action_builder.With_targets.add_directories ~directory_targets:[ pkg_markdown_dir ] in - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] Adding markdown generation rule for package: %s, target dir: %s" - (Package.Name.to_string pkg) (Path.Build.to_string pkg_markdown_dir)) - ]) in add_rule sctx rule) in let* () = Memo.parallel_iter libs ~f:(setup_lib_markdown_rules sctx) in add_format_alias_deps ctx Markdown (Pkg pkg) all_odocs - ) in setup_pkg_rules_def "setup-package-markdown-rules" f ;; @@ -1237,21 +1194,10 @@ let setup_package_aliases_format sctx (pkg : Package.t) (output : Output_format. let ctx = Super_context.context sctx in match (output : Output_format.t) with | Markdown -> - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] setup_package_aliases_format called for Markdown, package: %s" (Package.Name.to_string (Package.name pkg))) - ]) in let* is_markdown_supported = supports_doc_markdown sctx in if not is_markdown_supported - then ( - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] Markdown not supported, skipping alias for package: %s" (Package.Name.to_string (Package.name pkg))) - ]) in - Memo.return () - ) + then Memo.return () else ( - let () = Console.print_user_message (User_message.make [ - Pp.text (Printf.sprintf "[DEBUG] Markdown supported, creating alias for package: %s" (Package.Name.to_string (Package.name pkg))) - ]) in let name = Package.name pkg in let alias = let pkg_dir = Package.dir pkg in @@ -1420,23 +1366,12 @@ let gen_rules sctx ~dir rest = >>> setup_toplevel_index_rule sctx Html >>> setup_toplevel_index_rule sctx Json) | [ "_markdown" ] -> - let () = Console.print_user_message (User_message.make [ - Pp.text "[DEBUG] gen_rules called for _markdown directory" - ]) in let* packages = Dune_load.packages () in let ctx = Super_context.context sctx in let* is_markdown_supported = supports_doc_markdown sctx in if not is_markdown_supported - then ( - let () = Console.print_user_message (User_message.make [ - Pp.text "[DEBUG] Markdown not supported, returning no_rules for _markdown" - ]) in - Memo.return Gen_rules.no_rules - ) + then Memo.return Gen_rules.no_rules else ( - let () = Console.print_user_message (User_message.make [ - Pp.text "[DEBUG] Markdown supported, setting up markdown rules for all packages" - ]) in let all_package_dirs = Package.Name.Map.to_list packages |> List.map ~f:(fun (_, (pkg : Package.t)) -> From 2a4775e6bf2ebedb255b37484db4c8ad7995c9f1 Mon Sep 17 00:00:00 2001 From: David Sancho Moreno Date: Thu, 16 Oct 2025 01:35:06 +0800 Subject: [PATCH 20/20] Abstract shell and format_alias --- src/dune_rules/odoc.ml | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/dune_rules/odoc.ml b/src/dune_rules/odoc.ml index 8644d7fd41f..b98843419f9 100644 --- a/src/dune_rules/odoc.ml +++ b/src/dune_rules/odoc.ml @@ -162,6 +162,12 @@ module Output_format = struct ;; end +let output_dir_for_format ctx format target = + match (format : Output_format.t) with + | Html | Json -> Paths.html ctx target + | Markdown -> Paths.markdown ctx target +;; + module Dep : sig (** [format_alias output ctx target] returns the alias that depends on all targets produced by odoc for [target] in output format [output]. *) @@ -180,15 +186,7 @@ module Dep : sig These dependencies may be used using the [deps] function *) val setup_deps : Context.t -> target -> Path.Set.t -> unit Memo.t end = struct - let format_alias f ctx m = - let dir = - match (f : Output_format.t) with - | Html | Json -> Paths.html ctx m - | Markdown -> Paths.markdown ctx m - in - Output_format.alias f ~dir - ;; - + let format_alias f ctx m = Output_format.alias f ~dir:(output_dir_for_format ctx f m) let alias = Alias.make (Alias.Name.of_string ".odoc-all") let deps ctx pkg requires = @@ -1122,6 +1120,15 @@ let setup_lib_markdown_rules sctx lib = Memo.With_implicit_output.exec setup_lib_markdown_rules_def (sctx, lib)) ;; +let markdown_shell_command odoc_path all_odocs ~markdown_root = + List.map all_odocs ~f:(fun odoc -> + let odocl_rel = + Path.reach (Path.build odoc.odocl_file) ~from:(Path.build markdown_root) + in + Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel) + |> String.concat ~sep:" && " +;; + let setup_pkg_markdown_rules_def = let f (sctx, pkg) = let ctx = Super_context.context sctx in @@ -1150,16 +1157,7 @@ let setup_pkg_markdown_rules_def = let open Action_builder.O in let* odoc_prog = odoc_program sctx (Context.build_dir ctx) in let odoc_path = Action.Prog.ok_exn odoc_prog |> Path.to_string in - let shell_cmd = - List.map all_odocs ~f:(fun odoc -> - let odocl_rel = - Path.reach - (Path.build odoc.odocl_file) - ~from:(Path.build markdown_root) - in - Printf.sprintf "%s markdown-generate -o . %s" odoc_path odocl_rel) - |> String.concat ~sep:" && " - in + let shell_cmd = markdown_shell_command odoc_path all_odocs ~markdown_root in let* () = List.map all_odocs ~f:(fun odoc -> Action_builder.path (Path.build odoc.odocl_file))