diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 5e2f716491d..698fcf5a940 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -638,7 +638,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) v.mkApp(vPrimOp, vPrimOp); return addConstant(primOp.name, v, { .type = nThunk, // FIXME - .doc = primOp.doc, + .doc = primOp.doc.c_str(), }); } @@ -665,13 +665,14 @@ std::optional EvalState::getDoc(Value & v) { if (v.isPrimOp()) { auto v2 = &v; - if (auto * doc = v2->primOp->doc) + auto & doc = v2->primOp->doc; + if (doc != "") return Doc { .pos = {}, .name = v2->primOp->name, .arity = v2->primOp->arity, .args = v2->primOp->args, - .doc = doc, + .doc = doc.c_str(), }; } return {}; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f15d1965307..3e99bdc6691 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -67,7 +67,7 @@ struct PrimOp /** * Optional free-form documentation about the primop. */ - const char * doc = nullptr; + const std::string doc = ""; /** * Implementation of the primop. diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 0c3e36750e8..fde02223bc3 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) +libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) $(CMARK_LIBS) ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index ede0f5d740f..3a6a0e867b3 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -8,6 +8,7 @@ #include "registry.hh" #include "tarball.hh" #include "url.hh" +#include "cmark-cpp.hh" #include "value-to-json.hh" #include "fetch-to-store.hh" @@ -197,92 +198,144 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, static RegisterPrimOp primop_fetchTree({ .name = "fetchTree", .args = {"input"}, - .doc = R"( - Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with: - - - the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path) - - the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash - - backend-specific metadata (currently not documented). - - *input* must be an attribute set with the following attributes: - - - `type` (String, required) - - One of the [supported source types](#source-types). - This determines other required and allowed input attributes. - - - `narHash` (String, optional) - - The `narHash` parameter can be used to substitute the source of the tree. - It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. - If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. - - A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. - That is, `fetchTree` is idempotent. - - Downloads are cached in `$XDG_CACHE_HOME/nix`. - The remote source will be fetched from the network if both are true: - - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store - - > **Note** - > - > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. - - - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) - - ## Source types - - The following source types and associated input attributes are supported. - - - - The following input types are still subject to change: - - - `"path"` - - `"github"` - - `"gitlab"` - - `"sourcehut"` - - `"mercurial"` - - *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). - The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. - - > **Example** - > - > Fetch a GitHub repository using the attribute set representation: - > - > ```nix - > builtins.fetchTree { - > type = "github"; - > owner = "NixOS"; - > repo = "nixpkgs"; - > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - > } - > ``` - > - > This evaluates to the following attribute set: - > - > ```nix - > { - > lastModified = 1686503798; - > lastModifiedDate = "20230611171638"; - > narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; - > outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; - > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - > shortRev = "ae2e6b3"; - > } - > ``` - - > **Example** - > - > Fetch the same GitHub repository using the URL-like syntax: - > - > ```nix - > builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" - > ``` - )", + .doc = []() -> std::string { + using namespace cmark; + + // Stores strings referenced by AST. Deallocate after rendering. + std::vector textArena; + + auto root = node_new(CMARK_NODE_DOCUMENT); + + auto & before = textArena.emplace_back(stripIndentation(R"( + Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with: + + - the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path) + - the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash + - backend-specific metadata (currently not documented). + + *input* must be an attribute set with the following attributes: + + - `type` (String, required) + + One of the [supported source types](#source-types). + This determines other required and allowed input attributes. + + - `narHash` (String, optional) + + The `narHash` parameter can be used to substitute the source of the tree. + It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. + If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. + + A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. + That is, `fetchTree` is idempotent. + + Downloads are cached in `$XDG_CACHE_HOME/nix`. + The remote source will be fetched from the network if both are true: + - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store + + > **Note** + > + > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. + + - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) + + ## Source types + + The following source types and associated input attributes are supported. + + + )")); + parse_document(*root, before, CMARK_OPT_DEFAULT); + + auto & schemes = node_append_child(*root, node_new(CMARK_NODE_LIST)); + + for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) { + auto & s = node_append_child(schemes, node_new(CMARK_NODE_ITEM)); + { + auto & name_p = node_append_child(s, node_new(CMARK_NODE_PARAGRAPH)); + auto & name = node_append_child(name_p, node_new(CMARK_NODE_TEXT)); + node_set_literal(name, schemeName.data()); + } + parse_document(s, scheme->schemeDescription(), CMARK_OPT_DEFAULT); + + auto & attrs = node_append_child(s, node_new(CMARK_NODE_LIST)); + for (const auto & [attrName, attribute] : scheme->allowedAttrs()) { + auto & a = node_append_child(attrs, node_new(CMARK_NODE_ITEM)); + { + auto & name_info = node_append_child(a, node_new(CMARK_NODE_PARAGRAPH)); + { + auto & name = node_append_child(name_info, node_new(CMARK_NODE_CODE)); + auto & name_t = textArena.emplace_back(attrName); + node_set_literal(name, name_t.c_str()); + } + auto & info = node_append_child(name_info, node_new(CMARK_NODE_TEXT)); + auto & header = textArena.emplace_back(std::string { } + + " (" + attribute.type + + ", " + (attribute.required ? "required" : "optional") + + ")"); + node_set_literal(info, header.c_str()); + } + { + auto & doc = textArena.emplace_back(stripIndentation(attribute.doc)); + parse_document(a, doc, CMARK_OPT_DEFAULT); + } + } + } + + auto & after = textArena.emplace_back(stripIndentation(R"( + The following input types are still subject to change: + + - `"path"` + - `"github"` + - `"gitlab"` + - `"sourcehut"` + - `"mercurial"` + + *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + + > **Example** + > + > Fetch a GitHub repository using the attribute set representation: + > + > ```nix + > builtins.fetchTree { + > type = "github"; + > owner = "NixOS"; + > repo = "nixpkgs"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > } + > ``` + > + > This evaluates to the following attribute set: + > + > ```nix + > { + > lastModified = 1686503798; + > lastModifiedDate = "20230611171638"; + > narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + > outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > shortRev = "ae2e6b3"; + > } + > ``` + + > **Example** + > + > Fetch the same GitHub repository using the URL-like syntax: + > + > ```nix + > builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" + > ``` + )")); + parse_document(*root, after, CMARK_OPT_DEFAULT); + + auto p = render_commonmark(*root, CMARK_OPT_DEFAULT, 0); + assert(p); + return { &*p }; + }(), .fun = prim_fetchTree, .experimentalFeature = Xp::FetchTree, }); diff --git a/src/libutil/cmark-cpp.hh b/src/libutil/cmark-cpp.hh new file mode 100644 index 00000000000..f45fe134b67 --- /dev/null +++ b/src/libutil/cmark-cpp.hh @@ -0,0 +1,85 @@ +#pragma once +///@file + +#include "types.hh" +#include "util.hh" + +#include + +namespace nix::cmark { + +using Node = struct cmark_node; +using NodeType = cmark_node_type; +using ListType = cmark_list_type; + +using Iter = struct cmark_iter; + +struct Deleter +{ + void operator () (Node * ptr) { cmark_node_free(ptr); } + void operator () (Iter * ptr) { cmark_iter_free(ptr); } +}; + +template +using UniquePtr = std::unique_ptr; + +static inline void parse_document(Node & root, std::string_view s, int options) +{ + cmark_parser * parser = cmark_parser_new_with_mem_into_root( + options, + cmark_get_default_mem_allocator(), + &root); + cmark_parser_feed(parser, s.data(), s.size()); + (void) cmark_parser_finish(parser); + cmark_parser_free(parser); +} + +static inline UniquePtr parse_document(std::string_view s, int options) +{ + return UniquePtr { + cmark_parse_document(s.data(), s.size(), options) + }; +} + +static inline std::unique_ptr render_commonmark(Node & root, int options, int width) +{ + return std::unique_ptr { + cmark_render_commonmark(&root, options, width) + }; +} + +static inline std::unique_ptr render_xml(Node & root, int options) +{ + return std::unique_ptr { + cmark_render_xml(&root, options) + }; +} + +static inline UniquePtr node_new(NodeType type) +{ + return UniquePtr { + cmark_node_new(type) + }; +} + +/** + * The parent takes ownership + */ +static inline Node & node_append_child(Node & node, UniquePtr child) +{ + auto status = (bool) cmark_node_append_child(&node, &*child); + assert(status); + return *child.release(); +} + +static inline bool node_set_literal(Node & node, const char * content) +{ + return (bool) cmark_node_set_literal(&node, content); +} + +static inline bool node_set_list_type(Node & node, ListType type) +{ + return (bool) cmark_node_set_list_type(&node, type); +} + +} diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 11a0431da89..78af680995c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -344,6 +344,16 @@ template overloaded(Ts...) -> overloaded; std::string showBytes(uint64_t bytes); +/** + * For using `std::unique` with C functions. + */ +struct FreeDeleter +{ + template + void operator()(T *p) const { std::free(p); } +}; + + /** * Provide an addition operator between strings and string_views * inexplicably omitted from the standard library. diff --git a/src/nix/main.cc b/src/nix/main.cc index b1716552442..5b98924d7aa 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -424,7 +424,7 @@ void mainWrapped(int argc, char * * argv) auto b = nlohmann::json::object(); if (!builtin.value->isPrimOp()) continue; auto primOp = builtin.value->primOp; - if (!primOp->doc) continue; + if (primOp->doc == "") continue; b["arity"] = primOp->arity; b["args"] = primOp->args; b["doc"] = trim(stripIndentation(primOp->doc));