Skip to content

Commit

Permalink
Render fetcher info in primop again
Browse files Browse the repository at this point in the history
  • Loading branch information
Ericson2314 committed Jan 17, 2024
1 parent 2eec79b commit 80aae23
Show file tree
Hide file tree
Showing 7 changed files with 225 additions and 92 deletions.
7 changes: 4 additions & 3 deletions src/libexpr/eval.cc
Original file line number Diff line number Diff line change
Expand Up @@ -657,7 +657,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
v.mkApp(vPrimOp, vPrimOp);
return addConstant(primOp.name, v, {
.type = nThunk, // FIXME
.doc = primOp.doc,
.doc = primOp.doc.c_str(),
});
}

Expand All @@ -684,13 +684,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{
if (v.isPrimOp()) {
auto v2 = &v;
if (auto * doc = v2->primOp->doc)
auto & doc = v2->primOp->doc;
if (doc != "")
return Doc {
.pos = {},
.name = v2->primOp->name,
.arity = v2->primOp->arity,
.args = v2->primOp->args,
.doc = doc,
.doc = doc.c_str(),
};
}
return {};
Expand Down
2 changes: 1 addition & 1 deletion src/libexpr/eval.hh
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ struct PrimOp
/**
* Optional free-form documentation about the primop.
*/
const char * doc = nullptr;
const std::string doc = "";

/**
* Implementation of the primop.
Expand Down
2 changes: 1 addition & 1 deletion src/libexpr/local.mk
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib

libexpr_LIBS = libutil libstore libfetchers

libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) $(CMARK_LIBS)
ifdef HOST_LINUX
libexpr_LDFLAGS += -ldl
endif
Expand Down
220 changes: 134 additions & 86 deletions src/libexpr/primops/fetchTree.cc
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include "registry.hh"
#include "tarball.hh"
#include "url.hh"
#include "cmark-cpp.hh"
#include "value-to-json.hh"

#include <ctime>
Expand Down Expand Up @@ -194,92 +195,139 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args,
static RegisterPrimOp primop_fetchTree({
.name = "fetchTree",
.args = {"input"},
.doc = R"(
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
- the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
- the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
*input* must be an attribute set with the following attributes:
- `type` (String, required)
One of the [supported source types](#source-types).
This determines other required and allowed input attributes.
- `narHash` (String, optional)
The `narHash` parameter can be used to substitute the source of the tree.
It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
That is, `fetchTree` is idempotent.
Downloads are cached in `$XDG_CACHE_HOME/nix`.
The remote source will be fetched from the network if both are true:
- A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
> **Note**
>
> [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
- There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
## Source types
The following source types and associated input attributes are supported.
<!-- TODO: It would be soooo much more predictable to work with (and
document) if `fetchTree` was a curried call with the first paramter for
`type` or an attribute like `builtins.fetchTree.git`! -->
The following input types are still subject to change:
- `"path"`
- `"github"`
- `"gitlab"`
- `"sourcehut"`
- `"mercurial"`
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
> **Example**
>
> Fetch a GitHub repository using the attribute set representation:
>
> ```nix
> builtins.fetchTree {
> type = "github";
> owner = "NixOS";
> repo = "nixpkgs";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> }
> ```
>
> This evaluates to the following attribute set:
>
> ```nix
> {
> lastModified = 1686503798;
> lastModifiedDate = "20230611171638";
> narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
> outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> shortRev = "ae2e6b3";
> }
> ```
> **Example**
>
> Fetch the same GitHub repository using the URL-like syntax:
>
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)",
.doc = []() -> std::string {
using namespace cmark;

// Stores strings referenced by AST. Deallocate after rendering.
std::vector<std::string> textArena;

auto root = node_new(CMARK_NODE_DOCUMENT);

auto & before = textArena.emplace_back(stripIndentation(R"(
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
- the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path)
- the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
*input* must be an attribute set with the following attributes:
- `type` (String, required)
One of the [supported source types](#source-types).
This determines other required and allowed input attributes.
- `narHash` (String, optional)
The `narHash` parameter can be used to substitute the source of the tree.
It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
That is, `fetchTree` is idempotent.
Downloads are cached in `$XDG_CACHE_HOME/nix`.
The remote source will be fetched from the network if both are true:
- A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
> **Note**
>
> [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
- There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
## Source types
The following source types and associated input attributes are supported.
<!-- TODO: It would be soooo much more predictable to work with (and
document) if `fetchTree` was a curried call with the first paramter for
`type` or an attribute like `builtins.fetchTree.git`! -->
)"));
node_append_child(*root, parse_document(before, CMARK_OPT_DEFAULT));

auto & schemes = node_append_child(*root, node_new(CMARK_NODE_LIST));

for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
auto & s = node_append_child(schemes, node_new(CMARK_NODE_ITEM));
{
auto & name_p = node_append_child(s, node_new(CMARK_NODE_PARAGRAPH));
auto & name = node_append_child(name_p, node_new(CMARK_NODE_TEXT));
node_set_literal(name, schemeName.data());
}
node_append_child(s, parse_document(scheme->schemeDescription(), CMARK_OPT_DEFAULT));

auto & attrs = node_append_child(s, node_new(CMARK_NODE_LIST));
for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
auto & a = node_append_child(attrs, node_new(CMARK_NODE_ITEM));
{
auto & name_p = node_append_child(a, node_new(CMARK_NODE_PARAGRAPH));
auto & name = node_append_child(name_p, node_new(CMARK_NODE_TEXT));
auto & header = textArena.emplace_back(attrName
+ " (" + attribute.type
+ ", " + (attribute.required ? "required" : "optional")
+ ")");
node_set_literal(name, header.c_str());
}
{
auto & doc = textArena.emplace_back(stripIndentation(attribute.doc));
node_append_child(a, parse_document(doc, CMARK_OPT_DEFAULT));
}
}
}

auto & after = textArena.emplace_back(stripIndentation(R"(
The following input types are still subject to change:
- `"path"`
- `"github"`
- `"gitlab"`
- `"sourcehut"`
- `"mercurial"`
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
> **Example**
>
> Fetch a GitHub repository using the attribute set representation:
>
> ```nix
> builtins.fetchTree {
> type = "github";
> owner = "NixOS";
> repo = "nixpkgs";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> }
> ```
>
> This evaluates to the following attribute set:
>
> ```nix
> {
> lastModified = 1686503798;
> lastModifiedDate = "20230611171638";
> narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
> outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> shortRev = "ae2e6b3";
> }
> ```
> **Example**
>
> Fetch the same GitHub repository using the URL-like syntax:
>
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)"));
node_append_child(*root, parse_document(after, CMARK_OPT_DEFAULT));

auto p = render_commonmark(*root, CMARK_OPT_DEFAULT, 0);
assert(p);
return { &*p };
}(),
.fun = prim_fetchTree,
.experimentalFeature = Xp::FetchTree,
});
Expand Down
74 changes: 74 additions & 0 deletions src/libutil/cmark-cpp.hh
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#pragma once
///@file

#include "types.hh"
#include "util.hh"

#include <cmark.h>

namespace nix::cmark {

using Node = struct cmark_node;
using NodeType = cmark_node_type;
using ListType = cmark_list_type;

using Iter = struct cmark_iter;

struct Deleter
{
void operator () (Node * ptr) { cmark_node_free(ptr); }
void operator () (Iter * ptr) { cmark_iter_free(ptr); }
};

template <typename T>
using UniquePtr = std::unique_ptr<Node, Deleter>;

static inline UniquePtr<Node> parse_document(std::string_view s, int options)
{
return UniquePtr<Node> {
cmark_parse_document(s.data(), s.size(), options)
};
}

static inline std::unique_ptr<char, FreeDeleter> render_commonmark(Node & root, int options, int width)
{
return std::unique_ptr<char, FreeDeleter> {
cmark_render_commonmark(&root, options, width)
};
}

static inline std::unique_ptr<char, FreeDeleter> render_xml(Node & root, int options)
{
return std::unique_ptr<char, FreeDeleter> {
cmark_render_xml(&root, options)
};
}

static inline UniquePtr<Node> node_new(NodeType type)
{
return UniquePtr<Node> {
cmark_node_new(type)
};
}

/**
* The parent takes ownership
*/
static inline Node & node_append_child(Node & node, UniquePtr<Node> child)
{
auto status = (bool) cmark_node_append_child(&node, &*child);
assert(status);
return *child.release();
}

static inline bool node_set_literal(Node & node, const char * content)
{
return (bool) cmark_node_set_literal(&node, content);
}

static inline bool node_set_list_type(Node & node, ListType type)
{
return (bool) cmark_node_set_list_type(&node, type);
}

}
10 changes: 10 additions & 0 deletions src/libutil/util.hh
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,16 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
std::string showBytes(uint64_t bytes);


/**
* For using `std::unique` with C functions.
*/
struct FreeDeleter
{
template <typename T>
void operator()(T *p) const { std::free(p); }
};


/**
* Provide an addition operator between strings and string_views
* inexplicably omitted from the standard library.
Expand Down
2 changes: 1 addition & 1 deletion src/nix/main.cc
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ void mainWrapped(int argc, char * * argv)
auto b = nlohmann::json::object();
if (!builtin.value->isPrimOp()) continue;
auto primOp = builtin.value->primOp;
if (!primOp->doc) continue;
if (primOp->doc == "") continue;
b["arity"] = primOp->arity;
b["args"] = primOp->args;
b["doc"] = trim(stripIndentation(primOp->doc));
Expand Down

0 comments on commit 80aae23

Please sign in to comment.