From d17d358d10dedc54fa167eb6c8369e2d5d81dcec Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 Jan 2024 13:21:38 -0500 Subject: [PATCH 1/4] Add `cmark` dependency `cmark` is a more robust library for constructing Markdown than `lowdown`. It is also more portable. Unfortunately it doesn't have a terminal mode, so we cannot get rid of lowdown yet. The `--enable-markdown` flag is renamed to `--enable-lowdown` accordingly. --- Makefile.config.in | 1 + configure.ac | 13 ++++++++----- package.nix | 8 +++++--- packaging/hydra.nix | 4 ++-- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 3100d207365..08189487070 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -4,6 +4,7 @@ BOOST_LDFLAGS = @BOOST_LDFLAGS@ BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ CC = @CC@ CFLAGS = @CFLAGS@ +CMARK_LIBS = @CMARK_LIBS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ CXXLTO = @CXXLTO@ diff --git a/configure.ac b/configure.ac index 198198dea26..2a4b234a449 100644 --- a/configure.ac +++ b/configure.ac @@ -372,21 +372,24 @@ PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest]) PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) +# Look for cmark library. +PKG_CHECK_MODULES([CMARK], [libcmark], [CXXFLAGS="$CMARK_CFLAGS $CXXFLAGS"]) + # Look for lowdown library. -AC_ARG_ENABLE([markdown], AS_HELP_STRING([--enable-markdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]), - enable_markdown=$enableval, enable_markdown=auto) -AS_CASE(["$enable_markdown"], +AC_ARG_ENABLE([lowdown], AS_HELP_STRING([--enable-lowdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]), + enable_lowdown=$enableval, enable_lowdown=auto) +AS_CASE(["$enable_lowdown"], [yes | auto], [ PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [ CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS" have_lowdown=1 AC_DEFINE(HAVE_LOWDOWN, 1, [Whether lowdown is available and should be used for Markdown rendering.]) ], [ - AS_IF([test "x$enable_markdown" == "xyes"], [AC_MSG_ERROR([--enable-markdown was specified, but lowdown was not found.])]) + AS_IF([test "x$enable_lowdown" == "xyes"], [AC_MSG_ERROR([--enable-lowdown was specified, but lowdown was not found.])]) ]) ], [no], [have_lowdown=], - [AC_MSG_ERROR([bad value "$enable_markdown" for --enable-markdown, must be one of: yes, no, auto])]) + [AC_MSG_ERROR([bad value "$enable_lowdown" for --enable-lowdown, must be one of: yes, no, auto])]) # Look for libgit2. diff --git a/package.nix b/package.nix index 8ab1846679c..6ae4d886b56 100644 --- a/package.nix +++ b/package.nix @@ -23,6 +23,7 @@ , libseccomp , libsodium , man +, cmark , lowdown , mdbook , mdbook-linkcheck @@ -79,7 +80,7 @@ , enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. -, enableMarkdown ? !stdenv.hostPlatform.isWindows +, enableLowdown ? !stdenv.hostPlatform.isWindows # Which interactive line editor library to use for Nix's repl. # @@ -228,7 +229,8 @@ in { toml11 xz ({ inherit readline editline; }.${readlineFlavor}) - ] ++ lib.optionals enableMarkdown [ + cmark + ] ++ lib.optionals enableLowdown [ lowdown ] ++ lib.optionals buildUnitTests [ gtest @@ -255,7 +257,7 @@ in { (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") - (lib.enableFeature enableMarkdown "markdown") + (lib.enableFeature enableLowdown "lowdown") (lib.enableFeature installUnitTests "install-unit-tests") (lib.withFeatureAs true "readline-flavor" readlineFlavor) ] ++ lib.optionals (!forDevShell) [ diff --git a/packaging/hydra.nix b/packaging/hydra.nix index cba1b25832a..d2bfb474e3a 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -90,9 +90,9 @@ in # Toggles some settings for better coverage. Windows needs these # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: + buildReadlineNoLowdown = forAllSystems (system: self.packages.${system}.nix.override { - enableMarkdown = false; + enableLowdown = false; readlineFlavor = "readline"; } ); From f9d7c8021d3e954652840b3280d4da7d7d003dfe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 26 Jan 2024 22:26:05 -0500 Subject: [PATCH 2/4] Update `cmark` source code so we can parser into an existing AST MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Needed until https://github.com/commonmark/cmark/pull/524 is released. • Added input 'cmark': 'github:commonmark/cmark/cd37711b8a08da67ba4e21a42614b86dd8def929' (2024-01-26) --- flake.lock | 17 +++++++++++++++++ flake.nix | 2 ++ packaging/dependencies.nix | 5 +++++ 3 files changed, 24 insertions(+) diff --git a/flake.lock b/flake.lock index b5d0b881c5c..c88f1124a41 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "cmark": { + "flake": false, + "locked": { + "lastModified": 1706295831, + "narHash": "sha256-nEI85W8w49ZVr17ycO+7aZvcgA3U2QphNZGrfQl2mSk=", + "owner": "commonmark", + "repo": "cmark", + "rev": "cd37711b8a08da67ba4e21a42614b86dd8def929", + "type": "github" + }, + "original": { + "owner": "commonmark", + "repo": "cmark", + "type": "github" + } + }, "flake-compat": { "flake": false, "locked": { @@ -128,6 +144,7 @@ }, "root": { "inputs": { + "cmark": "cmark", "flake-compat": "flake-compat", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", diff --git a/flake.nix b/flake.nix index 303779c2b18..5612de83476 100644 --- a/flake.nix +++ b/flake.nix @@ -6,6 +6,8 @@ inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; + # Until https://github.com/commonmark/cmark/pull/524 is released + inputs.cmark = { url = "github:commonmark/cmark"; flake = false; }; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index e5f4c0f9180..abe5b152111 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -140,6 +140,11 @@ scope: { ]; }); + cmark = pkgs.cmark.overrideAttrs (_: { + src = inputs.cmark; + version = inputs.lastModifiedDate; + }); + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { useMusl = true; enableStatic = true; From 448842ad67315f58b775c2a4fd7597d6fab82048 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 Jan 2024 17:54:58 -0500 Subject: [PATCH 3/4] Systematize `builtins.fetchTree` docs The renaming task is splatting everything together into markdown lists. The docs are temporily not rendered. --- src/libexpr/primops/fetchTree.cc | 131 ---------------------- src/libfetchers/fetchers.cc | 16 +-- src/libfetchers/fetchers.hh | 25 ++++- src/libfetchers/git.cc | 179 +++++++++++++++++++++++++++---- src/libfetchers/github.cc | 60 +++++++++-- src/libfetchers/indirect.cc | 28 ++++- src/libfetchers/mercurial.cc | 38 +++++-- src/libfetchers/path.cc | 33 ++++-- src/libfetchers/tarball.cc | 103 ++++++++++++++++-- src/nix/main.cc | 41 +++++-- 10 files changed, 441 insertions(+), 213 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index d2266e2bc57..4f6e76f343b 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -251,137 +251,6 @@ static RegisterPrimOp primop_fetchTree({ document) if `fetchTree` was a curried call with the first parameter for `type` or an attribute like `builtins.fetchTree.git`! --> - - `"file"` - - Place a plain file into the Nix store. - This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) - - - `url` (String, required) - - Supported protocols: - - - `https` - - > **Example** - > - > ```nix - > fetchTree { - > type = "file"; - > url = "https://example.com/index.html"; - > } - > ``` - - - `http` - - Insecure HTTP transfer for legacy sources. - - > **Warning** - > - > HTTP performs no encryption or authentication. - > Use a `narHash` known in advance to ensure the output has expected contents. - - - `file` - - A file on the local file system. - - > **Example** - > - > ```nix - > fetchTree { - > type = "file"; - > url = "file:///home/eelco/nix/README.md"; - > } - > ``` - - - `"tarball"` - - Download a tar archive and extract it into the Nix store. - This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) - - - `url` (String, required) - - > **Example** - > - > ```nix - > fetchTree { - > type = "tarball"; - > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; - > } - > ``` - - - `"git"` - - Fetch a Git tree and copy it to the Nix store. - This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). - - - `url` (String, required) - - The URL formats supported are the same as for Git itself. - - > **Example** - > - > ```nix - > fetchTree { - > type = "git"; - > url = "git@github.com:NixOS/nixpkgs.git"; - > } - > ``` - - > **Note** - > - > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. - - - `ref` (String, optional) - - By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. - - A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. - - Default: `"HEAD"` - - - `rev` (String, optional) - - A Git revision; a commit hash. - - Default: the tip of `ref` - - - `shallow` (Bool, optional) - - Make a shallow clone when fetching the Git tree. - When this is enabled, the options `ref` and `allRefs` have no effect anymore. - - Default: `true` - - - `submodules` (Bool, optional) - - Also fetch submodules if available. - - Default: `false` - - - `allRefs` (Bool, optional) - - By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. - - Whether to fetch all references (eg. branches and tags) of the repository. - With this argument being true, it's possible to load a `rev` from *any* `ref`. - (Without setting this option, only `rev`s from the specified `ref` are supported). - - Default: `false` - - - `lastModified` (Integer, optional) - - Unix timestamp of the fetched commit. - - If set, pass through the value to the output attribute set. - Otherwise, generated from the fetched Git tree. - - - `revCount` (Integer, optional) - - Number of revisions in the history of the Git repository before the fetched commit. - - If set, pass through the value to the output attribute set. - Otherwise, generated from the fetched Git tree. - The following input types are still subject to change: - `"path"` diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index b07e8cb6ee7..4604202d874 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -8,8 +8,6 @@ namespace nix::fetchers { -using InputSchemeMap = std::map>; - std::unique_ptr inputSchemes = nullptr; void registerInputScheme(std::shared_ptr && inputScheme) @@ -22,17 +20,9 @@ void registerInputScheme(std::shared_ptr && inputScheme) inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); } -nlohmann::json dumpRegisterInputSchemeInfo() { - using nlohmann::json; - - auto res = json::object(); - - for (auto & [name, scheme] : *inputSchemes) { - auto & r = res[name] = json::object(); - r["allowedAttrs"] = scheme->allowedAttrs(); - } - - return res; +const InputSchemeMap & getAllInputSchemes() +{ + return *inputSchemes; } Input Input::fromURL( diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index a5f9bdcc6d5..d0925e7bdf3 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -179,14 +179,26 @@ struct InputScheme */ virtual std::string_view schemeName() const = 0; + /** + * Longform description of this scheme, for documentation purposes. + */ + virtual std::string schemeDescription() const = 0; + + // TODO remove these defaults + struct AttributeInfo { + const char * type = "String"; + bool required = true; + const char * doc = ""; + }; + /** * Allowed attributes in an attribute set that is converted to an - * input. + * input, and documentation for each attribute. * - * `type` is not included from this set, because the `type` field is + * `type` is not included from this map, because the `type` field is parsed first to choose which scheme; `type` is always required. */ - virtual StringSet allowedAttrs() const = 0; + virtual std::map allowedAttrs() const = 0; virtual ParsedURL toURL(const Input & input) const; @@ -244,7 +256,12 @@ struct InputScheme void registerInputScheme(std::shared_ptr && fetcher); -nlohmann::json dumpRegisterInputSchemeInfo(); +using InputSchemeMap = std::map>; + +/** + * Use this for docs, not for finding a specific scheme + */ +const InputSchemeMap & getAllInputSchemes(); struct PublicKey { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 99d91919eb4..71415fb519f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -201,26 +201,169 @@ struct GitInputScheme : InputScheme return "git"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + return stripIndentation(R"( + Fetch a Git tree and copy it to the Nix store. + This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). + )"); + } + + std::map allowedAttrs() const override { return { - "url", - "ref", - "rev", - "shallow", - "submodules", - "exportIgnore", - "lastModified", - "revCount", - "narHash", - "allRefs", - "name", - "dirtyRev", - "dirtyShortRev", - "verifyCommit", - "keytype", - "publicKey", - "publicKeys", + { + "url", + { + .type = "String", + .required = true, + .doc = R"( + The URL formats supported are the same as for Git itself. + + > **Example** + > + > ```nix + > fetchTree { + > type = "git"; + > url = "git@github.com:NixOS/nixpkgs.git"; + > } + > ``` + + > **Note** + > + > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. + )", + }, + }, + { + "ref", + { + .type = "String", + .required = false, + .doc = R"( + By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. + + A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. + + Default: `"HEAD"` + )", + }, + }, + { + "rev", + { + .type = "String", + .required = false, + .doc = R"( + A Git revision; a commit hash. + + Default: the tip of `ref` + )", + }, + }, + { + "shallow", + { + .type = "Bool", + .required = false, + .doc = R"( + Make a shallow clone when fetching the Git tree. + When this is enabled, the options `ref` and `allRefs` have no effect anymore. + + Default: `true` + )", + }, + }, + { + "submodules", + { + .type = "Bool", + .required = false, + .doc = R"( + Also fetch submodules if available. + + Default: `false` + )", + }, + }, + { + "lastModified", + { + .type = "integer", + .required = false, + .doc = R"( + Unix timestamp of the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + )", + }, + }, + { + "revCount", + { + .type = "integer", + .required = false, + .doc = R"( + Number of revisions in the history of the Git repository before the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + )", + }, + }, + { + "narHash", + {}, + }, + { + "allRefs", + { + .type = "Bool", + .required = false, + .doc = R"( + By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. + + Whether to fetch all references (eg. branches and tags) of the repository. + With this argument being true, it's possible to load a `rev` from *any* `ref`. + (Without setting this option, only `rev`s from the specified `ref` are supported). + + Default: `false` + )", + }, + }, + { + "name", + {}, + }, + { + "dirtyRev", + {}, + }, + { + "dirtyShortRev", + {}, + }, + { + "exportIgnore", + {}, + }, + { + "verifyCommit", + {}, + }, + { + "keytype", + {}, + }, + { + "publicKey", + {}, + }, + { + "publicKeys", + {}, + }, }; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 308cff33a46..9146a2eac7e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -107,17 +107,41 @@ struct GitArchiveInputScheme : InputScheme return input; } - StringSet allowedAttrs() const override + std::map allowedAttrs() const override { return { - "owner", - "repo", - "ref", - "rev", - "narHash", - "lastModified", - "host", - "treeHash", + { + "owner", + {}, + }, + { + "repo", + {}, + }, + { + "ref", + {}, + }, + { + "rev", + {}, + }, + { + "narHash", + {}, + }, + { + "lastModified", + {}, + }, + { + "host", + {}, + }, + { + "treeHash", + {}, + }, }; } @@ -330,6 +354,12 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string_view schemeName() const override { return "github"; } + std::string schemeDescription() const override + { + // TODO + return ""; + } + std::optional> accessHeaderFromToken(const std::string & token) const override { // Github supports PAT/OAuth2 tokens and HTTP Basic @@ -413,6 +443,12 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string_view schemeName() const override { return "gitlab"; } + std::string schemeDescription() const override + { + // TODO + return ""; + } + std::optional> accessHeaderFromToken(const std::string & token) const override { // Gitlab supports 4 kinds of authorization, two of which are @@ -488,6 +524,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme { std::string_view schemeName() const override { return "sourcehut"; } + std::string schemeDescription() const override + { + // TODO + return ""; + } + std::optional> accessHeaderFromToken(const std::string & token) const override { // SourceHut supports both PAT and OAuth2. See diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 2e5cd82c78d..5d669f603a0 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -57,13 +57,31 @@ struct IndirectInputScheme : InputScheme return "indirect"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + // TODO + return ""; + } + + std::map allowedAttrs() const override { return { - "id", - "ref", - "rev", - "narHash", + { + "id", + {}, + }, + { + "ref", + {}, + }, + { + "rev", + {}, + }, + { + "narHash", + {}, + }, }; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 2c987f79d50..6e9501f3ef8 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -78,15 +78,39 @@ struct MercurialInputScheme : InputScheme return "hg"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + // TODO + return ""; + } + + std::map allowedAttrs() const override { return { - "url", - "ref", - "rev", - "revCount", - "narHash", - "name", + { + "url", + {}, + }, + { + "ref", + {}, + }, + { + "rev", + {}, + }, + { + "revCount", + {}, + }, + { + "narHash", + {}, + }, + { + "name", + {}, + }, }; } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index fca0df84b10..d9f7f734927 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -40,19 +40,40 @@ struct PathInputScheme : InputScheme return "path"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + // TODO + return ""; + } + + std::map allowedAttrs() const override { return { - "path", + { + "path", + {}, + }, /* Allow the user to pass in "fake" tree info attributes. This is useful for making a pinned tree work the same as the repository from which is exported (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - "rev", - "revCount", - "lastModified", - "narHash", + { + "rev", + {}, + }, + { + "revCount", + {}, + }, + { + "lastModified", + {}, + }, + { + "narHash", + {}, + }, }; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index aa5d61bc5a0..f339940135d 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -277,17 +277,85 @@ struct CurlInputScheme : InputScheme return input; } - StringSet allowedAttrs() const override + std::map allowedAttrs() const override { return { - "type", - "url", - "narHash", - "name", - "unpack", - "rev", - "revCount", - "lastModified", + { + "url", + { + .type = "String", + .required = true, + .doc = R"( + Supported protocols: + + - `https` + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "https://example.com/index.html"; + > } + > ``` + + - `http` + + Insecure HTTP transfer for legacy sources. + + > **Warning** + > + > HTTP performs no encryption or authentication. + > Use a `narHash` known in advance to ensure the output has expected contents. + + - `file` + + A file on the local file system. + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "file:///home/eelco/nix/README.md"; + > } + > ``` + + > **Example** + > + > ```nix + > fetchTree { + > type = "tarball"; + > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; + > } + > ``` + )", + }, + }, + { + "narHash", + {}, + }, + { + "name", + {}, + }, + { + "unpack", + {}, + }, + { + "rev", + {}, + }, + { + "revCount", + {}, + }, + { + "lastModified", + {}, + }, }; } @@ -322,6 +390,14 @@ struct FileInputScheme : CurlInputScheme { std::string_view schemeName() const override { return "file"; } + std::string schemeDescription() const override + { + return stripIndentation(R"( + Place a plain file into the Nix store. + This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) + )"); + } + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); @@ -356,6 +432,15 @@ struct TarballInputScheme : CurlInputScheme { std::string_view schemeName() const override { return "tarball"; } + std::string schemeDescription() const override + { + return stripIndentation(R"( + Download a tar archive and extract it into the Nix store. + This has the same underyling implementation as [`builtins.fetchTarball`](@doc + root@/language/builtins.md#builtins-fetchTarball) + )"); + } + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); diff --git a/src/nix/main.cc b/src/nix/main.cc index eff2d60a473..5403ced12e3 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -217,21 +217,40 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs std::string dumpCli() { - auto res = nlohmann::json::object(); + using nlohmann::json; + + auto res = json::object(); res["args"] = toJSON(); - auto stores = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); - auto & j = stores[storeName]; - j["doc"] = storeConfig->doc(); - j["settings"] = storeConfig->toJSON(); - j["experimentalFeature"] = storeConfig->experimentalFeature(); + { + auto & stores = res["stores"] = json::object(); + for (const auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + auto & j = stores[storeName]; + j["doc"] = storeConfig->doc(); + j["settings"] = storeConfig->toJSON(); + j["experimentalFeature"] = storeConfig->experimentalFeature(); + } } - res["stores"] = std::move(stores); - res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); + + { + auto & fetchers = res["fetchers"] = json::object(); + + for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) { + auto & s = fetchers[schemeName] = json::object(); + s["description"] = scheme->schemeDescription(); + auto & attrs = s["allowedAttrs"] = json::object(); + for (auto & [fieldName, field] : scheme->allowedAttrs()) { + auto & f = attrs[fieldName] = json::object(); + f["type"] = field.type; + f["required"] = field.required; + f["doc"] = stripIndentation(field.doc); + } + } + + }; return res.dump(); } From f66a76134df140f97ff1788a68f1dceb9371798f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Jan 2024 12:23:55 -0500 Subject: [PATCH 4/4] Render fetcher info in primop again --- src/libexpr/eval.cc | 7 +- src/libexpr/eval.hh | 2 +- src/libexpr/local.mk | 2 +- src/libexpr/primops/fetchTree.cc | 225 +++++++++++++++++++------------ src/libutil/cmark-cpp.hh | 85 ++++++++++++ src/libutil/util.hh | 10 ++ src/nix/main.cc | 2 +- 7 files changed, 241 insertions(+), 92 deletions(-) create mode 100644 src/libutil/cmark-cpp.hh diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9eae6078b1d..1489a2ab2a1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -500,7 +500,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) v.mkApp(vPrimOp, vPrimOp); return addConstant(primOp.name, v, { .type = nThunk, // FIXME - .doc = primOp.doc, + .doc = primOp.doc.c_str(), }); } @@ -527,13 +527,14 @@ std::optional EvalState::getDoc(Value & v) { if (v.isPrimOp()) { auto v2 = &v; - if (auto * doc = v2->primOp()->doc) + auto & doc = v2->primOp()->doc; + if (doc != "") return Doc { .pos = {}, .name = v2->primOp()->name, .arity = v2->primOp()->arity, .args = v2->primOp()->args, - .doc = doc, + .doc = doc.c_str(), }; } if (v.isLambda()) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f7ed6be83af..526515278d8 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -88,7 +88,7 @@ struct PrimOp /** * Optional free-form documentation about the primop. */ - const char * doc = nullptr; + const std::string doc = ""; /** * Add a trace item, `while calling the '' builtin` diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 68518e184b5..45a210a8464 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -20,7 +20,7 @@ libexpr_CXXFLAGS += \ libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) +libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) $(CMARK_LIBS) ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 4f6e76f343b..c3ef4d4bd52 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -8,6 +8,7 @@ #include "registry.hh" #include "tarball.hh" #include "url.hh" +#include "cmark-cpp.hh" #include "value-to-json.hh" #include "fetch-to-store.hh" @@ -210,92 +211,144 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, static RegisterPrimOp primop_fetchTree({ .name = "fetchTree", .args = {"input"}, - .doc = R"( - Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with: - - - the resulting fixed-output [store path](@docroot@/store/store-path.md) - - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash - - backend-specific metadata (currently not documented). - - *input* must be an attribute set with the following attributes: - - - `type` (String, required) - - One of the [supported source types](#source-types). - This determines other required and allowed input attributes. - - - `narHash` (String, optional) - - The `narHash` parameter can be used to substitute the source of the tree. - It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. - If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. - - A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. - That is, `fetchTree` is idempotent. - - Downloads are cached in `$XDG_CACHE_HOME/nix`. - The remote source will be fetched from the network if both are true: - - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store - - > **Note** - > - > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. - - - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) - - ## Source types - - The following source types and associated input attributes are supported. - - - - The following input types are still subject to change: - - - `"path"` - - `"github"` - - `"gitlab"` - - `"sourcehut"` - - `"mercurial"` - - *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). - The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. - - > **Example** - > - > Fetch a GitHub repository using the attribute set representation: - > - > ```nix - > builtins.fetchTree { - > type = "github"; - > owner = "NixOS"; - > repo = "nixpkgs"; - > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - > } - > ``` - > - > This evaluates to the following attribute set: - > - > ```nix - > { - > lastModified = 1686503798; - > lastModifiedDate = "20230611171638"; - > narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; - > outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; - > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - > shortRev = "ae2e6b3"; - > } - > ``` - - > **Example** - > - > Fetch the same GitHub repository using the URL-like syntax: - > - > ```nix - > builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" - > ``` - )", + .doc = []() -> std::string { + using namespace cmark; + + // Stores strings referenced by AST. Deallocate after rendering. + std::vector textArena; + + auto root = node_new(CMARK_NODE_DOCUMENT); + + auto & before = textArena.emplace_back(stripIndentation(R"( + Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with: + + - the resulting fixed-output [store path](@docroot@/store/store-path.md) + - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash + - backend-specific metadata (currently not documented). + + *input* must be an attribute set with the following attributes: + + - `type` (String, required) + + One of the [supported source types](#source-types). + This determines other required and allowed input attributes. + + - `narHash` (String, optional) + + The `narHash` parameter can be used to substitute the source of the tree. + It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. + If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. + + A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. + That is, `fetchTree` is idempotent. + + Downloads are cached in `$XDG_CACHE_HOME/nix`. + The remote source will be fetched from the network if both are true: + - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store + + > **Note** + > + > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. + + - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) + + ## Source types + + The following source types and associated input attributes are supported. + + + )")); + parse_document(*root, before, CMARK_OPT_DEFAULT); + + auto & schemes = node_append_child(*root, node_new(CMARK_NODE_LIST)); + + for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) { + auto & s = node_append_child(schemes, node_new(CMARK_NODE_ITEM)); + { + auto & name_p = node_append_child(s, node_new(CMARK_NODE_PARAGRAPH)); + auto & name = node_append_child(name_p, node_new(CMARK_NODE_TEXT)); + node_set_literal(name, schemeName.data()); + } + parse_document(s, scheme->schemeDescription(), CMARK_OPT_DEFAULT); + + auto & attrs = node_append_child(s, node_new(CMARK_NODE_LIST)); + for (const auto & [attrName, attribute] : scheme->allowedAttrs()) { + auto & a = node_append_child(attrs, node_new(CMARK_NODE_ITEM)); + { + auto & name_info = node_append_child(a, node_new(CMARK_NODE_PARAGRAPH)); + { + auto & name = node_append_child(name_info, node_new(CMARK_NODE_CODE)); + auto & name_t = textArena.emplace_back(attrName); + node_set_literal(name, name_t.c_str()); + } + auto & info = node_append_child(name_info, node_new(CMARK_NODE_TEXT)); + auto & header = textArena.emplace_back(std::string { } + + " (" + attribute.type + + ", " + (attribute.required ? "required" : "optional") + + ")"); + node_set_literal(info, header.c_str()); + } + { + auto & doc = textArena.emplace_back(stripIndentation(attribute.doc)); + parse_document(a, doc, CMARK_OPT_DEFAULT); + } + } + } + + auto & after = textArena.emplace_back(stripIndentation(R"( + The following input types are still subject to change: + + - `"path"` + - `"github"` + - `"gitlab"` + - `"sourcehut"` + - `"mercurial"` + + *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. + + > **Example** + > + > Fetch a GitHub repository using the attribute set representation: + > + > ```nix + > builtins.fetchTree { + > type = "github"; + > owner = "NixOS"; + > repo = "nixpkgs"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > } + > ``` + > + > This evaluates to the following attribute set: + > + > ```nix + > { + > lastModified = 1686503798; + > lastModifiedDate = "20230611171638"; + > narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + > outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > shortRev = "ae2e6b3"; + > } + > ``` + + > **Example** + > + > Fetch the same GitHub repository using the URL-like syntax: + > + > ```nix + > builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" + > ``` + )")); + parse_document(*root, after, CMARK_OPT_DEFAULT); + + auto p = render_commonmark(*root, CMARK_OPT_DEFAULT, 0); + assert(p); + return { &*p }; + }(), .fun = prim_fetchTree, .experimentalFeature = Xp::FetchTree, }); diff --git a/src/libutil/cmark-cpp.hh b/src/libutil/cmark-cpp.hh new file mode 100644 index 00000000000..f45fe134b67 --- /dev/null +++ b/src/libutil/cmark-cpp.hh @@ -0,0 +1,85 @@ +#pragma once +///@file + +#include "types.hh" +#include "util.hh" + +#include + +namespace nix::cmark { + +using Node = struct cmark_node; +using NodeType = cmark_node_type; +using ListType = cmark_list_type; + +using Iter = struct cmark_iter; + +struct Deleter +{ + void operator () (Node * ptr) { cmark_node_free(ptr); } + void operator () (Iter * ptr) { cmark_iter_free(ptr); } +}; + +template +using UniquePtr = std::unique_ptr; + +static inline void parse_document(Node & root, std::string_view s, int options) +{ + cmark_parser * parser = cmark_parser_new_with_mem_into_root( + options, + cmark_get_default_mem_allocator(), + &root); + cmark_parser_feed(parser, s.data(), s.size()); + (void) cmark_parser_finish(parser); + cmark_parser_free(parser); +} + +static inline UniquePtr parse_document(std::string_view s, int options) +{ + return UniquePtr { + cmark_parse_document(s.data(), s.size(), options) + }; +} + +static inline std::unique_ptr render_commonmark(Node & root, int options, int width) +{ + return std::unique_ptr { + cmark_render_commonmark(&root, options, width) + }; +} + +static inline std::unique_ptr render_xml(Node & root, int options) +{ + return std::unique_ptr { + cmark_render_xml(&root, options) + }; +} + +static inline UniquePtr node_new(NodeType type) +{ + return UniquePtr { + cmark_node_new(type) + }; +} + +/** + * The parent takes ownership + */ +static inline Node & node_append_child(Node & node, UniquePtr child) +{ + auto status = (bool) cmark_node_append_child(&node, &*child); + assert(status); + return *child.release(); +} + +static inline bool node_set_literal(Node & node, const char * content) +{ + return (bool) cmark_node_set_literal(&node, content); +} + +static inline bool node_set_list_type(Node & node, ListType type) +{ + return (bool) cmark_node_set_list_type(&node, type); +} + +} diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0fb6ff837ec..2ea7f35cf42 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -332,6 +332,16 @@ template overloaded(Ts...) -> overloaded; std::string showBytes(uint64_t bytes); +/** + * For using `std::unique` with C functions. + */ +struct FreeDeleter +{ + template + void operator()(T *p) const { std::free(p); } +}; + + /** * Provide an addition operator between strings and string_views * inexplicably omitted from the standard library. diff --git a/src/nix/main.cc b/src/nix/main.cc index 5403ced12e3..9631923f9bd 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -458,7 +458,7 @@ void mainWrapped(int argc, char * * argv) auto b = nlohmann::json::object(); if (!builtin.value->isPrimOp()) continue; auto primOp = builtin.value->primOp(); - if (!primOp->doc) continue; + if (primOp->doc == "") continue; b["args"] = primOp->args; b["doc"] = trim(stripIndentation(primOp->doc)); if (primOp->experimentalFeature)