Merge pull request NixOS#10412 from roberth/c-string-context

C API: Add `nix_string_realise`

src/libexpr-c/nix_api_expr_internal.h

@@ -35,4 +35,10 @@ struct nix_string_context
     nix::NixStringContext & ctx;
 };
 
+struct nix_realised_string
+{
+    std::string str;
+    std::vector<StorePath> storePaths;
+};
+
 #endif // NIX_API_EXPR_INTERNAL_H

src/libexpr-c/nix_api_value.cc

@@ -2,14 +2,17 @@
 #include "config.hh"
 #include "eval.hh"
 #include "globals.hh"
+#include "path.hh"
 #include "primops.hh"
 #include "value.hh"
 
 #include "nix_api_expr.h"
 #include "nix_api_expr_internal.h"
 #include "nix_api_util.h"
 #include "nix_api_util_internal.h"
+#include "nix_api_store_internal.h"
 #include "nix_api_value.h"
+#include "value/context.hh"
 
 #ifdef HAVE_BOEHMGC
 # include "gc/gc.h"
@@ -528,3 +531,55 @@ void nix_bindings_builder_free(BindingsBuilder * bb)
     delete (nix::BindingsBuilder *) bb;
 #endif
 }
+
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto &v = check_value_not_null(value);
+        nix::NixStringContext stringContext;
+        auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned();
+        nix::StorePathSet storePaths;
+        auto rewrites = state->state.realiseContext(stringContext, &storePaths);
+
+        auto s = nix::rewriteStrings(rawStr, rewrites);
+
+        // Convert to the C API StorePath type and convert to vector for index-based access
+        std::vector<StorePath> vec;
+        for (auto &sp : storePaths) {
+            vec.push_back(StorePath{sp});
+        }
+
+        return new nix_realised_string {
+            .str = s,
+            .storePaths = vec
+        };
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_realised_string_free(nix_realised_string * s)
+{
+    delete s;
+}
+
+size_t nix_realised_string_get_buffer_size(nix_realised_string * s)
+{
+    return s->str.size();
+}
+
+const char * nix_realised_string_get_buffer_start(nix_realised_string * s)
+{
+    return s->str.data();
+}
+
+size_t nix_realised_string_get_store_path_count(nix_realised_string * s)
+{
+    return s->storePaths.size();
+}
+
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, size_t i)
+{
+    return &s->storePaths[i];
+}

src/libexpr-c/nix_api_value.h

@@ -9,6 +9,7 @@
  */
 
 #include "nix_api_util.h"
+#include "nix_api_store.h"
 #include "stdbool.h"
 #include "stddef.h"
 #include "stdint.h"
@@ -69,6 +70,10 @@ typedef struct PrimOp PrimOp;
  */
 typedef struct ExternalValue ExternalValue;
 
+/** @brief String without placeholders, and realised store paths
+ */
+typedef struct nix_realised_string nix_realised_string;
+
 /** @defgroup primops
  * @brief Create your own primops
  * @{
@@ -167,7 +172,10 @@ const char * nix_get_typename(nix_c_context * context, const Value * value);
  */
 bool nix_get_bool(nix_c_context * context, const Value * value);
 
-/** @brief Get string
+/** @brief Get the raw string
+ *
+ * This may contain placeholders.
+ *
  * @param[out] context Optional, stores error information
  * @param[in] value Nix value to inspect
  * @return string
@@ -425,6 +433,56 @@ nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder,
 void nix_bindings_builder_free(BindingsBuilder * builder);
 /**@}*/
 
+/** @brief Realise a string context.
+ *
+ * This will
+ *  - realise the store paths referenced by the string's context, and
+ *  - perform the replacement of placeholders.
+ *  - create temporary garbage collection roots for the store paths, for
+ *    the lifetime of the current process.
+ *  - log to stderr
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value, which must be a string
+ * @param[in] state Nix evaluator state
+ * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false.
+                    You should set this to true when this call is part of a primop.
+                    You should set this to false when building for your application's purpose.
+ * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free
+ */
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD);
+
+/** @brief Start of the string
+ * @param[in] realised_string
+ * @return pointer to the start of the string. It may not be null-terminated.
+ */
+const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string);
+
+/** @brief Length of the string
+ * @param[in] realised_string
+ * @return length of the string in bytes
+ */
+size_t nix_realised_string_get_buffer_size(nix_realised_string * realised_string);
+
+/** @brief Number of realised store paths
+ * @param[in] realised_string
+ * @return number of realised store paths that were referenced by the string via its context
+ */
+size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_string);
+
+/** @brief Get a store path. The store paths are stored in an arbitrary order.
+ * @param[in] realised_string
+ * @param[in] index index of the store path, must be less than the count
+ * @return store path
+ */
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index);
+
+/** @brief Free a realised string
+ * @param[in] realised_string
+ */
+void nix_realised_string_free(nix_realised_string * realised_string);
+
+
 // cffi end
 #ifdef __cplusplus
 }

src/libexpr/eval.hh

@@ -733,10 +733,12 @@ public:
     bool fullGC();
 
     /**
-     * Realise the given context, and return a mapping from the placeholders
-     * used to construct the associated value to their final store path
+     * Realise the given context
+     * @param[in] context the context to realise
+     * @param[out] maybePaths if not nullptr, all built or referenced store paths will be added to this set
+     * @return a mapping from the placeholders used to construct the associated value to their final store path.
      */
-    [[nodiscard]] StringMap realiseContext(const NixStringContext & context);
+    [[nodiscard]] StringMap realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true);
 
     /* Call the binary path filter predicate used builtins.path etc. */
     bool callPathFilter(

src/libexpr/primops.cc

@@ -39,7 +39,7 @@ namespace nix {
  * Miscellaneous
  *************************************************************/
 
-StringMap EvalState::realiseContext(const NixStringContext & context)
+StringMap EvalState::realiseContext(const NixStringContext & context, StorePathSet * maybePathsOut, bool isIFD)
 {
     std::vector<DerivedPath::Built> drvs;
     StringMap res;
@@ -59,21 +59,23 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
             },
             [&](const NixStringContextElem::Opaque & o) {
                 auto ctxS = store->printStorePath(o.path);
-                res.insert_or_assign(ctxS, ctxS);
                 ensureValid(o.path);
+                if (maybePathsOut)
+                    maybePathsOut->emplace(o.path);
             },
             [&](const NixStringContextElem::DrvDeep & d) {
                 /* Treat same as Opaque */
                 auto ctxS = store->printStorePath(d.drvPath);
-                res.insert_or_assign(ctxS, ctxS);
                 ensureValid(d.drvPath);
+                if (maybePathsOut)
+                    maybePathsOut->emplace(d.drvPath);
             },
         }, c.raw);
     }
 
     if (drvs.empty()) return {};
 
-    if (!evalSettings.enableImportFromDerivation)
+    if (isIFD && !evalSettings.enableImportFromDerivation)
         error<EvalError>(
             "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
             drvs.begin()->to_string(*store)
@@ -90,6 +92,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
         auto outputs = resolveDerivedPath(*buildStore, drv, &*store);
         for (auto & [outputName, outputPath] : outputs) {
             outputsToCopyAndAllow.insert(outputPath);
+            if (maybePathsOut)
+                maybePathsOut->emplace(outputPath);
 
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
@@ -106,10 +110,13 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     }
 
     if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow);
-    for (auto & outputPath : outputsToCopyAndAllow) {
-        /* Add the output of this derivations to the allowed
-           paths. */
-        allowPath(outputPath);
+
+    if (isIFD) {
+        for (auto & outputPath : outputsToCopyAndAllow) {
+            /* Add the output of this derivations to the allowed
+            paths. */
+            allowPath(outputPath);
+        }
     }
 
     return res;

src/libstore-c/nix_api_store.cc

@@ -128,7 +128,19 @@ nix_err nix_store_realise(
     NIXC_CATCH_ERRS
 }
 
+void nix_store_path_name(const StorePath *store_path, void * callback, void * user_data)
+{
+    std::string_view name = store_path->path.name();
+    ((nix_get_string_callback) callback)(name.data(), name.size(), user_data);
+}
+
+
 void nix_store_path_free(StorePath * sp)
 {
     delete sp;
 }
+
+StorePath * nix_store_path_clone(const StorePath * p)
+{
+    return new StorePath{p->path};
+}

src/libstore-c/nix_api_store.h

@@ -90,6 +90,23 @@ nix_err nix_store_get_uri(nix_c_context * context, Store * store, void * callbac
  */
 StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path);
 
+/**
+ * @brief Get the path name (e.g. "name" in /nix/store/...-name)
+ *
+ * @param[in] store_path the path to get the name from
+ * @param[in] callback called with the name
+ * @param[in] user_data arbitrary data, passed to the callback when it's called.
+ */
+void nix_store_path_name(const StorePath *store_path, void * callback, void * user_data);
+
+/**
+ * @brief Copy a StorePath
+ *
+ * @param[in] p the path to copy
+ * @return a new StorePath
+ */
+StorePath * nix_store_path_clone(const StorePath * p);
+
 /** @brief Deallocate a StorePath
  *
  * Does not fail.
@@ -111,7 +128,9 @@ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath *
 /**
  * @brief Realise a Nix store path
  *
- * Blocking, calls callback once for each realised output
+ * Blocking, calls callback once for each realised output.
+ *
+ * @note When working with expressions, consider using e.g. nix_string_realise to get the output. `.drvPath` may not be accurate or available in the future. See https://github.com/NixOS/nix/issues/6507
  *
  * @param[out] context Optional, stores error information
  * @param[in] store Nix Store reference

src/libstore/build/derivation-goal.cc

@@ -1138,7 +1138,7 @@ void DerivationGoal::resolvedFinished()
 
 HookReply DerivationGoal::tryBuildHook()
 {
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
+    if (settings.buildHook.get().empty() || !worker.tryBuildHook || !useDerivation) return rpDecline;
 
     if (!worker.hook)
         worker.hook = std::make_unique<HookInstance>();

tests/unit/libexpr/local.mk

@@ -34,7 +34,7 @@ libexpr-tests_EXTRA_INCLUDES = \
 libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES)
 
 libexpr-tests_LIBS = \
-    libexpr-test-support libstore-test-support libutils-test-support \
+    libexpr-test-support libstore-test-support libutil-test-support \
     libexpr libexprc libfetchers libstore libstorec libutil libutilc
 
 libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock

tests/unit/libexpr/main.cc

@@ -0,0 +1,39 @@
+#include <gtest/gtest.h>
+#include <cstdlib>
+#include "globals.hh"
+#include "logging.hh"
+
+using namespace nix;
+
+int main (int argc, char **argv) {
+    if (argc > 1 && std::string_view(argv[1]) == "__build-remote") {
+        printError("test-build-remote: not supported in libexpr unit tests");
+        return 1;
+    }
+
+    // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook.
+    settings.buildHook = {};
+
+    #if __linux__ // should match the conditional around sandboxBuildDir declaration.
+
+    // When building and testing nix within the host's Nix sandbox, our store dir will be located in the host's sandboxBuildDir, e.g.:
+    // Host
+    //   storeDir = /nix/store
+    //   sandboxBuildDir = /build
+    // This process
+    //   storeDir = /build/foo/bar/store
+    //   sandboxBuildDir = /build
+    // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different sandboxBuildDir.
+    settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir";
+    #endif
+
+    #if __APPLE__
+    // Avoid this error, when already running in a sandbox:
+    // sandbox-exec: sandbox_apply: Operation not permitted
+    settings.sandboxMode = smDisabled;
+    setEnv("_NIX_TEST_NO_SANDBOX", "1");
+    #endif
+
+    ::testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}