Skip to content

Commit

Permalink
Merge pull request #10674 from nix-windows/local-store-on-windows
Browse files Browse the repository at this point in the history 
Build the local store on Windows
  • Loading branch information
@Ericson2314
Ericson2314 authored May 10, 2024
2 parents cb7224a + e0ff8da commit 87ab3c0
Show file tree
Hide file tree
Showing 20 changed files with 230 additions and 159 deletions.
12 changes: 6 additions & 6 deletions maintainers/flake-module.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,17 +241,17 @@
''^src/libstore/unix/build/worker\.hh$''
''^src/libstore/unix/builtins/fetchurl\.cc$''
''^src/libstore/unix/builtins/unpack-channel\.cc$''
''^src/libstore/unix/gc\.cc$''
''^src/libstore/gc\.cc$''
''^src/libstore/unix/local-overlay-store\.cc$''
''^src/libstore/unix/local-overlay-store\.hh$''
''^src/libstore/unix/local-store\.cc$''
''^src/libstore/unix/local-store\.hh$''
''^src/libstore/local-store\.cc$''
''^src/libstore/local-store\.hh$''
''^src/libstore/unix/lock\.cc$''
''^src/libstore/unix/lock\.hh$''
''^src/libstore/unix/optimise-store\.cc$''
''^src/libstore/optimise-store\.cc$''
''^src/libstore/unix/pathlocks\.cc$''
''^src/libstore/unix/posix-fs-canonicalise\.cc$''
''^src/libstore/unix/posix-fs-canonicalise\.hh$''
''^src/libstore/posix-fs-canonicalise\.cc$''
''^src/libstore/posix-fs-canonicalise\.hh$''
''^src/libstore/uds-remote-store\.cc$''
''^src/libstore/uds-remote-store\.hh$''
''^src/libstore/windows/build\.cc$''
Expand Down
2 changes: 1 addition & 1 deletion src/libfetchers/cache.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ struct CacheImpl : Cache
Key key,
Store & store,
Attrs value,
const StorePath & storePath)
const StorePath & storePath) override
{
/* Add the store prefix to the cache key to handle multiple
store prefixes. */
Expand Down
0 src/libstore/unix/ca-specific-schema.sql → src/libstore/ca-specific-schema.sql
View file
File renamed without changes.
88 changes: 54 additions & 34 deletions src/libstore/unix/gc.cc → src/libstore/gc.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

#if !defined(__linux__)
// For shelling out to lsof
# include "processes.hh"
# include "processes.hh"
#endif

#include <functional>
Expand All @@ -19,18 +19,20 @@
#include <climits>
#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#if HAVE_STATVFS
# include <sys/statvfs.h>
#endif
#ifndef _WIN32
# include <poll.h>
# include <sys/socket.h>
# include <sys/un.h>
#endif
#include <sys/types.h>
#include <sys/un.h>
#include <unistd.h>

namespace nix {

using namespace nix::unix;

static std::string gcSocketPath = "/gc-socket/socket";
static std::string gcRootsDir = "gcroots";

Expand Down Expand Up @@ -64,7 +66,7 @@ void LocalStore::createTempRootsFile()
/* Check whether the garbage collector didn't get in our
way. */
struct stat st;
if (fstat(fdTempRoots->get(), &st) == -1)
if (fstat(fromDescriptorReadOnly(fdTempRoots->get()), &st) == -1)
throw SysError("statting '%1%'", fnTempRoots);
if (st.st_size == 0) break;

Expand Down Expand Up @@ -108,7 +110,7 @@ void LocalStore::addTempRoot(const StorePath & path)
debug("connecting to '%s'", socketPath);
*fdRootsSocket = createUnixDomainSocket();
try {
nix::connect(fdRootsSocket->get(), socketPath);
nix::connect(toSocket(fdRootsSocket->get()), socketPath);
} catch (SysError & e) {
/* The garbage collector may have exited or not
created the socket yet, so we need to restart. */
Expand Down Expand Up @@ -166,12 +168,16 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
// those to keep the directory alive.
continue;
}
Path path = i.path();
Path path = i.path().string();

pid_t pid = std::stoi(name);

debug("reading temporary root file '%1%'", path);
AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666));
AutoCloseFD fd(toDescriptor(open(path.c_str(),
#ifndef _WIN32
O_CLOEXEC |
#endif
O_RDWR, 0666)));
if (!fd) {
/* It's okay if the file has disappeared. */
if (errno == ENOENT) continue;
Expand Down Expand Up @@ -240,8 +246,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R
unlink(path.c_str());
}
} else {
struct stat st2 = lstat(target);
if (!S_ISLNK(st2.st_mode)) return;
if (!std::filesystem::is_symlink(target)) return;
Path target2 = readLink(target);
if (isInStore(target2)) foundRoot(target, target2);
}
Expand Down Expand Up @@ -297,24 +302,25 @@ Roots LocalStore::findRoots(bool censor)
return roots;
}

typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots;
/**
* Key is a mere string because cannot has path with macOS's libc++
*/
typedef std::unordered_map<std::string, std::unordered_set<std::string>> UncheckedRoots;

static void readProcLink(const std::string & file, UncheckedRoots & roots)
static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots)
{
constexpr auto bufsiz = PATH_MAX;
char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH)
std::filesystem::path buf;
try {
buf = std::filesystem::read_symlink(file);
} catch (std::filesystem::filesystem_error & e) {
if (e.code() == std::errc::no_such_file_or_directory
|| e.code() == std::errc::permission_denied
|| e.code() == std::errc::no_such_process)
return;
throw SysError("reading symlink");
throw;
}
if (res == bufsiz) {
throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz));
}
if (res > 0 && buf[0] == '/')
roots[std::string(static_cast<char *>(buf), res)]
.emplace(file);
if (buf.is_absolute())
roots[buf.string()].emplace(file.string());
}

static std::string quoteRegexChars(const std::string & raw)
Expand Down Expand Up @@ -371,12 +377,12 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
}
fdDir.reset();

auto mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n");
std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name);
auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile.string()), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex))
unchecked[match[1]].emplace(mapFile);
unchecked[match[1]].emplace(mapFile.string());
}

auto envFile = fmt("/proc/%s/environ", ent->d_name);
Expand Down Expand Up @@ -407,7 +413,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
for (const auto & line : lsofLines) {
std::smatch match;
if (std::regex_match(line, match, lsofRegex))
unchecked[match[1]].emplace("{lsof}");
unchecked[match[1].str()].emplace("{lsof}");
}
} catch (ExecError & e) {
/* lsof not installed, lsof failed */
Expand Down Expand Up @@ -490,6 +496,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
createDirs(dirOf(socketPath));
auto fdServer = createUnixDomainSocket(socketPath, 0666);

// TODO nonblocking socket on windows?
#ifdef _WIN32
throw UnimplementedError("External GC client not implemented yet");
#else
if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1)
throw SysError("making socket '%1%' non-blocking", socketPath);

Expand Down Expand Up @@ -590,6 +600,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
if (serverThread.joinable()) serverThread.join();
});

#endif

/* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */
printInfo("finding garbage collector roots...");
Expand Down Expand Up @@ -623,8 +635,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
by another process. We need to be sure that we can acquire an
exclusive lock before deleting them. */
if (baseName.find("tmp-", 0) == 0) {
AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY);
if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) {
AutoCloseFD tmpDirFd = openDirectory(realPath);
if (!tmpDirFd || !lockFile(tmpDirFd.get(), ltWrite, false)) {
debug("skipping locked tempdir '%s'", realPath);
return;
}
Expand Down Expand Up @@ -857,7 +869,13 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
int64_t overhead = st.st_blocks * 512ULL;
int64_t overhead =
#ifdef _WIN32
0
#else
st.st_blocks * 512ULL
#endif
;

printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
Expand All @@ -870,6 +888,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)

void LocalStore::autoGC(bool sync)
{
#ifdef HAVE_STATVFS
static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE");

auto getAvail = [this]() -> uint64_t {
Expand Down Expand Up @@ -946,6 +965,7 @@ void LocalStore::autoGC(bool sync)
sync:
// Wait for the future outside of the state lock.
if (sync) future.get();
#endif
}


Expand Down
2 changes: 2 additions & 0 deletions src/libstore/globals.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,8 +424,10 @@ public:
Setting<bool> useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal",
"Whether SQLite should use WAL mode."};

#ifndef _WIN32
Setting<bool> syncBeforeRegistering{this, false, "sync-before-registering",
"Whether to call `sync()` before registering a path as valid."};
#endif

Setting<bool> useSubstitutes{
this, true, "substitute",
Expand Down
Loading

0 comments on commit 87ab3c0

Please sign in to comment.