build: add nix flake to set up a build environment (PROOF-537) (#38)

* work on shell * update catch2 * bump cuda rules * reformat * work on shell * fix up * work on flake * work on flake * tweak opts * disable compiler-rt * refactor * work on cuda support * work on cuda support * work * work on build support * get build passing * fix drivers * work on flake * clean up * document * undo function_ref change * undo function_ref changes * undo change
spaceandtimelabs · Oct 26, 2023 · 8de812b · 8de812b
1 parent 8f84edd
commit 8de812b
Show file tree

Hide file tree

Showing 10 changed files with 290 additions and 1 deletion.
diff --git a/.bazelrc b/.bazelrc
@@ -2,6 +2,7 @@ build --experimental_cc_implementation_deps
 
 build --copt -fPIC
 build --copt -std=c++20
+build --linkopt=-lm
 
 build:portable_glibc --define SXT_PORTABLE_GLIBC=1
 build:portable_glibc --action_env=BAZEL_LINKLIBS='-l%:libstdc++.a'

diff --git a/bazel/sxt_benchmark.bzl b/bazel/sxt_benchmark.bzl
@@ -89,6 +89,6 @@ def sxt_cc_benchmark(
             "//conditions:default": [],
         }),
         deps = deps,
-        linkopts = linkopts,
+        linkopts = linkopts + ["-lm"],
         visibility = visibility,
     )
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -0,0 +1,23 @@
+{
+  description = "blitzar build environment";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+  };
+
+  outputs = { self, nixpkgs, }:
+    let
+      system = "x86_64-linux";
+      pkgs = import nixpkgs { 
+        inherit system; 
+        config.allowUnfree = true;
+        config.cudaSupport = true;
+      };
+    in
+    {
+      formatter.${system} = pkgs.nixpkgs-fmt;
+      devShells.${system}.default = import ./nix/shell.nix {
+        inherit pkgs;
+      };
+    };
+}
diff --git a/nix/bazel.nix b/nix/bazel.nix
@@ -0,0 +1,34 @@
+{ pkgs, clang, clangpp }:
+let
+  # Set PATH so that it only includes things bazel needs.
+  path = pkgs.lib.strings.concatStringsSep ":" [
+    "${clang}/bin"
+    "${clangpp}/bin"
+    "${pkgs.gcc13}/bin"
+    "${pkgs.gcc13.libc.bin}/bin"
+    "${pkgs.binutils}/bin"
+    "${pkgs.coreutils}/bin"
+    "${pkgs.findutils}/bin"
+    "${pkgs.gnused}/bin"
+    "${pkgs.gnugrep}/bin"
+    "${pkgs.bash}/bin"
+  ];
+  bazel = "${pkgs.bazel_6}/bin/bazel";
+in
+pkgs.writeShellScriptBin "bazel" ''
+  if [[
+    "$1" == "build" ||
+    "$1" == "test" ||
+    "$1" == "run"
+  ]]; then
+    exec ${bazel} $1 \
+     --action_env CC=${clang} \
+     --action_env CXX=${clangpp} \
+     --action_env PATH="${path}" \
+     --action_env=BAZEL_LINKLIBS='-l%:libc++.a' \
+     --action_env=BAZEL_LINKOPTS=-static-libstdc++ \
+     ''${@:2}
+  else
+    exec ${bazel} $@
+  fi''
+
diff --git a/nix/clang-wrapper.nix b/nix/clang-wrapper.nix
@@ -0,0 +1,15 @@
+{ pkgs, clang, name }:
+# TODO(rnburn): Modify the build so that the driver will add these arguments automatically
+# and then we can drop this wrapper.
+pkgs.writers.writePython3Bin "${name}"
+{
+  flakeIgnore = [ "E501" ];
+} ''
+  import sys
+  import os
+  args_p = ['-stdlib=libc++'] + sys.argv[1:]
+  args_p = ['-isystem', '${clang}/include/x86_64-unknown-linux-gnu/c++/v1'] + args_p
+  args_p = ['-L', '${clang}/lib/x86_64-unknown-linux-gnu'] + args_p
+  compiler = '${clang}/bin/${name}'
+  os.execv(compiler, [compiler] + args_p)
+''
diff --git a/nix/clang.nix b/nix/clang.nix
@@ -0,0 +1,73 @@
+{ pkgs }:
+with pkgs;
+let
+  gccForLibs = gcc13.cc;
+in
+stdenvNoCC.mkDerivation {
+  name = "clang";
+  src = pkgs.fetchgit {
+    url = "https://github.com/llvm/llvm-project";
+    rev = "6768a3d";
+    hash = "sha256-xxw+v1VTT9W4WFGRVL3BGZpc1raeSV0vch5YksHzKck=";
+  };
+  nativeBuildInputs = [
+    cmake
+    perl
+    ninja
+    python3
+    git
+  ];
+  buildInputs = [
+    gcc13
+  ];
+  NIX_LDFLAGS = "-L${gccForLibs}/lib/gcc/${targetPlatform.config}/${gccForLibs.version} -L${gcc13.libc}/lib";
+  CFLAGS = "-B${gccForLibs}/lib/gcc/${targetPlatform.config}/${gccForLibs.version} -B${gcc13.libc}/lib";
+  patches = [
+    ./clang_driver.patch
+  ];
+  postPatch = ''
+    substituteInPlace clang/lib/Driver/ToolChains/Gnu.cpp \
+      --replace 'GLIBC_PATH_ABC123' '${gcc13.libc}/lib'
+  '';
+  configurePhase = pkgs.lib.strings.concatStringsSep " " [
+    "mkdir build; cd build;"
+    "cmake"
+    "-G \"Unix Makefiles\""
+    "-DGCC_INSTALL_PREFIX=${gccForLibs}"
+    "-DC_INCLUDE_DIRS=${gcc13.libc.dev}/include"
+    "-DLLVM_TARGETS_TO_BUILD=\"host;NVPTX\""
+    "-DLLVM_BUILTIN_TARGETS=\"x86_64-unknown-linux-gnu\""
+    "-DLLVM_RUNTIME_TARGETS=\"x86_64-unknown-linux-gnu\""
+    "-DLLVM_ENABLE_PROJECTS=\"clang;clang-tools-extra\""
+
+    # TODO(rnburn): build with compiler-rt so that we have access to
+    # sanitizers after this issue gets resolved: https://github.com/llvm/llvm-project/issues/69056#issuecomment-1781423887.
+    #"-DLLVM_ENABLE_RUNTIMES=\"libcxx;libcxxabi;libunwind;compiler-rt\""
+    "-DLLVM_ENABLE_RUNTIMES=\"libcxx;libcxxabi;libunwind\""
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_CMAKE_BUILD_TYPE=Release"
+
+    # libcxx
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_LIBCXX_ENABLE_SHARED=ON"
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_LIBCXX_ENABLE_STATIC=ON"
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_LIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON"
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_LIBCXX_STATICALLY_LINK_ABI_IN_STATIC_LIBRARY=ON"
+
+    # libcxxabi
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_LIBCXXABI_USE_LLVM_UNWINDER=ON"
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_LIBCXXABI_ENABLE_STATIC=ON"
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_LIBCXXABI_ENABLE_STATIC_UNWINDER=ON"
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_LIBCXXABI_STATICALLY_LINK_UNWINDER_IN_STATIC_LIBRARY=ON"
+
+    # libunwind
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_LIBUNWIND_ENABLE_STATIC=ON"
+
+    # compiler-rt
+    "-DRUNTIMES_x86_64-unknown-linux-gnu_COMPILER_RT_USE_LLVM_UNWINDER=ON"
+
+    "-DCMAKE_BUILD_TYPE=Release"
+    "-DCMAKE_INSTALL_PREFIX=\"$out\""
+    "../llvm"
+  ];
+  buildPhase = "make";
+  installPhase = "make install";
+}
diff --git a/nix/clang_driver.patch b/nix/clang_driver.patch
@@ -0,0 +1,58 @@
+diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
+index 7b2966f70bf6..e31145124c2d 100644
+--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
++++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
+@@ -1802,14 +1802,7 @@ enum class LibGccType { UnspecifiedLibGcc, StaticLibGcc, SharedLibGcc };
+
+ static LibGccType getLibGccType(const ToolChain &TC, const Driver &D,
+                                 const ArgList &Args) {
+-  if (Args.hasArg(options::OPT_static_libgcc) ||
+-      Args.hasArg(options::OPT_static) || Args.hasArg(options::OPT_static_pie) ||
+-      // The Android NDK only provides libunwind.a, not libunwind.so.
+-      TC.getTriple().isAndroid())
+-    return LibGccType::StaticLibGcc;
+-  if (Args.hasArg(options::OPT_shared_libgcc))
+-    return LibGccType::SharedLibGcc;
+-  return LibGccType::UnspecifiedLibGcc;
++  return LibGccType::StaticLibGcc;
+ }
+
+ // Gcc adds libgcc arguments in various ways:
+diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
+index cdd911af9a73..d629ff8abb27 100644
+--- a/clang/lib/Driver/ToolChains/Gnu.cpp
++++ b/clang/lib/Driver/ToolChains/Gnu.cpp
+@@ -467,8 +467,7 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
+     if (!Args.hasArg(options::OPT_shared) && !IsStaticPIE &&
+         !Args.hasArg(options::OPT_r)) {
+       CmdArgs.push_back("-dynamic-linker");
+-      CmdArgs.push_back(Args.MakeArgString(Twine(D.DyldPrefix) +
+-                                           ToolChain.getDynamicLinker(Args)));
++      CmdArgs.push_back(Args.MakeArgString(Twine("GLIBC_PATH_ABC123/ld-linux-x86-64.so.2")));
+     }
+   }
+
+@@ -490,10 +489,12 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
+           crt1 = "crt1.o";
+       }
+       if (crt1)
+-        CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crt1)));
++        CmdArgs.push_back(Args.MakeArgString(Twine("GLIBC_PATH_ABC123/") + crt1));
+
+-      CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crti.o")));
++      CmdArgs.push_back(Args.MakeArgString(Twine("GLIBC_PATH_ABC123/") + "crti.o"));
+     }
++    CmdArgs.push_back("-L");
++    CmdArgs.push_back("GLIBC_PATH_ABC123");
+
+     if (IsVE) {
+       CmdArgs.push_back("-z");
+@@ -671,7 +672,7 @@ void tools::gnutools::Linker::ConstructJob(Compilation &C, const JobAction &JA,
+         CmdArgs.push_back(Args.MakeArgString(P));
+       }
+       if (!isAndroid)
+-        CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtn.o")));
++        CmdArgs.push_back(Args.MakeArgString(Twine("GLIBC_PATH_ABC123/") + "crtn.o"));
+     }
+   }
+
diff --git a/nix/cuda.nix b/nix/cuda.nix
@@ -0,0 +1,33 @@
+{ pkgs }:
+with pkgs;
+let
+  toolkit = cudaPackages_12_2.cudatoolkit;
+  toolkit-lib = cudaPackages_12_2.cudatoolkit.lib;
+in
+pkgs.stdenvNoCC.mkDerivation {
+  name = "cudaWrapped";
+
+  buildInputs = [
+    toolkit
+    toolkit-lib
+  ];
+
+  unpackPhase = "true";
+  buildPhase = "";
+
+  installPhase = ''
+    mkdir $out
+    for f in `ls -1 ${toolkit}`
+    do
+      if [[ $f != "lib64" && $f != "lib" ]]; then
+        ln -s ${toolkit}/$f $out/$f
+      fi
+    done
+    mkdir $out/lib64
+    for f in `ls -1 ${toolkit}/lib`
+    do
+      ln -s ${toolkit}/lib/$f $out/lib64/$f
+    done
+    ln -s ${toolkit-lib}/lib/libcudart_static.a $out/lib64/libcudart_static.a
+  '';
+}
diff --git a/nix/shell.nix b/nix/shell.nix
@@ -0,0 +1,25 @@
+{ pkgs }:
+let
+  clang = import ./clang.nix { inherit pkgs; };
+  wrap-clang = import ./clang-wrapper.nix { inherit pkgs; inherit clang; name = "clang"; };
+  wrap-clangpp = import ./clang-wrapper.nix { inherit pkgs; inherit clang; name = "clang++"; };
+  bazel = import ./bazel.nix { inherit pkgs; clang = wrap-clang; clangpp = wrap-clangpp; };
+  cuda = import ./cuda.nix { inherit pkgs; };
+in
+with pkgs;
+mkShell {
+  buildInputs = [
+    pkgs.python3
+    bazel
+    wrap-clang
+    wrap-clangpp
+    clang
+    cuda
+  ];
+  LD_LIBRARY_PATH = lib.makeLibraryPath [
+    "/usr/lib/wsl"
+    linuxPackages.nvidia_x11
+  ];
+  shellHook = ''
+  '';
+}