diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 16dbda0..2378d23 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,16 +31,16 @@ jobs: strategy: fail-fast: false matrix: + os: [ubuntu-latest, macos-latest] # end-user packages to ci build for cache pkg: [aslp, bap-aslp, bap-primus, basil, retdec-uq-pac, asl-translator, gtirb-semantics, ddisasm, retdec5, remill, alive2-aslp, alive2-regehr, - planter, aslp_web, godbolt ] - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} defaults: run: { shell: "bash -ex -o pipefail {0}" } @@ -75,7 +75,7 @@ jobs: echo NIXPKGS_ALLOW_BROKEN="$("$broken" && echo 1 || echo 0)" > "$GITHUB_ENV" id: broken - - run: ./nix-cached.sh build --impure .#${{matrix.pkg}} + - run: ./nix-cached.sh build --impure -L .#${{matrix.pkg}} id: build continue-on-error: ${{ fromJSON(steps.broken.outputs.bool) }} @@ -84,7 +84,7 @@ jobs: run: | for test in $(nix eval --impure .#${{matrix.pkg}}.tests --json | jq -r keys[]); do t=.#${{matrix.pkg}}.tests.$test - ./nix-cached.sh build --impure "$t" --no-link + ./nix-cached.sh build --impure -L "$t" --no-link done continue-on-error: ${{ fromJSON(steps.broken.outputs.bool) }} if: steps.build.outcome == 'success' diff --git a/aslp/aslp-cpp.nix b/aslp/aslp-cpp.nix index 3b8fe6a..f667360 100644 --- a/aslp/aslp-cpp.nix +++ b/aslp/aslp-cpp.nix @@ -1,9 +1,12 @@ { stdenv +, clang17Stdenv , cmake , asli }: -stdenv.mkDerivation { +let stdenv' = if stdenv.isDarwin then clang17Stdenv else stdenv; + +in stdenv'.mkDerivation { pname = "aslp-cpp"; version = asli.version; diff --git a/aslp/overlay.nix b/aslp/overlay.nix index c871bed..01a7d11 100644 --- a/aslp/overlay.nix +++ b/aslp/overlay.nix @@ -7,7 +7,6 @@ final: prev: inherit (final.ocamlPackages_pac_5) aslp_web; overlay_ocamlPackages = ofinal: oprev: { - asli = ofinal.callPackage ./asli.nix { inherit (final) z3; ocaml_z3 = ofinal.z3; }; aslp = ofinal.asli; # .overrideAttrs { src = prev.lib.cleanSource ~/progs/aslp; } @@ -47,6 +46,8 @@ final: prev: } oprev.ocaml_pcre; + + mirage-crypto-rng = oprev.mirage-crypto-rng.overrideAttrs { doCheck = false; }; }; } diff --git a/bap/bap.nix b/bap/bap.nix new file mode 100644 index 0000000..6589fc0 --- /dev/null +++ b/bap/bap.nix @@ -0,0 +1,13 @@ +{ makeBinaryWrapper, symlinkJoin, orig-bap, z3 }: + +# XXX: fix in ocamlPackages.bap of upstream nixpkgs + +symlinkJoin { + name = "bap-wrapped"; + inherit (orig-bap) version; + paths = [ orig-bap ]; + nativeBuildInputs = [ makeBinaryWrapper ]; + postBuild = '' + wrapProgram $out/bin/bap --prefix DYLD_FALLBACK_LIBRARY_PATH : ${z3.lib}/lib + ''; +} diff --git a/bap/overlay.nix b/bap/overlay.nix index dc527b7..cc554d8 100644 --- a/bap/overlay.nix +++ b/bap/overlay.nix @@ -1,7 +1,8 @@ final: prev: { overlay_ocamlPackages = ofinal: oprev: { - bap = oprev.bap.overrideAttrs (p: { + bapWrapper = ofinal.callPackage ./bap.nix { orig-bap = oprev.bap ; z3 = prev.z3 ; }; + bap = ofinal.bapWrapper.overrideAttrs (p: { # configurePhase = '' # runHook preConfigure # echo old "$configureFlags" diff --git a/basil/0001-basil-protoc-version.patch b/basil/0001-basil-protoc-version.patch new file mode 100644 index 0000000..67cc3f8 --- /dev/null +++ b/basil/0001-basil-protoc-version.patch @@ -0,0 +1,13 @@ +diff --git a/build.sbt b/build.sbt +index f1e6339f..2d9ede97 100644 +--- a/build.sbt ++++ b/build.sbt +@@ -31,6 +31,8 @@ lazy val root = project + + scalacOptions ++= Seq("-deprecation", "-feature") + ++PB.protocExecutable := file("PROTOC_PLACEHOLDER") ++ + Compile / PB.targets := Seq( + scalapb.gen() -> (Compile / sourceManaged).value / "scalapb" + ) diff --git a/basil/basil.nix b/basil/basil.nix index b56bf29..9e91a3f 100644 --- a/basil/basil.nix +++ b/basil/basil.nix @@ -6,8 +6,15 @@ , jre , testers , basil +, protobuf }: +let + replaceProtocPlaceholder = '' + substituteInPlace build.sbt \ + --replace-fail 'PROTOC_PLACEHOLDER' '${lib.getExe protobuf}' + ''; +in mkSbtDerivation { pname = "basil"; version = "0.1.2-alpha-unstable-2024-10-30"; @@ -21,14 +28,29 @@ mkSbtDerivation { sha256 = "sha256-gAeoWKLG2xLIK6T0dwQA73YE6LqTO3EoDvbHUT73uSU="; }; - depsSha256 = "sha256-++gg+SKskDyaqHowNG2RPS7evuCzPYvvXMC4Rkp7b6U="; + patches = [ ./0001-basil-protoc-version.patch ] ; + + # we must run the command in both the main derivation + # and the dependency-generating derivation. + overrideDepsAttrs = depsfinal: depsprev: { + postPatch = replaceProtocPlaceholder; + }; + postPatch = replaceProtocPlaceholder; + + depsSha256 = "sha256-tDJuleKVLMPCZNJGNxokuScDOU4siLQEmM1FZff+5oM="; buildPhase = '' + runHook preBuild + javac -version sbt assembly + + runHook postBuild ''; installPhase = '' + runHook preInstall + mkdir -p $out/bin mkdir -p $out/share/basil @@ -47,6 +69,8 @@ mkSbtDerivation { makeWrapper "${lib.getExe jre}" $out/bin/basil \ --add-flags -jar \ --add-flags "$out/share/basil/$(basename $JAR)" + + runHook postInstall ''; meta = { diff --git a/flake.lock b/flake.lock index 16d5981..9a06f64 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1729880355, - "narHash": "sha256-RP+OQ6koQQLX5nw0NmcDrzvGL8HDLnyXt/jHhL1jwjM=", + "lastModified": 1732521221, + "narHash": "sha256-2ThgXBUXAE1oFsVATK1ZX9IjPcS4nKFOAjhPNKuiMn0=", "owner": "nixos", "repo": "nixpkgs", - "rev": "18536bf04cd71abd345f9579158841376fdd0c5a", + "rev": "4633a7c72337ea8fd23a4f2ba3972865e3ec685d", "type": "github" }, "original": { diff --git a/gtirb/0001-ddisasm-disable-concurrent-souffle.patch b/gtirb/0001-ddisasm-disable-concurrent-souffle.patch new file mode 100644 index 0000000..b93dcff --- /dev/null +++ b/gtirb/0001-ddisasm-disable-concurrent-souffle.patch @@ -0,0 +1,44 @@ +diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt +index 1db64e22..b1fd4848 100644 +--- a/src/CMakeLists.txt ++++ b/src/CMakeLists.txt +@@ -178,7 +178,7 @@ function(GENERATE_ARCH_CPP_SINGLE) + OUTPUT "${PARAM_PATH}.cpp" + WORKING_DIRECTORY "${SOUFFLE_DATALOG_DIR}" + COMMAND +- ${SOUFFLE} main.dl -g "${CMAKE_BINARY_DIR}/src/${PARAM_PATH}.cpp" -jauto ++ ${SOUFFLE} main.dl -g "${CMAKE_BINARY_DIR}/src/${PARAM_PATH}.cpp" + -MARCH_${PARAM_MARCH} ${SOUFFLE_PREPROCESSOR_ARG} ${SOUFFLE_PROFILING_ARG} + -N ddisasm_${ARCH} + DEPENDS ${DATALOG_BASE_SOURCES} ${PARAM_DATALOG_SOURCES}) +@@ -203,7 +203,7 @@ function(GENERATE_ARCH_CPP_MANY) + # Generate CPP code + execute_process( + COMMAND +- ${SOUFFLE} main.dl -G ${STAGING_PATH} -jauto -MARCH_${PARAM_MARCH} ++ ${SOUFFLE} main.dl -G ${STAGING_PATH} -MARCH_${PARAM_MARCH} + ${SOUFFLE_PREPROCESSOR_ARG} ${SOUFFLE_PROFILING_ARG} -N ddisasm_${ARCH} + WORKING_DIRECTORY "${SOUFFLE_DATALOG_DIR}") + +diff --git a/src/passes/CMakeLists.txt b/src/passes/CMakeLists.txt +index aa20b7d0..ad89a198 100644 +--- a/src/passes/CMakeLists.txt ++++ b/src/passes/CMakeLists.txt +@@ -70,7 +70,7 @@ add_custom_command( + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/passes" + COMMAND + ${SOUFFLE} ${NO_RETURN_DATALOG_MAIN} -g souffle_no_return.cpp -N +- souffle_no_return -jauto ${SOUFFLE_PREPROCESSOR_ARG} ++ souffle_no_return ${SOUFFLE_PREPROCESSOR_ARG} + DEPENDS ${NO_RETURN_DATALOG_SOURCES}) + + add_library(no_return_pass STATIC NoReturnPass.cpp ${NO_RETURN_CPP}) +@@ -114,7 +114,7 @@ add_custom_command( + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/src/passes" + COMMAND + ${SOUFFLE} ${FUNCTION_INFERENCE_DATALOG_MAIN} -g +- souffle_function_inference.cpp -N souffle_function_inference -jauto ++ souffle_function_inference.cpp -N souffle_function_inference + ${SOUFFLE_PREPROCESSOR_ARG} + DEPENDS ${FUNCTION_INFERENCE_DATALOG_SOURCES}) + diff --git a/gtirb/0001-gtirb-link-absl.patch b/gtirb/0001-gtirb-link-absl.patch new file mode 100644 index 0000000..07aec66 --- /dev/null +++ b/gtirb/0001-gtirb-link-absl.patch @@ -0,0 +1,31 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index e176b6a7..1c03fe8f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -427,11 +427,8 @@ endif() + # --------------------------------------------------------------------------- + # protobuf + # --------------------------------------------------------------------------- +-if(CL_API) +- find_package(Protobuf 3.7.0 REQUIRED) +-else() +- find_package(Protobuf 3.0.0 REQUIRED) +-endif() ++set(protobuf_MODULE_COMPATIBLE TRUE CACHE BOOL "") ++find_package(Protobuf REQUIRED CONFIG) + + if(Protobuf_VERSION VERSION_LESS 3.2) + add_definitions(-DPROTOBUF_SET_BYTES_LIMIT) +diff --git a/src/gtirb/proto/CMakeLists.txt b/src/gtirb/proto/CMakeLists.txt +index 74cf20a9..d53a8bd0 100644 +--- a/src/gtirb/proto/CMakeLists.txt ++++ b/src/gtirb/proto/CMakeLists.txt +@@ -45,7 +45,7 @@ endforeach(PROTO_H) + # Add proto library target + add_library(gtirb_proto STATIC ${PROTO_CPP_SOURCES} ${PROTO_COPIED_HEADERS}) + +-target_link_libraries(gtirb_proto ${Boost_LIBRARIES} ${Protobuf_LIBRARIES}) ++target_link_libraries(gtirb_proto ${Boost_LIBRARIES} protobuf::libprotobuf) + + target_compile_definitions(gtirb_proto PRIVATE GTIRB_${PROJECT_NAME}_EXPORTS) + diff --git a/gtirb/0001-gtirb_pprinter-include-map.patch b/gtirb/0001-gtirb_pprinter-include-map.patch new file mode 100644 index 0000000..3697141 --- /dev/null +++ b/gtirb/0001-gtirb_pprinter-include-map.patch @@ -0,0 +1,12 @@ +diff --git a/src/gtirb_pprinter/driver/parser.hpp b/src/gtirb_pprinter/driver/parser.hpp +index 2eca4fb..0741103 100644 +--- a/src/gtirb_pprinter/driver/parser.hpp ++++ b/src/gtirb_pprinter/driver/parser.hpp +@@ -4,6 +4,7 @@ + #include + #include + #include ++#include + + namespace fs = boost::filesystem; + diff --git a/gtirb/ddisasm.nix b/gtirb/ddisasm.nix index 9c3c1d6..f66e22a 100644 --- a/gtirb/ddisasm.nix +++ b/gtirb/ddisasm.nix @@ -2,6 +2,7 @@ , stdenv , fetchurl , fetchFromGitHub +, fetchzip , cmake , boost , lief @@ -11,12 +12,17 @@ , capstone-grammatech , souffle , ddisasm -, runCommandCC +, runCommand , testers , jq }: -stdenv.mkDerivation { +let + elf-test-files = fetchzip { + url = "https://gist.github.com/katrinafyi/8bcc7a6756b6f467a658e292181cdf8b/archive/453c9b2c5ebdca4d30816e26805b121a919dd150.tar.gz"; + hash = "sha256-xewqpzAR+rfAMM9Hn97gwzTrhpHONjltbIjhd15PaPw="; + }; +in stdenv.mkDerivation { pname = "ddisasm"; version = "0-unstable-2024-10-31"; @@ -32,14 +38,23 @@ stdenv.mkDerivation { url = "https://github.com/rina-forks/ddisasm/compare/main..determinism.patch"; hash = "sha256-xISyR7ptR2LfHZJAGUyXqANLIab9yZrQFh8wLbJLsx8="; }) - ]; + ] ++ lib.optional stdenv.isDarwin ./0001-ddisasm-disable-concurrent-souffle.patch; buildInputs = [ cmake boost lief gtirb gtirb-pprinter libehp ]; nativeBuildInputs = [ capstone-grammatech souffle ]; cmakeFlags = [ "-DDDISASM_ENABLE_TESTS=OFF" "-DDDISASM_GENERATE_MANY=ON" ]; - CXXFLAGS = [ "-includeset" ]; + postPatch = '' + ( + shopt -s globstar + substituteInPlace **/*.cpp **/*.h --replace-warn unordered_map map --replace-warn unordered_set set + substituteInPlace src/passes/Disassembler.cpp \ + --replace-fail \ + 'gtirb::UUID Uuid;' \ + 'gtirb::UUID Uuid{gtirb::Node::UUIDGenerator()};' + ) + ''; passthru.deterministic = ddisasm; @@ -49,16 +64,15 @@ stdenv.mkDerivation { version = "Disassemble"; }; - passthru.tests.ddisasm-deterministic = runCommandCC + passthru.tests.ddisasm-deterministic = runCommand "ddisasm-deterministic-test" { nativeBuildInputs = [ ddisasm.deterministic jq ]; } '' mkdir -p $out && cd $out - echo 'int main(void) { return 0; }' > a.c - $CC a.c + cp -v ${elf-test-files}/a.out . ddisasm a.out --json | jq -S > a1 ddisasm a.out --json | jq -S > a2 - diff -q a1 a2 + diff -u a1 a2 ''; meta = { diff --git a/gtirb/gtirb-pprinter.nix b/gtirb/gtirb-pprinter.nix index 2edd043..a58744a 100644 --- a/gtirb/gtirb-pprinter.nix +++ b/gtirb/gtirb-pprinter.nix @@ -22,6 +22,15 @@ stdenv.mkDerivation { hash = "sha256-9CZ+ndHX5f4rKbGXvCrqEg55Ep9JEkS/u//grdqTpTc="; }; + patches = [ ./0001-gtirb_pprinter-include-map.patch ]; + + postPatch = '' + ( + shopt -s globstar + substituteInPlace **/*.cpp **/*.hpp --replace-warn unordered_map map --replace-warn unordered_set set + ) + ''; + buildInputs = [ cmake python3 gtirb boost abseil-cpp gtest ]; nativeBuildInputs = [ capstone-grammatech ]; diff --git a/gtirb/gtirb.nix b/gtirb/gtirb.nix index 702a9f5..8c4855b 100644 --- a/gtirb/gtirb.nix +++ b/gtirb/gtirb.nix @@ -25,22 +25,34 @@ stdenv.mkDerivation { url = "https://github.com/rina-forks/gtirb/compare/master..det.patch"; hash = "sha256-86cRmnV5CL5DjOzFj+cJYUYKQpHQ6DsqnZDaMGa/kog="; }) - ]; + ] ++ lib.optional stdenv.isDarwin ./0001-gtirb-link-absl.patch; + + postPatch = '' + ( + shopt -u globstar + substituteInPlace include/gtirb/{CFG,Module}.hpp --replace-warn unordered_map map --replace-warn unordered_set set + ) + ''; nativeBuildInputs = [ ]; buildInputs = [ cmake python3 boost doxygen ]; propagatedBuildInputs = [ protobuf ]; - cmakeFlags = [ "-DGTIRB_ENABLE_TESTS=OFF" "-DGTIRB_PY_API=ON" ]; + cmakeFlags = [ + "-DGTIRB_ENABLE_TESTS=OFF" + "-DGTIRB_PY_API=ON" + "-DGTIRB_RUN_CLANG_TIDY=OFF" + # "-DCLANG_TIDY_EXE=${lib.getExe' clang-tools "clang-tidy"}" + ]; CXXFLAGS = "-includeset -Wno-error=unused-result -Wno-error=array-bounds"; preConfigure = '' substituteInPlace CMakeLists.txt \ - --replace '$'{PYTHON_VERSION} ${python3.version} + --replace-warn '$'{PYTHON_VERSION} ${python3.version} substituteInPlace src/CMakeLists.txt src/gtirb/proto/CMakeLists.txt \ - --replace 'DESTINATION lib' 'DESTINATION ''${CMAKE_INSTALL_LIBDIR}' \ - --replace 'DESTINATION include' 'DESTINATION ''${CMAKE_INSTALL_INCLUDEDIR}' + --replace-fail 'DESTINATION lib' 'DESTINATION ''${CMAKE_INSTALL_LIBDIR}' \ + --replace-warn 'DESTINATION include' 'DESTINATION ''${CMAKE_INSTALL_INCLUDEDIR}' ''; postInstall = '' diff --git a/llvm-translator/alive2-aslp.nix b/llvm-translator/alive2-aslp.nix index d388f45..78850f8 100644 --- a/llvm-translator/alive2-aslp.nix +++ b/llvm-translator/alive2-aslp.nix @@ -62,11 +62,11 @@ ''; }; - passthru.tests.run-arm-tv = runCommand "test-run-arm-tv" {} '' - rm -rfv logs logs-aslp - set +o pipefail - ${alive2-aslp.arm-tv-scripts}/bin/run-arm-tv.pl 2>&1 | tee /dev/stderr | grep 'please specify' - touch $out - ''; + # passthru.tests.run-arm-tv = runCommand "test-run-arm-tv" {} '' + # rm -rfv logs logs-aslp + # set +o pipefail + # ${alive2-aslp.arm-tv-scripts}/bin/run-arm-tv.pl 2>&1 | tee /dev/stderr | grep 'please specify' + # touch $out + # ''; }) diff --git a/llvm-translator/alive2-regehr.nix b/llvm-translator/alive2-regehr.nix index c36bbf8..69667bf 100644 --- a/llvm-translator/alive2-regehr.nix +++ b/llvm-translator/alive2-regehr.nix @@ -1,4 +1,6 @@ -{ alive2 +{ lib +, alive2 +, stdenv , llvmPackages , fetchFromGitHub }: @@ -15,5 +17,6 @@ }; patches = [ ]; - CXXFLAGS = prev.CXXFLAGS or "" + " -Wno-error=maybe-uninitialized"; + CXXFLAGS = (prev.CXXFLAGS or "") + + lib.optionalString (!stdenv.isDarwin) " -Wno-error=maybe-uninitialized"; }) diff --git a/llvm-translator/alive2.nix b/llvm-translator/alive2.nix index e1df3db..b6e4586 100644 --- a/llvm-translator/alive2.nix +++ b/llvm-translator/alive2.nix @@ -12,9 +12,13 @@ , clang , llvmPackages , git-am-shim +, overrideCC +, llvmPackages_17 }: -stdenv.mkDerivation { +let buildStdenv = if stdenv.isDarwin then overrideCC stdenv llvmPackages_17.clang else stdenv; in + +buildStdenv.mkDerivation { pname = "alive2"; version = "2022-10-26"; diff --git a/llvm-translator/llvm-custom.nix b/llvm-translator/llvm-custom.nix index f74b222..351ee6f 100644 --- a/llvm-translator/llvm-custom.nix +++ b/llvm-translator/llvm-custom.nix @@ -15,10 +15,14 @@ let doCheck = false; # install Target .inc files from build directory for lifter project. - postFixup = '' - cd /build/$sourceRoot/build/lib && file Target + # at this point, we should within the build artifact directory + postBuild = '' + mkdir -p $dev/include + pushd ./lib + file Target find Target -name '*.inc' -print0 \ | xargs -0 -I{} cp -v --no-clobber --parents {} $dev/include + popd ''; }); diff --git a/llvm-translator/remill.nix b/llvm-translator/remill.nix index e46f9e4..efec31e 100644 --- a/llvm-translator/remill.nix +++ b/llvm-translator/remill.nix @@ -64,7 +64,7 @@ stdenv.mkDerivation (self: sleigh = sleigh' self; nativeBuildInputs = [ cmake ]; - buildInputs = [ self.sleigh llvm xed glog gtest abseil-cpp glibc_multi ]; + buildInputs = [ self.sleigh llvm glog gtest abseil-cpp xed glibc_multi ]; outputs = [ "out" "dev" "lib" ]; @@ -140,4 +140,12 @@ stdenv.mkDerivation (self: hardeningDisable = [ "zerocallusedregs" ]; + meta = with lib; { + description = "Library for lifting machine code to LLVM bitcode"; + homepage = "https://github.com/lifting-bits/remill"; + license = licenses.asl20; + maintainers = with maintainers; [ katrinafyi ]; + broken = stdenv.isAarch64; # XXX: problems with xed (fixable) and libcxx (unsolved) + }; + }) diff --git a/llvm-translator/retdec-uq-pac.nix b/llvm-translator/retdec-uq-pac.nix index 47c221d..1b1093f 100644 --- a/llvm-translator/retdec-uq-pac.nix +++ b/llvm-translator/retdec-uq-pac.nix @@ -2,7 +2,7 @@ retdec.overrideAttrs (final: prev: { cmakeFlags = prev.cmakeFlags ++ [ "-DRETDEC_DEV_TOOLS=1" ]; - patches = [ + patches = prev.patches ++ [ (fetchurl { url = "https://gist.githubusercontent.com/katrinafyi/c33f6f9ccaad4420f76f84e6cb219fe0/raw/0001-emit-names-for-capstone-intrinsics.patch"; hash = "sha256-LJjzLw2R8ckB10jL92RH/kHggX2oQrAZNQUsonM6ciQ="; diff --git a/llvm-translator/retdec5.nix b/llvm-translator/retdec5.nix index 70972fa..63c5ee6 100644 --- a/llvm-translator/retdec5.nix +++ b/llvm-translator/retdec5.nix @@ -1,9 +1,12 @@ { stdenv , fetchFromGitHub +, fetchpatch , fetchzip +, writeText , lib , openssl , cmake +, ninja , autoconf , automake , libtool @@ -14,11 +17,16 @@ , perl , python3 , ncurses +, time +, upx , gtest , libffi , libxml2 , zlib +, buildEnv , enableTests ? true +, buildDevTools ? true +, compileYaraPatterns ? false }: let @@ -28,25 +36,36 @@ let owner = "capstone-engine"; repo = "capstone"; rev = "5.0-rc2"; - sha256 = "sha256-nB7FcgisBa8rRDS3k31BbkYB+tdqA6Qyj9hqCnFW+ME="; + hash = "sha256-nB7FcgisBa8rRDS3k31BbkYB+tdqA6Qyj9hqCnFW+ME="; }; llvm = fetchFromGitHub { owner = "avast-tl"; repo = "llvm"; rev = "2a1f3d8a97241c6e91710be8f84cf3cf80c03390"; - sha256 = "sha256-+v1T0VI9R92ed9ViqsfYZMJtPCjPHCr4FenoYdLuFOU="; + hash = "sha256-+v1T0VI9R92ed9ViqsfYZMJtPCjPHCr4FenoYdLuFOU="; }; yaracpp = fetchFromGitHub { owner = "VirusTotal"; repo = "yara"; rev = "v4.2.0-rc1"; - sha256 = "sha256-WcN6ClYO2d+/MdG06RHx3kN0o0WVAY876dJiG7CwJ8w="; + hash = "sha256-WcN6ClYO2d+/MdG06RHx3kN0o0WVAY876dJiG7CwJ8w="; }; yaramod = fetchFromGitHub { owner = "avast"; repo = "yaramod"; - rev = "a367d910ae79698e64e99d8414695281723cd34b"; - sha256 = "sha256-mnjYQOn/Z37XAtW8YsfPewM9t1WYsyjivTnmRwYWSQ0="; + rev = "aa06dd408c492a8f4488774caf2ee105ccc23ab5"; + hash = "sha256-wiD4A+6Jsd7bzGo8mmWnkb7l2XcjEgLCK/A4iW9Zt/A="; + + # yaramod builds yet more dependencies. these need to be installed to "lib" + postFetch = '' + ( + cd $out + shopt -s globstar + substituteInPlace src/CMakeLists.txt deps/**/CMakeLists.txt \ + --replace-quiet CMAKE_ARGS 'CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=lib' \ + --replace-quiet '$'{CMAKE_INSTALL_LIBDIR} lib + ) + ''; }; keystone = fetchFromGitHub { # only for tests @@ -55,14 +74,14 @@ let rev = "d7ba8e378e5284e6384fc9ecd660ed5f6532e922"; sha256 = "1yzw3v8xvxh1rysh97y0i8y9svzbglx2zbsqjhrfx18vngh0x58f"; }; - # googletest imported from nixpkgs retdec-support-version = "2019-03-08"; retdec-support = { rev = retdec-support-version; } // # for checking the version against the expected version fetchzip { url = "https://github.com/avast-tl/retdec-support/releases/download/${retdec-support-version}/retdec-support_${retdec-support-version}.tar.xz"; - hash = "sha256-paeNrxXTE7swuKjP+sN42xnCYS7x5Y5CcUe7tyzsLxs="; + hash = "sha256-t1tx4MfLW/lwtbO5JQ1nrFBIOeMclq+0dENuXW+ahIM="; + stripRoot = false; }; check-dep = name: dep: @@ -82,20 +101,41 @@ let CAPSTONE = capstone; LLVM = llvm; YARA = yaracpp; - YARAMOD = yaramod; + YARAMOD = yaramod // { rev = ""; }; SUPPORT_PKG = retdec-support; } // lib.optionalAttrs enableTests { KEYSTONE = keystone; + GOOGLETEST = gtest.src // { rev = ""; }; # empty rev bypasses dep version checking }; + + # overwrite install-share.py to copy instead of download. + # we use this so the copy happens at the right time in the build, + # otherwise, the build process cleans the directory. + install-share = + writeText + "install-share.py" + '' + import os, sys, shutil, subprocess + + install_path, arch_url, sha256hash_ref, version = sys.argv[1:] + support_dir = os.path.join(install_path, 'share', 'retdec', 'support') + + assert os.path.isdir(arch_url), "nix install-share.py expects a path for support url" + + os.makedirs(support_dir, exist_ok=True) + shutil.copytree(arch_url, support_dir, dirs_exist_ok=True) + subprocess.check_call(['chmod', '-R', 'u+w', support_dir]) + ''; in stdenv.mkDerivation (self: { pname = "retdec"; - # If you update this you will also need to adjust the versions of the updated dependencies. You can do this by first just updating retdec - # itself and trying to build it. The build should fail and tell you which dependencies you have to upgrade to which versions. + # If you update this you will also need to adjust the versions of the updated dependencies. # I've notified upstream about this problem here: # https://github.com/avast-tl/retdec/issues/412 - # gcc is pinned to gcc8 in all-packages.nix. That should probably be re-evaluated on update. + # + # The dependencies and their sources are listed in this file: + # https://github.com/avast/retdec/blob/master/cmake/deps.cmake version = "5.0"; src = fetchFromGitHub { @@ -105,8 +145,22 @@ stdenv.mkDerivation (self: { sha256 = "sha256-H4e+aSgdBBbG6X6DzHGiDEIASPwBVNVsfHyeBTQLAKI="; }; + patches = [ + # gcc 13 compatibility: https://github.com/avast/retdec/pull/1153 + (fetchpatch { + url = "https://github.com/avast/retdec/commit/dbaab2c3d17b1eae22c581e8ab6bfefadf4ef6ae.patch"; + hash = "sha256-YqHYPGAGWT4x6C+CpsOSsOIZ+NPM2FBQtGQFs74OUIQ="; + }) + # aarch64 compatibility: https://github.com/avast/retdec/pull/1195 + (fetchpatch { + url = "https://github.com/avast/retdec/commit/93233e79a0b14e6aeeee6b117a1c93feba88577b.patch"; + hash = "sha256-0VT7J9mcrneUo80BSi+unqCGdNNp4HaRx615M6hm0n8="; + }) + ]; + nativeBuildInputs = [ cmake + ninja autoconf automake libtool @@ -127,70 +181,116 @@ stdenv.mkDerivation (self: { ] ++ lib.optional self.doInstallCheck gtest; cmakeFlags = [ - "-DRETDEC_TESTS=${if self.doInstallCheck then "ON" else "OFF"}" # build tests - ] ++ lib.mapAttrsToList (k: v: "-D${k}_URL=${v}") deps; - - patches = [ ]; - - # fix for a gcc13 change. - env.CXXFLAGS = "-include cstdint"; + (lib.cmakeBool "RETDEC_TESTS" self.doInstallCheck) # build tests + (lib.cmakeBool "RETDEC_DEV_TOOLS" buildDevTools) # build tools e.g. capstone2llvmir, retdectool + (lib.cmakeBool "RETDEC_COMPILE_YARA" compileYaraPatterns) # build and install compiled patterns + ] ++ lib.mapAttrsToList (k: v: lib.cmakeFeature "${k}_URL" "${v}") deps; preConfigure = - lib.concatStringsSep "\n" (lib.mapAttrsToList check-dep deps) + lib.concatLines (lib.mapAttrsToList check-dep deps) + '' - mkdir -p "$out/share/retdec" - cp --no-preserve=mode -r ${retdec-support} "$out/share/retdec/support" # write permission needed during install - - # the CMakeLists assume CMAKE_INSTALL_BINDIR, etc are path components but in Nix, they are absolute. - # therefore, we need to remove the unnecessary CMAKE_INSTALL_PREFIX prepend. - substituteInPlace ./CMakeLists.txt \ - --replace "''$"{CMAKE_INSTALL_PREFIX}/"''$"{RETDEC_INSTALL_BIN_DIR} "''$"{CMAKE_INSTALL_FULL_BINDIR} \ - --replace "''$"{CMAKE_INSTALL_PREFIX}/"''$"{RETDEC_INSTALL_LIB_DIR} "''$"{CMAKE_INSTALL_FULL_LIBDIR} \ - --replace "''$"{CMAKE_INSTALL_PREFIX}/"''$"{RETDEC_INSTALL_SUPPORT_DIR} "''$"{RETDEC_INSTALL_SUPPORT_DIR} \ - - # similarly for yaramod. here, we fix the LIBDIR to lib64. for whatever reason, only "lib64" works. - substituteInPlace deps/yaramod/CMakeLists.txt \ - --replace "''$"{YARAMOD_INSTALL_DIR}/"''$"{CMAKE_INSTALL_LIBDIR} "''$"{YARAMOD_INSTALL_DIR}/lib64 \ - --replace CMAKE_ARGS 'CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=lib64' - - # yara needs write permissions in the generated source directory. - echo ${lib.escapeShellArg '' - ExternalProject_Add_Step( - yara chmod WORKING_DIRECTORY ''${YARA_DIR} - DEPENDEES download COMMAND chmod -R u+rw . - ) - ''} >> deps/yara/CMakeLists.txt - - # patch gtest to use the system package - gtest=deps/googletest/CMakeLists.txt - old="$(cat $gtest)" - (echo 'find_package(GTest REQUIRED)'; echo "$old") > $gtest - sed -i 's/ExternalProject_[^(]\+[(]/ set(IGNORED /g' $gtest - - substituteInPlace $gtest \ - --replace '$'{GTEST_LIB} "GTest::gtest"\ - --replace '$'{GMOCK_LIB} "GTest::gmock"\ - --replace '$'{GTEST_MAIN_LIB} "GTest::gtest_main"\ - --replace '$'{GMOCK_MAIN_LIB} "GTest::gmock_main" - - # without git history, there's no chance of these tests passing. - substituteInPlace tests/utils/version_tests.cpp \ - --replace VersionTests DISABLED_VersionTests + cp -v ${install-share} ./support/install-share.py + + # the CMakeLists assume CMAKE_INSTALL_BINDIR, etc are path components but in Nix, they are absolute. + # therefore, we need to remove the unnecessary CMAKE_INSTALL_PREFIX prepend. + substituteInPlace ./CMakeLists.txt \ + --replace-fail "''$"{CMAKE_INSTALL_PREFIX}/"''$"{RETDEC_INSTALL_BIN_DIR} "''$"{CMAKE_INSTALL_FULL_BINDIR} \ + --replace-fail "''$"{CMAKE_INSTALL_PREFIX}/"''$"{RETDEC_INSTALL_LIB_DIR} "''$"{CMAKE_INSTALL_FULL_LIBDIR} \ + + # --replace "''$"{CMAKE_INSTALL_PREFIX}/"''$"{RETDEC_INSTALL_SUPPORT_DIR} "''$"{RETDEC_INSTALL_SUPPORT_DIR} + # note! Nix does not set CMAKE_INSTALL_DATADIR to an absolute path, so this replacement would be incorrect + + # similarly for yaramod. + substituteInPlace deps/yaramod/CMakeLists.txt \ + --replace-fail "''$"{YARAMOD_INSTALL_DIR}/"''$"{CMAKE_INSTALL_LIBDIR} "''$"{YARAMOD_INSTALL_DIR}/lib + + # yara needs write permissions in the generated source directory. + echo ' + ExternalProject_Add_Step( + yara chmod WORKING_DIRECTORY ''${YARA_DIR} + DEPENDEES download COMMAND chmod -R u+w . + ) + ' >> deps/yara/CMakeLists.txt + + # yara: to support building retdec with ninja + substituteInPlace deps/yara/CMakeLists.txt \ + --replace-fail '$'{YARA_CONFIGURE_ARGS} ' ''${YARA_CONFIGURE_ARGS} --disable-dependency-tracking' \ + --replace-fail '$'{YARA_MAKE_PROGRAM} $(command -v make) + + # all vendored dependencies must build their libs into the "lib" subdirectory. + # retdec's install phase will copy these into the correct Nix output. + substituteInPlace deps/*/CMakeLists.txt \ + --replace-quiet CMAKE_ARGS 'CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=lib' + + substituteInPlace src/utils/CMakeLists.txt \ + --replace-warn '$'{RETDEC_GIT_VERSION_TAG} ${self.version} \ + --replace-warn '$'{RETDEC_GIT_COMMIT_HASH} ${self.src.rev} + + + # tests: without git history, there is no chance these tests will pass. + substituteInPlace tests/utils/version_tests.cpp \ + --replace-warn VersionTests DISABLED_VersionTests + + + # scripts: patch paths + substituteInPlace scripts/retdec-utils.py \ + --replace-fail /usr/bin/time ${time} \ + --replace-fail /usr/local/bin/gtime ${time} + substituteInPlace scripts/retdec-unpacker.py \ + --replace-fail "'upx'" "'${upx}'" ''; + # build first to make sure YARA_DIR has u+w permissions? + preBuild = '' + ninja yara + ''; + doInstallCheck = enableTests; installCheckPhase = '' - ${python3.interpreter} "$out/bin/retdec-tests-runner.py" + ${python3.interpreter} "''${!outputBin}/bin/retdec-tests-runner.py" rm -rf $out/bin/__pycache__ ''; + + # static-code patterns are split out since they are rarely needed by end users. + # from retdec's wiki: + # However, these are mainly for older compilers used for testing purposes. + # If you want to remove code from newer compilers or some custom libraries, + # you have to create your own signature files. + # see: https://github.com/avast/retdec/wiki/Removing-Statically-Linked-Code + postFixup = '' + mkdir -p $dev/lib/retdec + mv -v $out/share/retdec/cmake $dev/lib/retdec + + yara_dir=share/retdec/support/generic/yara_patterns + mkdir -p $patterns/$yara_dir + mv -v $out/$yara_dir/static-code $patterns/$yara_dir + ''; + + outputs = [ "out" "lib" "dev" "patterns" ]; + + passthru.full = buildEnv { + name = "${self.pname}-full-${self.version}"; + inherit (self.finalPackage) meta; + paths = [ self.finalPackage.out self.finalPackage.patterns ]; + postBuild = '' + config=$out/share/retdec/decompiler-config.json + cp -v --remove-destination $(readlink -f $config) $config + + static_code=support/generic/yara_patterns/static-code/ + substituteInPlace $config \ + --replace-fail ./$static_code $out/share/retdec/$static_code + ''; + }; + + passthru.deps = deps; + meta = with lib; { - description = "A retargetable machine-code decompiler based on LLVM"; + description = "Retargetable machine-code decompiler based on LLVM"; homepage = "https://retdec.com"; license = licenses.mit; - maintainers = with maintainers; [ dtzWill timokau ]; - platforms = [ "x86_64-linux" "i686-linux" ]; + maintainers = with maintainers; [ katrinafyi ]; }; }) diff --git a/overlay.nix b/overlay.nix index 2f1b27f..6f05f37 100644 --- a/overlay.nix +++ b/overlay.nix @@ -20,8 +20,28 @@ let # llvm-translator packages overlay_ocamlPackages = ofinal: oprev: { - ocaml-llvm-14 = ofinal.callPackage ./llvm-translator/ocaml-llvm.nix { libllvm = final.llvmPackages_14.libllvm; }; - asl-translator = ofinal.callPackage ./llvm-translator/asl-translator.nix { llvm = ofinal.ocaml-llvm-14; }; + # ctypes and ctypes-foreign v0.22.0 do not build on macOS + ctypes = oprev.ctypes.overrideAttrs (old: { + version = "0.23.0"; + src = prev.fetchFromGitHub { + owner = "ocamllabs"; + repo = "ocaml-ctypes"; + rev = "0.23.0"; + hash = "sha256-fZfTsOMppHiI7BVvgICVt/9ofGFAfYjXzHSDA7L4vZk="; + }; + }); + ctypes-foreign = oprev.ctypes-foreign.override (old: { + ctypes = ofinal.ctypes; + }); + + ocaml-llvm-14 = ofinal.callPackage ./llvm-translator/ocaml-llvm.nix { + libllvm = final.llvmPackages_14.libllvm; + ctypes = ofinal.ctypes; + ctypes-foreign = ofinal.ctypes-foreign ; + }; + asl-translator = ofinal.callPackage ./llvm-translator/asl-translator.nix { + llvm = ofinal.ocaml-llvm-14; + }; }; inherit (final.ocamlPackages_pac) asl-translator;