From 343767d8079ac6fcd1ef725912d6b7004fb128d4 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Mon, 12 Aug 2024 13:33:02 +0200 Subject: [PATCH 1/6] nix: dinamycally include rocksdb drammatically improve first compilation by avoiding to recompile rocksdb. It may also improve performance. --- flake.nix | 17 +++++++++++++++-- rocksdb-overlay.nix | 13 +++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 rocksdb-overlay.nix diff --git a/flake.nix b/flake.nix index 2199c7625..6a16330d2 100644 --- a/flake.nix +++ b/flake.nix @@ -20,7 +20,10 @@ flake-utils.lib.eachDefaultSystem (system: let - overlays = [ (import rust-overlay) ]; + overlays = [ + (import rust-overlay) + (import ./rocksdb-overlay.nix) + ]; pkgs = import nixpkgs { inherit system overlays; }; @@ -28,7 +31,7 @@ craneLib = (crane.mkLib pkgs).overrideToolchain rustToolchain; - src = craneLib.cleanCargoSource ./.; + src = craneLib.cleanCargoSource ./.; nativeBuildInputs = with pkgs; [ rustToolchain clang ]; # required only at build time buildInputs = with pkgs; [ ]; # also required at runtime @@ -39,6 +42,11 @@ ELEMENTSD_SKIP_DOWNLOAD = true; BITCOIND_SKIP_DOWNLOAD = true; ELECTRUMD_SKIP_DOWNLOAD = true; + + # link rocksdb dynamically + ROCKSDB_INCLUDE_DIR = "${pkgs.rocksdb}/include"; + ROCKSDB_LIB_DIR = "${pkgs.rocksdb}/lib"; + }; cargoArtifacts = craneLib.buildDepsOnly commonArgs; bin = craneLib.buildPackage (commonArgs // { @@ -80,6 +88,11 @@ devShells.default = mkShell { inputsFrom = [ bin ]; LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; # for rocksdb + + # to link rocksdb dynamically + ROCKSDB_INCLUDE_DIR = "${pkgs.rocksdb}/include"; + ROCKSDB_LIB_DIR = "${pkgs.rocksdb}/lib"; + }; } ); diff --git a/rocksdb-overlay.nix b/rocksdb-overlay.nix new file mode 100644 index 000000000..722ba7a7f --- /dev/null +++ b/rocksdb-overlay.nix @@ -0,0 +1,13 @@ +final: prev: { + + rocksdb = prev.rocksdb.overrideAttrs (oldAttrs: rec { + version = "8.1.1"; + + src = final.fetchFromGitHub { + owner = "facebook"; + repo = oldAttrs.pname; + rev = "v${version}"; + hash = "sha256-79hRtc5QSWLLyjRGCmuYZSoIc9IcIsnl8UCinz2sVw4="; + }; + }); +} From 99d22baacf195ca6f0b1804b3eae19fd3cb80d34 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Mon, 12 Aug 2024 13:44:33 +0200 Subject: [PATCH 2/6] nix: set up direnv users with direnv+nix have the env setup just by entering the dir --- .envrc | 8 ++++++++ .gitignore | 1 + 2 files changed, 9 insertions(+) create mode 100644 .envrc diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..6b6394b2f --- /dev/null +++ b/.envrc @@ -0,0 +1,8 @@ +#!/bin/bash + +if ! has nix_direnv_version || ! nix_direnv_version 2.2.1; then + source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.2.1/direnvrc" "sha256-zelF0vLbEl5uaqrfIzbgNzJWGmLzCmYAkInj/LNxvKs=" +fi + +watch_file rust-toolchain.toml +use flake diff --git a/.gitignore b/.gitignore index 8ed8a0203..9e717b212 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ target *~ *.pyc result +.direnv From 73497a25c9dd058658a8aeb42685f66064160acd Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Mon, 12 Aug 2024 13:53:35 +0200 Subject: [PATCH 3/6] nix: skip exec download in nix they don't work anyway why bother --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index 6a16330d2..96c961d38 100644 --- a/flake.nix +++ b/flake.nix @@ -88,6 +88,9 @@ devShells.default = mkShell { inputsFrom = [ bin ]; LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; # for rocksdb + ELEMENTSD_SKIP_DOWNLOAD = true; + BITCOIND_SKIP_DOWNLOAD = true; + ELECTRUMD_SKIP_DOWNLOAD = true; # to link rocksdb dynamically ROCKSDB_INCLUDE_DIR = "${pkgs.rocksdb}/include"; From 025f4537959598f77af75efc263ff6494a7b894e Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Mon, 12 Aug 2024 13:59:26 +0200 Subject: [PATCH 4/6] define also needed components in rust-toolchain.toml --- rust-toolchain.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 7897a24d1..64aead883 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,10 @@ [toolchain] channel = "1.75.0" +components = [ + "cargo", + "clippy", + "rust-src", + "rust-std", + "rustc", + "rustfmt" +] From 92f0801b9b504043367be65dd4c9dd902c68fe0b Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Mon, 12 Aug 2024 14:20:46 +0200 Subject: [PATCH 5/6] nix: avoid env vars duplication --- flake.nix | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/flake.nix b/flake.nix index 96c961d38..d23f3735f 100644 --- a/flake.nix +++ b/flake.nix @@ -36,8 +36,7 @@ nativeBuildInputs = with pkgs; [ rustToolchain clang ]; # required only at build time buildInputs = with pkgs; [ ]; # also required at runtime - commonArgs = { - inherit src buildInputs nativeBuildInputs; + envVars = { LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; ELEMENTSD_SKIP_DOWNLOAD = true; BITCOIND_SKIP_DOWNLOAD = true; @@ -46,8 +45,12 @@ # link rocksdb dynamically ROCKSDB_INCLUDE_DIR = "${pkgs.rocksdb}/include"; ROCKSDB_LIB_DIR = "${pkgs.rocksdb}/lib"; - }; + + commonArgs = { + inherit src buildInputs nativeBuildInputs; + } // envVars; + cargoArtifacts = craneLib.buildDepsOnly commonArgs; bin = craneLib.buildPackage (commonArgs // { inherit cargoArtifacts; @@ -63,7 +66,6 @@ doCheck = false; }); - in with pkgs; { @@ -84,19 +86,9 @@ program = "${bin}/bin/electrs"; }; - - devShells.default = mkShell { + devShells.default = mkShell (envVars // { inputsFrom = [ bin ]; - LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; # for rocksdb - ELEMENTSD_SKIP_DOWNLOAD = true; - BITCOIND_SKIP_DOWNLOAD = true; - ELECTRUMD_SKIP_DOWNLOAD = true; - - # to link rocksdb dynamically - ROCKSDB_INCLUDE_DIR = "${pkgs.rocksdb}/include"; - ROCKSDB_LIB_DIR = "${pkgs.rocksdb}/lib"; - - }; + }); } ); } From 98c2768d687538b579037a65212ecbe1b0dbf1b1 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Mon, 12 Aug 2024 14:31:19 +0200 Subject: [PATCH 6/6] use exe_path instead of downloaded_exe_path exe_path continue to look at downloaded_exe_path unless a specific env var is specified --- tests/common.rs | 2 +- tests/electrum.rs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/common.rs b/tests/common.rs index 78932aa50..c85ebf7d9 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -57,7 +57,7 @@ impl TestRunner { } // Setup node - let node = NodeD::with_conf(noded::downloaded_exe_path().unwrap(), &node_conf).unwrap(); + let node = NodeD::with_conf(noded::exe_path().unwrap(), &node_conf).unwrap(); #[cfg(not(feature = "liquid"))] let (node_client, params) = (&node.client, &node.params); diff --git a/tests/electrum.rs b/tests/electrum.rs index d35d91eb3..504d16ca8 100644 --- a/tests/electrum.rs +++ b/tests/electrum.rs @@ -23,8 +23,7 @@ fn test_electrum() -> Result<()> { let server_arg = format!("{}:t", electrum_addr.to_string()); electrum_wallet_conf.args = vec!["-v", "--server", &server_arg]; electrum_wallet_conf.view_stdout = true; - let electrum_wallet = - ElectrumD::with_conf(electrumd::downloaded_exe_path()?, &electrum_wallet_conf)?; + let electrum_wallet = ElectrumD::with_conf(electrumd::exe_path()?, &electrum_wallet_conf)?; let notify_wallet = || { electrum_server.notify();