From ef74ba5a21833931fca62c6e1690a98c0243326c Mon Sep 17 00:00:00 2001 From: Alistair Date: Mon, 23 Sep 2024 20:17:06 +0100 Subject: [PATCH] chore(nix): add octez distribution to nix shell and integration tests --- crates/jstzd/tests/dummy.rs | 11 +++ flake.lock | 89 +++++++++++++++++++ flake.nix | 66 +++++++++++++- nix/crates.nix | 10 +-- .../0001-fix-octez-rust-deps-for-nix.patch | 23 +++++ 5 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 nix/patches/0001-fix-octez-rust-deps-for-nix.patch diff --git a/crates/jstzd/tests/dummy.rs b/crates/jstzd/tests/dummy.rs index afb88e02..d08a980d 100644 --- a/crates/jstzd/tests/dummy.rs +++ b/crates/jstzd/tests/dummy.rs @@ -1,6 +1,17 @@ use jstzd::main; +use tokio::process::Command; #[tokio::test] async fn test_main() { main().await.unwrap(); } + +#[tokio::test] +async fn test_octez_client() { + let output = Command::new("octez-client") + .arg("--version") + .output() + .await + .unwrap(); + println!("octez-client --version: {:?}", output); +} diff --git a/flake.lock b/flake.lock index 4e190b6e..03861118 100644 --- a/flake.lock +++ b/flake.lock @@ -73,12 +73,101 @@ "type": "github" } }, + "octez-v21": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ], + "opam-nix-integration": [ + "opam-nix-integration" + ], + "opam-repository": "opam-repository", + "rust-overlay": [ + "rust-overlay" + ] + }, + "locked": { + "lastModified": 1726495500, + "narHash": "sha256-LxIn3uvqOrNdlcUnbSvHGnVTn47rREvVqdvA4+AL8D0=", + "owner": "tezos", + "repo": "tezos", + "rev": "44643df827aa2d745c3d8a09a6649fe175ff77d7", + "type": "gitlab" + }, + "original": { + "owner": "tezos", + "ref": "octez-v21.0-rc2", + "repo": "tezos", + "type": "gitlab" + } + }, + "opam-nix-integration": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ], + "opam-repository": "opam-repository_2" + }, + "locked": { + "lastModified": 1716561961, + "narHash": "sha256-SgeZItYuUZV7WSaoX4G7J1ZpmwgeI3ocQCLP95E+O1U=", + "owner": "vapourismo", + "repo": "opam-nix-integration", + "rev": "0f98236c75cdb436be7669ccaa249264456baa37", + "type": "github" + }, + "original": { + "owner": "vapourismo", + "repo": "opam-nix-integration", + "type": "github" + } + }, + "opam-repository": { + "flake": false, + "locked": { + "lastModified": 1723381368, + "narHash": "sha256-fN8jaeSfK8lbIQcvPXk96FeH8i7sjGjkwqnf0C9ZCog=", + "owner": "ocaml", + "repo": "opam-repository", + "rev": "0f4d0ee5b69b496a4e26f305891c31400f0b4b5f", + "type": "github" + }, + "original": { + "owner": "ocaml", + "repo": "opam-repository", + "type": "github" + } + }, + "opam-repository_2": { + "flake": false, + "locked": { + "lastModified": 1688548241, + "narHash": "sha256-cmk1kCLtcL9GJplfl4J7t0r8g0N25O4G0ctycV4eyPo=", + "owner": "ocaml", + "repo": "opam-repository", + "rev": "860072345fa2f234ff10ffeee88db6906e138e92", + "type": "github" + }, + "original": { + "owner": "ocaml", + "repo": "opam-repository", + "type": "github" + } + }, "root": { "inputs": { "crane": "crane", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", "npm-buildpackage": "npm-buildpackage", + "octez-v21": "octez-v21", + "opam-nix-integration": "opam-nix-integration", "rust-overlay": "rust-overlay", "treefmt": "treefmt" } diff --git a/flake.nix b/flake.nix index 6aa09e1b..7b473388 100644 --- a/flake.nix +++ b/flake.nix @@ -32,6 +32,23 @@ url = "github:serokell/nix-npm-buildpackage"; inputs.nixpkgs.follows = "nixpkgs"; }; + + # Octez + + # We explicitly have opam-nix-integration as an input to avoid having two versions of nixpkgs + opam-nix-integration = { + url = "github:vapourismo/opam-nix-integration"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "flake-utils"; + }; + + octez-v21 = { + url = "gitlab:tezos/tezos/octez-v21.0-rc2"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-utils.follows = "flake-utils"; + inputs.rust-overlay.follows = "rust-overlay"; + inputs.opam-nix-integration.follows = "opam-nix-integration"; + }; }; outputs = inputs: @@ -43,6 +60,52 @@ overlays = [(import ./nix/overlay.nix) (import rust-overlay) npm-buildpackage.overlays.default]; }; + # Build octez release for this system + # + # TODO(https://linear.app/tezos/issue/JSTZ-152): + # This patch here should be upstreamed to tezos/tezos + octez = octez-v21.packages.${system}.default.overrideAttrs (old: let + rustToolchain = pkgs.rust-bin.fromRustupToolchainFile "${old.src}/rust-toolchain"; + rustPlatform = pkgs.makeRustPlatform { + rustc = rustToolchain; + cargo = rustToolchain; + }; + in { + patches = + (old.patches or []) + ++ [ + ./nix/patches/0001-fix-octez-rust-deps-for-nix.patch + ]; + + # Network access for fetching cargo dependencies is disabled in sandboxed + # builds. Instead we need to explicitly fetch the dependencies. Nixpkgs + # provides two ways to do this: + # + # - `fetchCargoTarball` fetches the dependencies using `cargo vendor` + # It requires an explicit `hash`. + # + # - `importCargoLock` parses the `Cargo.lock` file and fetches each + # dependency using `fetchurl`. It doesn't require an explicit `hash`. + # + # The latter is slower but doesn't require an explicit `hash` and is therefore + # more maintainable (since this derivation isn't built in CI). + cargoDeps = rustPlatform.importCargoLock { + lockFile = "${old.src}/src/rust_deps/Cargo.lock"; + }; + cargoRoot = "src/rust_deps"; + + nativeBuildInputs = + (old.nativeBuildInputs or []) + ++ [ + # See https://nixos.org/manual/nixpkgs/stable/#compiling-non-rust-packages-that-include-rust-code + # for more information. + # + # `cargoSetupHook` configures cargo to vendor dependencies using `cargoDeps`. + rustToolchain + rustPlatform.cargoSetupHook + ]; + }); + clangNoArch = if pkgs.stdenv.isDarwin then @@ -61,7 +124,7 @@ else pkgs.clang; rust-toolchain = pkgs.callPackage ./nix/rust-toolchain.nix {}; - crates = pkgs.callPackage ./nix/crates.nix {inherit crane rust-toolchain;}; + crates = pkgs.callPackage ./nix/crates.nix {inherit crane rust-toolchain octez;}; js-packages = pkgs.callPackage ./nix/js-packages.nix {}; fmt = treefmt.lib.evalModule pkgs { @@ -136,6 +199,7 @@ # Code coverage cargo-llvm-cov + octez ] ++ lib.optionals stdenv.isLinux [pkg-config openssl.dev] ++ lib.optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [Security SystemConfiguration]); diff --git a/nix/crates.nix b/nix/crates.nix index f4ec7386..6c1942d1 100644 --- a/nix/crates.nix +++ b/nix/crates.nix @@ -4,6 +4,7 @@ lib, stdenv, rust-toolchain, + octez, }: let craneLib = (crane.mkLib pkgs).overrideToolchain (_: rust-toolchain); @@ -114,33 +115,32 @@ in { cargo-test-unit = craneLib.cargoNextest (commonWorkspace // { - cargoArtifacts = cargoDeps; # Run the unit tests cargoNextestExtraArg = "--bins --lib"; }); cargo-test-int = craneLib.cargoNextest (commonWorkspace // { - cargoArtifacts = cargoDeps; + buildInputs = commonWorkspace.buildInputs ++ [octez]; + doCheck = true; # Run the integration tests # # FIXME(): # Don't run the `jstz_api` integration tests until they've been paralellized # # Note: --workspace is required for --exclude. Once --exclude is removed, remove --workspace - cargoNextestExtraArg = "--workspace --test \"*\" --exclude \"jstz_api\""; + cargoNextestExtraArgs = "--workspace --test \"*\" --exclude \"jstz_api\""; }); cargo-llvm-cov = craneLib.cargoLlvmCov (commonWorkspace // { - cargoArtifacts = cargoDeps; + buildInputs = commonWorkspace.buildInputs ++ [octez]; # Generate coverage reports for codecov cargoLlvmCovExtraArgs = "--workspace --exclude-from-test \"jstz_api\" --codecov --output-path $out"; }); cargo-clippy = craneLib.cargoClippy (commonWorkspace // { - cargoArtifacts = cargoDeps; cargoClippyExtraArgs = "--all-targets -- --deny warnings"; }); }; diff --git a/nix/patches/0001-fix-octez-rust-deps-for-nix.patch b/nix/patches/0001-fix-octez-rust-deps-for-nix.patch new file mode 100644 index 00000000..4e58d5bb --- /dev/null +++ b/nix/patches/0001-fix-octez-rust-deps-for-nix.patch @@ -0,0 +1,23 @@ +diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml +index f4336a077a..6ab49b65cf 100644 +--- a/manifest/product_octez.ml ++++ b/manifest/product_octez.ml +@@ -522,7 +522,7 @@ let octez_rust_deps = + [S "source_tree"; S "../riscv"]; + [S "source_tree"; S "../kernel_sdk"]; + ]; +- [S "action"; [S "no-infer"; [S "bash"; S "./build.sh"]]]; ++ [S "action"; [S "no-infer"; [S "system"; S "bash ./build.sh"]]]; + ]; + ] + +diff --git a/src/rust_deps/dune b/src/rust_deps/dune +index de48dbfde8..20934e2e21 100644 +--- a/src/rust_deps/dune ++++ b/src/rust_deps/dune +@@ -22,4 +22,4 @@ + (source_tree src) + (source_tree ../riscv) + (source_tree ../kernel_sdk)) +- (action (no-infer (bash ./build.sh)))) ++ (action (no-infer (system "bash ./build.sh"))))