From dfd95ec4bdfdc9807131d922dede427c65a057b5 Mon Sep 17 00:00:00 2001 From: jordy25519 Date: Wed, 11 Dec 2024 11:59:38 +0800 Subject: [PATCH] Tidy up build.rs Improve CI Update drift-ffi-sys commit --- .github/workflows/build.yml | 16 ++- .github/workflows/release.yml | 2 +- README.md | 11 +- build.rs | 251 +++++++++++++++++++-------------- crates/drift-ffi-sys | 2 +- crates/drift-idl-gen/README.md | 2 + res/drift.json | 2 +- 7 files changed, 172 insertions(+), 114 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8d98e02..7fb0898 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,21 @@ jobs: runs-on: ubicloud steps: - name: Check out - uses: actions/checkout@v2 + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Cache Rust toolchain + uses: actions/cache@v3 + with: + path: | + ~/.rustup + ~/.cargo/bin + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-rust-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-rust- - name: Config rust toolchain run: | rustup show active-toolchain diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 91fd30c..0f4e33c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,7 +34,7 @@ jobs: - name: Publish drift-rs run: | # add libdrift_ffi_sys - curl -L https://github.com/user-attachments/files/17160233/libdrift_ffi_sys.so.zip > ffi.zip + curl -L https://github.com/drift-labs/drift-ffi-sys/releases/download/v2.103.0/libdrift_ffi_sys.so.zip > ffi.zip unzip ffi.zip sudo mv libdrift_ffi_sys.so $CARGO_DRIFT_FFI_PATH rm ffi.zip # clean up for git diff --git a/README.md b/README.md index 9b18a5e..f7950f1 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Experimental, high performance Rust SDK for building offchain clients for [Drift drift-rs = "1.0.0-alpha.4" # build from source (also builds and links 'libdrift_ffi_sys') -drift-rs = { git = "https://github.com/drift-labs/drift-rs", tag = "v1.0.0-alpha.4" } +drift-rs = { git = "https://github.com/drift-labs/drift-rs", tag = "v1.0.0-alpha.5" } ``` _*_`drift-rs` uses drift program over ffi. @@ -63,17 +63,16 @@ rustup override set 1.81.0-x86_64-apple-darwin ⚠️ the default toolchain is incompatible due to memory layout differences between solana program (BPF) and aarch64 and will fail at runtime with deserialization errors like: `InvalidSize`. ## Local Development -drift-rs links to the drift program crate via FFI, build from source or optionally install from [drift-ffi-sys](https://github.com/drift-labs/drift-ffi-sys/releases) +drift-rs links to the drift program crate via FFI, build from source (default) or optionally install from [drift-ffi-sys](https://github.com/drift-labs/drift-ffi-sys/releases) ```bash -# Build from source +# Build from source (default) CARGO_DRIFT_FFI_STATIC=1 # Provide a prebuilt drift_ffi_sys lib CARGO_DRIFT_FFI_PATH="/path/to/libdrift_ffi_sys" ``` ## Development -## Update IDL types -1) copy updated IDL to `res/drift.json` from protocol-v2 branch +## Updating IDL types +1) copy the updated IDL to `res/drift.json` from protocol-v2 branch 2) `cargo check` 3) commit changes - diff --git a/build.rs b/build.rs index e2b7eb4..f527f43 100644 --- a/build.rs +++ b/build.rs @@ -1,132 +1,175 @@ -use std::{collections::HashMap, fs::File, io::Write, path::Path}; +use std::{collections::HashMap, path::Path}; const LIB: &str = "libdrift_ffi_sys"; +const SUPPORTED_PLATFORMS: &[(&str, &str, &str)] = &[ + ("apple", "x86_64-apple-darwin", "dylib"), + ("linux", "x86_64-unknown-linux-gnu", "so"), +]; +const FFI_TOOLCHAIN_VERSION: &str = "1.76.0"; + +fn main() -> Result<(), Box> { + let current_dir = std::env::current_dir()?.canonicalize()?; + + // Generate IDL types from 'res/drift.json' + let idl_source_path = current_dir.join("res/drift.json"); + let idl_mod_path = current_dir.join("crates/src/drift_idl.rs"); + generate_idl_types(&idl_source_path, idl_mod_path.as_path())?; + + // Only build FFI lib if static or no lib path provided + if should_build_from_source() { + build_ffi_lib(¤t_dir)?; + } + + link_library()?; + Ok(()) +} + +fn generate_idl_types(idl_source_path: &Path, idl_mod_path: &Path) -> Result<(), Box> { + let idl_mod_rs = drift_idl_gen::generate_rust_types(&idl_source_path) + .map_err(|err| format!("generating IDL failed: {err:?}"))?; + + std::fs::write(&idl_mod_path, idl_mod_rs)?; + Ok(()) +} -fn main() { - let current_dir = std::env::current_dir().unwrap().canonicalize().unwrap(); - - // Generate rust types from anchor IDL - let idl_source_path = ¤t_dir.join(Path::new("res/drift.json")); - let idl_mod_rs = match drift_idl_gen::generate_rust_types(idl_source_path) { - Ok(idl_mod_rs) => idl_mod_rs, - Err(err) => panic!("generating IDL failed: {err:?}"), - }; - let idl_mod_path = current_dir.join(Path::new("crates/src/drift_idl.rs")); - let mut file = File::create(&idl_mod_path).expect("create IDL .rs"); - file.write_all(idl_mod_rs.as_bytes()) - .expect("wrote IDL .rs"); - - if std::env::var("CARGO_DRIFT_FFI_STATIC").is_ok() +fn should_build_from_source() -> bool { + std::env::var("CARGO_DRIFT_FFI_STATIC").is_ok() || std::env::var("CARGO_DRIFT_FFI_PATH").is_err() - { - // Build + Link FFI crate from source - println!("{LIB}: building from source..."); - let drift_ffi_sys_crate = current_dir.join(Path::new("crates/drift-ffi-sys")); - - // the x86_64 target must exist - let host_target = std::env::var("TARGET").unwrap(); - let (lib_target, lib_ext) = if host_target.contains("apple") { - ("x86_64-apple-darwin", "dylib") - } else if host_target.contains("linux") { - ("x86_64-unknown-linux-gnu", "so") - } else { - eprintln!("Unsupported host platform: {host_target}, please open an issue at: https://github.com/drift-labs/drift-rs/issues"); - fail_build(); - }; - - // "RUSTC" is set as the cargo version of the main SDK build, it must be unset for the ffi build - // "CARGO*" envs are also configured for the main SDK build - // https://users.rust-lang.org/t/switching-toolchains-in-build-rs-use-nightly-for-data-generation-and-stable-for-main-compilation/114443/5 - let ffi_build_envs: HashMap = std::env::vars() - .filter(|(k, _v)| !k.starts_with("CARGO") && !k.starts_with("RUSTC")) - .collect(); - println!("{ffi_build_envs:?}"); - let profile = std::env::var("PROFILE").expect("cargo PROFILE set"); - - // force drift-ffi-sys to build with specific toolchain (arch=x86_64, version=<=1.76.0) - // this ensures zero copy deserialization works correctly with the onchain data layout - let ffi_toolchain = format!("1.76.0-{lib_target}"); - let installed_toolchains_query = std::process::Command::new("rustup") - .args(["toolchain", "list"]) - .output() - .expect("rustup installed"); - if !installed_toolchains_query.status.success() { - println!("Check 'rustup' is installed and discoverable in system PATH"); - } - let installed_toolchains = String::from_utf8_lossy(&installed_toolchains_query.stdout); - if !installed_toolchains.contains(&ffi_toolchain) { - eprintln!("Required toolchain: {ffi_toolchain} is missing. Run: 'rustup install {ffi_toolchain}' to install and retry the build"); - fail_build(); - } +} - // install the dylib to system path - let libffi_out_path = - drift_ffi_sys_crate.join(Path::new(&format!("target/{profile}/{LIB}.{lib_ext}"))); - - // Build ffi crate and link - let mut ffi_build = std::process::Command::new("rustup"); - ffi_build - .env_clear() - .envs(ffi_build_envs) - .current_dir(drift_ffi_sys_crate.clone()) - .args(["run", &ffi_toolchain, "cargo", "build"]); - - match profile.as_str() { - "debug" => (), - "release" => { - ffi_build.arg("--release"); - } - custom => { - ffi_build.arg(format!("--profile={custom}")); - } - } +fn build_ffi_lib(current_dir: &Path) -> Result<(), Box> { + println!("cargo:warning={LIB}: building from source..."); - let output = ffi_build.output().expect("drift-ffi-sys built"); - if !output.status.success() { - eprintln!(" {}", String::from_utf8_lossy(output.stderr.as_slice())); - fail_build(); - } + let host_target = std::env::var("TARGET")?; + let (lib_target, lib_ext) = get_platform_details(&host_target)?; + + verify_toolchain(lib_target)?; + + // Build the library + let profile = std::env::var("PROFILE")?; + let drift_ffi_sys_crate = current_dir.join("crates/drift-ffi-sys"); - if !output.status.success() { - eprintln!( - "{LIB} could not be installed: {}", - String::from_utf8_lossy(output.stderr.as_slice()) - ); + build_with_toolchain(&drift_ffi_sys_crate, lib_target, &profile)?; + install_library(&drift_ffi_sys_crate, &profile, lib_ext)?; + + Ok(()) +} + +fn get_platform_details( + host_target: &str, +) -> Result<(&'static str, &'static str), Box> { + for (platform, target, ext) in SUPPORTED_PLATFORMS { + if host_target.contains(platform) { + return Ok((target, ext)); } + } + + println!("cargo:warning=Unsupported host platform: {host_target}"); + println!( + "cargo:warning=Please open an issue at: https://github.com/drift-labs/drift-rs/issues" + ); + Err("Unsupported platform".into()) +} - if let Ok(out_dir) = std::env::var("OUT_DIR") { - let _output = std::process::Command::new("cp") - .args([ - libffi_out_path.to_str().expect("ffi build path"), - out_dir.as_str(), - ]) - .output() - .expect("install ok"); - println!("{LIB}: searching for lib at: {out_dir}"); - println!("cargo:rustc-link-search=native={out_dir}"); +fn verify_toolchain(lib_target: &str) -> Result<(), Box> { + let ffi_toolchain = format!("{FFI_TOOLCHAIN_VERSION}-{lib_target}"); + let output = std::process::Command::new("rustup") + .args(["toolchain", "list"]) + .output()?; + + if !output.status.success() { + return Err("Failed to query rustup toolchains".into()); + } + + let installed_toolchains = String::from_utf8_lossy(&output.stdout); + if !installed_toolchains.contains(&ffi_toolchain) { + println!("cargo:warning=Required toolchain {ffi_toolchain} is missing"); + println!("cargo:warning=Run: 'rustup install {ffi_toolchain}' to install"); + return Err("Missing required toolchain".into()); + } + + Ok(()) +} + +fn build_with_toolchain( + drift_ffi_sys_crate: &Path, + lib_target: &str, + profile: &str, +) -> Result<(), Box> { + // Filter out cargo and rustc environment variables + let ffi_build_envs: HashMap = std::env::vars() + .filter(|(k, _v)| !k.starts_with("CARGO") && !k.starts_with("RUSTC")) + .collect(); + + let ffi_toolchain = format!("{FFI_TOOLCHAIN_VERSION}-{lib_target}"); + let mut ffi_build = std::process::Command::new("rustup"); + ffi_build + .env_clear() + .envs(ffi_build_envs) + .current_dir(drift_ffi_sys_crate) + .args(["run", &ffi_toolchain, "cargo", "build"]); + + match profile { + "debug" => (), + "release" => { + ffi_build.arg("--release"); } + custom => { + ffi_build.arg(format!("--profile={custom}")); + } + } - let _output = std::process::Command::new("ln") + let output = ffi_build.output()?; + if !output.status.success() { + println!("cargo:warning={}", String::from_utf8_lossy(&output.stderr)); + return Err("FFI build failed".into()); + } + + Ok(()) +} + +fn install_library( + drift_ffi_sys_crate: &Path, + profile: &str, + lib_ext: &str, +) -> Result<(), Box> { + let libffi_out_path = drift_ffi_sys_crate.join(format!("target/{profile}/{LIB}.{lib_ext}")); + + if let Ok(out_dir) = std::env::var("OUT_DIR") { + std::process::Command::new("cp") + .args([ + libffi_out_path.to_str().ok_or("Invalid path")?, + out_dir.as_str(), + ]) + .output()?; + println!("cargo:warning={LIB}: searching for lib at: {out_dir}"); + println!("cargo:rustc-link-search=native={out_dir}"); + } else { + // Install to system library path + std::process::Command::new("ln") .args([ "-sf", - libffi_out_path.to_str().expect("ffi build path"), + libffi_out_path.to_str().ok_or("Invalid path")?, "/usr/local/lib/", ]) - .output() - .expect("install ok"); + .output()?; - println!("{LIB}: searching for lib at: /usr/local/lib"); + println!("cargo:warning={LIB}: searching for lib at: /usr/local/lib"); println!("cargo:rustc-link-search=native=/usr/local/lib"); } + Ok(()) +} + +fn link_library() -> Result<(), Box> { if let Ok(lib_path) = std::env::var("CARGO_DRIFT_FFI_PATH") { - println!("{LIB}: searching for lib at: {lib_path}"); println!("cargo:rustc-link-search=native={lib_path}"); } println!("cargo:rustc-link-lib=dylib=drift_ffi_sys"); + Ok(()) } fn fail_build() -> ! { - eprintln!("{LIB} build failed"); + println!("cargo:warning={LIB} build failed"); std::process::exit(1); } diff --git a/crates/drift-ffi-sys b/crates/drift-ffi-sys index 9332730..14e4499 160000 --- a/crates/drift-ffi-sys +++ b/crates/drift-ffi-sys @@ -1 +1 @@ -Subproject commit 9332730fb3668d938ce3f2a9e9b62d692ffedb64 +Subproject commit 14e4499a346b2839bbcce759135395c990a81543 diff --git a/crates/drift-idl-gen/README.md b/crates/drift-idl-gen/README.md index 10b0a0f..a4e36a0 100644 --- a/crates/drift-idl-gen/README.md +++ b/crates/drift-idl-gen/README.md @@ -1,5 +1,7 @@ # drfit-idl-gen +⚠️ for drift-rs there's no need to run this manually. The `build.rs` script will trigger it. + Generates rust anchor structs from IDL json This is implemented rather than another project for a couple reasons: diff --git a/res/drift.json b/res/drift.json index 7c2d534..7ec66c7 100644 --- a/res/drift.json +++ b/res/drift.json @@ -12066,7 +12066,7 @@ }, { "name": "hash", - "type": "string", + "type": "String", "index": false }, {