Skip to content

Commit

Permalink
Tidy up build.rs
Browse files Browse the repository at this point in the history
Improve CI
Update drift-ffi-sys commit
  • Loading branch information
jordy25519 committed Dec 11, 2024
1 parent 343f298 commit dfd95ec
Show file tree
Hide file tree
Showing 7 changed files with 172 additions and 114 deletions.
16 changes: 15 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,21 @@ jobs:
runs-on: ubicloud
steps:
- name: Check out
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Cache Rust toolchain
uses: actions/cache@v3
with:
path: |
~/.rustup
~/.cargo/bin
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-rust-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-rust-
- name: Config rust toolchain
run: |
rustup show active-toolchain
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- name: Publish drift-rs
run: |
# add libdrift_ffi_sys
curl -L https://github.com/user-attachments/files/17160233/libdrift_ffi_sys.so.zip > ffi.zip
curl -L https://github.com/drift-labs/drift-ffi-sys/releases/download/v2.103.0/libdrift_ffi_sys.so.zip > ffi.zip
unzip ffi.zip
sudo mv libdrift_ffi_sys.so $CARGO_DRIFT_FFI_PATH
rm ffi.zip # clean up for git
Expand Down
11 changes: 5 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Experimental, high performance Rust SDK for building offchain clients for [Drift
drift-rs = "1.0.0-alpha.4"

# build from source (also builds and links 'libdrift_ffi_sys')
drift-rs = { git = "https://github.com/drift-labs/drift-rs", tag = "v1.0.0-alpha.4" }
drift-rs = { git = "https://github.com/drift-labs/drift-rs", tag = "v1.0.0-alpha.5" }
```

_*_`drift-rs` uses drift program over ffi.
Expand Down Expand Up @@ -63,17 +63,16 @@ rustup override set 1.81.0-x86_64-apple-darwin
⚠️ the default toolchain is incompatible due to memory layout differences between solana program (BPF) and aarch64 and will fail at runtime with deserialization errors like: `InvalidSize`.

## Local Development
drift-rs links to the drift program crate via FFI, build from source or optionally install from [drift-ffi-sys](https://github.com/drift-labs/drift-ffi-sys/releases)
drift-rs links to the drift program crate via FFI, build from source (default) or optionally install from [drift-ffi-sys](https://github.com/drift-labs/drift-ffi-sys/releases)
```bash
# Build from source
# Build from source (default)
CARGO_DRIFT_FFI_STATIC=1
# Provide a prebuilt drift_ffi_sys lib
CARGO_DRIFT_FFI_PATH="/path/to/libdrift_ffi_sys"
```
## Development

## Update IDL types
1) copy updated IDL to `res/drift.json` from protocol-v2 branch
## Updating IDL types
1) copy the updated IDL to `res/drift.json` from protocol-v2 branch
2) `cargo check`
3) commit changes

251 changes: 147 additions & 104 deletions build.rs
Original file line number Diff line number Diff line change
@@ -1,132 +1,175 @@
use std::{collections::HashMap, fs::File, io::Write, path::Path};
use std::{collections::HashMap, path::Path};

const LIB: &str = "libdrift_ffi_sys";
const SUPPORTED_PLATFORMS: &[(&str, &str, &str)] = &[
("apple", "x86_64-apple-darwin", "dylib"),
("linux", "x86_64-unknown-linux-gnu", "so"),
];
const FFI_TOOLCHAIN_VERSION: &str = "1.76.0";

fn main() -> Result<(), Box<dyn std::error::Error>> {
let current_dir = std::env::current_dir()?.canonicalize()?;

// Generate IDL types from 'res/drift.json'
let idl_source_path = current_dir.join("res/drift.json");
let idl_mod_path = current_dir.join("crates/src/drift_idl.rs");
generate_idl_types(&idl_source_path, idl_mod_path.as_path())?;

// Only build FFI lib if static or no lib path provided
if should_build_from_source() {
build_ffi_lib(&current_dir)?;
}

link_library()?;
Ok(())
}

fn generate_idl_types(idl_source_path: &Path, idl_mod_path: &Path) -> Result<(), Box<dyn std::error::Error>> {
let idl_mod_rs = drift_idl_gen::generate_rust_types(&idl_source_path)
.map_err(|err| format!("generating IDL failed: {err:?}"))?;

std::fs::write(&idl_mod_path, idl_mod_rs)?;
Ok(())
}

fn main() {
let current_dir = std::env::current_dir().unwrap().canonicalize().unwrap();

// Generate rust types from anchor IDL
let idl_source_path = &current_dir.join(Path::new("res/drift.json"));
let idl_mod_rs = match drift_idl_gen::generate_rust_types(idl_source_path) {
Ok(idl_mod_rs) => idl_mod_rs,
Err(err) => panic!("generating IDL failed: {err:?}"),
};
let idl_mod_path = current_dir.join(Path::new("crates/src/drift_idl.rs"));
let mut file = File::create(&idl_mod_path).expect("create IDL .rs");
file.write_all(idl_mod_rs.as_bytes())
.expect("wrote IDL .rs");

if std::env::var("CARGO_DRIFT_FFI_STATIC").is_ok()
fn should_build_from_source() -> bool {
std::env::var("CARGO_DRIFT_FFI_STATIC").is_ok()
|| std::env::var("CARGO_DRIFT_FFI_PATH").is_err()
{
// Build + Link FFI crate from source
println!("{LIB}: building from source...");
let drift_ffi_sys_crate = current_dir.join(Path::new("crates/drift-ffi-sys"));

// the x86_64 target must exist
let host_target = std::env::var("TARGET").unwrap();
let (lib_target, lib_ext) = if host_target.contains("apple") {
("x86_64-apple-darwin", "dylib")
} else if host_target.contains("linux") {
("x86_64-unknown-linux-gnu", "so")
} else {
eprintln!("Unsupported host platform: {host_target}, please open an issue at: https://github.com/drift-labs/drift-rs/issues");
fail_build();
};

// "RUSTC" is set as the cargo version of the main SDK build, it must be unset for the ffi build
// "CARGO*" envs are also configured for the main SDK build
// https://users.rust-lang.org/t/switching-toolchains-in-build-rs-use-nightly-for-data-generation-and-stable-for-main-compilation/114443/5
let ffi_build_envs: HashMap<String, String> = std::env::vars()
.filter(|(k, _v)| !k.starts_with("CARGO") && !k.starts_with("RUSTC"))
.collect();
println!("{ffi_build_envs:?}");
let profile = std::env::var("PROFILE").expect("cargo PROFILE set");

// force drift-ffi-sys to build with specific toolchain (arch=x86_64, version=<=1.76.0)
// this ensures zero copy deserialization works correctly with the onchain data layout
let ffi_toolchain = format!("1.76.0-{lib_target}");
let installed_toolchains_query = std::process::Command::new("rustup")
.args(["toolchain", "list"])
.output()
.expect("rustup installed");
if !installed_toolchains_query.status.success() {
println!("Check 'rustup' is installed and discoverable in system PATH");
}
let installed_toolchains = String::from_utf8_lossy(&installed_toolchains_query.stdout);
if !installed_toolchains.contains(&ffi_toolchain) {
eprintln!("Required toolchain: {ffi_toolchain} is missing. Run: 'rustup install {ffi_toolchain}' to install and retry the build");
fail_build();
}
}

// install the dylib to system path
let libffi_out_path =
drift_ffi_sys_crate.join(Path::new(&format!("target/{profile}/{LIB}.{lib_ext}")));

// Build ffi crate and link
let mut ffi_build = std::process::Command::new("rustup");
ffi_build
.env_clear()
.envs(ffi_build_envs)
.current_dir(drift_ffi_sys_crate.clone())
.args(["run", &ffi_toolchain, "cargo", "build"]);

match profile.as_str() {
"debug" => (),
"release" => {
ffi_build.arg("--release");
}
custom => {
ffi_build.arg(format!("--profile={custom}"));
}
}
fn build_ffi_lib(current_dir: &Path) -> Result<(), Box<dyn std::error::Error>> {
println!("cargo:warning={LIB}: building from source...");

let output = ffi_build.output().expect("drift-ffi-sys built");
if !output.status.success() {
eprintln!(" {}", String::from_utf8_lossy(output.stderr.as_slice()));
fail_build();
}
let host_target = std::env::var("TARGET")?;
let (lib_target, lib_ext) = get_platform_details(&host_target)?;

verify_toolchain(lib_target)?;

// Build the library
let profile = std::env::var("PROFILE")?;
let drift_ffi_sys_crate = current_dir.join("crates/drift-ffi-sys");

if !output.status.success() {
eprintln!(
"{LIB} could not be installed: {}",
String::from_utf8_lossy(output.stderr.as_slice())
);
build_with_toolchain(&drift_ffi_sys_crate, lib_target, &profile)?;
install_library(&drift_ffi_sys_crate, &profile, lib_ext)?;

Ok(())
}

fn get_platform_details(
host_target: &str,
) -> Result<(&'static str, &'static str), Box<dyn std::error::Error>> {
for (platform, target, ext) in SUPPORTED_PLATFORMS {
if host_target.contains(platform) {
return Ok((target, ext));
}
}

println!("cargo:warning=Unsupported host platform: {host_target}");
println!(
"cargo:warning=Please open an issue at: https://github.com/drift-labs/drift-rs/issues"
);
Err("Unsupported platform".into())
}

if let Ok(out_dir) = std::env::var("OUT_DIR") {
let _output = std::process::Command::new("cp")
.args([
libffi_out_path.to_str().expect("ffi build path"),
out_dir.as_str(),
])
.output()
.expect("install ok");
println!("{LIB}: searching for lib at: {out_dir}");
println!("cargo:rustc-link-search=native={out_dir}");
fn verify_toolchain(lib_target: &str) -> Result<(), Box<dyn std::error::Error>> {
let ffi_toolchain = format!("{FFI_TOOLCHAIN_VERSION}-{lib_target}");
let output = std::process::Command::new("rustup")
.args(["toolchain", "list"])
.output()?;

if !output.status.success() {
return Err("Failed to query rustup toolchains".into());
}

let installed_toolchains = String::from_utf8_lossy(&output.stdout);
if !installed_toolchains.contains(&ffi_toolchain) {
println!("cargo:warning=Required toolchain {ffi_toolchain} is missing");
println!("cargo:warning=Run: 'rustup install {ffi_toolchain}' to install");
return Err("Missing required toolchain".into());
}

Ok(())
}

fn build_with_toolchain(
drift_ffi_sys_crate: &Path,
lib_target: &str,
profile: &str,
) -> Result<(), Box<dyn std::error::Error>> {
// Filter out cargo and rustc environment variables
let ffi_build_envs: HashMap<String, String> = std::env::vars()
.filter(|(k, _v)| !k.starts_with("CARGO") && !k.starts_with("RUSTC"))
.collect();

let ffi_toolchain = format!("{FFI_TOOLCHAIN_VERSION}-{lib_target}");
let mut ffi_build = std::process::Command::new("rustup");
ffi_build
.env_clear()
.envs(ffi_build_envs)
.current_dir(drift_ffi_sys_crate)
.args(["run", &ffi_toolchain, "cargo", "build"]);

match profile {
"debug" => (),
"release" => {
ffi_build.arg("--release");
}
custom => {
ffi_build.arg(format!("--profile={custom}"));
}
}

let _output = std::process::Command::new("ln")
let output = ffi_build.output()?;
if !output.status.success() {
println!("cargo:warning={}", String::from_utf8_lossy(&output.stderr));
return Err("FFI build failed".into());
}

Ok(())
}

fn install_library(
drift_ffi_sys_crate: &Path,
profile: &str,
lib_ext: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let libffi_out_path = drift_ffi_sys_crate.join(format!("target/{profile}/{LIB}.{lib_ext}"));

if let Ok(out_dir) = std::env::var("OUT_DIR") {
std::process::Command::new("cp")
.args([
libffi_out_path.to_str().ok_or("Invalid path")?,
out_dir.as_str(),
])
.output()?;
println!("cargo:warning={LIB}: searching for lib at: {out_dir}");
println!("cargo:rustc-link-search=native={out_dir}");
} else {
// Install to system library path
std::process::Command::new("ln")
.args([
"-sf",
libffi_out_path.to_str().expect("ffi build path"),
libffi_out_path.to_str().ok_or("Invalid path")?,
"/usr/local/lib/",
])
.output()
.expect("install ok");
.output()?;

println!("{LIB}: searching for lib at: /usr/local/lib");
println!("cargo:warning={LIB}: searching for lib at: /usr/local/lib");
println!("cargo:rustc-link-search=native=/usr/local/lib");
}

Ok(())
}

fn link_library() -> Result<(), Box<dyn std::error::Error>> {
if let Ok(lib_path) = std::env::var("CARGO_DRIFT_FFI_PATH") {
println!("{LIB}: searching for lib at: {lib_path}");
println!("cargo:rustc-link-search=native={lib_path}");
}
println!("cargo:rustc-link-lib=dylib=drift_ffi_sys");
Ok(())
}

fn fail_build() -> ! {
eprintln!("{LIB} build failed");
println!("cargo:warning={LIB} build failed");
std::process::exit(1);
}
2 changes: 1 addition & 1 deletion crates/drift-ffi-sys
Submodule drift-ffi-sys updated 2 files
+7 −6 Cargo.lock
+2 −2 Cargo.toml
2 changes: 2 additions & 0 deletions crates/drift-idl-gen/README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# drfit-idl-gen

⚠️ for drift-rs there's no need to run this manually. The `build.rs` script will trigger it.

Generates rust anchor structs from IDL json
This is implemented rather than another project for a couple reasons:

Expand Down
2 changes: 1 addition & 1 deletion res/drift.json
Original file line number Diff line number Diff line change
Expand Up @@ -12066,7 +12066,7 @@
},
{
"name": "hash",
"type": "string",
"type": "String",
"index": false
},
{
Expand Down

0 comments on commit dfd95ec

Please sign in to comment.