From 9b320f8aecce30b4073b37eb317757391326cd30 Mon Sep 17 00:00:00 2001 From: DemosChiang <1239865849@qq.com> Date: Mon, 20 May 2024 14:22:59 +0800 Subject: [PATCH 1/2] refactor:key handover --- Cargo.lock | 8 + Cargo.toml | 1 + scripts/docker/ceseal/gramine/handover.ts | 4 +- .../docker/ceseal/gramine/handover/Cargo.toml | 10 + .../docker/ceseal/gramine/handover/src/arg.rs | 51 ++++ .../ceseal/gramine/handover/src/error.rs | 22 ++ .../ceseal/gramine/handover/src/main.rs | 281 ++++++++++++++++++ 7 files changed, 375 insertions(+), 2 deletions(-) create mode 100644 scripts/docker/ceseal/gramine/handover/Cargo.toml create mode 100644 scripts/docker/ceseal/gramine/handover/src/arg.rs create mode 100644 scripts/docker/ceseal/gramine/handover/src/error.rs create mode 100644 scripts/docker/ceseal/gramine/handover/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 0e780890..29efa512 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4836,6 +4836,14 @@ dependencies = [ "thiserror", ] +[[package]] +name = "handover" +version = "0.1.0" +dependencies = [ + "clap", + "tokio", +] + [[package]] name = "hash-db" version = "0.15.2" diff --git a/Cargo.toml b/Cargo.toml index 7c6373e7..e10142f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "pallets/mq/runtime-api", "standalone/chain/*", "standalone/teeworker/cifrost", + "scripts/docker/ceseal/gramine/handover", ] resolver = "2" diff --git a/scripts/docker/ceseal/gramine/handover.ts b/scripts/docker/ceseal/gramine/handover.ts index ee8cb32c..b76dd88c 100644 --- a/scripts/docker/ceseal/gramine/handover.ts +++ b/scripts/docker/ceseal/gramine/handover.ts @@ -126,7 +126,7 @@ if (await exists(path.join(currentPath, "data/protected_files/runtime-data.seal" } let previousVersion: number | undefined = await confirmPreviousVersion(); - +//Otherwise, confirm whether a previous version exists. If there is no previous version, no handover is required, back up the current version to the backup directory and exit. if (previousVersion === undefined) { log("No previous version, no need to handover!"); @@ -135,7 +135,7 @@ if (previousVersion === undefined) { Deno.exit(0); } - +//If the current version is the same as the previous version, there is no need to hand over and exit directly. if (currentVersion == previousVersion) { log("same version, no need to handover") Deno.exit(0); diff --git a/scripts/docker/ceseal/gramine/handover/Cargo.toml b/scripts/docker/ceseal/gramine/handover/Cargo.toml new file mode 100644 index 00000000..efa82d49 --- /dev/null +++ b/scripts/docker/ceseal/gramine/handover/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "handover" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = { workspace = true, features = ["derive"] } +tokio = { workspace = true, features = ["full"] } diff --git a/scripts/docker/ceseal/gramine/handover/src/arg.rs b/scripts/docker/ceseal/gramine/handover/src/arg.rs new file mode 100644 index 00000000..ce7cdeb6 --- /dev/null +++ b/scripts/docker/ceseal/gramine/handover/src/arg.rs @@ -0,0 +1,51 @@ +use clap::Parser; + +#[derive(Parser, Debug)] +#[command( + about = "xxx", + version, + author +)] +pub struct Args { + #[arg( + long, + help = "The backup path of the each version of ceseal", + default_value = "/opt/ceseal/backups" + )] + pub previous_version_ceseal_path: String, + + #[arg( + long, + help = "The backup path of the current version of ceseal", + default_value = "/opt/ceseal/releases/current" + )] + pub current_version_ceseal_path: String, + + #[arg( + long, + help = "ceseal log path for detect the status of ceseal", + default_value = "/tmp/ceseal.log" + )] + pub ceseal_log_path: String, + + #[arg( + long, + help = "the relative path where each version of ceseal stores sealed runtime_data", + default_value = "data/protected_files/runtime-data.seal" + )] + pub ceseal_runtime_data_seal_path: String, + + #[arg( + long, + help = "the relative path where each version of ceseal stores checkpoint file", + default_value = "data/storage_files" + )] + pub ceseal_storage_files_path: String, + + #[arg( + long, + help = "old ceseal start on this port", + default_value = "1888" + )] + pub previous_ceseal_port: String, +} \ No newline at end of file diff --git a/scripts/docker/ceseal/gramine/handover/src/error.rs b/scripts/docker/ceseal/gramine/handover/src/error.rs new file mode 100644 index 00000000..83f73198 --- /dev/null +++ b/scripts/docker/ceseal/gramine/handover/src/error.rs @@ -0,0 +1,22 @@ +use std::{error, fmt}; + +#[derive(Debug)] +pub enum Error { + StartCesealFailed(String), + RedirectCesealLogFailed(String), + DetectCesealRunningStatueFailed(String), + PreviousVersionFailed(String) +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::StartCesealFailed(e) => write!(f, "{:?}", e), + Error::RedirectCesealLogFailed(e) => write!(f, "{:?}", e), + Error::DetectCesealRunningStatueFailed(e) => write!(f, "{:?}", e), + Error::PreviousVersionFailed(e) => write!(f, "{:?}", e), + } + } +} + +impl error::Error for Error {} diff --git a/scripts/docker/ceseal/gramine/handover/src/main.rs b/scripts/docker/ceseal/gramine/handover/src/main.rs new file mode 100644 index 00000000..6f0898f7 --- /dev/null +++ b/scripts/docker/ceseal/gramine/handover/src/main.rs @@ -0,0 +1,281 @@ +mod arg; +mod error; +use arg::Args; +use clap::Parser; +use core::panic; +use error::Error; +use std::{path::Path, process::Stdio}; +use tokio::{ + io::{AsyncBufReadExt, BufReader, BufWriter}, + process::{Child, ChildStdout, Command}, +}; + +#[tokio::main] +async fn main() { + let args = Args::parse(); + + let mut current_version: u64 = 0; + let mut previous_version: u64 = 0; + match tokio::fs::read_link(&args.current_version_ceseal_path).await { + Ok(real_path) => + if let Some(path_str) = real_path.to_str() { + current_version = path_str + .split("/") + .last() + .expect("no last version number") + .parse::() + .expect("parse current ceseal version from str to u64 failed!"); + } else { + panic!("can't get real path of current ceseal"); + }, + Err(e) => panic!("Error reading symlink {}: {}", args.current_version_ceseal_path, e), + } + log(format!("Current version: {}", current_version)); + + // Get the path to the current Ceseal version and check whether it has been initialized. + let current_ceseal_runtime_data_path = + Path::new(&args.current_version_ceseal_path).join(&args.ceseal_runtime_data_seal_path); + if current_ceseal_runtime_data_path.exists() { + log(format!("runtime-data.seal exists, no need to handover")); + return + } + + let current_ceseal_backup_path = Path::new(&args.previous_version_ceseal_path); + match confirm_previous_ceseal_version( + args.previous_version_ceseal_path.clone(), + args.ceseal_runtime_data_seal_path.clone(), + current_version, + ) { + Ok(ver) => { + // Otherwise, confirm whether a previous version exists. If there is no previous version, no handover is + // required, back up the current version to the backup directory and exit. + if ver == 0 { + log(format!("No previous version, no need to handover!")); + + let current_ceseal_backup_path = current_ceseal_backup_path.join(current_version.to_string()); + if let Err(err) = tokio::fs::copy(args.current_version_ceseal_path, current_ceseal_backup_path).await { + panic!("Error backing up current version: {}", err); + } + return + } + //If the current version is the same as the previous version, there is no need to hand over and exit + // directly. + if ver == current_version { + log(format!("same version, no need to handover")); + return + } + previous_version = ver; + }, + Err(e) => { + panic!("confirm_previous_ceseal_version error :{:?}", e); + }, + }; + + let previous_ceseal_path = Path::new(&args.previous_version_ceseal_path).join(previous_version.to_string()); + log(format!("Previous ${previous_version}")); + let current_ceseal_storage_path = + Path::new(&args.current_version_ceseal_path).join(&args.ceseal_storage_files_path); + let previous_ceseal_storage_path = previous_ceseal_path.join(&args.ceseal_storage_files_path); + + tokio::fs::remove_file(&args.ceseal_log_path) + .await + .expect("remove old ceseal log file fail"); + + //start old ceseal + let mut old_process = start_previous_ceseal(previous_ceseal_path.to_str().unwrap().to_string(), args.previous_ceseal_port.clone()) + .await + .expect("start previous ceseal fail"); + redirect_ceseal_runtime_log(old_process.stdout.take().expect("previous ceseal log output in invaid!"), args.ceseal_log_path.clone()) + .await + .expect("redirect ceseal runtime log fail"); + + //wait for old ceseal went well + wait_for_ceseal_to_run_successfully(args.ceseal_log_path.clone()) + .await + .expect("wait for ceseal log fail"); + log(format!("previous ceseal started!")); + + ensure_data_dir(current_ceseal_storage_path.parent().unwrap().to_str().unwrap()) + .expect("ensure current data dir fail"); + + + //start current version ceseal + let command = Command::new("/opt/ceseal/releases/current/gramine-sgx") + .args(&["ceseal", &format!("--request-handover-from=http://localhost:{:?}",args.previous_ceseal_port.clone())]) + .current_dir(&args.current_version_ceseal_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); + match command { + Ok(child) => { + let output = child.wait_with_output().await.expect("Failed to wait for command to execute"); + let code = output.status.code().unwrap_or(-1); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + log(format!("Exit code: {}", code)); + log(format!("Stdout:\n{}", stdout)); + log(format!("Stderr:\n{}", stderr)); + + if code != 0 { + log("Handover failed".to_string()); + return + } + }, + Err(e) => panic!("Error executing current ceseal command: {}", e), + } + old_process.kill().await.expect("old ceseal stop fail"); + + match tokio::fs::remove_dir_all(¤t_ceseal_storage_path).await { + Ok(_) => log("Removed previous storage successfully.".to_string()), + Err(e) => eprintln!("Error removing previous storage: {}", e), + } + match tokio::fs::copy(&previous_ceseal_storage_path, ¤t_ceseal_storage_path).await { + Ok(_) => log("Copied checkpoint from previous successfully.".to_string()), + Err(e) => eprintln!("Error copying checkpoint from previous: {}", e), + } + + // 备份当前版本到备份目录 + match tokio::fs::copy(&args.current_version_ceseal_path, ¤t_ceseal_backup_path).await { + Ok(_) => log(format!("Backed up current version successfully to {:?}", current_ceseal_backup_path)), + Err(e) => eprintln!("Error backing up current version: {}", e), + } + +} + +pub async fn start_previous_ceseal(previous_ceseal_path: String, port: String) -> Result { + let extra_args: &[&str] = &vec!["--port", &port]; + + let mut cmd = Command::new(Path::new(&previous_ceseal_path).join("start.sh")); + cmd.stdin(Stdio::piped()) + .stdout(Stdio::piped()) + // .stderr(Stdio::piped()) + .env("SKIP_AESMD", "1") + .env("EXTRA_OPTS", extra_args.join(" ")); + + let child = cmd.spawn().map_err(|e| Error::StartCesealFailed(e.to_string()))?; + + Ok(child) +} + +pub async fn redirect_ceseal_runtime_log(stdout: ChildStdout, log_path: String) -> Result<(), Error> { + //redirect process log into new created log file + let log_path = Path::new(&log_path); + let log_file = tokio::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(log_path) + .await + .map_err(|e| Error::RedirectCesealLogFailed(e.to_string()))?; + let mut log_writer = BufWriter::new(log_file); + + let mut reader = BufReader::new(stdout); + + tokio::spawn(async move { + tokio::io::copy(&mut reader, &mut log_writer) + .await + .expect("Error piping stdout to log file"); + }); + Ok(()) +} + +pub async fn wait_for_ceseal_to_run_successfully(log_path: String) -> Result<(), Error> { + let log_file = tokio::fs::File::open(log_path) + .await + .map_err(|e| Error::DetectCesealRunningStatueFailed(e.to_string()))?; + let mut reader = BufReader::new(log_file); + let mut line = String::new(); + loop { + match reader.read_line(&mut line).await { + Ok(bytes_read) if bytes_read > 0 => { + log(format!("ceseal log is :{}", line)); + + if line.contains("Ceseal internal server will listening on") { + return Ok(()) + } + line.clear(); + }, + Ok(_) => break, + Err(err) => return Err(Error::DetectCesealRunningStatueFailed(err.to_string())), + } + } + Err(Error::DetectCesealRunningStatueFailed("no log of normal startup of ceseal was detected".to_string())) +} + +pub fn confirm_previous_ceseal_version( + previous_version_ceseal_path: String, + ceseal_runtime_data_seal_path: String, + current_version: u64, +) -> Result { + let entries = + std::fs::read_dir(&previous_version_ceseal_path).map_err(|e| Error::PreviousVersionFailed(e.to_string()))?; + + let mut versiont_list: Vec = Vec::new(); + for entry in entries { + if let Ok(entry) = entry { + let path = entry.path(); + if path.is_file() { + if let Some(file_name) = path.file_name() { + versiont_list.push( + file_name + .to_str() + .ok_or(Error::PreviousVersionFailed(format!( + "error file appears in the path {:?}", + &previous_version_ceseal_path + )))? + .to_string() + .parse::() + .map_err(|e| Error::PreviousVersionFailed(e.to_string()))?, + ) + } + } + } + } + versiont_list.sort_by(|a, b| b.cmp(a)); + + let mut previous_version = 0; + for version in versiont_list { + if previous_version == 0 || previous_version < version { + if version >= current_version { + continue + } else if !Path::new(&previous_version_ceseal_path) + .join(&version.to_string()) + .join(&ceseal_runtime_data_seal_path) + .exists() + { + log(format!("no runtime-data.seal found in ${version}, skip")); + continue + } + previous_version = version; + break + } + } + + Ok(previous_version) +} + +fn ensure_data_dir(data_dir: &str) -> Result<(), std::io::Error> { + if !Path::new(data_dir).exists() { + std::fs::create_dir_all(data_dir)?; + } + + // Create the protected_files subdirectory if it does not exist + let protected_files_dir = Path::new(data_dir).join("protected_files"); + if !protected_files_dir.exists() { + std::fs::create_dir_all(&protected_files_dir)?; + } + + // Create the storage_files subdirectory if it does not exist + let storage_files_dir = Path::new(data_dir).join("storage_files"); + if !storage_files_dir.exists() { + std::fs::create_dir_all(&storage_files_dir)?; + } + + Ok(()) +} + +const LOG_PREFIX: &str = "[Handover🤝]"; +fn log(log_text: String) { + println!("{:?} {:?}", LOG_PREFIX, log_text) +} From 9891f45fb8133cd80e365351cabd216b2d9936c9 Mon Sep 17 00:00:00 2001 From: Demos Chiang <1239865849@qq.com> Date: Thu, 23 May 2024 12:48:27 +0000 Subject: [PATCH 2/2] fix:Fix handover errors and correct logic --- Cargo.lock | 5 +- Cargo.toml | 1 + crates/cestory/src/ceseal_service.rs | 5 +- .../docker/ceseal/gramine/handover/Cargo.toml | 1 + .../docker/ceseal/gramine/handover/src/arg.rs | 28 +- .../ceseal/gramine/handover/src/error.rs | 4 +- .../ceseal/gramine/handover/src/main.rs | 360 ++++++++++++------ 7 files changed, 283 insertions(+), 121 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29efa512..cf789e49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4842,6 +4842,7 @@ version = "0.1.0" dependencies = [ "clap", "tokio", + "walkdir", ] [[package]] @@ -14057,9 +14058,9 @@ checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", diff --git a/Cargo.toml b/Cargo.toml index e10142f7..63afaa2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -408,6 +408,7 @@ webpki = { git = "https://github.com/rustls/webpki", version = "=0.102.0-alpha.3 "alloc", "ring", ], rev = "2ed9a4324f48c2c46ffdd7dc9d3eb315af25fce2" } # Release version no-std has bug +walkdir = "2.5.0" # webpki = { version = "0.102.0", package = "rustls-webpki", default-features = false, features = ["alloc", "ring"] } # ---- Generic crates end ---- diff --git a/crates/cestory/src/ceseal_service.rs b/crates/cestory/src/ceseal_service.rs index 73ef5d1a..5d28abbf 100644 --- a/crates/cestory/src/ceseal_service.rs +++ b/crates/cestory/src/ceseal_service.rs @@ -367,8 +367,8 @@ impl CesealApi for RpcSe .get_ceseal_bin_added_at(&runtime_hash) .ok_or_else(|| from_display("Client ceseal not allowed on chain"))?; - if my_runtime_timestamp >= req_runtime_timestamp { - return Err(Status::internal("No handover for old ceseal")) + if my_runtime_timestamp >= req_runtime_timestamp { + return Err(Status::internal("Same ceseal version or rollback ,No local handover provided")) } } else { info!("Skip ceseal timestamp check in dev mode"); @@ -684,6 +684,7 @@ impl Ceseal { Ok(pb::SyncedTo { synced_to: last_block }) } + //Check whether checkpoint file is used and save it regularly fn maybe_take_checkpoint(&mut self) -> anyhow::Result<()> { if !self.args.enable_checkpoint { return Ok(()) diff --git a/scripts/docker/ceseal/gramine/handover/Cargo.toml b/scripts/docker/ceseal/gramine/handover/Cargo.toml index efa82d49..d020eec4 100644 --- a/scripts/docker/ceseal/gramine/handover/Cargo.toml +++ b/scripts/docker/ceseal/gramine/handover/Cargo.toml @@ -8,3 +8,4 @@ edition = "2021" [dependencies] clap = { workspace = true, features = ["derive"] } tokio = { workspace = true, features = ["full"] } +walkdir = { workspace = true } \ No newline at end of file diff --git a/scripts/docker/ceseal/gramine/handover/src/arg.rs b/scripts/docker/ceseal/gramine/handover/src/arg.rs index ce7cdeb6..4d5d9879 100644 --- a/scripts/docker/ceseal/gramine/handover/src/arg.rs +++ b/scripts/docker/ceseal/gramine/handover/src/arg.rs @@ -23,17 +23,31 @@ pub struct Args { #[arg( long, - help = "ceseal log path for detect the status of ceseal", - default_value = "/tmp/ceseal.log" + help = "ceseal home", + default_value = "/opt/ceseal/data" )] - pub ceseal_log_path: String, + pub ceseal_data_path: String, #[arg( long, - help = "the relative path where each version of ceseal stores sealed runtime_data", - default_value = "data/protected_files/runtime-data.seal" + help = "Ceseal log path for detect the status of previous ceseal", + default_value = "/tmp/pre_ceseal.log" )] - pub ceseal_runtime_data_seal_path: String, + pub previous_ceseal_log_path: String, + + #[arg( + long, + help = "Ceseal log path for detect the status of new ceseal", + default_value = "/tmp/new_ceseal.log" + )] + pub new_ceseal_log_path: String, + + #[arg( + long, + help = "The relative path where each version of ceseal stores protected files", + default_value = "data/protected_files" + )] + pub ceseal_protected_files_path: String, #[arg( long, @@ -47,5 +61,5 @@ pub struct Args { help = "old ceseal start on this port", default_value = "1888" )] - pub previous_ceseal_port: String, + pub previous_ceseal_port: u64, } \ No newline at end of file diff --git a/scripts/docker/ceseal/gramine/handover/src/error.rs b/scripts/docker/ceseal/gramine/handover/src/error.rs index 83f73198..5976f191 100644 --- a/scripts/docker/ceseal/gramine/handover/src/error.rs +++ b/scripts/docker/ceseal/gramine/handover/src/error.rs @@ -5,7 +5,8 @@ pub enum Error { StartCesealFailed(String), RedirectCesealLogFailed(String), DetectCesealRunningStatueFailed(String), - PreviousVersionFailed(String) + PreviousVersionFailed(String), + CopyDirectory(String) } impl fmt::Display for Error { @@ -15,6 +16,7 @@ impl fmt::Display for Error { Error::RedirectCesealLogFailed(e) => write!(f, "{:?}", e), Error::DetectCesealRunningStatueFailed(e) => write!(f, "{:?}", e), Error::PreviousVersionFailed(e) => write!(f, "{:?}", e), + Error::CopyDirectory(e) => write!(f, "{:?}", e), } } } diff --git a/scripts/docker/ceseal/gramine/handover/src/main.rs b/scripts/docker/ceseal/gramine/handover/src/main.rs index 6f0898f7..9e862c6d 100644 --- a/scripts/docker/ceseal/gramine/handover/src/main.rs +++ b/scripts/docker/ceseal/gramine/handover/src/main.rs @@ -13,7 +13,6 @@ use tokio::{ #[tokio::main] async fn main() { let args = Args::parse(); - let mut current_version: u64 = 0; let mut previous_version: u64 = 0; match tokio::fs::read_link(&args.current_version_ceseal_path).await { @@ -33,29 +32,35 @@ async fn main() { log(format!("Current version: {}", current_version)); // Get the path to the current Ceseal version and check whether it has been initialized. - let current_ceseal_runtime_data_path = - Path::new(&args.current_version_ceseal_path).join(&args.ceseal_runtime_data_seal_path); + let current_ceseal_runtime_data_path = Path::new(&args.current_version_ceseal_path) + .join(&args.ceseal_protected_files_path) + .join("runtime-data.seal"); if current_ceseal_runtime_data_path.exists() { log(format!("runtime-data.seal exists, no need to handover")); return } - let current_ceseal_backup_path = Path::new(&args.previous_version_ceseal_path); + let current_ceseal_backup_path = Path::new(&args.previous_version_ceseal_path).join(current_version.to_string()); match confirm_previous_ceseal_version( args.previous_version_ceseal_path.clone(), - args.ceseal_runtime_data_seal_path.clone(), + args.ceseal_protected_files_path.clone(), + args.ceseal_data_path.clone(), current_version, - ) { + ) + .await + { Ok(ver) => { - // Otherwise, confirm whether a previous version exists. If there is no previous version, no handover is - // required, back up the current version to the backup directory and exit. - if ver == 0 { - log(format!("No previous version, no need to handover!")); - - let current_ceseal_backup_path = current_ceseal_backup_path.join(current_version.to_string()); - if let Err(err) = tokio::fs::copy(args.current_version_ceseal_path, current_ceseal_backup_path).await { + // Anyway, back up the current version + if !current_ceseal_backup_path.exists() { + if let Err(err) = + copy_directory(Path::new(&args.current_version_ceseal_path), ¤t_ceseal_backup_path).await + { panic!("Error backing up current version: {}", err); } + } + + if ver == 0 { + log(format!("No previous version, no need to handover!")); return } //If the current version is the same as the previous version, there is no need to hand over and exit @@ -73,74 +78,77 @@ async fn main() { let previous_ceseal_path = Path::new(&args.previous_version_ceseal_path).join(previous_version.to_string()); log(format!("Previous ${previous_version}")); - let current_ceseal_storage_path = - Path::new(&args.current_version_ceseal_path).join(&args.ceseal_storage_files_path); - let previous_ceseal_storage_path = previous_ceseal_path.join(&args.ceseal_storage_files_path); - tokio::fs::remove_file(&args.ceseal_log_path) - .await - .expect("remove old ceseal log file fail"); + if let Err(err) = tokio::fs::remove_file(&args.previous_ceseal_log_path).await { + log(format!("remove old ceseal log file fail : {err}")) + }; + if let Err(err) = tokio::fs::remove_file(&args.new_ceseal_log_path).await { + log(format!("remove new ceseal log file fail : {err}")) + }; //start old ceseal - let mut old_process = start_previous_ceseal(previous_ceseal_path.to_str().unwrap().to_string(), args.previous_ceseal_port.clone()) - .await - .expect("start previous ceseal fail"); - redirect_ceseal_runtime_log(old_process.stdout.take().expect("previous ceseal log output in invaid!"), args.ceseal_log_path.clone()) - .await - .expect("redirect ceseal runtime log fail"); - - //wait for old ceseal went well - wait_for_ceseal_to_run_successfully(args.ceseal_log_path.clone()) - .await - .expect("wait for ceseal log fail"); + let mut old_process = start_previous_ceseal( + previous_ceseal_path.to_str().unwrap().to_string(), + args.previous_ceseal_port.to_string(), + ) + .await + .expect("start previous ceseal fail"); + redirect_ceseal_runtime_log( + old_process.stdout.take().expect("previous ceseal log output in invaid!"), + args.previous_ceseal_log_path.clone(), + ) + .await + .expect("redirect ceseal runtime log fail"); + + //wait for preivious ceseal went well + wait_for_ceseal_to_run_successfully( + args.previous_ceseal_log_path.clone(), + "Ceseal internal server will listening on", + ) + .await + .expect("wait for previous ceseal log fail"); log(format!("previous ceseal started!")); + let current_ceseal_storage_path = + Path::new(&args.current_version_ceseal_path).join(&args.ceseal_storage_files_path); + let previous_ceseal_storage_path = previous_ceseal_path.join(&args.ceseal_storage_files_path); ensure_data_dir(current_ceseal_storage_path.parent().unwrap().to_str().unwrap()) + .await .expect("ensure current data dir fail"); - //start current version ceseal let command = Command::new("/opt/ceseal/releases/current/gramine-sgx") - .args(&["ceseal", &format!("--request-handover-from=http://localhost:{:?}",args.previous_ceseal_port.clone())]) - .current_dir(&args.current_version_ceseal_path) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn(); + .args(&["ceseal", &format!("--request-handover-from=http://localhost:{}", args.previous_ceseal_port)]) + .current_dir(&args.current_version_ceseal_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); match command { - Ok(child) => { - let output = child.wait_with_output().await.expect("Failed to wait for command to execute"); - let code = output.status.code().unwrap_or(-1); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - - log(format!("Exit code: {}", code)); - log(format!("Stdout:\n{}", stdout)); - log(format!("Stderr:\n{}", stderr)); - - if code != 0 { - log("Handover failed".to_string()); - return - } - }, - Err(e) => panic!("Error executing current ceseal command: {}", e), - } + Ok(mut child) => { + redirect_ceseal_runtime_log( + child.stdout.take().expect("new ceseal log output in invaid!"), + args.new_ceseal_log_path.clone(), + ) + .await + .expect("redirect ceseal runtime log fail"); + wait_for_ceseal_to_run_successfully(args.new_ceseal_log_path, "Handover done") + .await + .expect("wait for new ceseal log fail"); + log(format!("handover success!")); + }, + Err(e) => panic!("Error executing current ceseal command: {}", e), + } old_process.kill().await.expect("old ceseal stop fail"); + kill_previous_ceseal(previous_version).await; match tokio::fs::remove_dir_all(¤t_ceseal_storage_path).await { - Ok(_) => log("Removed previous storage successfully.".to_string()), - Err(e) => eprintln!("Error removing previous storage: {}", e), - } - match tokio::fs::copy(&previous_ceseal_storage_path, ¤t_ceseal_storage_path).await { - Ok(_) => log("Copied checkpoint from previous successfully.".to_string()), - Err(e) => eprintln!("Error copying checkpoint from previous: {}", e), - } - - // 备份当前版本到备份目录 - match tokio::fs::copy(&args.current_version_ceseal_path, ¤t_ceseal_backup_path).await { - Ok(_) => log(format!("Backed up current version successfully to {:?}", current_ceseal_backup_path)), - Err(e) => eprintln!("Error backing up current version: {}", e), - } - + Ok(_) => log("Removed current storage successfully.".to_string()), + Err(e) => eprintln!("Error removing previous storage: {}", e), + } + match copy_directory(&previous_ceseal_storage_path, ¤t_ceseal_storage_path).await { + Ok(_) => log("Copied checkpoint from previous successfully.".to_string()), + Err(e) => panic!("Error copying checkpoint from previous: {}", e), + } } pub async fn start_previous_ceseal(previous_ceseal_path: String, port: String) -> Result { @@ -180,8 +188,10 @@ pub async fn redirect_ceseal_runtime_log(stdout: ChildStdout, log_path: String) Ok(()) } -pub async fn wait_for_ceseal_to_run_successfully(log_path: String) -> Result<(), Error> { - let log_file = tokio::fs::File::open(log_path) +pub async fn wait_for_ceseal_to_run_successfully(log_path: String, flag: &str) -> Result<(), Error> { + let sleep_for_ceseal_running = tokio::time::Duration::from_secs(5); + let mut sleep_times = 3; + let log_file = tokio::fs::File::open(&log_path) .await .map_err(|e| Error::DetectCesealRunningStatueFailed(e.to_string()))?; let mut reader = BufReader::new(log_file); @@ -189,46 +199,60 @@ pub async fn wait_for_ceseal_to_run_successfully(log_path: String) -> Result<(), loop { match reader.read_line(&mut line).await { Ok(bytes_read) if bytes_read > 0 => { - log(format!("ceseal log is :{}", line)); + log(format!("{}:{}", &log_path, line)); - if line.contains("Ceseal internal server will listening on") { + if line.contains(flag) { return Ok(()) } line.clear(); }, - Ok(_) => break, + Ok(_) => { + if sleep_times > 0 { + tokio::time::sleep(sleep_for_ceseal_running).await; + sleep_times -= 1; + continue + } + return Err(Error::DetectCesealRunningStatueFailed("ceseal log has no content".to_string())) + }, Err(err) => return Err(Error::DetectCesealRunningStatueFailed(err.to_string())), } } - Err(Error::DetectCesealRunningStatueFailed("no log of normal startup of ceseal was detected".to_string())) } -pub fn confirm_previous_ceseal_version( +pub async fn confirm_previous_ceseal_version( previous_version_ceseal_path: String, - ceseal_runtime_data_seal_path: String, + ceseal_protected_files_path: String, + ceseal_data_path: String, current_version: u64, ) -> Result { - let entries = - std::fs::read_dir(&previous_version_ceseal_path).map_err(|e| Error::PreviousVersionFailed(e.to_string()))?; - + if !Path::new(&previous_version_ceseal_path).exists() { + tokio::fs::create_dir_all(&previous_version_ceseal_path) + .await + .map_err(|e| Error::PreviousVersionFailed(e.to_string()))?; + } + let mut entries = tokio::fs::read_dir(&previous_version_ceseal_path) + .await + .map_err(|e| Error::PreviousVersionFailed(e.to_string()))?; let mut versiont_list: Vec = Vec::new(); - for entry in entries { - if let Ok(entry) = entry { - let path = entry.path(); - if path.is_file() { - if let Some(file_name) = path.file_name() { - versiont_list.push( - file_name - .to_str() - .ok_or(Error::PreviousVersionFailed(format!( - "error file appears in the path {:?}", - &previous_version_ceseal_path - )))? - .to_string() - .parse::() - .map_err(|e| Error::PreviousVersionFailed(e.to_string()))?, - ) - } + while let Some(entry) = entries + .next_entry() + .await + .map_err(|e| Error::PreviousVersionFailed(e.to_string()))? + { + let path = entry.path(); + if path.is_dir() { + if let Some(file_name) = path.file_name() { + versiont_list.push( + file_name + .to_str() + .ok_or(Error::PreviousVersionFailed(format!( + "error file appears in the path {:?}", + &previous_version_ceseal_path + )))? + .to_string() + .parse::() + .map_err(|e| Error::PreviousVersionFailed(e.to_string()))?, + ) } } } @@ -239,43 +263,161 @@ pub fn confirm_previous_ceseal_version( if previous_version == 0 || previous_version < version { if version >= current_version { continue - } else if !Path::new(&previous_version_ceseal_path) - .join(&version.to_string()) - .join(&ceseal_runtime_data_seal_path) - .exists() - { - log(format!("no runtime-data.seal found in ${version}, skip")); - continue } previous_version = version; break } } + //Everytime publish next version of ceseal will be detect overhere + //Let's backup the previous runtime-data.seal file into 'backups ceseal's data' first + if previous_version != 0 { + let previous_runtime_data = Path::new(&previous_version_ceseal_path) + .join(&previous_version.to_string()) + .join(&ceseal_protected_files_path) + .join("runtime-data.seal"); + if !previous_runtime_data.exists() { + log(format!("no runtime-data.seal found in version {previous_version}, sync from {ceseal_data_path} now")); + //Since the runtimedata generated by ceseal was not saved to the '/opt/ceseal/backups' path after exiting + // when it was started for the first time, it needs to be synchronized here to avoid data loss from data to + // backup. + copy_directory( + &Path::new(&ceseal_data_path).join(&previous_version.to_string()), + &Path::new(&previous_version_ceseal_path) + .join(&previous_version.to_string()) + .join( + &ceseal_protected_files_path + .split("/") + .next() + .unwrap() + .parse::() + .unwrap(), + ), + ) + .await?; + } + } Ok(previous_version) } -fn ensure_data_dir(data_dir: &str) -> Result<(), std::io::Error> { +async fn ensure_data_dir(data_dir: &str) -> Result<(), std::io::Error> { if !Path::new(data_dir).exists() { - std::fs::create_dir_all(data_dir)?; + tokio::fs::create_dir_all(data_dir).await?; } // Create the protected_files subdirectory if it does not exist let protected_files_dir = Path::new(data_dir).join("protected_files"); if !protected_files_dir.exists() { - std::fs::create_dir_all(&protected_files_dir)?; + tokio::fs::create_dir_all(&protected_files_dir).await?; + log("create protected file for current ceseal...".to_string()) } - // Create the storage_files subdirectory if it does not exist let storage_files_dir = Path::new(data_dir).join("storage_files"); if !storage_files_dir.exists() { - std::fs::create_dir_all(&storage_files_dir)?; + tokio::fs::create_dir_all(&storage_files_dir).await?; + log("create storage file for current ceseal...".to_string()) } - Ok(()) } const LOG_PREFIX: &str = "[Handover🤝]"; fn log(log_text: String) { - println!("{:?} {:?}", LOG_PREFIX, log_text) + println!("{} {}", LOG_PREFIX, log_text) +} + +use walkdir::WalkDir; +async fn copy_directory(source: &Path, destination: &Path) -> Result<(), Error> { + let mut tasks = vec![]; + for entry in WalkDir::new(source).into_iter().filter_map(Result::ok) { + let path = entry.path(); + let relative_path = path.strip_prefix(source).map_err(|e| Error::CopyDirectory(e.to_string()))?; + let dest_path = destination.join(relative_path); + + if path.is_dir() { + tokio::fs::create_dir_all(&dest_path) + .await + .map_err(|e| Error::CopyDirectory(e.to_string()))?; + } else if path.is_file() { + let path = path.to_path_buf(); + let dest_path = dest_path.to_path_buf(); + let task = tokio::spawn(async move { tokio::fs::copy(&path, &dest_path).await }); + tasks.push(task); + } + } + + for task in tasks { + let _ = task.await; + } + Ok(()) +} + +pub async fn kill_previous_ceseal(version: u64) { + let cmd = + format!("ps -eaf | grep \"backups/{}/cruntime/sgx/loader\" | grep -v \"grep\" | awk '{{print $2}}'", version); + + let process = Command::new("bash") + .arg("-c") + .arg(cmd) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("Failed to spawn process"); + + let output = process.wait_with_output().await.expect("Failed to read output"); + + if output.status.success() { + let pid_str = std::str::from_utf8(&output.stdout).expect("Failed to parse output as UTF-8"); + if let Ok(pid) = pid_str.trim().parse::() { + log(format!("kill the previous version {} ceseal pid: {}", version, pid)); + Command::new("kill") + .arg("-9") + .arg(pid.to_string()) + .status() + .await + .expect("Failed to kill process"); + } + } else { + let error_str = std::str::from_utf8(&output.stderr).expect("Failed to parse error as UTF-8"); + log(format!("{}", error_str)); + } +} + +#[cfg(test)] +mod test { + use super::*; + use tokio::test; + #[test] + async fn start_old_ceseal() { + let args = Args::parse(); + let previous_version = 1; + let previous_ceseal_path = Path::new(&args.previous_version_ceseal_path).join(previous_version.to_string()); + log(format!("Previous ${previous_version}")); + + if let Err(err) = tokio::fs::remove_file(&args.previous_ceseal_log_path).await { + log(format!("remove old ceseal log file fail : {err}")) + }; + + //start old ceseal + let mut old_process = start_previous_ceseal( + previous_ceseal_path.to_str().unwrap().to_string(), + args.previous_ceseal_port.to_string(), + ) + .await + .expect("start previous ceseal fail"); + redirect_ceseal_runtime_log( + old_process.stdout.take().expect("previous ceseal log output in invaid!"), + args.previous_ceseal_log_path.clone(), + ) + .await + .expect("redirect ceseal runtime log fail"); + + //wait for old ceseal went well + wait_for_ceseal_to_run_successfully( + args.previous_ceseal_log_path.clone(), + "Ceseal internal server will listening on", + ) + .await + .expect("wait for ceseal log fail"); + log(format!("previous ceseal started!")); + } }