diff --git a/Cargo.lock b/Cargo.lock index d79284a2a4..bd802afee0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1524,6 +1524,7 @@ dependencies = [ "iml-fs", "iml-systemd", "iml-wire-types", + "serde_json", "thiserror", "tokio", ] diff --git a/iml-system-test-utils/Cargo.toml b/iml-system-test-utils/Cargo.toml index 31e928c774..bee4959c14 100644 --- a/iml-system-test-utils/Cargo.toml +++ b/iml-system-test-utils/Cargo.toml @@ -11,5 +11,6 @@ iml-cmd = { version = "0.2.0", path = "../iml-cmd" } iml-fs = { version = "0.2", path = "../iml-fs" } iml-systemd = { version = "0.2.0", path = "../iml-systemd" } iml-wire-types = { version = "0.2.0", path = "../iml-wire-types" } +serde_json = "1.0" tokio = {version="0.2", features=["process", "macros", "fs"]} thiserror = "1.0" \ No newline at end of file diff --git a/iml-system-test-utils/src/ssh.rs b/iml-system-test-utils/src/ssh.rs index 41fe6c6223..708b0eb22c 100644 --- a/iml-system-test-utils/src/ssh.rs +++ b/iml-system-test-utils/src/ssh.rs @@ -5,6 +5,7 @@ use futures::future::try_join_all; use iml_cmd::{CheckedChildExt, CheckedCommandExt, CmdError}; use std::{ + collections::BTreeSet, process::{Output, Stdio}, str, }; @@ -186,3 +187,25 @@ pub async fn create_iml_diagnostics<'a, 'b>( scp_parallel(hosts, "/var/tmp/sosreport*", "/tmp").await } + +pub async fn get_host_bindings<'a, 'b>(hosts: &'b [&'a str]) -> Result, CmdError> { + let hosts_output: Vec<(&str, Output)> = + ssh_exec_parallel(hosts, "cat /etc/multipath/bindings").await?; + + let wwids = hosts_output + .into_iter() + .map(|(_, output)| { + let stdout = + str::from_utf8(&output.stdout).expect("Couldn't parse multipath bindings file."); + + stdout + .lines() + .map(|line| line.split(' ').last().expect("Couldn't parse WWID").into()) + .collect::>() + }) + .fold(BTreeSet::::new(), |acc, xs| { + acc.union(&xs).cloned().collect() + }); + + Ok(wwids) +} diff --git a/iml-system-test-utils/src/vagrant.rs b/iml-system-test-utils/src/vagrant.rs index dedaf49d8a..4d1dcc908a 100644 --- a/iml-system-test-utils/src/vagrant.rs +++ b/iml-system-test-utils/src/vagrant.rs @@ -8,6 +8,7 @@ use crate::{ }; use futures::future::try_join_all; use iml_cmd::{CheckedCommandExt, CmdError}; +use iml_wire_types::Volume; use std::{ collections::{BTreeSet, HashMap}, env, str, @@ -320,6 +321,48 @@ pub async fn setup_iml_install( Ok(()) } +pub async fn get_iml_devices(config: &ClusterConfig) -> Result, CmdError> { + let output = run_vm_command(config.manager, "iml devices list -d json") + .await? + .checked_output() + .await?; + + let data_str = str::from_utf8(&output.stdout).expect("Couldn't parse devices information."); + let volumes: Vec = + serde_json::from_str(data_str).expect("Couldn't serialize devices information."); + + let labels: BTreeSet = volumes.into_iter().map(|v| v.label).collect(); + + Ok(labels) +} + +pub async fn wait_for_all_devices(max_tries: i32, config: &ClusterConfig) -> Result<(), CmdError> { + let mut count = 1; + let wwids: BTreeSet = ssh::get_host_bindings(&config.storage_servers()[..]).await?; + println!("Comparing wwids from api to bindings: {:?}", wwids); + + let iml_devices: BTreeSet = get_iml_devices(config).await?; + let mut all_volumes_accounted_for = wwids.is_subset(&iml_devices); + + println!("Comparing iml devices to bindings files."); + println!("iml_devices: {:?}", iml_devices); + println!("binding wwids: {:?}", wwids); + + while !all_volumes_accounted_for && count < max_tries { + delay_for(Duration::from_secs(5)).await; + + let iml_devices: BTreeSet = get_iml_devices(config).await?; + all_volumes_accounted_for = wwids.is_subset(&iml_devices); + count += 1; + + println!("Comparing iml devices to bindings files."); + println!("iml_devices: {:?}", iml_devices); + println!("binding wwids: {:?}", wwids); + } + + Ok(()) +} + pub async fn setup_deploy_servers( config: &ClusterConfig, setup_config: &SetupConfigType, @@ -327,6 +370,8 @@ pub async fn setup_deploy_servers( ) -> Result<(), CmdError> { setup_iml_install(&config.all(), &setup_config, &config).await?; + wait_for_all_devices(10, config).await?; + for (profile, hosts) in server_map { run_vm_command( config.manager, @@ -355,6 +400,9 @@ pub async fn add_docker_servers( config: &ClusterConfig, server_map: &HashMap, ) -> Result<(), CmdError> { + // Need to wait for devices on docker as well. + //wait_for_all_devices(10, config).await?; + iml::server_add(&server_map).await?; halt()