diff --git a/Cargo.lock b/Cargo.lock index 20ba8776a..f8390bd4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,7 +148,7 @@ dependencies = [ "chrono", "hmac", "log", - "rand", + "rand 0.8.5", "serde", "serde_json", "sha2", @@ -619,6 +619,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -726,7 +735,7 @@ dependencies = [ "lazy_static", "log", "nix 0.27.1", - "rand", + "rand 0.8.5", "retry", "semver 1.0.22", "serde", @@ -917,7 +926,7 @@ dependencies = [ "nix 0.26.4", "openssl", "pretty_env_logger", - "rand", + "rand 0.8.5", "reqwest", "serde", "serde_yaml", @@ -938,7 +947,7 @@ dependencies = [ "log", "nix 0.26.4", "openssl", - "rand", + "rand 0.8.5", "secrecy", "serde_bytes", "sys-info", @@ -1033,7 +1042,7 @@ dependencies = [ "hex", "log", "openssl", - "rand", + "rand 0.8.5", "regex", "tokio", "tss-esapi", @@ -1049,11 +1058,14 @@ dependencies = [ "fdo-http-wrapper", "fdo-store", "fdo-util", + "flate2", "hex", "log", "openssl", "serde", "serde_yaml", + "tar", + "tempdir", "thiserror", "tokio", "warp", @@ -1174,6 +1186,28 @@ dependencies = [ "serde_yaml", ] +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "windows-sys 0.52.0", +] + +[[package]] +name = "flate2" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1204,6 +1238,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.30" @@ -2242,7 +2282,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -2408,6 +2448,19 @@ dependencies = [ "scheduled-thread-pool", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -2416,7 +2469,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -2426,9 +2479,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", ] +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.4" @@ -2438,6 +2506,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -2476,6 +2553,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "reqwest" version = "0.11.27" @@ -3003,12 +3089,33 @@ dependencies = [ "libc", ] +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.10.1" @@ -3035,7 +3142,7 @@ dependencies = [ "percent-encoding", "pest", "pest_derive", - "rand", + "rand 0.8.5", "regex", "serde", "serde_json", @@ -3288,7 +3395,7 @@ dependencies = [ "http", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror", "url", diff --git a/db/src/lib.rs b/db/src/lib.rs index 9b1773900..e55d5a9f5 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -56,6 +56,9 @@ where /// Gets an OV fn get_ov(guid: &str, conn: &mut T) -> Result; + /// Returns all the OVs in the DB + fn get_all_ovs(conn: &mut T) -> Result>; + /// Deletes an OV fn delete_ov(guid: &str, conn: &mut T) -> Result<()>; @@ -101,6 +104,9 @@ where /// Gets an OV fn get_ov(guid: &str, conn: &mut T) -> Result; + /// Returns all the OVs in the DB + fn get_all_ovs(conn: &mut T) -> Result>; + /// Deletes an OV fn delete_ov(guid: &str, conn: &mut T) -> Result<()>; diff --git a/db/src/postgres.rs b/db/src/postgres.rs index cd8f6531f..48c3c208f 100644 --- a/db/src/postgres.rs +++ b/db/src/postgres.rs @@ -113,6 +113,13 @@ impl DBStoreOwner for PostgresOwnerDB { Ok(result) } + fn get_all_ovs(conn: &mut PgConnection) -> Result> { + let result = super::schema::owner_vouchers::dsl::owner_vouchers + .select(OwnerOV::as_select()) + .load(conn)?; + Ok(result) + } + fn delete_ov(guid: &str, conn: &mut PgConnection) -> Result<()> { diesel::delete(owner_vouchers::dsl::owner_vouchers) .filter(super::schema::owner_vouchers::guid.eq(guid)) @@ -222,6 +229,13 @@ impl DBStoreRendezvous for PostgresRendezvousDB { Ok(result) } + fn get_all_ovs(conn: &mut PgConnection) -> Result> { + let result = super::schema::rendezvous_vouchers::dsl::rendezvous_vouchers + .select(RendezvousOV::as_select()) + .load(conn)?; + Ok(result) + } + fn delete_ov(guid: &str, conn: &mut PgConnection) -> Result<()> { diesel::delete(rendezvous_vouchers::dsl::rendezvous_vouchers) .filter(super::schema::rendezvous_vouchers::guid.eq(guid)) diff --git a/db/src/sqlite.rs b/db/src/sqlite.rs index 1a49bb481..1d28801d0 100644 --- a/db/src/sqlite.rs +++ b/db/src/sqlite.rs @@ -115,6 +115,13 @@ impl DBStoreOwner for SqliteOwnerDB { Ok(result) } + fn get_all_ovs(conn: &mut SqliteConnection) -> Result> { + let result = super::schema::owner_vouchers::dsl::owner_vouchers + .select(OwnerOV::as_select()) + .load(conn)?; + Ok(result) + } + fn delete_ov(guid: &str, conn: &mut SqliteConnection) -> Result<()> { diesel::delete(owner_vouchers::dsl::owner_vouchers) .filter(super::schema::owner_vouchers::guid.eq(guid)) @@ -224,6 +231,13 @@ impl DBStoreRendezvous for SqliteRendezvousDB { Ok(result) } + fn get_all_ovs(conn: &mut SqliteConnection) -> Result> { + let result = super::schema::rendezvous_vouchers::dsl::rendezvous_vouchers + .select(RendezvousOV::as_select()) + .load(conn)?; + Ok(result) + } + fn delete_ov(guid: &str, conn: &mut SqliteConnection) -> Result<()> { diesel::delete(rendezvous_vouchers::dsl::rendezvous_vouchers) .filter(super::schema::rendezvous_vouchers::guid.eq(guid)) diff --git a/manufacturing-server/Cargo.toml b/manufacturing-server/Cargo.toml index 0da5591f8..014a862d7 100644 --- a/manufacturing-server/Cargo.toml +++ b/manufacturing-server/Cargo.toml @@ -17,6 +17,9 @@ warp = "0.3.6" log = "0.4" hex = "0.4" serde_yaml = "0.9" +tar = "0.4.41" +flate2 = "1.0.31" +tempdir = "0.3.7" fdo-data-formats = { path = "../data-formats", version = "0.5.0" } fdo-http-wrapper = { path = "../http-wrapper", version = "0.5.0", features = ["server"] } diff --git a/manufacturing-server/src/main.rs b/manufacturing-server/src/main.rs index c99bd1eac..58c76f4ea 100644 --- a/manufacturing-server/src/main.rs +++ b/manufacturing-server/src/main.rs @@ -1,26 +1,32 @@ use std::collections::BTreeMap; use std::convert::{TryFrom, TryInto}; -use std::fs; +use std::fs::{self, File}; +use std::io::Read; use std::str::FromStr; use std::sync::Arc; +use fdo_data_formats::{constants::ErrorCode, ProtocolVersion}; +use fdo_store::Store; + +use warp::{Filter, Rejection}; + use anyhow::{bail, Context, Error, Result}; use openssl::{ pkey::{PKey, Private}, x509::X509, }; use serde_yaml::Value; +use tempdir::TempDir; use tokio::signal::unix::{signal, SignalKind}; -use warp::Filter; +use warp::reply::Response; use fdo_data_formats::{ constants::{KeyStorageType, MfgStringType, PublicKeyType, RendezvousVariable}, ownershipvoucher::OwnershipVoucher, publickey::{PublicKey, X5Chain}, types::{Guid, RendezvousInfo}, - ProtocolVersion, + Serializable, }; -use fdo_store::Store; use fdo_util::servers::{ configuration::manufacturing_server::{DiunSettings, ManufacturingServerSettings}, settings_for, yaml_to_cbor, OwnershipVoucherStoreMetadataKey, @@ -56,7 +62,7 @@ struct ManufacturingServiceUD { session_store: Arc, ownership_voucher_store: Box< dyn Store< - fdo_store::WriteOnlyOpen, + fdo_store::ReadWriteOpen, Guid, OwnershipVoucher, OwnershipVoucherStoreMetadataKey, @@ -268,7 +274,51 @@ async fn main() -> Result<()> { // Initialize handlers let hello = warp::get().map(|| "Hello from the manufacturing server"); - let handler_ping = fdo_http_wrapper::server::ping_handler(); + let ud = user_data.clone(); + let handler_export = warp::get() + .and(warp::path("export").map(move || (ud.clone())).and_then( + |ud: Arc| async move { + match ud.ownership_voucher_store.load_all_data().await { + Ok(ovs) => Ok(ovs), + Err(_) => { + return Err(Rejection::from(fdo_http_wrapper::server::Error::new( + ErrorCode::InternalServerError, + fdo_data_formats::constants::MessageType::Invalid, + "Error loading ownership vouchers", + ))) + } + } + }, + )) + .map(|ovs: Vec| { + let tmp_dir = TempDir::new("manufacturer-server-ovs").unwrap(); + for ov in ovs { + let file_path = tmp_dir.path().join(ov.header().guid().to_string()); + let tmp_file = File::create(file_path).unwrap(); + OwnershipVoucher::serialize_to_writer(&ov, &tmp_file).unwrap(); + } + let tmp_dir_archive = TempDir::new("manufacturer-server-ovs-archive").unwrap(); + let tar_gz = File::create(tmp_dir_archive.path().join("ovs.tar.gz")).unwrap(); + let mut tar = tar::Builder::new(tar_gz); + tar.append_dir_all(".", tmp_dir).unwrap(); + tar.finish().unwrap(); + let mut file = File::open(tmp_dir_archive.path().join("ovs.tar.gz")).unwrap(); + let mut data: Vec = Vec::new(); + match file.read_to_end(&mut data) { + Err(why) => { + println!("Error: {:?}", why); + return Response::new(String::new().into()); + } + Ok(_) => { + let mut res = Response::new(data.into()); + res.headers_mut().insert( + "Content-Type", + warp::http::header::HeaderValue::from_static("application/x-tar"), + ); + return res; + } + } + }); // DI let handler_di_app_start = fdo_http_wrapper::server::fdo_request_filter( @@ -307,7 +357,7 @@ async fn main() -> Result<()> { let routes = warp::post() .and( hello - .or(handler_ping) + .or(fdo_http_wrapper::server::ping_handler()) // DI .or(handler_di_app_start) .or(handler_di_set_hmac) @@ -316,6 +366,7 @@ async fn main() -> Result<()> { .or(handler_diun_request_key_parameters) .or(handler_diun_provide_key), ) + .or(handler_export) .recover(fdo_http_wrapper::server::handle_rejection) .with(warp::log("manufacturing-server")); diff --git a/owner-onboarding-server/src/main.rs b/owner-onboarding-server/src/main.rs index 6f7ab2af7..d38c7af78 100644 --- a/owner-onboarding-server/src/main.rs +++ b/owner-onboarding-server/src/main.rs @@ -117,6 +117,7 @@ async fn _handle_report_to_rendezvous(udt: &OwnerServiceUDT, ov: &OwnershipVouch } async fn report_to_rendezvous(udt: OwnerServiceUDT) -> Result<()> { + // TODO: this below (query_data vs query_ovs_db) should be abstracted into the store's Filter's query stuff match udt.ownership_voucher_store.query_data().await { Ok(mut ft) => { ft.neq( diff --git a/store/src/directory.rs b/store/src/directory.rs index 98abf6244..4c4a65a04 100644 --- a/store/src/directory.rs +++ b/store/src/directory.rs @@ -1,6 +1,7 @@ use std::collections::HashSet; use std::convert::TryInto; use std::fs::{self, File}; +use std::io; use std::marker::PhantomData; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; @@ -206,6 +207,27 @@ where V: Serializable + Send + Sync + Clone + 'static, MKT: crate::MetadataLocalKey + 'static, { + async fn load_all_data(&self) -> Result, StoreError> { + let entries = fs::read_dir(&self.directory) + .map_err(|e| StoreError::Unspecified(format!("Error reading store directory: {e:?}")))? + .map(|res| res.map(|e| e.path())) + .collect::, io::Error>>() + .map_err(|e| { + StoreError::Unspecified(format!("Error collecting store directory entries: {e:?}")) + })?; + let mut items = Vec::::new(); + for entry in entries { + let file = match File::open(&entry) { + Err(e) => return Err(StoreError::Unspecified(format!("Error opening file: {e}"))), + Ok(f) => f, + }; + items.push(V::deserialize_from_reader(&file).map_err(|e| { + StoreError::Unspecified(format!("Error deserializing value: {e:?}")) + })?); + } + Ok(items) + } + async fn load_data(&self, key: &K) -> Result, StoreError> { let path = self.get_path(key); log::trace!("Attempting to load data from {}", path.display()); diff --git a/store/src/lib.rs b/store/src/lib.rs index dbcbd3a02..5ff017f45 100644 --- a/store/src/lib.rs +++ b/store/src/lib.rs @@ -144,6 +144,14 @@ where type QueryResult = Result>, StoreError>; pub trait Store: Send + Sync { + fn load_all_data<'life0, 'async_trait>( + &'life0 self, + ) -> Pin, StoreError>> + 'async_trait + Send>> + where + 'life0: 'async_trait, + Self: 'async_trait, + OT: Readable; + fn load_data<'life0, 'life1, 'async_trait>( &'life0 self, key: &'life1 K, diff --git a/store/src/pg.rs b/store/src/pg.rs index 777738070..b99bfc668 100644 --- a/store/src/pg.rs +++ b/store/src/pg.rs @@ -86,6 +86,22 @@ where V: Serializable + Send + Sync + Clone + 'static, MKT: crate::MetadataLocalKey + 'static, { + async fn load_all_data(&self) -> Result, StoreError> { + let conn = &mut self + .connection_pool + .get() + .expect("Couldn't establish a connection"); + let entries = fdo_db::postgres::PostgresManufacturerDB::get_all_ovs(conn) + .expect("Error selecting OVs"); + let mut items = Vec::::new(); + for entry in entries { + items.push(V::deserialize_from_reader(&mut &entry.contents[..]).map_err(|e| { + StoreError::Unspecified(format!("Error deserializing value: {e:?}")) + })?); + } + Ok(items) + } + async fn load_data(&self, key: &K) -> Result, StoreError> { let conn = &mut self .connection_pool @@ -234,6 +250,9 @@ where } } +// TODO: this whole implementation uses OwnershipVoucher but the store interface +// has been made to work with different objects (generics indeed). Think about Sessions too +// This has to be changed to work with everything, like the directory store. #[async_trait] impl Store for PostgresOwnerStore where @@ -242,6 +261,22 @@ where V: Serializable + Send + Sync + Clone + 'static, MKT: crate::MetadataLocalKey + 'static, { + async fn load_all_data(&self) -> Result, StoreError> { + let conn = &mut self + .connection_pool + .get() + .expect("Couldn't establish a connection"); + let entries = + fdo_db::postgres::PostgresOwnerDB::get_all_ovs(conn).expect("Error selecting OVs"); + let mut items = Vec::::new(); + for entry in entries { + items.push(V::deserialize_from_reader(&mut &entry.contents[..]).map_err(|e| { + StoreError::Unspecified(format!("Error deserializing value: {e:?}")) + })?); + } + Ok(items) + } + async fn load_data(&self, key: &K) -> Result, StoreError> { let conn = &mut self .connection_pool @@ -464,6 +499,22 @@ where V: Serializable + Send + Sync + Clone + 'static, MKT: crate::MetadataLocalKey + 'static, { + async fn load_all_data(&self) -> Result, StoreError> { + let conn = &mut self + .connection_pool + .get() + .expect("Couldn't establish a connection"); + let entries = + fdo_db::postgres::PostgresRendezvousDB::get_all_ovs(conn).expect("Error selecting OVs"); + let mut items = Vec::::new(); + for entry in entries { + items.push(V::deserialize_from_reader(&mut &entry.contents[..]).map_err(|e| { + StoreError::Unspecified(format!("Error deserializing value: {e:?}")) + })?); + } + Ok(items) + } + async fn load_data(&self, key: &K) -> Result, StoreError> { let conn = &mut self .connection_pool diff --git a/store/src/sqlite.rs b/store/src/sqlite.rs index b99e5397b..adef18520 100644 --- a/store/src/sqlite.rs +++ b/store/src/sqlite.rs @@ -86,6 +86,22 @@ where V: Serializable + Send + Sync + Clone + 'static, MKT: crate::MetadataLocalKey + 'static, { + async fn load_all_data(&self) -> Result, StoreError> { + let conn = &mut self + .connection_pool + .get() + .expect("Couldn't establish a connection"); + let entries = fdo_db::sqlite::SqliteManufacturerDB::get_all_ovs(conn) + .expect("Error selecting OVs"); + let mut items = Vec::::new(); + for entry in entries { + items.push(V::deserialize_from_reader(&mut &entry.contents[..]).map_err(|e| { + StoreError::Unspecified(format!("Error deserializing value: {e:?}")) + })?); + } + Ok(items) + } + async fn load_data(&self, key: &K) -> Result, StoreError> { let conn = &mut self .connection_pool @@ -243,6 +259,22 @@ where V: Serializable + Send + Sync + Clone + 'static, MKT: crate::MetadataLocalKey + 'static, { + async fn load_all_data(&self) -> Result, StoreError> { + let conn = &mut self + .connection_pool + .get() + .expect("Couldn't establish a connection"); + let entries = fdo_db::sqlite::SqliteOwnerDB::get_all_ovs(conn) + .expect("Error selecting OVs"); + let mut items = Vec::::new(); + for entry in entries { + items.push(V::deserialize_from_reader(&mut &entry.contents[..]).map_err(|e| { + StoreError::Unspecified(format!("Error deserializing value: {e:?}")) + })?); + } + Ok(items) + } + async fn load_data(&self, key: &K) -> Result, StoreError> { let conn = &mut self .connection_pool @@ -461,6 +493,22 @@ where V: Serializable + Send + Sync + Clone + 'static, MKT: crate::MetadataLocalKey + 'static, { + async fn load_all_data(&self) -> Result, StoreError> { + let conn = &mut self + .connection_pool + .get() + .expect("Couldn't establish a connection"); + let entries = fdo_db::sqlite::SqliteRendezvousDB::get_all_ovs(conn) + .expect("Error selecting OVs"); + let mut items = Vec::::new(); + for entry in entries { + items.push(V::deserialize_from_reader(&mut &entry.contents[..]).map_err(|e| { + StoreError::Unspecified(format!("Error deserializing value: {e:?}")) + })?); + } + Ok(items) + } + async fn load_data(&self, key: &K) -> Result, StoreError> { let conn = &mut self .connection_pool