From a793ecfbd1dc0c205606c5032531bac1d6461e9f Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Fri, 6 Sep 2024 12:01:15 -0600 Subject: [PATCH 1/3] Add postinstall --- .../services/rust-service/package.md | 3 + rust/cargo-psibase/src/package.rs | 85 ++++++++++++------- rust/psibase/src/block.rs | 2 + rust/test_package/service/Cargo.toml | 1 + rust/test_package/service/src/lib.rs | 2 + 5 files changed, 62 insertions(+), 31 deletions(-) diff --git a/doc/psidk/src/development/services/rust-service/package.md b/doc/psidk/src/development/services/rust-service/package.md index 28b974c64..14f2fe3f6 100644 --- a/doc/psidk/src/development/services/rust-service/package.md +++ b/doc/psidk/src/development/services/rust-service/package.md @@ -11,6 +11,7 @@ The available fields are: - `server`: May be present on any crate that builds a service. The value is a crate which will handle HTTP requests sent to this service. The other crate will be built and included in the current package. - `plugin`: May be present on any crate that builds a service. The value is a crate that should be built with `cargo component` and uploaded as `/plugin.wasm` - `flags`: [Flags](../../../specifications/data-formats/package.md#serviceservicejson) for the service. +- `postinstall`: An array of actions to run when the package is installed. May be specified on the top-level crate or on any service. Actions from a single `postinstall` will be run in order. The order of actions from multiple crates is unspecified. - `dependencies`: Additional packages, not build by cargo, that the package depends on. Example: @@ -29,6 +30,8 @@ flags = [] server = "example" # Plugin for the front end plugin = "example-plugin" +# Run the service's init action +postinstall = [{sender="tpack", service="tpack", method="init", rawData="0000"}] [package.metadata.psibase.dependencies] HttpServer = "0.12.0" diff --git a/rust/cargo-psibase/src/package.rs b/rust/cargo-psibase/src/package.rs index 9560ce676..1ec19267c 100644 --- a/rust/cargo-psibase/src/package.rs +++ b/rust/cargo-psibase/src/package.rs @@ -2,8 +2,8 @@ use crate::{build, build_plugin, Args, SERVICE_POLYFILL}; use anyhow::anyhow; use cargo_metadata::{Metadata, Node, Package, PackageId}; use psibase::{ - AccountNumber, Checksum256, ExactAccountNumber, Meta, PackageInfo, PackageRef, PackagedService, - ServiceInfo, + AccountNumber, Action, Checksum256, ExactAccountNumber, Meta, PackageInfo, PackageRef, + PackagedService, ServiceInfo, }; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -24,6 +24,7 @@ pub struct PsibaseMetadata { flags: Vec, dependencies: HashMap, services: Option>, + postinstall: Option>, } impl PsibaseMetadata { @@ -104,6 +105,7 @@ fn should_build_package( meta: &Meta, services: &[(&String, ServiceInfo, PathBuf)], data: &[(&String, &str, PathBuf)], + postinstall: &[Action], ) -> Result { let timestamp = if let Ok(metadata) = filename.metadata() { metadata.modified()? @@ -153,6 +155,12 @@ fn should_build_package( if existing_data != new_data { return Ok(true); } + // check postinstall + let mut existing_postinstall = Vec::new(); + existing.postinstall(&mut existing_postinstall)?; + if &existing_postinstall[..] != postinstall { + return Ok(true); + } Ok(false) } @@ -168,6 +176,7 @@ pub async fn build_package( let mut services = Vec::new(); let mut plugins = Vec::new(); let mut data_files = Vec::new(); + let mut postinstall: Vec = Vec::new(); let get_dep = get_dep_type(|service, dep| { let r = metadata.resolved.get(&service.id.repr.as_str()).unwrap(); @@ -203,6 +212,12 @@ pub async fn build_package( visited.insert(root); queue.push(root); } + // Add postinstall from the root whether it is a service or not + if !visited.contains(root) { + if let Some(actions) = &meta.postinstall { + postinstall.extend_from_slice(actions.as_slice()); + } + } } else { Err(anyhow!("Cannot package a virtual workspace"))? } @@ -243,6 +258,9 @@ pub async fn build_package( plugins.push((plugin, &package.name, "/plugin.wasm", &id.repr)); } services.push((&package.name, info, &package.id.repr)); + if let Some(actions) = &pmeta.postinstall { + postinstall.extend_from_slice(actions.as_slice()); + } } } @@ -300,35 +318,40 @@ pub async fn build_package( let out_dir = get_package_dir(args, metadata); let out_path = out_dir.join(package_name.clone() + ".psi"); - let mut file = if should_build_package(&out_path, &meta, &service_wasms, &data_files)? { - std::fs::create_dir_all(&out_dir)?; - let mut out = ZipWriter::new( - OpenOptions::new() - .read(true) - .write(true) - .create(true) - .truncate(true) - .open(&out_path)?, - ); - let options: FileOptions = FileOptions::default(); - out.start_file("meta.json", options)?; - write!(out, "{}", &serde_json::to_string(&meta)?)?; - for (service, info, path) in service_wasms { - out.start_file(format!("service/{}.wasm", service), options)?; - std::io::copy(&mut File::open(path)?, &mut out)?; - out.start_file(format!("service/{}.json", service), options)?; - write!(out, "{}", &serde_json::to_string(&info)?)?; - } - for (service, dest, src) in data_files { - out.start_file(format!("data/{}{}", service, dest), options)?; - std::io::copy(&mut File::open(src)?, &mut out)?; - } - let mut file = out.finish()?; - file.rewind()?; - file - } else { - File::open(out_path)? - }; + let mut file = + if should_build_package(&out_path, &meta, &service_wasms, &data_files, &postinstall)? { + std::fs::create_dir_all(&out_dir)?; + let mut out = ZipWriter::new( + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(&out_path)?, + ); + let options: FileOptions = FileOptions::default(); + out.start_file("meta.json", options)?; + write!(out, "{}", &serde_json::to_string(&meta)?)?; + for (service, info, path) in service_wasms { + out.start_file(format!("service/{}.wasm", service), options)?; + std::io::copy(&mut File::open(path)?, &mut out)?; + out.start_file(format!("service/{}.json", service), options)?; + write!(out, "{}", &serde_json::to_string(&info)?)?; + } + for (service, dest, src) in data_files { + out.start_file(format!("data/{}{}", service, dest), options)?; + std::io::copy(&mut File::open(src)?, &mut out)?; + } + if !postinstall.is_empty() { + out.start_file("script/postinstall.json", options)?; + write!(out, "{}", &serde_json::to_string(&postinstall)?)?; + } + let mut file = out.finish()?; + file.rewind()?; + file + } else { + File::open(out_path)? + }; // Calculate the package checksum let mut hasher = Sha256::new(); std::io::copy(&mut file, &mut hasher)?; diff --git a/rust/psibase/src/block.rs b/rust/psibase/src/block.rs index a82e8481e..77d940bed 100644 --- a/rust/psibase/src/block.rs +++ b/rust/psibase/src/block.rs @@ -29,6 +29,8 @@ pub type BlockNum = u32; Deserialize, SimpleObject, InputObject, + PartialEq, + Eq, )] #[fracpack(fracpack_mod = "fracpack")] #[to_key(psibase_mod = "crate")] diff --git a/rust/test_package/service/Cargo.toml b/rust/test_package/service/Cargo.toml index 67624f591..2c966dccf 100644 --- a/rust/test_package/service/Cargo.toml +++ b/rust/test_package/service/Cargo.toml @@ -10,6 +10,7 @@ publish = false [package.metadata.psibase] server = "r-tpack" +postinstall = [{sender="tpack", service="tpack", method="init", rawData="0000"}] [package.metadata.psibase.dependencies] HttpServer = "0.12.0" diff --git a/rust/test_package/service/src/lib.rs b/rust/test_package/service/src/lib.rs index 6bb21d612..5de2e042d 100644 --- a/rust/test_package/service/src/lib.rs +++ b/rust/test_package/service/src/lib.rs @@ -5,6 +5,8 @@ mod service { fn foo(value: i32) -> i32 { value } + #[action] + fn init() {} } #[psibase::test_case(packages("TestPackage"))] From d50dc6057ef69ff80389c2f23f2249740ceade5a Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Fri, 6 Sep 2024 14:11:58 -0600 Subject: [PATCH 2/3] Add ui upload --- .../services/rust-service/package.md | 3 ++ rust/cargo-psibase/src/package.rs | 53 +++++++++++++++++-- rust/test_package/query/Cargo.toml | 1 + rust/test_package/ui/index.html | 5 ++ 4 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 rust/test_package/ui/index.html diff --git a/doc/psidk/src/development/services/rust-service/package.md b/doc/psidk/src/development/services/rust-service/package.md index 14f2fe3f6..882d48cba 100644 --- a/doc/psidk/src/development/services/rust-service/package.md +++ b/doc/psidk/src/development/services/rust-service/package.md @@ -12,6 +12,7 @@ The available fields are: - `plugin`: May be present on any crate that builds a service. The value is a crate that should be built with `cargo component` and uploaded as `/plugin.wasm` - `flags`: [Flags](../../../specifications/data-formats/package.md#serviceservicejson) for the service. - `postinstall`: An array of actions to run when the package is installed. May be specified on the top-level crate or on any service. Actions from a single `postinstall` will be run in order. The order of actions from multiple crates is unspecified. +- `data`: An array of `{src, dst}` paths to upload to the service. `src` is relative to the location of Cargo.toml. If `src` is a directory, its contents will be included recursively. - `dependencies`: Additional packages, not build by cargo, that the package depends on. Example: @@ -32,6 +33,8 @@ server = "example" plugin = "example-plugin" # Run the service's init action postinstall = [{sender="tpack", service="tpack", method="init", rawData="0000"}] +# Upload the UI +data = [{src = "ui/", dst = "/"}] [package.metadata.psibase.dependencies] HttpServer = "0.12.0" diff --git a/rust/cargo-psibase/src/package.rs b/rust/cargo-psibase/src/package.rs index 1ec19267c..cdc82a29f 100644 --- a/rust/cargo-psibase/src/package.rs +++ b/rust/cargo-psibase/src/package.rs @@ -14,6 +14,12 @@ use std::io::{BufReader, Seek, Write}; use std::path::{Path, PathBuf}; use zip::write::{FileOptions, ZipWriter}; +#[derive(Serialize, Deserialize, Default)] +struct DataFiles { + src: String, + dst: String, +} + #[derive(Serialize, Deserialize, Default)] #[serde(default)] pub struct PsibaseMetadata { @@ -25,6 +31,7 @@ pub struct PsibaseMetadata { dependencies: HashMap, services: Option>, postinstall: Option>, + data: Vec, } impl PsibaseMetadata { @@ -104,7 +111,7 @@ fn should_build_package( filename: &Path, meta: &Meta, services: &[(&String, ServiceInfo, PathBuf)], - data: &[(&String, &str, PathBuf)], + data: &[(&String, String, PathBuf)], postinstall: &[Action], ) -> Result { let timestamp = if let Ok(metadata) = filename.metadata() { @@ -148,7 +155,7 @@ fn should_build_package( return Ok(true); } let account: AccountNumber = service.parse()?; - new_data.push((account.value, *dest)); + new_data.push((account.value, dest.as_str())); } existing_data.sort(); new_data.sort(); @@ -164,6 +171,35 @@ fn should_build_package( Ok(false) } +fn add_files<'a>( + service: &'a String, + src: &Path, + dest: &String, + out: &mut Vec<(&'a String, String, PathBuf)>, +) -> Result<(), anyhow::Error> { + if src.is_file() { + out.push((service, dest.clone(), src.to_path_buf())); + } else if src.is_dir() { + for entry in src.read_dir()? { + let entry = entry?; + add_files( + service, + &entry.path(), + &(dest.clone() + + "/" + + entry.file_name().to_str().ok_or_else(|| { + anyhow!( + "non-unicode file name: {}", + entry.file_name().to_string_lossy() + ) + })?), + out, + )?; + } + } + Ok(()) +} + pub async fn build_package( args: &Args, metadata: &MetadataIndex<'_>, @@ -257,6 +293,17 @@ pub async fn build_package( }; plugins.push((plugin, &package.name, "/plugin.wasm", &id.repr)); } + for data in &pmeta.data { + let src = package.manifest_path.parent().unwrap().join(&data.src); + let mut dest = data.dst.clone(); + if !dest.starts_with('/') { + dest = "/".to_string() + &dest; + } + if dest.ends_with('/') { + dest.pop(); + } + add_files(&package.name, src.as_std_path(), &dest, &mut data_files)?; + } services.push((&package.name, info, &package.id.repr)); if let Some(actions) = &pmeta.postinstall { postinstall.extend_from_slice(actions.as_slice()); @@ -313,7 +360,7 @@ pub async fn build_package( plugin ))? }; - data_files.push((service, path, paths.pop().unwrap())) + data_files.push((service, path.to_string(), paths.pop().unwrap())) } let out_dir = get_package_dir(args, metadata); diff --git a/rust/test_package/query/Cargo.toml b/rust/test_package/query/Cargo.toml index db6df3a13..7822a2cfe 100644 --- a/rust/test_package/query/Cargo.toml +++ b/rust/test_package/query/Cargo.toml @@ -10,6 +10,7 @@ publish = false [package.metadata.psibase] plugin = "tpack-plugin" +data = [{src = "../ui", dst = "/"}] [dependencies] psibase = { path = "../../psibase", version = "0.12.0" } diff --git a/rust/test_package/ui/index.html b/rust/test_package/ui/index.html new file mode 100644 index 000000000..350147c29 --- /dev/null +++ b/rust/test_package/ui/index.html @@ -0,0 +1,5 @@ + + + Lorem ipsum dolor sit amet + + From 63defd159e116d270b21de3d429697ebdddaf315 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Fri, 6 Sep 2024 14:18:25 -0600 Subject: [PATCH 3/3] Don't search for files until after building all the services --- rust/cargo-psibase/src/package.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rust/cargo-psibase/src/package.rs b/rust/cargo-psibase/src/package.rs index cdc82a29f..8a500f94b 100644 --- a/rust/cargo-psibase/src/package.rs +++ b/rust/cargo-psibase/src/package.rs @@ -212,7 +212,8 @@ pub async fn build_package( let mut services = Vec::new(); let mut plugins = Vec::new(); let mut data_files = Vec::new(); - let mut postinstall: Vec = Vec::new(); + let mut postinstall = Vec::new(); + let mut data_sources = Vec::new(); let get_dep = get_dep_type(|service, dep| { let r = metadata.resolved.get(&service.id.repr.as_str()).unwrap(); @@ -302,7 +303,7 @@ pub async fn build_package( if dest.ends_with('/') { dest.pop(); } - add_files(&package.name, src.as_std_path(), &dest, &mut data_files)?; + data_sources.push((&package.name, src, dest)); } services.push((&package.name, info, &package.id.repr)); if let Some(actions) = &pmeta.postinstall { @@ -363,6 +364,10 @@ pub async fn build_package( data_files.push((service, path.to_string(), paths.pop().unwrap())) } + for (service, src, dest) in data_sources { + add_files(service, src.as_std_path(), &dest, &mut data_files)?; + } + let out_dir = get_package_dir(args, metadata); let out_path = out_dir.join(package_name.clone() + ".psi"); let mut file =