diff --git a/Cargo.lock b/Cargo.lock index 4dc6ab3c..4a44ad35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -744,7 +744,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.4", + "thiserror 2.0.6", "tokio", "tokio-util", "tower-service", @@ -833,9 +833,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "jobserver", "libc", @@ -1168,9 +1168,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "ff" @@ -1910,9 +1910,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", @@ -2193,20 +2193,20 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 1.0.69", + "thiserror 2.0.6", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" +checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" dependencies = [ "pest", "pest_generator", @@ -2214,9 +2214,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" +checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" dependencies = [ "pest", "pest_meta", @@ -2227,9 +2227,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" +checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" dependencies = [ "once_cell", "pest", @@ -2446,7 +2446,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.19", "socket2", - "thiserror 2.0.4", + "thiserror 2.0.6", "tokio", "tracing", ] @@ -2465,7 +2465,7 @@ dependencies = [ "rustls 0.23.19", "rustls-pki-types", "slab", - "thiserror 2.0.4", + "thiserror 2.0.6", "tinyvec", "tracing", "web-time", @@ -2689,15 +2689,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2961,6 +2961,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3214,11 +3223,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.4" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ - "thiserror-impl 2.0.4", + "thiserror-impl 2.0.6", ] [[package]] @@ -3234,9 +3243,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.4" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" dependencies = [ "proc-macro2", "quote", @@ -3396,6 +3405,40 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap 2.7.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tonic" version = "0.12.3" @@ -3762,6 +3805,8 @@ dependencies = [ "serde", "serde_json", "sha256", + "tokio", + "toml", "tonic", "tracing", "vorpal-schema", @@ -3777,6 +3822,7 @@ dependencies = [ "async_zip", "filetime", "futures-lite", + "ignore", "sanitize-filename", "sha256", "tokio", @@ -3785,7 +3831,6 @@ dependencies = [ "tracing", "uuid", "vorpal-schema", - "walkdir", ] [[package]] @@ -3844,9 +3889,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -3855,13 +3900,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -3870,9 +3914,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.47" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", @@ -3883,9 +3927,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3893,9 +3937,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", @@ -3906,15 +3950,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "web-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -4091,6 +4135,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/README.md b/README.md index 66cb2fa4..a0ea5b1a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,27 @@ Vorpal distributively builds and ships software reliably using BYOL (bring-you-o Below are examples of building a Rust application with different configuration languages: +### Rust + +```rust +use anyhow::Result; +use vorpal_schema::vorpal::config::v0::Config; +use vorpal_sdk::config::{artifact::language::rust::rust_artifact, get_context}; + +#[tokio::main] +async fn main() -> Result<()> { + let context = &mut get_context().await?; + + let artifact = rust_artifact(context, "vorpal").await?; + + context + .run(Config { + artifacts: vec![artifact], + }) + .await +} +``` + ### Go ```go @@ -17,102 +38,50 @@ package main import ( "context" - "github.com/vorpal_schema/vorpal/config/v0" - "github.com/vorpal_sdk/config/artifact" - "github.com/vorpal_sdk/config/cli" -) + "log" -// 1. Create a function that returns a populated configuration -func config(ctx *cli.ContextConfig) (*config.Config, error) { - // NOTE: custom logic can be added anywhere in this function - - // 2. Define artifact parameters - artifactExcludes := []string{".env", ".packer", ".vagrant", "script"} - artifactName := "vorpal" - artifactSystems := artifact.AddSystems([]string{"aarch64-linux", "aarch64-macos"}) + "github.com/vorpal-sdk/vorpal" + "github.com/vorpal-sdk/vorpal/config" + "github.com/vorpal-sdk/vorpal/config/artifact/language/rust" +) - // 3. Create artifact (rust) - artifact, err := artifact.RustArtifact(ctx, artifactExcludes, artifactName, artifactSystems) +func main() { + context, err := config.GetContext(context.Background()) if err != nil { - return nil, err + log.Fatal(err) } - // 4. Return config with artifact - return &config.Config{ - Artifacts: []config.Artifact{artifact}, - }, nil -} - -func main() { - ctx := context.Background() - if err := cli.Execute(ctx, config); err != nil { - panic(err) + artifact, err := rust.Artifact(context, "vorpal") + if err != nil { + log.Fatal(err) } -} -``` - -### Rust - -```rust -use anyhow::Result; -use vorpal_schema::vorpal::config::v0::Config; -use vorpal_sdk::config::{ - artifact::{add_systems, language::rust}, - cli::execute, - ContextConfig, -}; -// 1. Create a function that returns a populated configuration -fn config(context: &mut ContextConfig) -> Result { - // NOTE: custom logic can be added anywhere in this function - - // 2. Define artifact parameters - let artifact_excludes = vec![".env", ".packer", ".vagrant", "script"]; - let artifact_name = "vorpal"; - let artifact_systems = add_systems(vec!["aarch64-linux", "aarch64-macos"])?; - - // 3. Create artifact (rust) - let artifact = rust::artifact(context, artifact_excludes, artifact_name, artifact_systems)?; - - // 4. Return config with artifact - Ok(Config { - artifacts: vec![artifact], + err = context.Run(config.Config{ + Artifacts: []config.Artifact{artifact}, }) -} - -#[tokio::main] -async fn main() -> Result<()> { - // 5. Execute the configuration - execute(config).await + if err != nil { + log.Fatal(err) + } } ``` ### TypeScript ```typescript -import { ContextConfig, execute, addSystems, rust } from '@vorpal/config'; -import { Config } from '@vorpal/schema'; +import { getContext } from '@vorpal/sdk'; +import { rustArtifact } from '@vorpal/sdk/config/artifact/language/rust'; -// 1. Create a function that returns a populated configuration -function config(context: ContextConfig): Config { - // NOTE: custom logic can be added anywhere in this function +async function main() { + const context = await getContext(); - // 2. Define artifact parameters - const artifactExcludes = ['.env', '.packer', '.vagrant', 'script']; - const artifactName = 'vorpal'; - const artifactSystems = addSystems(['aarch64-linux', 'aarch64-macos']); + const artifact = await rustArtifact(context, 'vorpal'); - // 3. Create artifact (rust) - const artifact = rust.artifact(context, artifactExcludes, artifactName, artifactSystems); - - // 4. Return config with artifact - return { + await context.run({ artifacts: [artifact], - }; + })); } -// 5. Execute the configuration -await execute(config); +main().catch(console.error); ``` ## Design @@ -184,19 +153,70 @@ These steps guide how to compile from source code and test Vorpal by building it 4. Check configuration: ```bash -./dist/vorpal config +./dist/vorpal config --artifact "" ``` 5. Build artifacts: ```bash -./dist/vorpal build +./dist/vorpal build --artifact "" ``` -## Sandboxes +## Artifacts + +Vorpal uses `artifacts` to describe every aspect of your software in the language of your choice: + +```rust +Artifact { + // name of artifact + name: "example".to_string(), + + // artifacts for this artifact + artifacts: vec![], + + // source paths for artifact + sources: vec![ + ArtifactSource { + excludes: vec![], // optional, to remove files + hash: None, // optional, to track changes + includes: vec![], // optional, to only use files + name: "example", // required, unique per source + path: ".", // required, relative location to context + } + ], + + // steps of artifact (in order) + steps: vec![ + ArtifactStep { + entrypoint: Some("/bin/bash"), // required, host path for command (can be artifact) + arguments: vec![], // optional, arguements for entrypoint + environments: vec![], // optional, environment variables for step + script: Some("echo \"hello, world!\" > $VORPAL_OUTPUT/hello_world.txt"), // optional, script passed to executor + }, + ], + + // systems for artifact + systems: vec!["aarch64-linux", "aarch64-macos"], +}; +``` -Offical sandboxes maintained by the Vorpal development team that provide reproducibile environments. +Artifacts can be wrapped in language functions and/or modules to be shared within projects or organizations providing centrally managed and reusable configurations with domain-specific overrides (see examples in overview). -### Linux +### Sources + +Coming soon. + +### Steps + +Steps provided by the SDKs are maintained to provide reproducibile cross-platform environments for them. These environments include strictly maintained low-level dependencies that are used as a wrapper for each step. + +> [!NOTE] +> Vorpal enables developers to create their own build steps instead of using the SDKs which are provided to handle "common" scenarios. + +#### Linux + +On Linux, developers can run steps in a community maintained sandbox which is isolated similiar to containers. + +The following are included in the sandbox: - `bash` - `binutils` @@ -231,10 +251,14 @@ Offical sandboxes maintained by the Vorpal development team that provide reprodu - `xz` - `zlib` -### macOS +#### macOS + +Coming soon. + +#### Windows Coming soon. -### Windows +### Systems Coming soon. diff --git a/cli/src/worker.rs b/cli/src/worker.rs index ca0289bc..62ebe467 100644 --- a/cli/src/worker.rs +++ b/cli/src/worker.rs @@ -22,7 +22,7 @@ use vorpal_store::{ copy_files, get_artifact_archive_path, get_artifact_path, get_file_paths, get_private_key_path, get_source_archive_path, }, - temps::create_temp_dir, + temps::create_sandbox_dir, }; const DEFAULT_CHUNKS_SIZE: usize = 8192; // default grpc limit @@ -168,7 +168,7 @@ pub async fn build( } let mut sandbox_fetches = vec![]; - let sandbox_path = create_temp_dir().await?; + let sandbox_path = create_sandbox_dir().await?; for artifact_source in &artifact.sources { let handle = tokio::spawn(fetch_source( diff --git a/config/src/main.rs b/config/src/main.rs index 9d70f066..d95239da 100644 --- a/config/src/main.rs +++ b/config/src/main.rs @@ -1,31 +1,16 @@ use anyhow::Result; use vorpal_schema::vorpal::config::v0::Config; -use vorpal_sdk::config::{ - artifact::{add_systems, language::rust}, - cli::execute, - ContextConfig, -}; - -// 1. Create a function that returns a populated configuration -fn config(context: &mut ContextConfig) -> Result { - // NOTE: custom logic can be added anywhere in this function - - // 2. Define artifact parameters - let artifact_excludes = vec![".env", ".packer", ".vagrant", "script"]; - let artifact_name = "vorpal"; - let artifact_systems = add_systems(vec!["aarch64-linux", "aarch64-macos"])?; - - // 3. Create artifact (rust) - let artifact = rust::artifact(context, artifact_excludes, artifact_name, artifact_systems)?; - - // 4. Return config with artifact - Ok(Config { - artifacts: vec![artifact], - }) -} +use vorpal_sdk::config::{artifact::language::rust::rust_artifact, get_context}; #[tokio::main] async fn main() -> Result<()> { - // 5. Execute the configuration - execute(config).await + let context = &mut get_context().await?; + + let artifact = rust_artifact(context, "vorpal").await?; + + context + .run(Config { + artifacts: vec![artifact], + }) + .await } diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index 4b7e4984..3249e0cd 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -10,6 +10,8 @@ indoc = { default-features = false, version = "2" } serde = { features = ["serde_derive"], version = "1" } serde_json = { default-features = false, features = ["std"], version = "1" } sha256 = { default-features = false, version = "1" } +tokio = { default-features = false, version = "1" } +toml = "0.8.19" tonic = { default-features = false, version = "0" } tracing = { default-features = false, version = "0" } vorpal-schema = { default-features = false, path = "../schema" } diff --git a/sdk/src/config/artifact/language/rust.rs b/sdk/src/config/artifact/language/rust.rs new file mode 100644 index 00000000..e30abbdc --- /dev/null +++ b/sdk/src/config/artifact/language/rust.rs @@ -0,0 +1,225 @@ +use crate::config::{ + artifact::{ + add_artifact, get_artifact_envkey, + toolchain::{cargo, protoc, rustc}, + }, + ConfigContext, +}; +use anyhow::{bail, Result}; +use indoc::formatdoc; +use serde::Deserialize; +use std::fs; +use std::path::Path; +use toml::from_str; +use vorpal_schema::vorpal::artifact::v0::{ArtifactEnvironment, ArtifactId, ArtifactSource}; + +#[derive(Debug, Deserialize)] +struct RustArtifactCargoToml { + bin: Option>, + workspace: Option, +} + +#[derive(Debug, Deserialize)] +struct RustArtifactCargoTomlBinary { + name: String, + path: String, +} + +#[derive(Debug, Deserialize)] +struct RustArtifactCargoTomlWorkspace { + members: Option>, +} + +fn read_cargo_toml(path: &str) -> Result { + let contents = fs::read_to_string(path).expect("Failed to read Cargo.toml"); + Ok(from_str(&contents).expect("Failed to parse Cargo.toml")) +} + +pub async fn rust_artifact<'a>(context: &mut ConfigContext, name: &'a str) -> Result { + // Get toolchain artifacts + + let cargo = cargo::artifact(context).await?; + let rustc = rustc::artifact(context).await?; + let protoc = protoc::artifact(context).await?; + + // Get the source path + + let source = "."; + let source_path = Path::new(source).to_path_buf(); + + if !source_path.exists() { + bail!("Artifact `source.{}.path` not found: {:?}", name, source); + } + + // Load root cargo.toml + + let source_cargo_path = source_path.join("Cargo.toml"); + + if !source_cargo_path.exists() { + bail!("Cargo.toml not found: {:?}", source_cargo_path); + } + + let source_cargo = read_cargo_toml(source_cargo_path.to_str().unwrap())?; + + // Get list of binary targets + + let mut workspaces = vec![]; + let mut workspaces_bin_names = vec![]; + let mut workspaces_targets = vec![]; + + if let Some(workspace) = source_cargo.workspace { + if let Some(members) = workspace.members { + for member in members { + let member_path = source_path.join(member.clone()); + let member_cargo_path = member_path.join("Cargo.toml"); + + if !member_cargo_path.exists() { + bail!("Cargo.toml not found: {:?}", member_cargo_path); + } + + let member_cargo = read_cargo_toml(member_cargo_path.to_str().unwrap())?; + + let mut member_target_paths = vec![]; + + if let Some(bins) = member_cargo.bin { + for bin in bins { + member_target_paths.push(format!("{}/{}", member, bin.path)); + workspaces_bin_names.push(bin.name); + } + } + + if member_target_paths.is_empty() { + member_target_paths.push(format!("{}/src/lib.rs", member)); + } + + for member_path in member_target_paths { + workspaces_targets.push(member_path); + } + + workspaces.push(member); + } + } + } + + // Set default systems + + let systems = vec![ + "aarch64-linux", + "aarch64-macos", + "x86_64-linux", + "x86_64-macos", + ]; + + // Create vendor artifact + + let mut vendor_tomls = vec!["Cargo.toml".to_string(), "Cargo.lock".to_string()]; + + for workspace in workspaces.iter() { + vendor_tomls.push(format!("{}/Cargo.toml", workspace)); + } + + let vendor = add_artifact( + context, + vec![cargo.clone(), rustc.clone()], + vec![ + ArtifactEnvironment { + key: "HOME".to_string(), + value: "$VORPAL_WORKSPACE/home".to_string(), + }, + ArtifactEnvironment { + key: "PATH".to_string(), + value: format!( + "{cargo}/bin:{rustc}/bin", + cargo = get_artifact_envkey(&cargo), + rustc = get_artifact_envkey(&rustc) + ), + }, + ], + format!("{}-vendor", name).as_str(), + formatdoc! {" + mkdir -pv $HOME + + pushd ./source/{source} + + target_paths=({target_paths}) + + for target_path in ${{target_paths[@]}}; do + mkdir -pv \"$(dirname \"${{target_path}}\")\" + touch \"${{target_path}}\" + done + + mkdir -p \"$VORPAL_OUTPUT/vendor\" + + cargo_vendor=$(cargo vendor --versioned-dirs $VORPAL_OUTPUT/vendor) + + echo \"$cargo_vendor\" > \"$VORPAL_OUTPUT/config.toml\"", + source = name, + target_paths = workspaces_targets.join(" "), + }, + vec![ArtifactSource { + excludes: vec![], + hash: None, + includes: vendor_tomls, + name: name.to_string(), + path: source.to_string(), + }], + systems.clone(), + ) + .await?; + + // Create artifact + + // TODO: implement artifact for 'check` to pre-bake the vendor cache + + add_artifact( + context, + vec![cargo.clone(), protoc.clone(), rustc.clone(), vendor.clone()], + vec![ + ArtifactEnvironment { + key: "HOME".to_string(), + value: "$VORPAL_WORKSPACE/home".to_string(), + }, + ArtifactEnvironment { + key: "PATH".to_string(), + value: format!( + "{cargo}/bin:{rustc}/bin:{protoc}/bin", + cargo = get_artifact_envkey(&cargo), + rustc = get_artifact_envkey(&rustc), + protoc = get_artifact_envkey(&protoc), + ), + }, + ], + name, + formatdoc! {" + mkdir -pv $HOME + + pushd ./source/{name} + + mkdir -p .cargo + + ln -sv \"{vendor_envkey}/config.toml\" .cargo/config.toml + + cargo build --offline --release + cargo test --offline --release + + mkdir -p \"$VORPAL_OUTPUT/bin\" + + bin_names=({bin_names}) + + for bin_name in ${{bin_names[@]}}; do + cp -v \"target/release/${{bin_name}}\" \"$VORPAL_OUTPUT/bin/\" + done", + bin_names = workspaces_bin_names.join(" "), + vendor_envkey = get_artifact_envkey(&vendor), + }, + vec![ArtifactSource { + excludes: vec![], + hash: None, + includes: vec![], + name: name.to_string(), + path: source.to_string(), + }], + systems, + ) + .await +} diff --git a/sdk/src/config/artifact/language/rust/mod.rs b/sdk/src/config/artifact/language/rust/mod.rs deleted file mode 100644 index 5e7de836..00000000 --- a/sdk/src/config/artifact/language/rust/mod.rs +++ /dev/null @@ -1,156 +0,0 @@ -use crate::config::{ - artifact::{ - add_artifact, add_artifact_source, get_artifact_envkey, - toolchain::{cargo, protoc, rustc}, - }, - ContextConfig, -}; -use anyhow::Result; -use indoc::formatdoc; -use vorpal_schema::vorpal::artifact::v0::{ArtifactEnvironment, ArtifactId, ArtifactSystem}; - -pub fn artifact<'a>( - context: &mut ContextConfig, - excludes: Vec<&'a str>, - name: &'a str, - systems: Vec, -) -> Result { - let cargo = cargo::artifact(context)?; - let rustc = rustc::artifact(context)?; - let protoc = protoc::artifact(context)?; - - let path = "."; - - let targets = systems.iter().map(|s| (*s).into()).collect::>(); - - let vendors_source = add_artifact_source( - vec![], - None, - vec![ - "Cargo.lock".to_string(), - "Cargo.toml".to_string(), - "cli/Cargo.toml".to_string(), - "config/Cargo.toml".to_string(), - "notary/Cargo.toml".to_string(), - "registry/Cargo.toml".to_string(), - "schema/Cargo.toml".to_string(), - "sdk/Cargo.toml".to_string(), - "store/Cargo.toml".to_string(), - "worker/Cargo.toml".to_string(), - ], - name.to_string(), - path.to_string(), - )?; - - let vendors = add_artifact( - context, - vec![cargo.clone(), rustc.clone()], - vec![ - ArtifactEnvironment { - key: "HOME".to_string(), - value: "$VORPAL_WORKSPACE/home".to_string(), - }, - ArtifactEnvironment { - key: "PATH".to_string(), - value: format!( - "{cargo}/bin:{rustc}/bin", - cargo = get_artifact_envkey(&cargo), - rustc = get_artifact_envkey(&rustc) - ), - }, - ], - format!("{}-cargo-vendor", name).as_str(), - formatdoc! {" - mkdir -pv $HOME - - dirs=(\"cli/src\" \"config/src\" \"notary/src\" \"registry/src\" \"schema/src\" \"sdk/src\" \"store/src\" \"worker/src\") - - pushd ./source/{source} - - for dir in \"${{dirs[@]}}\"; do - mkdir -p \"$dir\" - done - - for dir in \"${{dirs[@]}}\"; do - if [[ \"$dir\" == \"cli/src\" || \"$dir\" == \"config/src\" ]]; then - touch \"$dir/main.rs\" - else - touch \"$dir/lib.rs\" - fi - done - - mkdir -p \"$VORPAL_OUTPUT/vendor\" - - export CARGO_VENDOR=$(cargo vendor --versioned-dirs $VORPAL_OUTPUT/vendor) - - echo \"$CARGO_VENDOR\" > \"$VORPAL_OUTPUT/config.toml\"", - source = name, - }, - vec![vendors_source], - targets.clone(), - )?; - - // TODO: implement artifact for 'check` to pre-bake the vendor cache - - let mut artifact_excludes = vec![ - ".git".to_string(), - ".gitignore".to_string(), - "target".to_string(), - ]; - - artifact_excludes.extend(excludes.iter().map(|e| e.to_string())); - - let artifact_source = add_artifact_source( - artifact_excludes.clone(), - None, - vec![], - name.to_string(), - path.to_string(), - )?; - - add_artifact( - context, - vec![ - cargo.clone(), - protoc.clone(), - rustc.clone(), - vendors.clone(), - ], - vec![ - ArtifactEnvironment { - key: "HOME".to_string(), - value: "$VORPAL_WORKSPACE/home".to_string(), - }, - ArtifactEnvironment { - key: "PATH".to_string(), - value: format!( - "{cargo}/bin:{rustc}/bin:{protoc}/bin", - cargo = get_artifact_envkey(&cargo), - rustc = get_artifact_envkey(&rustc), - protoc = get_artifact_envkey(&protoc), - ), - }, - ], - name, - formatdoc! {" - mkdir -pv $HOME - - pushd ./source/{name} - - mkdir -p .cargo - - ln -sv \"{vendor_cache}/config.toml\" .cargo/config.toml - - cargo build --offline --release - - cargo test --offline --release - - mkdir -p \"$VORPAL_OUTPUT/bin\" - - cp -pr ./target/release/. $VORPAL_OUTPUT/.", - vendor_cache = get_artifact_envkey(&vendors), - }, - vec![artifact_source], - targets, - ) -} diff --git a/sdk/src/config/artifact/mod.rs b/sdk/src/config/artifact/mod.rs index 2ec22127..5f6bbd42 100644 --- a/sdk/src/config/artifact/mod.rs +++ b/sdk/src/config/artifact/mod.rs @@ -3,7 +3,7 @@ use crate::config::{ steps::{bash, bwrap}, toolchain::linux::{debian, vorpal}, }, - ContextConfig, + ConfigContext, }; use anyhow::{bail, Result}; use std::path::Path; @@ -11,57 +11,64 @@ use vorpal_schema::vorpal::artifact::v0::{ Artifact, ArtifactEnvironment, ArtifactId, ArtifactSource, ArtifactSystem, ArtifactSystem::{Aarch64Linux, Aarch64Macos, X8664Linux, X8664Macos}, }; -use vorpal_store::{hashes::hash_files, paths::get_file_paths}; +use vorpal_store::paths::get_file_paths; pub mod language; pub mod steps; pub mod toolchain; pub fn get_artifact_envkey(artifact: &ArtifactId) -> String { - let artifact_key = artifact.name.to_lowercase().replace("-", "_"); - format!("$VORPAL_ARTIFACT_{}", artifact_key).to_string() + format!( + "$VORPAL_ARTIFACT_{}", + artifact.name.to_lowercase().replace("-", "_") + ) + .to_string() } -pub fn add_systems(systems: Vec<&str>) -> Result> { - let mut build_systems = vec![]; - - for system in systems { - match system { - "aarch64-linux" => build_systems.push(Aarch64Linux), - "aarch64-macos" => build_systems.push(Aarch64Macos), - "x86_64-linux" => build_systems.push(X8664Linux), - "x86_64-macos" => build_systems.push(X8664Macos), - _ => bail!("Unsupported system: {}", system), - } - } - - Ok(build_systems) -} - -pub fn add_artifact_source( - excludes: Vec, - hash: Option, - includes: Vec, - name: String, - path: String, +pub async fn add_artifact_source( + context: &mut ConfigContext, + source: ArtifactSource, ) -> Result { - // TODO: add support for downloading sources + // TODO: add support for 'remote' sources - let source_path = Path::new(&path).to_path_buf(); + let source_path = Path::new(&source.path).to_path_buf(); if !source_path.exists() { - bail!("Artifact `source.{}.path` not found: {:?}", name, path); + bail!( + "Artifact `source.{}.path` not found: {:?}", + source.name, + source.path + ); } - let source_files = get_file_paths(&source_path, excludes.clone(), includes.clone())?; - - let source_hash = hash_files(source_files)?; + let source_files = get_file_paths( + &source_path, + source.excludes.clone(), + source.includes.clone(), + )?; + + let source_hash = match context.get_source_hash( + source_files.clone(), + source.name.clone(), + source_path.clone(), + ) { + Some(hash) => hash.clone(), + None => { + context + .add_source_hash( + source_files.clone(), + source.name.clone(), + source_path.clone(), + ) + .await? + } + }; - if let Some(hash) = hash.clone() { + if let Some(hash) = source.hash.clone() { if hash != source_hash { bail!( "Artifact `source.{}.hash` mismatch: {} != {}", - name, + source.name, hash, source_hash ); @@ -69,27 +76,53 @@ pub fn add_artifact_source( } Ok(ArtifactSource { - excludes, + excludes: source.excludes, hash: Some(source_hash), - includes, - name, - path, + includes: source.includes, + name: source.name, + path: source.path, }) } -pub fn add_artifact( - context: &mut ContextConfig, +pub fn add_artifact_systems(systems: Vec<&str>) -> Result> { + let mut build_systems = vec![]; + + for system in systems { + match system { + "aarch64-linux" => build_systems.push(Aarch64Linux), + "aarch64-macos" => build_systems.push(Aarch64Macos), + "x86_64-linux" => build_systems.push(X8664Linux), + "x86_64-macos" => build_systems.push(X8664Macos), + _ => bail!("Unsupported system: {}", system), + } + } + + Ok(build_systems) +} + +// cross-platform sandboxed artifact + +pub async fn add_artifact( + context: &mut ConfigContext, artifacts: Vec, environments: Vec, name: &str, script: String, sources: Vec, - systems: Vec, + systems: Vec<&str>, ) -> Result { - let build_target = context.get_target(); + // Setup artifacts + + let mut build_artifacts = vec![]; + + for artifact in artifacts { + build_artifacts.push(artifact); + } // Setup environments + let build_target = context.get_target(); + let mut build_environments = vec![]; if build_target == Aarch64Linux || build_target == X8664Linux { @@ -149,19 +182,22 @@ pub fn add_artifact( build_environments.push(env); } - let mut build_artifacts = vec![]; - let mut build_steps = vec![]; + // Setup sources - // Setup artifacts + let mut build_sources = vec![]; - for artifact in artifacts { - build_artifacts.push(artifact); + for source in sources.clone().into_iter() { + let source = add_artifact_source(context, source).await?; + + build_sources.push(source); } // Setup steps + let mut build_steps = vec![]; + if build_target == Aarch64Linux || build_target == X8664Linux { - let linux_debian = debian::artifact(context)?; + let linux_debian = debian::artifact(context).await?; let linux_vorpal = vorpal::artifact(context, &linux_debian)?; build_artifacts.push(linux_vorpal.clone()); @@ -179,12 +215,17 @@ pub fn add_artifact( build_steps.push(bash(build_environments.clone(), script)); } - // Setup artifacts + // Setup systems + + let systems = add_artifact_systems(systems)?; + let systems = systems.iter().map(|s| (*s).into()).collect::>(); + + // Add artifact to context context.add_artifact(Artifact { artifacts: build_artifacts.clone(), name: name.to_string(), - sources, + sources: build_sources, steps: build_steps, systems, }) diff --git a/sdk/src/config/artifact/toolchain/cargo.rs b/sdk/src/config/artifact/toolchain/cargo.rs index b236b9f9..9b62d287 100644 --- a/sdk/src/config/artifact/toolchain/cargo.rs +++ b/sdk/src/config/artifact/toolchain/cargo.rs @@ -1,6 +1,6 @@ use crate::config::{ artifact::{add_artifact, get_artifact_envkey}, - ContextConfig, + ConfigContext, }; use anyhow::{bail, Result}; use indoc::formatdoc; @@ -9,14 +9,14 @@ use vorpal_schema::vorpal::artifact::v0::{ ArtifactSystem::{Aarch64Linux, Aarch64Macos, UnknownSystem, X8664Linux, X8664Macos}, }; -pub fn artifact(context: &mut ContextConfig) -> Result { +pub async fn artifact(context: &mut ConfigContext) -> Result { let name = "cargo"; let systems = vec![ - Aarch64Linux.into(), - Aarch64Macos.into(), - X8664Linux.into(), - X8664Macos.into(), + "aarch64-linux", + "aarch64-macos", + "x86_64-linux", + "x86_64-macos", ]; let source = add_artifact( @@ -40,7 +40,8 @@ pub fn artifact(context: &mut ContextConfig) -> Result { }, vec![], systems.clone(), - )?; + ) + .await?; add_artifact( context, @@ -54,4 +55,5 @@ pub fn artifact(context: &mut ContextConfig) -> Result { vec![], systems, ) + .await } diff --git a/sdk/src/config/artifact/toolchain/linux/debian.rs b/sdk/src/config/artifact/toolchain/linux/debian.rs index aba59855..fed834a9 100644 --- a/sdk/src/config/artifact/toolchain/linux/debian.rs +++ b/sdk/src/config/artifact/toolchain/linux/debian.rs @@ -3,16 +3,16 @@ use crate::config::{ add_artifact_source, get_artifact_envkey, steps::{bash, docker}, }, - ContextConfig, + ConfigContext, }; use anyhow::Result; use indoc::formatdoc; use vorpal_schema::vorpal::artifact::v0::{ - Artifact, ArtifactEnvironment, ArtifactId, + Artifact, ArtifactEnvironment, ArtifactId, ArtifactSource, ArtifactSystem::{Aarch64Linux, X8664Linux}, }; -pub fn artifact(context: &mut ContextConfig) -> Result { +pub async fn artifact(context: &mut ConfigContext) -> Result { let environments = vec![ArtifactEnvironment { key: "PATH".to_string(), value: "/usr/bin:/bin:/usr/sbin:/sbin".to_string(), @@ -21,15 +21,19 @@ pub fn artifact(context: &mut ContextConfig) -> Result { let systems = vec![Aarch64Linux.into(), X8664Linux.into()]; let source = add_artifact_source( - vec![], - None, - vec![ - "Dockerfile".to_string(), - "script/version_check.sh".to_string(), - ], - "docker".to_string(), - ".".to_string(), - )?; + context, + ArtifactSource { + excludes: vec![], + hash: None, + includes: vec![ + "Dockerfile".to_string(), + "script/version_check.sh".to_string(), + ], + name: "docker".to_string(), + path: ".".to_string(), + }, + ) + .await?; let source = context.add_artifact(Artifact { artifacts: vec![], diff --git a/sdk/src/config/artifact/toolchain/linux/vorpal/mod.rs b/sdk/src/config/artifact/toolchain/linux/vorpal/mod.rs index 8a867522..ad3122c9 100644 --- a/sdk/src/config/artifact/toolchain/linux/vorpal/mod.rs +++ b/sdk/src/config/artifact/toolchain/linux/vorpal/mod.rs @@ -1,4 +1,4 @@ -use crate::config::artifact::{get_artifact_envkey, steps, ContextConfig}; +use crate::config::artifact::{get_artifact_envkey, steps, ConfigContext}; use anyhow::Result; use vorpal_schema::vorpal::artifact::v0::{ Artifact, ArtifactEnvironment, ArtifactId, @@ -8,7 +8,7 @@ use vorpal_schema::vorpal::artifact::v0::{ mod script; mod source; -pub fn artifact(context: &mut ContextConfig, linux_debian: &ArtifactId) -> Result { +pub fn artifact(context: &mut ConfigContext, linux_debian: &ArtifactId) -> Result { let bash = source::bash(context, linux_debian)?; let binutils = source::binutils(context, linux_debian)?; let bison = source::bison(context, linux_debian)?; diff --git a/sdk/src/config/artifact/toolchain/linux/vorpal/source.rs b/sdk/src/config/artifact/toolchain/linux/vorpal/source.rs index bb1f10b1..fb057ce4 100644 --- a/sdk/src/config/artifact/toolchain/linux/vorpal/source.rs +++ b/sdk/src/config/artifact/toolchain/linux/vorpal/source.rs @@ -1,4 +1,4 @@ -use crate::config::artifact::{get_artifact_envkey, steps::bwrap, ContextConfig}; +use crate::config::artifact::{get_artifact_envkey, steps::bwrap, ConfigContext}; use anyhow::Result; use indoc::formatdoc; use vorpal_schema::vorpal::artifact::v0::{ @@ -36,7 +36,7 @@ fn new_artifact_source( } } -pub fn bash(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result { +pub fn bash(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("bash"), @@ -51,7 +51,7 @@ pub fn bash(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn binutils(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("binutils"), @@ -66,7 +66,7 @@ pub fn binutils(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn bison(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("bison"), @@ -81,7 +81,7 @@ pub fn bison(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn coreutils(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("coreutils"), @@ -96,7 +96,7 @@ pub fn coreutils(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn curl(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("curl"), @@ -111,7 +111,7 @@ pub fn curl(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn curl_cacert(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("openssl-cacert"), @@ -126,7 +126,7 @@ pub fn curl_cacert(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn diffutils(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("diffutils"), @@ -141,7 +141,7 @@ pub fn diffutils(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn file(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("file"), @@ -156,7 +156,7 @@ pub fn file(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn findutils(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("findutils"), @@ -171,7 +171,7 @@ pub fn findutils(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn gawk(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("gawk"), @@ -188,7 +188,7 @@ pub fn gawk(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn gcc(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("gcc"), @@ -213,7 +213,7 @@ pub fn gcc(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn gettext(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("gettext"), @@ -228,7 +228,7 @@ pub fn gettext(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn glibc_patch(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("glibc-patch"), @@ -244,7 +244,7 @@ pub fn glibc_patch(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { @@ -267,7 +267,7 @@ pub fn glibc( )) } -pub fn grep(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result { +pub fn grep(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("grep"), @@ -282,7 +282,7 @@ pub fn grep(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn gzip(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("gzip"), @@ -297,7 +297,7 @@ pub fn gzip(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn libidn2(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("libidn2"), @@ -312,7 +312,7 @@ pub fn libidn2(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn libpsl(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("libpsl"), @@ -327,7 +327,7 @@ pub fn libpsl(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn libunistring(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("libunistring"), @@ -342,7 +342,7 @@ pub fn libunistring(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result< )) } -pub fn linux_headers(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result { +pub fn linux_headers(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("linux-headers"), @@ -357,7 +357,7 @@ pub fn linux_headers(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result )) } -pub fn m4(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result { +pub fn m4(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("m4"), @@ -372,7 +372,7 @@ pub fn m4(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn make(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("make"), @@ -387,7 +387,7 @@ pub fn make(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn ncurses(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("ncurses"), @@ -402,7 +402,7 @@ pub fn ncurses(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn openssl(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("openssl"), @@ -417,7 +417,7 @@ pub fn openssl(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn patch(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("patch"), @@ -432,7 +432,7 @@ pub fn patch(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn perl(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("perl"), @@ -446,7 +446,7 @@ pub fn perl(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn python(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("python"), @@ -461,7 +461,7 @@ pub fn python(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn sed(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("sed"), @@ -476,7 +476,7 @@ pub fn sed(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn tar(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("tar"), @@ -491,7 +491,7 @@ pub fn tar(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn texinfo(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("texinfo"), @@ -506,7 +506,7 @@ pub fn texinfo(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn unzip_patch_fixes(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("unzip-patch-fixes"), @@ -519,7 +519,7 @@ pub fn unzip_patch_fixes(context: &mut ContextConfig, rootfs: &ArtifactId) -> Re )) } -pub fn unzip_patch_gcc14(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result { +pub fn unzip_patch_gcc14(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("unzip-patch-gcc14"), @@ -533,7 +533,7 @@ pub fn unzip_patch_gcc14(context: &mut ContextConfig, rootfs: &ArtifactId) -> Re } pub fn unzip( - context: &mut ContextConfig, + context: &mut ConfigContext, rootfs: &ArtifactId, patch_fixes: &ArtifactId, patch_gcc14: &ArtifactId, @@ -560,7 +560,7 @@ pub fn unzip( )) } -pub fn util_linux(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result { +pub fn util_linux(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("util-linux"), @@ -575,7 +575,7 @@ pub fn util_linux(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn xz(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("xz"), @@ -590,7 +590,7 @@ pub fn xz(context: &mut ContextConfig, rootfs: &ArtifactId) -> Result Result { +pub fn zlib(context: &mut ConfigContext, rootfs: &ArtifactId) -> Result { context.add_artifact(new_artifact_source( vec![], new_source_name("zlib"), diff --git a/sdk/src/config/artifact/toolchain/protoc.rs b/sdk/src/config/artifact/toolchain/protoc.rs index 046c6b93..58556e8f 100644 --- a/sdk/src/config/artifact/toolchain/protoc.rs +++ b/sdk/src/config/artifact/toolchain/protoc.rs @@ -1,4 +1,4 @@ -use crate::config::{artifact::add_artifact, ContextConfig}; +use crate::config::{artifact::add_artifact, ConfigContext}; use anyhow::{bail, Result}; use indoc::formatdoc; use vorpal_schema::vorpal::artifact::v0::{ @@ -6,14 +6,14 @@ use vorpal_schema::vorpal::artifact::v0::{ ArtifactSystem::{Aarch64Linux, Aarch64Macos, UnknownSystem, X8664Linux, X8664Macos}, }; -pub fn artifact(context: &mut ContextConfig) -> Result { +pub async fn artifact(context: &mut ConfigContext) -> Result { let name = "protoc"; let systems = vec![ - Aarch64Linux.into(), - Aarch64Macos.into(), - X8664Linux.into(), - X8664Macos.into(), + "aarch64-linux", + "aarch64-macos", + "x86_64-linux", + "x86_64-macos", ]; add_artifact( @@ -39,5 +39,5 @@ pub fn artifact(context: &mut ContextConfig) -> Result { }, vec![], systems, - ) + ).await } diff --git a/sdk/src/config/artifact/toolchain/rust_std.rs b/sdk/src/config/artifact/toolchain/rust_std.rs index e672a43c..c5b8392c 100644 --- a/sdk/src/config/artifact/toolchain/rust_std.rs +++ b/sdk/src/config/artifact/toolchain/rust_std.rs @@ -1,6 +1,6 @@ use crate::config::{ artifact::{add_artifact, get_artifact_envkey}, - ContextConfig, + ConfigContext, }; use anyhow::{bail, Result}; use indoc::formatdoc; @@ -9,7 +9,7 @@ use vorpal_schema::vorpal::artifact::v0::{ ArtifactSystem::{Aarch64Linux, Aarch64Macos, UnknownSystem, X8664Linux, X8664Macos}, }; -pub fn artifact(context: &mut ContextConfig) -> Result { +pub async fn artifact(context: &mut ConfigContext) -> Result { let name = "rust-std"; let target = match context.get_target() { @@ -21,10 +21,10 @@ pub fn artifact(context: &mut ContextConfig) -> Result { }; let systems = vec![ - Aarch64Linux.into(), - Aarch64Macos.into(), - X8664Linux.into(), - X8664Macos.into(), + "aarch64-linux", + "aarch64-macos", + "x86_64-linux", + "x86_64-macos", ]; let version = "1.78.0"; @@ -42,7 +42,8 @@ pub fn artifact(context: &mut ContextConfig) -> Result { }, vec![], systems.clone(), - )?; + ) + .await?; add_artifact( context, @@ -56,4 +57,5 @@ pub fn artifact(context: &mut ContextConfig) -> Result { vec![], systems, ) + .await } diff --git a/sdk/src/config/artifact/toolchain/rustc.rs b/sdk/src/config/artifact/toolchain/rustc.rs index 02bcfc2c..a68175a5 100644 --- a/sdk/src/config/artifact/toolchain/rustc.rs +++ b/sdk/src/config/artifact/toolchain/rustc.rs @@ -1,6 +1,6 @@ use crate::config::{ artifact::{add_artifact, get_artifact_envkey, toolchain::rust_std}, - ContextConfig, + ConfigContext, }; use anyhow::{bail, Result}; use indoc::formatdoc; @@ -9,16 +9,16 @@ use vorpal_schema::vorpal::artifact::v0::{ ArtifactSystem::{Aarch64Linux, Aarch64Macos, UnknownSystem, X8664Linux, X8664Macos}, }; -pub fn artifact(context: &mut ContextConfig) -> Result { - let rust_std = rust_std::artifact(context)?; +pub async fn artifact(context: &mut ConfigContext) -> Result { + let rust_std = rust_std::artifact(context).await?; let name = "rustc"; let systems = vec![ - Aarch64Linux.into(), - Aarch64Macos.into(), - X8664Linux.into(), - X8664Macos.into(), + "aarch64-linux", + "aarch64-macos", + "x86_64-linux", + "x86_64-macos", ]; let target = match context.get_target() { @@ -44,7 +44,8 @@ pub fn artifact(context: &mut ContextConfig) -> Result { }, vec![], systems.clone(), - )?; + ) + .await?; add_artifact( context, @@ -63,4 +64,5 @@ pub fn artifact(context: &mut ContextConfig) -> Result { vec![], systems, ) + .await } diff --git a/sdk/src/config/cli.rs b/sdk/src/config/cli.rs index 7aaf9609..6e9f4e74 100644 --- a/sdk/src/config/cli.rs +++ b/sdk/src/config/cli.rs @@ -1,58 +1,6 @@ -use crate::{config::ContextConfig, service}; +use crate::config::ConfigContext; use anyhow::Result; use clap::{Parser, Subcommand}; use std::env::consts::{ARCH, OS}; use tracing::Level; -use vorpal_schema::{ - get_artifact_system, vorpal::artifact::v0::ArtifactSystem, vorpal::config::v0::Config, -}; - -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -#[command(propagate_version = true)] -pub struct Cli { - #[command(subcommand)] - command: Command, -} - -fn get_default_system() -> String { - format!("{}-{}", ARCH, OS) -} - -#[derive(Subcommand)] -enum Command { - Start { - #[clap(default_value_t = Level::INFO, global = true, long)] - level: Level, - - #[clap(long, short)] - port: u16, - - #[arg(default_value_t = get_default_system(), long, short)] - target: String, - }, -} - -pub type ConfigFunction = fn(context: &mut ContextConfig) -> Result; - -pub async fn execute(config: ConfigFunction) -> Result<()> { - let args = Cli::parse(); - - match args.command { - Command::Start { port, target, .. } => { - let target = get_artifact_system::(&target); - - if target == ArtifactSystem::UnknownSystem { - return Err(anyhow::anyhow!("Invalid target system")); - } - - let mut config_context = ContextConfig::new(target); - - let config = config(&mut config_context)?; - - service::listen(config_context, config, port).await?; - } - } - - Ok(()) -} +use vorpal_schema::{get_artifact_system, vorpal::artifact::v0::ArtifactSystem}; diff --git a/sdk/src/config/mod.rs b/sdk/src/config/mod.rs index 8d803522..0503cf9f 100644 --- a/sdk/src/config/mod.rs +++ b/sdk/src/config/mod.rs @@ -1,25 +1,73 @@ -use crate::service; +use crate::config::service::ConfigServer; use anyhow::Result; use clap::{Parser, Subcommand}; use serde::{Deserialize, Serialize}; use sha256::digest; use std::collections::HashMap; use std::env::consts::{ARCH, OS}; +use std::path::PathBuf; +use tokio::fs::{create_dir_all, remove_dir_all}; +use tonic::transport::Server; use tracing::Level; use vorpal_schema::{ get_artifact_system, vorpal::{ artifact::v0::{Artifact, ArtifactId, ArtifactSystem}, - config::v0::Config, + config::v0::{config_service_server::ConfigServiceServer, Config}, }, }; +use vorpal_store::{hashes::hash_files, paths::copy_files, temps::create_sandbox_dir}; pub mod artifact; -pub mod cli; +pub mod service; -#[derive(Debug, Default)] -pub struct ContextConfig { +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +#[command(propagate_version = true)] +pub struct Cli { + #[command(subcommand)] + command: Command, +} + +fn get_default_system() -> String { + format!("{}-{}", ARCH, OS) +} + +#[derive(Subcommand)] +enum Command { + Start { + #[clap(default_value_t = Level::INFO, global = true, long)] + level: Level, + + #[clap(long, short)] + port: u16, + + #[arg(default_value_t = get_default_system(), long, short)] + target: String, + }, +} + +pub async fn get_context() -> Result { + let args = Cli::parse(); + + match args.command { + Command::Start { port, target, .. } => { + let target = get_artifact_system::(&target); + + if target == ArtifactSystem::UnknownSystem { + return Err(anyhow::anyhow!("Invalid target system")); + } + + Ok(ConfigContext::new(port, target)) + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct ConfigContext { artifact: HashMap, + pub port: u16, + source_hash: HashMap, system: ArtifactSystem, } @@ -28,10 +76,28 @@ pub struct ArtifactMetadata { pub system: ArtifactSystem, } -impl ContextConfig { - pub fn new(system: ArtifactSystem) -> Self { +fn get_source_key_digest(path: PathBuf, files: Vec) -> Result { + let mut relative_paths = vec![]; + + for file in files { + let relative_path = file.strip_prefix(&path).map_err(|e| anyhow::anyhow!(e))?; + + relative_paths.push(relative_path.display().to_string()); + } + + let relative_paths = relative_paths.join("\n"); + + let source_hash = digest(relative_paths.as_bytes()); + + Ok(source_hash) +} + +impl ConfigContext { + pub fn new(port: u16, system: ArtifactSystem) -> Self { Self { artifact: HashMap::new(), + port, + source_hash: HashMap::new(), system, } } @@ -64,61 +130,75 @@ impl ContextConfig { pub fn get_artifact(&self, hash: &str, name: &str) -> Option<&Artifact> { let artifact_key = format!("{}-{}", name, hash); - self.artifact.get(&artifact_key) } - pub fn get_target(&self) -> ArtifactSystem { - self.system - } -} + pub async fn add_source_hash( + &mut self, + files: Vec, + name: String, + path: PathBuf, + ) -> Result { + let source_key_digest = get_source_key_digest(path.clone(), files.clone())?; + let source_key = format!("{}-{}", name, source_key_digest); -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -#[command(propagate_version = true)] -pub struct Cli { - #[command(subcommand)] - command: Command, -} + if !self.source_hash.contains_key(&source_key) { + let sandbox_path = create_sandbox_dir().await?; -fn get_default_system() -> String { - format!("{}-{}", ARCH, OS) -} + create_dir_all(&sandbox_path) + .await + .map_err(|e| anyhow::anyhow!(e))?; -#[derive(Subcommand)] -enum Command { - Start { - #[clap(default_value_t = Level::INFO, global = true, long)] - level: Level, + let sandbox_files = copy_files(&path, files.clone(), &sandbox_path).await?; - #[clap(long, short)] - port: u16, + let source_hash = hash_files(sandbox_files.clone())?; - #[arg(default_value_t = get_default_system(), long, short)] - target: String, - }, -} + self.source_hash + .insert(source_key.clone(), source_hash.clone()); -pub type ConfigFunction = fn(context: &mut ContextConfig) -> Result; + remove_dir_all(&sandbox_path) + .await + .map_err(|e| anyhow::anyhow!(e))?; -pub async fn execute(config: ConfigFunction) -> Result<()> { - let args = Cli::parse(); + return Ok(source_hash); + } - match args.command { - Command::Start { port, target, .. } => { - let target = get_artifact_system::(&target); + let source_hash = self.source_hash.get(&source_key).unwrap(); - if target == ArtifactSystem::UnknownSystem { - return Err(anyhow::anyhow!("Invalid target system")); - } + Ok(source_hash.clone()) + } - let mut config_context = ContextConfig::new(target); + pub fn get_source_hash( + &self, + files: Vec, + name: String, + path: PathBuf, + ) -> Option<&String> { + let source_key_digest = get_source_key_digest(path, files).ok()?; + let source_key = format!("{}-{}", name, source_key_digest); - let config = config(&mut config_context)?; + self.source_hash.get(&source_key) + } - service::listen(config_context, config, port).await?; - } + pub fn get_target(&self) -> ArtifactSystem { + self.system } - Ok(()) + pub async fn run(&self, config: Config) -> Result<()> { + let addr = format!("[::]:{}", self.port) + .parse() + .expect("failed to parse address"); + + let context = self.clone(); + + let config_service = ConfigServiceServer::new(ConfigServer::new(context, config)); + + println!("Config server listening on {}", addr); + + Server::builder() + .add_service(config_service) + .serve(addr) + .await + .map_err(|e| anyhow::anyhow!("failed to serve: {}", e)) + } } diff --git a/sdk/src/service/hash.rs b/sdk/src/config/service/hash.rs similarity index 100% rename from sdk/src/service/hash.rs rename to sdk/src/config/service/hash.rs diff --git a/sdk/src/service/mod.rs b/sdk/src/config/service/mod.rs similarity index 56% rename from sdk/src/service/mod.rs rename to sdk/src/config/service/mod.rs index 1c746c04..a147e744 100644 --- a/sdk/src/service/mod.rs +++ b/sdk/src/config/service/mod.rs @@ -1,22 +1,18 @@ -use crate::config::ContextConfig; +use crate::config::ConfigContext; use anyhow::Result; -use tonic::transport::Server; use vorpal_schema::vorpal::{ artifact::v0::{Artifact, ArtifactId}, - config::v0::{ - config_service_server::{ConfigService, ConfigServiceServer}, - Config, ConfigRequest, - }, + config::v0::{config_service_server::ConfigService, Config, ConfigRequest}, }; #[derive(Debug, Default)] pub struct ConfigServer { - pub context: ContextConfig, + pub context: ConfigContext, pub config: Config, } impl ConfigServer { - pub fn new(context: ContextConfig, config: Config) -> Self { + pub fn new(context: ConfigContext, config: Config) -> Self { Self { context, config } } } @@ -47,21 +43,3 @@ impl ConfigService for ConfigServer { Ok(tonic::Response::new(artifact.unwrap().clone())) } } - -pub async fn listen(context: ContextConfig, config: Config, port: u16) -> Result<()> { - let addr = format!("[::]:{}", port) - .parse() - .expect("failed to parse address"); - - let config_service = ConfigServiceServer::new(ConfigServer::new(context, config)); - - println!("Config server listening on {}", addr); - - Server::builder() - .add_service(config_service) - .serve(addr) - .await - .expect("failed to start worker server"); - - Ok(()) -} diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index 11bd8fe2..ef68c369 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -1,2 +1 @@ pub mod config; -pub mod service; diff --git a/store/Cargo.toml b/store/Cargo.toml index 5248b7dc..e017a12d 100644 --- a/store/Cargo.toml +++ b/store/Cargo.toml @@ -9,6 +9,7 @@ async-compression = { default-features = false, features = ["bzip2", "gzip", "to async_zip = { default-features = false, features = ["deflate", "tokio"], version = "0" } filetime = { version = "0" } futures-lite = { default-features = false, version = "2" } +ignore = { default-features = false, version = "0" } sanitize-filename = { default-features = false, version = "0" } sha256 = { default-features = false, version = "1" } tokio = { default-features = false, version = "1" } @@ -17,4 +18,3 @@ tokio-util = { default-features = false, features = ["compat"], version = "0" } tracing = { default-features = false, version = "0" } uuid = { default-features = false, features = ["std", "v7"], version = "1" } vorpal-schema = { default-features = false, path = "../schema" } -walkdir = { default-features = false, version = "2" } diff --git a/store/src/archives.rs b/store/src/archives.rs index 71553d7d..6e5bb17e 100644 --- a/store/src/archives.rs +++ b/store/src/archives.rs @@ -1,6 +1,6 @@ use crate::{ paths::{get_file_paths, set_paths_timestamps}, - temps::create_temp_file, + temps::create_sandbox_file, }; use anyhow::{Error, Result}; use async_compression::tokio::{ @@ -24,7 +24,7 @@ pub async fn compress_zstd( source_files: &[PathBuf], output_path: &PathBuf, ) -> Result { - let temp_file = create_temp_file(Some("tar.zst")) + let temp_file = create_sandbox_file(Some("tar.zst")) .await .expect("Failed to create temp file"); diff --git a/store/src/paths.rs b/store/src/paths.rs index 5926175c..8e4a9b0d 100644 --- a/store/src/paths.rs +++ b/store/src/paths.rs @@ -1,10 +1,10 @@ use anyhow::{bail, Error, Result}; use filetime::{set_file_times, set_symlink_file_times, FileTime}; +use ignore::WalkBuilder; use std::path::{Path, PathBuf}; use tokio::fs::{copy, create_dir_all, metadata, symlink}; use tracing::info; use uuid::Uuid; -use walkdir::WalkDir; // Store paths @@ -97,12 +97,27 @@ pub fn get_file_paths( excludes: Vec, includes: Vec, ) -> Result> { - let excludes_paths = excludes + let mut excludes_paths = excludes .into_iter() .map(|i| Path::new(&i).to_path_buf()) .collect::>(); - let mut files: Vec = WalkDir::new(source_path) + // Exclude git directory + + excludes_paths.push(Path::new(".git").to_path_buf()); + + let walker = WalkBuilder::new(source_path) + .standard_filters(false) // Don't use default filters + .git_exclude(true) + .git_global(true) + .git_ignore(true) + .hidden(false) + .ignore(false) + .parents(true) + .require_git(false) + .build(); + + let mut files: Vec = walker .into_iter() .filter_map(|entry| { let entry = entry.ok()?; @@ -160,8 +175,8 @@ pub async fn set_paths_timestamps(paths: &[PathBuf]) -> Result<(), Error> { pub async fn copy_files( source_path: &PathBuf, source_path_files: Vec, - destination_path: &Path, -) -> Result<()> { + target_path: &Path, +) -> Result> { if source_path_files.is_empty() { bail!("no source files found"); } @@ -177,7 +192,7 @@ pub async fn copy_files( let metadata = metadata(src).await.expect("failed to read metadata"); - let dest = destination_path.join(src.strip_prefix(source_path).unwrap()); + let dest = target_path.join(src.strip_prefix(source_path).unwrap()); if metadata.is_dir() { create_dir_all(dest).await.expect("create directory fail"); @@ -197,11 +212,11 @@ pub async fn copy_files( } } - let artifact_paths = get_file_paths(&destination_path.to_path_buf(), vec![], vec![])?; + let target_path_files = get_file_paths(&target_path.to_path_buf(), vec![], vec![])?; - set_paths_timestamps(&artifact_paths).await?; + set_paths_timestamps(&target_path_files).await?; - Ok(()) + Ok(target_path_files) } pub async fn setup_paths() -> Result<()> { diff --git a/store/src/temps.rs b/store/src/temps.rs index 4390d93c..d754344b 100644 --- a/store/src/temps.rs +++ b/store/src/temps.rs @@ -3,7 +3,7 @@ use anyhow::{anyhow, Result}; use std::path::PathBuf; use tokio::fs::{create_dir_all, File}; -pub async fn create_temp_dir() -> Result { +pub async fn create_sandbox_dir() -> Result { let dir_path = paths::get_temp_path(); create_dir_all(&dir_path) @@ -13,7 +13,7 @@ pub async fn create_temp_dir() -> Result { Ok(dir_path) } -pub async fn create_temp_file(extension: Option<&str>) -> Result { +pub async fn create_sandbox_file(extension: Option<&str>) -> Result { let mut file_path = paths::get_temp_path(); if let Some(extension) = extension { diff --git a/worker/src/artifact.rs b/worker/src/artifact.rs index ae1f1eaf..062ca8dd 100644 --- a/worker/src/artifact.rs +++ b/worker/src/artifact.rs @@ -33,7 +33,7 @@ use vorpal_schema::{ }, }, }; -use vorpal_store::temps::create_temp_dir; +use vorpal_store::temps::create_sandbox_dir; use vorpal_store::{ archives::{compress_zstd, unpack_zstd}, paths::{ @@ -380,7 +380,7 @@ impl ArtifactService for ArtifactServer { // Create workspace - let workspace_path = match create_temp_dir().await { + let workspace_path = match create_sandbox_dir().await { Ok(path) => path, Err(err) => { if let Err(err) = tx