From 53982ad51fa29cd1f3e4202150d19d26c9ed83d0 Mon Sep 17 00:00:00 2001 From: Shadaj Laddad Date: Fri, 5 Jan 2024 14:36:49 -0800 Subject: [PATCH] Update Hydroflow --- Cargo.lock | 91 ++++++++++++---------- README.md | 2 +- flow/examples/first_ten_distributed.rs | 6 +- flow/examples/first_ten_distributed_gcp.rs | 6 +- flow/src/first_ten.rs | 11 ++- flow/src/first_ten_distributed.rs | 31 ++++---- 6 files changed, 76 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e84c9b2..5386382 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,7 +149,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "async-task" -version = "4.6.0" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d90cd0b264dfdd8eb5bad0a2c217c1f88fa96a8573f40e7b12de23fb468f46" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" @@ -197,7 +197,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -380,9 +380,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -416,6 +416,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dunce" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" + [[package]] name = "dyn-clone" version = "1.0.16" @@ -609,7 +615,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -698,7 +704,7 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hydro_deploy" version = "0.5.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "anyhow", "async-channel 1.9.0", @@ -709,6 +715,7 @@ dependencies = [ "async-trait", "bytes", "cargo_metadata", + "dunce", "dyn-clone", "futures", "futures-core", @@ -728,7 +735,7 @@ dependencies = [ [[package]] name = "hydroflow" version = "0.5.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "bincode", "byteorder", @@ -759,7 +766,7 @@ dependencies = [ [[package]] name = "hydroflow_cli_integration" version = "0.3.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "async-recursion", "async-trait", @@ -775,7 +782,7 @@ dependencies = [ [[package]] name = "hydroflow_lang" version = "0.5.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "auto_impl", "itertools", @@ -786,13 +793,13 @@ dependencies = [ "serde", "serde_json", "slotmap", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] name = "hydroflow_plus" version = "0.5.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "bincode", "hydroflow", @@ -802,13 +809,13 @@ dependencies = [ "quote", "serde", "stageleft", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] name = "hydroflow_plus_cli_integration" version = "0.5.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "async-channel 1.9.0", "hydro_deploy", @@ -816,7 +823,7 @@ dependencies = [ "serde", "stageleft", "stageleft_tool", - "syn 2.0.46", + "syn 2.0.48", "tokio", ] @@ -893,7 +900,7 @@ dependencies = [ [[package]] name = "lattices" version = "0.5.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "cc-traits", "sealed", @@ -1147,7 +1154,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1228,7 +1235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1267,9 +1274,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.74" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" +checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708" dependencies = [ "unicode-ident", ] @@ -1277,7 +1284,7 @@ dependencies = [ [[package]] name = "pusherator" version = "0.0.3" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "either", "variadics", @@ -1357,7 +1364,7 @@ checksum = "5fddb4f8d99b0a2ebafc65a87a69a7b9875e4b1ae1f00db265d300ef7f28bccc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1449,7 +1456,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1478,14 +1485,14 @@ checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] name = "serde_json" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -1591,37 +1598,37 @@ dependencies = [ [[package]] name = "stageleft" version = "0.1.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", "stageleft_macro", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] name = "stageleft_macro" version = "0.1.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "lazy_static", "proc-macro-crate", "proc-macro2", "quote", "sha256", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] name = "stageleft_tool" version = "0.1.0" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "proc-macro2", "quote", "sha256", - "syn 2.0.46", + "syn 2.0.48", "syn-inline-mod", ] @@ -1638,9 +1645,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.46" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -1654,7 +1661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fa6dca1fdb7b2ed46dd534a326725419d4fb10f23d8c85a8b2860e5eb25d0f9" dependencies = [ "proc-macro2", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1687,7 +1694,7 @@ checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1717,7 +1724,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1783,7 +1790,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1816,7 +1823,7 @@ checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "variadics" version = "0.0.2" -source = "git+https://github.com/hydro-project/hydroflow.git#a3a41470c786e5fcf75b266aa2326a70cd86876f" +source = "git+https://github.com/hydro-project/hydroflow.git#a0af314a032096fc94b9f4aabb21aadc8184fb30" dependencies = [ "sealed", ] @@ -1866,7 +1873,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -1888,7 +1895,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/README.md b/README.md index de3b099..c5cd40e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ The template includes two sample programs, `first_ten` and `first_ten_distribute `first_ten` demonstrates how to use Hydroflow+ to create dataflow programs for a single machine, and can be launched by running `cargo run -p flow --bin first_ten`. -`first_ten_distributed` demonstrates how to use Hydroflow+ to create dataflow programs for a distributed system, and can be launched by running `cargo run -p flow --example first_ten_distributed`. Note the use of `--example` here because `src/bin/first_ten_distributed.rs` contains the binary that will be launched on each node, whereas `examples/first_ten_distributed.rs` contains a deployment script for connecting the nodes together. +`first_ten_distributed` demonstrates how to use Hydroflow+ to create dataflow programs for a distributed system, and can be launched by running `cargo run -p flow --example first_ten_distributed`. Note the use of `--example` here because `src/bin/first_ten_distributed.rs` contains the binary that will be launched for each process, whereas `examples/first_ten_distributed.rs` contains a deployment script for connecting the processes together. This template also comes with an example of deploying the `first_ten_distributed` flow to Google Cloud. To deploy, you will need to install the [Google Cloud SDK](https://cloud.google.com/sdk/docs/install) and [Terraform](https://developer.hashicorp.com/terraform/install). Then, authenticate with Google Cloud and launch the deployment script with your project ID as an argument: diff --git a/flow/examples/first_ten_distributed.rs b/flow/examples/first_ten_distributed.rs index ea8d274..d80d311 100644 --- a/flow/examples/first_ten_distributed.rs +++ b/flow/examples/first_ten_distributed.rs @@ -1,15 +1,15 @@ use hydro_deploy::{Deployment, HydroflowCrate}; -use hydroflow_plus_cli_integration::CLIDeployNodeBuilder; +use hydroflow_plus_cli_integration::DeployProcessSpec; #[tokio::main] async fn main() { let mut deployment = Deployment::new(); let localhost = deployment.Localhost(); - let builder = hydroflow_plus::GraphBuilder::new(); + let builder = hydroflow_plus::FlowBuilder::new(); flow::first_ten_distributed::first_ten_distributed( &builder, - &CLIDeployNodeBuilder::new(|| { + &DeployProcessSpec::new(|| { deployment.add_service( HydroflowCrate::new(".", localhost.clone()) .bin("first_ten_distributed") diff --git a/flow/examples/first_ten_distributed_gcp.rs b/flow/examples/first_ten_distributed_gcp.rs index 98bda83..6ed6803 100644 --- a/flow/examples/first_ten_distributed_gcp.rs +++ b/flow/examples/first_ten_distributed_gcp.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use hydro_deploy::{gcp::GCPNetwork, Deployment, HydroflowCrate}; -use hydroflow_plus_cli_integration::CLIDeployNodeBuilder; +use hydroflow_plus_cli_integration::DeployProcessSpec; use tokio::sync::RwLock; #[tokio::main] @@ -13,10 +13,10 @@ async fn main() { let mut deployment = Deployment::new(); let vpc = Arc::new(RwLock::new(GCPNetwork::new(&gcp_project, None))); - let builder = hydroflow_plus::GraphBuilder::new(); + let builder = hydroflow_plus::FlowBuilder::new(); flow::first_ten_distributed::first_ten_distributed( &builder, - &CLIDeployNodeBuilder::new(|| { + &DeployProcessSpec::new(|| { let host = deployment.GCPComputeEngineHost( gcp_project.clone(), "e2-micro", diff --git a/flow/src/first_ten.rs b/flow/src/first_ten.rs index efabd87..236f93a 100644 --- a/flow/src/first_ten.rs +++ b/flow/src/first_ten.rs @@ -1,20 +1,19 @@ -use hydroflow_plus::node::*; use hydroflow_plus::*; use stageleft::*; pub fn first_ten<'a, D: LocalDeploy<'a>>( - graph: &'a GraphBuilder<'a, D>, - node_builder: &impl NodeBuilder<'a, D>, + graph: &'a FlowBuilder<'a, D>, + process_spec: &impl ProcessSpec<'a, D>, ) { - let node = graph.node(node_builder); + let process = graph.process(process_spec); - let numbers = node.source_iter(q!(0..10)); + let numbers = process.source_iter(q!(0..10)); numbers.for_each(q!(|n| println!("{}", n))); } #[stageleft::entry] pub fn first_ten_runtime<'a>( - graph: &'a GraphBuilder<'a, SingleGraph>, + graph: &'a FlowBuilder<'a, SingleProcessGraph>, ) -> impl Quoted<'a, Hydroflow<'a>> { first_ten(graph, &()); graph.build_single() diff --git a/flow/src/first_ten_distributed.rs b/flow/src/first_ten_distributed.rs index 26e150a..02e2a98 100644 --- a/flow/src/first_ten_distributed.rs +++ b/flow/src/first_ten_distributed.rs @@ -1,20 +1,19 @@ -use hydroflow_plus::node::*; use hydroflow_plus::*; use stageleft::*; pub fn first_ten_distributed<'a, D: Deploy<'a>>( - graph: &'a GraphBuilder<'a, D>, - node_builder: &impl NodeBuilder<'a, D>, -) -> D::Node { - let node = graph.node(node_builder); - let second_node = graph.node(node_builder); + graph: &'a FlowBuilder<'a, D>, + process_spec: &impl ProcessSpec<'a, D>, +) -> D::Process { + let process = graph.process(process_spec); + let second_process = graph.process(process_spec); - let numbers = node.source_iter(q!(0..10)); + let numbers = process.source_iter(q!(0..10)); numbers - .send_bincode(&second_node) + .send_bincode(&second_process) .for_each(q!(|n| println!("{}", n))); - second_node + second_process } use hydroflow_plus::util::cli::HydroCLI; @@ -22,7 +21,7 @@ use hydroflow_plus_cli_integration::{CLIRuntime, HydroflowPlusMeta}; #[stageleft::entry] pub fn first_ten_distributed_runtime<'a>( - graph: &'a GraphBuilder<'a, CLIRuntime>, + graph: &'a FlowBuilder<'a, CLIRuntime>, cli: RuntimeData<&'a HydroCLI>, ) -> impl Quoted<'a, Hydroflow<'a>> { let _ = first_ten_distributed(graph, &cli); @@ -34,17 +33,17 @@ pub fn first_ten_distributed_runtime<'a>( mod tests { use hydro_deploy::{Deployment, HydroflowCrate}; use hydroflow_plus::futures::StreamExt; - use hydroflow_plus_cli_integration::{CLIDeployNodeBuilder, DeployCrateWrapper}; + use hydroflow_plus_cli_integration::{DeployCrateWrapper, DeployProcessSpec}; #[tokio::test] async fn first_ten_distributed() { let mut deployment = Deployment::new(); let localhost = deployment.Localhost(); - let builder = hydroflow_plus::GraphBuilder::new(); - let second_node = super::first_ten_distributed( + let builder = hydroflow_plus::FlowBuilder::new(); + let second_process = super::first_ten_distributed( &builder, - &CLIDeployNodeBuilder::new(|| { + &DeployProcessSpec::new(|| { deployment.add_service( HydroflowCrate::new(".", localhost.clone()) .bin("first_ten_distributed") @@ -55,12 +54,12 @@ mod tests { deployment.deploy().await.unwrap(); - let second_node_stdout = second_node.stdout().await; + let second_process_stdout = second_process.stdout().await; deployment.start().await.unwrap(); assert_eq!( - second_node_stdout.take(10).collect::>().await, + second_process_stdout.take(10).collect::>().await, vec!["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] ); }