feat: ISB(jetstream) writer framework (#2160)

Signed-off-by: Vigith Maurice <[email protected]> Signed-off-by: Yashash H L <[email protected]> Co-authored-by: Yashash H L <[email protected]>
numaproj · Oct 16, 2024 · 3d6e47f · 3d6e47f
1 parent f81062c
commit 3d6e47f
Show file tree

Hide file tree

Showing 18 changed files with 1,190 additions and 37 deletions.
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
diff --git a/rust/numaflow-core/Cargo.toml b/rust/numaflow-core/Cargo.toml
@@ -3,6 +3,10 @@ name = "numaflow-core"
 version = "0.1.0"
 edition = "2021"
 
+[features]
+nats-tests = []
+all-tests = ["nats-tests"]
+
 [dependencies]
 axum = "0.7.5"
 axum-server = { version = "0.7.1", features = ["tls-rustls"] }
@@ -37,6 +41,7 @@ kube = "0.95.0"
 log = "0.4.22"
 futures = "0.3.30"
 pin-project = "1.1.5"
+async-nats = "0.37.0"
 
 [dev-dependencies]
 tempfile = "3.11.0"

diff --git a/rust/numaflow-core/src/error.rs b/rust/numaflow-core/src/error.rs
@@ -28,9 +28,15 @@ pub enum Error {
     #[error("Config Error - {0}")]
     Config(String),
 
-    #[error("ServerInfoError Error - {0}")]
+    #[error("ServerInfo Error - {0}")]
     ServerInfo(String),
 
+    #[error("Proto Error - {0}")]
+    Proto(String),
+
+    #[error("ISB Error - {0}")]
+    ISB(String),
+
     #[error("OneShot Receiver Error - {0}")]
     ActorPatternRecv(String),
 }

diff --git a/rust/numaflow-core/src/lib.rs b/rust/numaflow-core/src/lib.rs
@@ -36,3 +36,8 @@ mod transformer;
 
 /// Reads from a stream.
 mod reader;
+
+/// [Pipeline]
+///
+/// [Pipeline]: https://numaflow.numaproj.io/core-concepts/pipeline/
+mod pipeline;