chore: reorganize rust code (#1922)

Signed-off-by: Vigith Maurice <[email protected]>
Signed-off-by: Yashash H L <[email protected]>
Co-authored-by: Vigith Maurice <[email protected]>

.github/workflows/ci.yaml

@@ -107,16 +107,19 @@ jobs:
       - name: Install grcov
         run: cargo install grcov
 
+      - name: Install Protobuf Compiler
+        run: sudo apt-get install -y protobuf-compiler
+
       - name: Test Rust
-        working-directory: ./serving
+        working-directory: ./rust
         run: |
-          CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='./target/debug/coverage/cargo-test-%p-%m.profraw' cargo test --all-features
+          CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='./target/debug/coverage/cargo-test-%p-%m.profraw' cargo test --all-features --workspace --all 
           grcov . -s ./target/debug/coverage/ --binary-path ./target/debug/ -t lcov --branch --ignore-not-existing -o ./target/debug/coverage/lcov.info
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4
         with:
-          files: ./test/profile.cov,./serving/target/debug/coverage/lcov.info
+          files: ./test/profile.cov,./rust/target/debug/coverage/lcov.info
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 

Dockerfile

@@ -20,42 +20,44 @@ RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/ca
 RUN apt-get update
 RUN apt-get install protobuf-compiler -y
 
-RUN cargo new serve
+RUN cargo new numaflow
 # Create a new empty shell project
-WORKDIR /serve
-RUN cargo new servesink
-COPY ./serving/servesink/Cargo.toml ./servesink/
+WORKDIR /numaflow
 
-RUN cargo new extras/upstreams
-COPY ./serving/extras/upstreams/Cargo.toml ./extras/upstreams/
+RUN cargo new servesink
+COPY ./rust/servesink/Cargo.toml ./servesink/
 
 RUN cargo new backoff
-COPY ./serving/backoff/Cargo.toml ./backoff/
+COPY ./rust/backoff/Cargo.toml ./backoff/
 
 RUN cargo new numaflow-models
-COPY ./serving/numaflow-models/Cargo.toml ./numaflow-models/
+COPY ./rust/numaflow-models/Cargo.toml ./numaflow-models/
+
+RUN cargo new monovertex
+COPY ./rust/monovertex/Cargo.toml ./monovertex/
 
-RUN cargo new source-sink
-COPY ./serving/source-sink/Cargo.toml ./source-sink/
+RUN cargo new serving
+COPY ./rust/serving/Cargo.toml ./serving/Cargo.toml
 
 # Copy all Cargo.toml and Cargo.lock files for caching dependencies
-COPY ./serving/Cargo.toml ./serving/Cargo.lock ./
+COPY ./rust/Cargo.toml ./rust/Cargo.lock ./
 
-# Build only the dependencies to cache them
-RUN cargo build --release
+# Build to cache dependencies
+RUN mkdir -p src/bin && echo "fn main() {}" > src/bin/main.rs && \
+    cargo build --workspace --all --release
 
 # Copy the actual source code files of the main project and the subprojects
-COPY ./serving/src ./src
-COPY ./serving/servesink/src ./servesink/src
-COPY ./serving/extras/upstreams/src ./extras/upstreams/src
-COPY ./serving/backoff/src ./backoff/src
-COPY ./serving/numaflow-models/src ./numaflow-models/src
-COPY ./serving/source-sink/src ./source-sink/src
-COPY ./serving/source-sink/build.rs ./source-sink/build.rs
-COPY ./serving/source-sink/proto ./source-sink/proto
+COPY ./rust/src ./src
+COPY ./rust/servesink/src ./servesink/src
+COPY ./rust/backoff/src ./backoff/src
+COPY ./rust/numaflow-models/src ./numaflow-models/src
+COPY ./rust/serving/src ./serving/src
+COPY ./rust/monovertex/src ./monovertex/src
+COPY ./rust/monovertex/build.rs ./monovertex/build.rs
+COPY ./rust/monovertex/proto ./monovertex/proto
 
 # Build the real binaries
-RUN touch src/main.rs servesink/src/main.rs numaflow-models/src/main.rs source-sink/src/main.rs && \
+RUN touch src/bin/main.rs && \
     cargo build --workspace --all --release
 
 ####################################################################################################
@@ -70,10 +72,8 @@ RUN apt-get update && apt-get install -y libssl3
 COPY --from=base /bin/numaflow /bin/numaflow
 COPY ui/build /ui/build
 
-COPY --from=extension-base /serve/target/release/serve /bin/serve
-COPY --from=extension-base /serve/target/release/sourcer-sinker /bin/sourcer-sinker
-
-COPY ./serving/config config
+COPY --from=extension-base /numaflow/target/release/numaflow /bin/numaflow-rs
+COPY ./rust/serving/config config
 
 ENTRYPOINT [ "/bin/numaflow" ]
 

Makefile

@@ -195,7 +195,7 @@ codegen:
 	$(MAKE) manifests
 	rm -rf ./vendor
 	go mod tidy
-	$(MAKE) --directory serving/numaflow-models generate
+	$(MAKE) --directory rust/numaflow-models generate
 
 clean:
 	-rm -rf ${CURRENT_DIR}/dist

pkg/apis/numaflow/v1alpha1/mono_vertex_types.go

@@ -403,7 +403,7 @@ func (mvspec MonoVertexSpec) DeepCopyWithoutReplicas() MonoVertexSpec {
 
 func (mvspec MonoVertexSpec) buildContainers(req getContainerReq) []corev1.Container {
 	mainContainer := containerBuilder{}.
-		init(req).command(MonoVertexBinary).build()
+		init(req).command(NumaflowRustBinary).args("--monovertex").build()
 
 	containers := []corev1.Container{mainContainer}
 	if mvspec.Source.UDSource != nil { // Only support UDSource for now.

pkg/apis/numaflow/v1alpha1/vertex_types.go

@@ -50,8 +50,7 @@ const (
 	VertexTypeReduceUDF VertexType = "ReduceUDF"
 )
 
-const ServingBinary = "/bin/serve"
-const MonoVertexBinary = "/bin/sourcer-sinker"
+const NumaflowRustBinary = "/bin/numaflow-rs"
 
 // +genclient
 // +kubebuilder:object:root=true
@@ -350,8 +349,9 @@ func (v Vertex) getServingContainer(req GetVertexPodSpecReq) (corev1.Container,
 		Env:             req.Env,
 		Image:           req.Image,
 		ImagePullPolicy: req.PullPolicy,
-		Command:         []string{ServingBinary}, // we use the same image, but we execute the extension binary
+		Command:         []string{NumaflowRustBinary}, // we use the same image, but we execute the extension binary
 		Resources:       req.DefaultResources,
+		Args:            []string{"--serving"},
 	}
 
 	// set the common envs

rust/Cargo.toml

@@ -0,0 +1,19 @@
+workspace = { members = ["backoff", "numaflow-models", "servesink", "serving", "monovertex"] }
+
+[[bin]]
+name = "numaflow"
+path = "src/bin/main.rs"
+
+[package]
+name = "numaflow"
+version = "0.1.0"
+edition = "2021"
+
+
+[dependencies]
+tokio = "1.39.2"
+backoff = { path = "backoff" }
+servesink = { path = "servesink" }
+serving = { path = "serving" }
+monovertex = { path = "monovertex" }
+tracing = "0.1.40"