diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e21c675fd..875ad8fe82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## v1.4.2 (2024-12-11) + + * [c9dc38f4](https://github.com/numaproj/numaflow/commit/c9dc38f4cce2b5db598536a7539f2a35febcf1ca) Update manifests to v1.4.2 + * [fea792b3](https://github.com/numaproj/numaflow/commit/fea792b36bd342adcdcdd96768b6fdd68921bfd2) fix: set max decode size of proto message (#2275) + +### Contributors + + * Sidhant Kohli + ## v1.4.1 (2024-12-05) * [346f2a73](https://github.com/numaproj/numaflow/commit/346f2a7321d158fa9ce9392cfdcc76d671d6f577) Update manifests to v1.4.1 diff --git a/Dockerfile b/Dockerfile index bded35d1f9..48d9d68959 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,8 +69,6 @@ COPY --from=base /bin/numaflow /bin/numaflow COPY --from=base /bin/numaflow-rs /bin/numaflow-rs COPY ui/build /ui/build -COPY ./rust/serving/config config - ENTRYPOINT [ "/bin/numaflow" ] #################################################################################################### @@ -89,4 +87,4 @@ RUN chmod +x /bin/e2eapi #################################################################################################### FROM scratch AS e2eapi COPY --from=testbase /bin/e2eapi . -ENTRYPOINT ["/e2eapi"] \ No newline at end of file +ENTRYPOINT ["/e2eapi"] diff --git a/config/advanced-install/namespaced-numaflow-server.yaml b/config/advanced-install/namespaced-numaflow-server.yaml index 9d34601fd3..7cb350b073 100644 --- a/config/advanced-install/namespaced-numaflow-server.yaml +++ b/config/advanced-install/namespaced-numaflow-server.yaml @@ -143,33 +143,71 @@ data: # example for local prometheus service # url: http://prometheus-operated.monitoring.svc.cluster.local:9090 patterns: + - name: vertex_gauge + object: vertex + title: Vertex Pending Messages + description: This query is the total number of pending messages for the vertex + expr: | + sum($metric_name{$filters}) by ($dimension, period) + params: + - name: start_time + required: false + - name: end_time + required: false + metrics: + - metric_name: vertex_pending_messages + required_filters: + - namespace + - pipeline + - vertex + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_gauge object: mono-vertex title: Pending Messages Lag description: This query is the total number of pending messages for the mono vertex expr: | - $metric_name{$filters} + sum($metric_name{$filters}) by ($dimension, period) params: - name: start_time required: false - name: end_time required: false metrics: - - metric_name: monovtx_pending - required_filters: - - namespace - - mvtx_name - dimensions: - - name: pod - filters: - - name: pod - required: false - - name: period - required: false - - name: mono-vertex - filters: - - name: period - required: false + - metric_name: monovtx_pending + required_filters: + - namespace + - mvtx_name + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: mono-vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_histogram object: mono-vertex title: Processing Time Latency @@ -192,11 +230,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false @@ -206,15 +240,11 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false - # Add histogram metrics similar to the pattern above + - name: vertex_throughput object: vertex title: Vertex Throughput and Message Rates @@ -235,12 +265,11 @@ data: - vertex dimensions: - name: vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod filters: - name: pod required: false + - name: mono_vertex_throughput object: mono-vertex title: Mono-Vertex Throughput and Message Rates @@ -260,11 +289,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false diff --git a/config/advanced-install/numaflow-server.yaml b/config/advanced-install/numaflow-server.yaml index bca4a13bbf..fcb283f11e 100644 --- a/config/advanced-install/numaflow-server.yaml +++ b/config/advanced-install/numaflow-server.yaml @@ -150,33 +150,71 @@ data: # example for local prometheus service # url: http://prometheus-operated.monitoring.svc.cluster.local:9090 patterns: + - name: vertex_gauge + object: vertex + title: Vertex Pending Messages + description: This query is the total number of pending messages for the vertex + expr: | + sum($metric_name{$filters}) by ($dimension, period) + params: + - name: start_time + required: false + - name: end_time + required: false + metrics: + - metric_name: vertex_pending_messages + required_filters: + - namespace + - pipeline + - vertex + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_gauge object: mono-vertex title: Pending Messages Lag description: This query is the total number of pending messages for the mono vertex expr: | - $metric_name{$filters} + sum($metric_name{$filters}) by ($dimension, period) params: - name: start_time required: false - name: end_time required: false metrics: - - metric_name: monovtx_pending - required_filters: - - namespace - - mvtx_name - dimensions: - - name: pod - filters: - - name: pod - required: false - - name: period - required: false - - name: mono-vertex - filters: - - name: period - required: false + - metric_name: monovtx_pending + required_filters: + - namespace + - mvtx_name + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: mono-vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_histogram object: mono-vertex title: Processing Time Latency @@ -199,11 +237,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false @@ -213,15 +247,11 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false - # Add histogram metrics similar to the pattern above + - name: vertex_throughput object: vertex title: Vertex Throughput and Message Rates @@ -242,12 +272,11 @@ data: - vertex dimensions: - name: vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod filters: - name: pod required: false + - name: mono_vertex_throughput object: mono-vertex title: Mono-Vertex Throughput and Message Rates @@ -267,11 +296,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false diff --git a/config/base/numaflow-server/numaflow-server-metrics-proxy-config.yaml b/config/base/numaflow-server/numaflow-server-metrics-proxy-config.yaml index d97370e392..fe634f5f17 100644 --- a/config/base/numaflow-server/numaflow-server-metrics-proxy-config.yaml +++ b/config/base/numaflow-server/numaflow-server-metrics-proxy-config.yaml @@ -9,33 +9,71 @@ data: # example for local prometheus service # url: http://prometheus-operated.monitoring.svc.cluster.local:9090 patterns: + - name: vertex_gauge + object: vertex + title: Vertex Pending Messages + description: This query is the total number of pending messages for the vertex + expr: | + sum($metric_name{$filters}) by ($dimension, period) + params: + - name: start_time + required: false + - name: end_time + required: false + metrics: + - metric_name: vertex_pending_messages + required_filters: + - namespace + - pipeline + - vertex + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_gauge object: mono-vertex title: Pending Messages Lag description: This query is the total number of pending messages for the mono vertex expr: | - $metric_name{$filters} + sum($metric_name{$filters}) by ($dimension, period) params: - name: start_time required: false - name: end_time required: false metrics: - - metric_name: monovtx_pending - required_filters: - - namespace - - mvtx_name - dimensions: - - name: pod - filters: - - name: pod - required: false - - name: period - required: false - - name: mono-vertex - filters: - - name: period - required: false + - metric_name: monovtx_pending + required_filters: + - namespace + - mvtx_name + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: mono-vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_histogram object: mono-vertex title: Processing Time Latency @@ -58,11 +96,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false @@ -72,15 +106,11 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false - # Add histogram metrics similar to the pattern above + - name: vertex_throughput object: vertex title: Vertex Throughput and Message Rates @@ -101,12 +131,11 @@ data: - vertex dimensions: - name: vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod filters: - name: pod required: false + - name: mono_vertex_throughput object: mono-vertex title: Mono-Vertex Throughput and Message Rates @@ -126,11 +155,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false \ No newline at end of file diff --git a/config/install.yaml b/config/install.yaml index 72df249f0f..69fbd4ca4d 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -28563,33 +28563,71 @@ data: # example for local prometheus service # url: http://prometheus-operated.monitoring.svc.cluster.local:9090 patterns: + - name: vertex_gauge + object: vertex + title: Vertex Pending Messages + description: This query is the total number of pending messages for the vertex + expr: | + sum($metric_name{$filters}) by ($dimension, period) + params: + - name: start_time + required: false + - name: end_time + required: false + metrics: + - metric_name: vertex_pending_messages + required_filters: + - namespace + - pipeline + - vertex + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_gauge object: mono-vertex title: Pending Messages Lag description: This query is the total number of pending messages for the mono vertex expr: | - $metric_name{$filters} + sum($metric_name{$filters}) by ($dimension, period) params: - name: start_time required: false - name: end_time required: false metrics: - - metric_name: monovtx_pending - required_filters: - - namespace - - mvtx_name - dimensions: - - name: pod - filters: - - name: pod - required: false - - name: period - required: false - - name: mono-vertex - filters: - - name: period - required: false + - metric_name: monovtx_pending + required_filters: + - namespace + - mvtx_name + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: mono-vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_histogram object: mono-vertex title: Processing Time Latency @@ -28612,11 +28650,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false @@ -28626,15 +28660,11 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false - # Add histogram metrics similar to the pattern above + - name: vertex_throughput object: vertex title: Vertex Throughput and Message Rates @@ -28655,12 +28685,11 @@ data: - vertex dimensions: - name: vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod filters: - name: pod required: false + - name: mono_vertex_throughput object: mono-vertex title: Mono-Vertex Throughput and Message Rates @@ -28680,11 +28709,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml index 1ae302f71f..810422a7cc 100644 --- a/config/namespace-install.yaml +++ b/config/namespace-install.yaml @@ -28451,33 +28451,71 @@ data: # example for local prometheus service # url: http://prometheus-operated.monitoring.svc.cluster.local:9090 patterns: + - name: vertex_gauge + object: vertex + title: Vertex Pending Messages + description: This query is the total number of pending messages for the vertex + expr: | + sum($metric_name{$filters}) by ($dimension, period) + params: + - name: start_time + required: false + - name: end_time + required: false + metrics: + - metric_name: vertex_pending_messages + required_filters: + - namespace + - pipeline + - vertex + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_gauge object: mono-vertex title: Pending Messages Lag description: This query is the total number of pending messages for the mono vertex expr: | - $metric_name{$filters} + sum($metric_name{$filters}) by ($dimension, period) params: - name: start_time required: false - name: end_time required: false metrics: - - metric_name: monovtx_pending - required_filters: - - namespace - - mvtx_name - dimensions: - - name: pod - filters: - - name: pod - required: false - - name: period - required: false - - name: mono-vertex - filters: - - name: period - required: false + - metric_name: monovtx_pending + required_filters: + - namespace + - mvtx_name + dimensions: + - name: pod + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: pod + required: false + - name: period + required: false + - name: mono-vertex + # expr: optional expression for prometheus query + # overrides the default expression + filters: + - name: period + required: false + - name: mono_vertex_histogram object: mono-vertex title: Processing Time Latency @@ -28500,11 +28538,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false @@ -28514,15 +28548,11 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false - # Add histogram metrics similar to the pattern above + - name: vertex_throughput object: vertex title: Vertex Throughput and Message Rates @@ -28543,12 +28573,11 @@ data: - vertex dimensions: - name: vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod filters: - name: pod required: false + - name: mono_vertex_throughput object: mono-vertex title: Mono-Vertex Throughput and Message Rates @@ -28568,11 +28597,7 @@ data: - mvtx_name dimensions: - name: mono-vertex - # expr: optional expression for prometheus query - # overrides the default expression - name: pod - # expr: optional expression for prometheus query - # overrides the default expression filters: - name: pod required: false diff --git a/go.mod b/go.mod index b5b7d57160..3783ba46da 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/mock v1.6.0 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 @@ -48,11 +49,11 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.27.0 + golang.org/x/crypto v0.31.0 golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc golang.org/x/net v0.29.0 golang.org/x/oauth2 v0.21.0 - golang.org/x/sync v0.8.0 + golang.org/x/sync v0.10.0 golang.org/x/tools v0.24.0 google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 google.golang.org/grpc v1.66.0 @@ -118,7 +119,6 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/gorilla/handlers v1.5.2 // indirect @@ -200,9 +200,9 @@ require ( go.mongodb.org/mongo-driver v1.15.0 // indirect golang.org/x/arch v0.7.0 // indirect golang.org/x/mod v0.20.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/term v0.24.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 48b6fba0ed..65f904db62 100644 --- a/go.sum +++ b/go.sum @@ -691,8 +691,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -806,8 +806,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -867,15 +867,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -888,8 +888,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1164,4 +1164,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+s sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= \ No newline at end of file diff --git a/pkg/daemon/server/daemon_server.go b/pkg/daemon/server/daemon_server.go index 768f789dde..351b2607ce 100644 --- a/pkg/daemon/server/daemon_server.go +++ b/pkg/daemon/server/daemon_server.go @@ -42,6 +42,7 @@ import ( "github.com/numaproj/numaflow/pkg/daemon/server/service" server "github.com/numaproj/numaflow/pkg/daemon/server/service/rater" "github.com/numaproj/numaflow/pkg/isbsvc" + "github.com/numaproj/numaflow/pkg/metrics" jsclient "github.com/numaproj/numaflow/pkg/shared/clients/nats" redisclient "github.com/numaproj/numaflow/pkg/shared/clients/redis" "github.com/numaproj/numaflow/pkg/shared/logging" @@ -156,7 +157,9 @@ func (ds *daemonServer) Run(ctx context.Context) error { go ds.exposeMetrics(ctx) version := numaflow.GetVersion() - pipeline_info.WithLabelValues(version.Version, version.Platform, ds.pipeline.Name).Set(1) + // TODO: clean it up in v1.6 + deprecatedPipelineInfo.WithLabelValues(version.Version, version.Platform, ds.pipeline.Name).Set(1) + metrics.BuildInfo.WithLabelValues(v1alpha1.ComponentDaemon, ds.pipeline.Name, version.Version, version.Platform).Set(1) log.Infof("Daemon server started successfully on %s", address) <-ctx.Done() diff --git a/pkg/daemon/server/metrics.go b/pkg/daemon/server/metrics.go index c27bdeedb0..6c31e5e727 100644 --- a/pkg/daemon/server/metrics.go +++ b/pkg/daemon/server/metrics.go @@ -23,10 +23,11 @@ import ( ) var ( - pipeline_info = promauto.NewGaugeVec(prometheus.GaugeOpts{ + // Deprecated: Use pkg/metrics.BuildInfo instead. + deprecatedPipelineInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ Subsystem: "pipeline", Name: "build_info", - Help: "A metric with a constant value '1', labeled by Numaflow binary version and platform, as well as the pipeline name", + Help: "A metric with a constant value '1', labeled by Numaflow binary version and platform, as well as the pipeline name. Deprecated: Use build_info instead", }, []string{metrics.LabelVersion, metrics.LabelPlatform, metrics.LabelPipeline}) // Pipeline processing lag, max(watermark) - min(watermark) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 2754983481..53605eca2c 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -41,6 +41,11 @@ const ( ) var ( + BuildInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "build_info", + Help: "A metric with a constant value '1', labeled by Numaflow binary version, platform, and other information", + }, []string{LabelComponent, LabelComponentName, LabelVersion, LabelPlatform}) + SDKInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "sdk_info", Help: "A metric with a constant value '1', labeled by SDK information such as version, language, and type", diff --git a/pkg/mvtxdaemon/server/daemon_server.go b/pkg/mvtxdaemon/server/daemon_server.go index a8f05f64a7..09f4e3bd77 100644 --- a/pkg/mvtxdaemon/server/daemon_server.go +++ b/pkg/mvtxdaemon/server/daemon_server.go @@ -38,6 +38,7 @@ import ( "github.com/numaproj/numaflow" "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" "github.com/numaproj/numaflow/pkg/apis/proto/mvtxdaemon" + "github.com/numaproj/numaflow/pkg/metrics" "github.com/numaproj/numaflow/pkg/mvtxdaemon/server/service" rateServer "github.com/numaproj/numaflow/pkg/mvtxdaemon/server/service/rater" "github.com/numaproj/numaflow/pkg/shared/logging" @@ -108,7 +109,9 @@ func (ds *daemonServer) Run(ctx context.Context) error { }() version := numaflow.GetVersion() - monoVertexInfo.WithLabelValues(version.Version, version.Platform, ds.monoVtx.Name).Set(1) + // Todo: clean it up in v1.6 + deprecatedMonoVertexInfo.WithLabelValues(version.Version, version.Platform, ds.monoVtx.Name).Set(1) + metrics.BuildInfo.WithLabelValues(v1alpha1.ComponentMonoVertexDaemon, ds.monoVtx.Name, version.Version, version.Platform).Set(1) log.Infof("MonoVertex daemon server started successfully on %s", address) <-ctx.Done() diff --git a/pkg/mvtxdaemon/server/metrics.go b/pkg/mvtxdaemon/server/metrics.go index f3c0c30796..e06ea5906d 100644 --- a/pkg/mvtxdaemon/server/metrics.go +++ b/pkg/mvtxdaemon/server/metrics.go @@ -24,9 +24,10 @@ import ( ) var ( - monoVertexInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ + // Deprecated: Use pkg/metrics.BuildInfo instead. + deprecatedMonoVertexInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ Subsystem: "monovtx", Name: "build_info", - Help: "A metric with a constant value '1', labeled by Numaflow binary version and platform, as well as the mono vertex name", + Help: "A metric with a constant value '1', labeled by Numaflow binary version and platform, as well as the mono vertex name. Deprecated: Use build_info instead", }, []string{metrics.LabelVersion, metrics.LabelPlatform, metrics.LabelMonoVertexName}) ) diff --git a/pkg/sinks/forward/forward.go b/pkg/sinks/forward/forward.go index ac60ecddf7..15b14daf3f 100644 --- a/pkg/sinks/forward/forward.go +++ b/pkg/sinks/forward/forward.go @@ -230,7 +230,7 @@ func (df *DataForward) forwardAChunk(ctx context.Context) error { return nil } - // if the validation passed, we will publish the watermark to all the toBuffer partitions. + // if the validation passed, we will publish the idle watermark to SINK OT even though we do not use it today. idlehandler.PublishIdleWatermark(ctx, df.sinkWriter.GetPartitionIdx(), df.sinkWriter, df.wmPublisher, df.idleManager, df.opts.logger, df.vertexName, df.pipelineName, dfv1.VertexTypeSink, df.vertexReplica, wmb.Watermark(time.UnixMilli(processorWMB.Watermark))) return nil } @@ -271,7 +271,7 @@ func (df *DataForward) forwardAChunk(ctx context.Context) error { } // write the messages to the sink - writeOffsets, fallbackMessages, err := df.writeToSink(ctx, df.sinkWriter, writeMessages, false) + _, fallbackMessages, err := df.writeToSink(ctx, df.sinkWriter, writeMessages, false) // error will not be nil only when we get ctx.Done() if err != nil { df.opts.logger.Errorw("failed to write to sink", zap.Error(err)) @@ -292,19 +292,13 @@ func (df *DataForward) forwardAChunk(ctx context.Context) error { } } - // FIXME: offsets are not supported for sink, so len(writeOffsets) > 0 will always fail - // in sink we don't drop any messages - // so len(dataMessages) should be the same as len(writeOffsets) - // if len(writeOffsets) is greater than 0, publish normal watermark - // if len(writeOffsets) is 0, meaning we only have control messages, - // we should not publish anything: the next len(readMessage) check will handle this idling situation - if len(writeOffsets) > 0 { - df.wmPublisher.PublishWatermark(processorWM, nil, int32(0)) - // reset because the toBuffer is no longer idling - df.idleManager.MarkActive(df.fromBufferPartition.GetPartitionIdx(), df.sinkWriter.GetName()) - } + // Always publish the watermark to SINK OT even though we do not use it today. + // There's no offset returned from sink writer. + df.wmPublisher.PublishWatermark(processorWM, nil, int32(0)) + // reset because the toBuffer is no longer idling + df.idleManager.MarkActive(df.fromBufferPartition.GetPartitionIdx(), df.sinkWriter.GetName()) - df.opts.logger.Debugw("write to sink completed") + df.opts.logger.Debugw("Write to sink completed") ackStart := time.Now() err = df.ackFromBuffer(ctx, readOffsets) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ec51332105..a210284fcd 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -17,18 +17,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.3" @@ -38,12 +26,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -71,12 +53,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" -[[package]] -name = "arraydeque" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" - [[package]] name = "async-nats" version = "0.35.1" @@ -388,9 +364,6 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" -dependencies = [ - "serde", -] [[package]] name = "block-buffer" @@ -503,60 +476,12 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "config" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf" -dependencies = [ - "async-trait", - "convert_case", - "json5", - "nom", - "pathdiff", - "ron", - "rust-ini", - "serde", - "serde_json", - "toml", - "yaml-rust2", -] - [[package]] name = "const-oid" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom", - "once_cell", - "tiny-keccak", -] - -[[package]] -name = "convert_case" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "core-foundation" version = "0.9.4" @@ -607,12 +532,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - [[package]] name = "crypto-common" version = "0.1.6" @@ -697,15 +616,6 @@ dependencies = [ "syn 2.0.90", ] -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - [[package]] name = "dtoa" version = "1.0.9" @@ -984,31 +894,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] - [[package]] name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -[[package]] -name = "hashlink" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" -dependencies = [ - "hashbrown 0.14.5", -] - [[package]] name = "headers" version = "0.4.0" @@ -1493,17 +1384,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json5" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" -dependencies = [ - "pest", - "pest_derive", - "serde", -] - [[package]] name = "jsonpath-rust" version = "0.5.1" @@ -1873,6 +1753,7 @@ dependencies = [ "semver", "serde", "serde_json", + "serving", "tempfile", "thiserror 2.0.3", "tokio", @@ -1954,16 +1835,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ordered-multimap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" -dependencies = [ - "dlv-list", - "hashbrown 0.14.5", -] - [[package]] name = "overload" version = "0.1.1" @@ -1999,12 +1870,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - [[package]] name = "pem" version = "3.0.4" @@ -2630,28 +2495,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "ron" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" -dependencies = [ - "base64 0.21.7", - "bitflags 2.6.0", - "serde", - "serde_derive", -] - -[[package]] -name = "rust-ini" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0698206bcb8882bf2a9ecb4c1e7785db57ff052297085a6efd4fe42302068a" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2955,15 +2798,6 @@ dependencies = [ "syn 2.0.90", ] -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3011,7 +2845,6 @@ dependencies = [ "backoff", "base64 0.22.1", "chrono", - "config", "hyper-util", "numaflow-models", "parking_lot", @@ -3319,15 +3152,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.7.6" @@ -3458,40 +3282,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "toml" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" -dependencies = [ - "indexmap 2.7.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "tonic" version = "0.12.3" @@ -3718,12 +3508,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - [[package]] name = "unicode-width" version = "0.1.14" @@ -4133,15 +3917,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" -dependencies = [ - "memchr", -] - [[package]] name = "winreg" version = "0.50.0" @@ -4164,17 +3939,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" -[[package]] -name = "yaml-rust2" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8902160c4e6f2fb145dbe9d6760a75e3c9522d8bf796ed7047c85919ac7115f8" -dependencies = [ - "arraydeque", - "encoding_rs", - "hashlink", -] - [[package]] name = "yasna" version = "0.5.2" diff --git a/rust/numaflow-core/Cargo.toml b/rust/numaflow-core/Cargo.toml index 73c15c489c..b4688a135b 100644 --- a/rust/numaflow-core/Cargo.toml +++ b/rust/numaflow-core/Cargo.toml @@ -17,6 +17,7 @@ tracing.workspace = true numaflow-pulsar.workspace = true numaflow-models.workspace = true numaflow-pb.workspace = true +serving.workspace = true backoff.workspace = true axum.workspace = true axum-server.workspace = true diff --git a/rust/numaflow-core/src/config/components.rs b/rust/numaflow-core/src/config/components.rs index 9b26c1d5d7..f17331ddaa 100644 --- a/rust/numaflow-core/src/config/components.rs +++ b/rust/numaflow-core/src/config/components.rs @@ -3,6 +3,8 @@ pub(crate) mod source { const DEFAULT_SOURCE_SOCKET: &str = "/var/run/numaflow/source.sock"; const DEFAULT_SOURCE_SERVER_INFO_FILE: &str = "/var/run/numaflow/sourcer-server-info"; + use std::collections::HashMap; + use std::env; use std::{fmt::Debug, time::Duration}; use bytes::Bytes; @@ -31,6 +33,7 @@ pub(crate) mod source { Generator(GeneratorConfig), UserDefined(UserDefinedConfig), Pulsar(PulsarSourceConfig), + Serving(serving::Settings), } impl From> for SourceType { @@ -96,6 +99,55 @@ pub(crate) mod source { } } + impl TryFrom> for SourceType { + type Error = Error; + // FIXME: Currently, the same settings comes from user-defined settings and env variables. + // We parse both, with user-defined values having higher precedence. + // There should be only one option (user-defined) to define the settings. + fn try_from(cfg: Box) -> Result { + let env_vars = env::vars().collect::>(); + + let mut settings: serving::Settings = env_vars + .try_into() + .map_err(|e: serving::Error| Error::Config(e.to_string()))?; + + settings.tid_header = cfg.msg_id_header_key; + + if let Some(auth) = cfg.auth { + if let Some(token) = auth.token { + let secret = crate::shared::create_components::get_secret_from_volume( + &token.name, + &token.key, + ) + .map_err(|e| Error::Config(format!("Reading API auth token secret: {e:?}")))?; + settings.api_auth_token = Some(secret); + } else { + tracing::warn!("Authentication token for Serving API is specified, but the secret is empty"); + }; + } + + if let Some(ttl) = cfg.store.ttl { + if ttl.is_negative() { + return Err(Error::Config(format!( + "TTL value for the store can not be negative. Provided value = {ttl:?}" + ))); + } + let ttl: std::time::Duration = ttl.into(); + let ttl_secs = ttl.as_secs() as u32; + // TODO: Identify a minimum value + if ttl_secs < 1 { + return Err(Error::Config(format!( + "TTL value for the store must not be less than 1 second. Provided value = {ttl:?}" + ))); + } + settings.redis.ttl_secs = Some(ttl_secs); + } + settings.redis.addr = cfg.store.url; + + Ok(SourceType::Serving(settings)) + } + } + impl TryFrom> for SourceType { type Error = Error; diff --git a/rust/numaflow-core/src/config/monovertex.rs b/rust/numaflow-core/src/config/monovertex.rs index 686e284615..c5f4f2622b 100644 --- a/rust/numaflow-core/src/config/monovertex.rs +++ b/rust/numaflow-core/src/config/monovertex.rs @@ -1,4 +1,3 @@ -use crate::config::monovertex::sink::SinkType; use std::time::Duration; use base64::prelude::BASE64_STANDARD; @@ -14,6 +13,7 @@ use crate::config::components::transformer::{ }; use crate::config::components::{sink, source}; use crate::config::get_vertex_replica; +use crate::config::monovertex::sink::SinkType; use crate::error::Error; use crate::Result; diff --git a/rust/numaflow-core/src/config/pipeline.rs b/rust/numaflow-core/src/config/pipeline.rs index c9d02e632c..6c0a4a08bc 100644 --- a/rust/numaflow-core/src/config/pipeline.rs +++ b/rust/numaflow-core/src/config/pipeline.rs @@ -1,4 +1,3 @@ -use crate::config::components::sink::SinkType; use std::collections::HashMap; use std::env; use std::time::Duration; @@ -10,6 +9,7 @@ use serde_json::from_slice; use crate::config::components::metrics::MetricsConfig; use crate::config::components::sink::SinkConfig; +use crate::config::components::sink::SinkType; use crate::config::components::source::SourceConfig; use crate::config::components::transformer::{TransformerConfig, TransformerType}; use crate::config::get_vertex_replica; diff --git a/rust/numaflow-core/src/metrics.rs b/rust/numaflow-core/src/metrics.rs index 045b3817b9..866e58f2c2 100644 --- a/rust/numaflow-core/src/metrics.rs +++ b/rust/numaflow-core/src/metrics.rs @@ -78,6 +78,7 @@ const PENDING: &str = "pending"; // processing times as timers const E2E_TIME: &str = "processing_time"; const READ_TIME: &str = "read_time"; +const WRITE_TIME: &str = "write_time"; const TRANSFORM_TIME: &str = "time"; const ACK_TIME: &str = "ack_time"; const SINK_TIME: &str = "time"; @@ -226,9 +227,11 @@ pub(crate) struct PipelineForwarderMetrics { pub(crate) ack_total: Family, Counter>, pub(crate) ack_time: Family, Histogram>, pub(crate) write_total: Family, Counter>, + pub(crate) write_time: Family, Histogram>, pub(crate) read_bytes_total: Family, Counter>, pub(crate) processed_time: Family, Histogram>, pub(crate) pending: Family, Gauge>, + pub(crate) dropped_total: Family, Counter>, } pub(crate) struct PipelineISBMetrics { @@ -395,6 +398,10 @@ impl PipelineMetrics { }), pending: Family::, Gauge>::default(), write_total: Family::, Counter>::default(), + write_time: Family::, Histogram>::new_with_constructor( + || Histogram::new(exponential_buckets_range(100.0, 60000000.0 * 15.0, 10)), + ), + dropped_total: Family::, Counter>::default(), }, isb: PipelineISBMetrics { paf_resolution_time: @@ -442,6 +449,21 @@ impl PipelineMetrics { "Number of pending messages", metrics.forwarder.pending.clone(), ); + forwarder_registry.register( + SINK_WRITE_TOTAL, + "Total number of Data Messages Written", + metrics.forwarder.write_total.clone(), + ); + forwarder_registry.register( + DROPPED_TOTAL, + "Total number of dropped messages", + metrics.forwarder.dropped_total.clone(), + ); + forwarder_registry.register( + WRITE_TIME, + "Time taken to write data", + metrics.forwarder.write_time.clone(), + ); metrics } } diff --git a/rust/numaflow-core/src/pipeline.rs b/rust/numaflow-core/src/pipeline.rs index a8deaa7b64..d59cd182b6 100644 --- a/rust/numaflow-core/src/pipeline.rs +++ b/rust/numaflow-core/src/pipeline.rs @@ -1,4 +1,3 @@ -use crate::pipeline::pipeline::isb::BufferReaderConfig; use std::time::Duration; use async_nats::jetstream::Context; @@ -13,6 +12,7 @@ use crate::metrics::{PipelineContainerState, UserDefinedContainerState}; use crate::pipeline::forwarder::source_forwarder; use crate::pipeline::isb::jetstream::reader::JetstreamReader; use crate::pipeline::isb::jetstream::writer::JetstreamWriter; +use crate::pipeline::pipeline::isb::BufferReaderConfig; use crate::shared::create_components; use crate::shared::create_components::create_sink_writer; use crate::shared::metrics::start_metrics_server; diff --git a/rust/numaflow-core/src/pipeline/isb/jetstream/reader.rs b/rust/numaflow-core/src/pipeline/isb/jetstream/reader.rs index 3773228906..4513cb9182 100644 --- a/rust/numaflow-core/src/pipeline/isb/jetstream/reader.rs +++ b/rust/numaflow-core/src/pipeline/isb/jetstream/reader.rs @@ -260,14 +260,15 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use super::*; - use crate::message::{Message, MessageID}; use async_nats::jetstream; use async_nats::jetstream::{consumer, stream}; use bytes::BytesMut; use chrono::Utc; use tokio::time::sleep; + use super::*; + use crate::message::{Message, MessageID}; + #[cfg(feature = "nats-tests")] #[tokio::test] async fn test_jetstream_read() { diff --git a/rust/numaflow-core/src/pipeline/isb/jetstream/writer.rs b/rust/numaflow-core/src/pipeline/isb/jetstream/writer.rs index 969f343ab1..a99d43856d 100644 --- a/rust/numaflow-core/src/pipeline/isb/jetstream/writer.rs +++ b/rust/numaflow-core/src/pipeline/isb/jetstream/writer.rs @@ -4,16 +4,6 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use crate::config::pipeline::isb::BufferFullStrategy; -use crate::config::pipeline::ToVertexConfig; -use crate::error::Error; -use crate::message::{IntOffset, Message, Offset}; -use crate::metrics::{pipeline_isb_metric_labels, pipeline_metrics}; -use crate::pipeline::isb::jetstream::Stream; -use crate::tracker::TrackerHandle; -use crate::Result; - -use crate::shared::forward; use async_nats::jetstream::consumer::PullConsumer; use async_nats::jetstream::context::PublishAckFuture; use async_nats::jetstream::publish::PublishAck; @@ -28,6 +18,16 @@ use tokio_stream::StreamExt; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; +use crate::config::pipeline::isb::BufferFullStrategy; +use crate::config::pipeline::ToVertexConfig; +use crate::error::Error; +use crate::message::{IntOffset, Message, Offset}; +use crate::metrics::{pipeline_isb_metric_labels, pipeline_metrics}; +use crate::pipeline::isb::jetstream::Stream; +use crate::shared::forward; +use crate::tracker::TrackerHandle; +use crate::Result; + const DEFAULT_RETRY_INTERVAL_MILLIS: u64 = 10; const DEFAULT_REFRESH_INTERVAL_SECS: u64 = 1; @@ -461,9 +461,6 @@ pub(crate) struct ResolveAndPublishResult { #[cfg(test)] mod tests { - use crate::pipeline::pipeline::isb::BufferWriterConfig; - use numaflow_models::models::ForwardConditions; - use numaflow_models::models::TagConditions; use std::collections::HashMap; use std::time::Instant; @@ -472,9 +469,12 @@ mod tests { use async_nats::jetstream::{consumer, stream}; use bytes::BytesMut; use chrono::Utc; + use numaflow_models::models::ForwardConditions; + use numaflow_models::models::TagConditions; use super::*; use crate::message::{Message, MessageID, ReadAck}; + use crate::pipeline::pipeline::isb::BufferWriterConfig; #[cfg(feature = "nats-tests")] #[tokio::test] diff --git a/rust/numaflow-core/src/shared/create_components.rs b/rust/numaflow-core/src/shared/create_components.rs index 26516e34d9..0e5fade691 100644 --- a/rust/numaflow-core/src/shared/create_components.rs +++ b/rust/numaflow-core/src/shared/create_components.rs @@ -74,7 +74,7 @@ pub(crate) async fn create_sink_writer( grpc::create_rpc_channel(ud_config.socket_path.clone().into()).await?, ) .max_encoding_message_size(ud_config.grpc_max_message_size) - .max_encoding_message_size(ud_config.grpc_max_message_size); + .max_decoding_message_size(ud_config.grpc_max_message_size); grpc::wait_until_sink_ready(cln_token, &mut sink_grpc_client).await?; ( SinkWriterBuilder::new( @@ -129,7 +129,7 @@ pub(crate) async fn create_sink_writer( grpc::create_rpc_channel(ud_config.socket_path.clone().into()).await?, ) .max_encoding_message_size(ud_config.grpc_max_message_size) - .max_encoding_message_size(ud_config.grpc_max_message_size); + .max_decoding_message_size(ud_config.grpc_max_message_size); grpc::wait_until_sink_ready(cln_token, &mut sink_grpc_client).await?; Ok(( @@ -178,7 +178,7 @@ pub async fn create_transformer( grpc::create_rpc_channel(ud_transformer.socket_path.clone().into()).await?, ) .max_encoding_message_size(ud_transformer.grpc_max_message_size) - .max_encoding_message_size(ud_transformer.grpc_max_message_size); + .max_decoding_message_size(ud_transformer.grpc_max_message_size); grpc::wait_until_transformer_ready(&cln_token, &mut transformer_grpc_client).await?; return Ok(( Some( @@ -242,7 +242,7 @@ pub async fn create_source( grpc::create_rpc_channel(udsource_config.socket_path.clone().into()).await?, ) .max_encoding_message_size(udsource_config.grpc_max_message_size) - .max_encoding_message_size(udsource_config.grpc_max_message_size); + .max_decoding_message_size(udsource_config.grpc_max_message_size); grpc::wait_until_source_ready(&cln_token, &mut source_grpc_client).await?; let (ud_read, ud_ack, ud_lag) = new_source(source_grpc_client.clone(), batch_size, read_timeout).await?; @@ -266,6 +266,9 @@ pub async fn create_source( None, )) } + SourceType::Serving(_) => { + unimplemented!("Serving as built-in source is not yet implemented") + } } } @@ -329,7 +332,7 @@ mod tests { #[tonic::async_trait] impl sink::Sinker for InMemorySink { - async fn sink(&self, mut _input: mpsc::Receiver) -> Vec { + async fn sink(&self, _input: mpsc::Receiver) -> Vec { vec![] } } diff --git a/rust/numaflow-core/src/shared/forward.rs b/rust/numaflow-core/src/shared/forward.rs index 11b9195ccb..050902328e 100644 --- a/rust/numaflow-core/src/shared/forward.rs +++ b/rust/numaflow-core/src/shared/forward.rs @@ -1,7 +1,8 @@ -use numaflow_models::models::ForwardConditions; use std::hash::{DefaultHasher, Hasher}; use std::sync::Arc; +use numaflow_models::models::ForwardConditions; + /// Checks if the message should to written to downstream vertex based the conditions /// and message tags. If not tags are provided by there are edge conditions present, we will /// still forward to all vertices. @@ -61,9 +62,10 @@ fn check_operator_condition( #[cfg(test)] mod tests { - use super::*; use numaflow_models::models::TagConditions; + use super::*; + #[tokio::test] async fn test_evaluate_write_condition_no_conditions() { let result = should_forward(None, None); diff --git a/rust/numaflow-core/src/shared/grpc.rs b/rust/numaflow-core/src/shared/grpc.rs index d6246b60a6..3500524f02 100644 --- a/rust/numaflow-core/src/shared/grpc.rs +++ b/rust/numaflow-core/src/shared/grpc.rs @@ -15,7 +15,7 @@ use tokio_util::sync::CancellationToken; use tonic::transport::{Channel, Endpoint}; use tonic::Request; use tower::service_fn; -use tracing::info; +use tracing::{info, warn}; use crate::error; use crate::error::Error; @@ -90,13 +90,21 @@ pub(crate) fn prost_timestamp_from_utc(t: DateTime) -> Option { pub(crate) async fn create_rpc_channel(socket_path: PathBuf) -> error::Result { const RECONNECT_INTERVAL: u64 = 1000; - const MAX_RECONNECT_ATTEMPTS: usize = 5; + const MAX_RECONNECT_ATTEMPTS: usize = 60; let interval = fixed::Interval::from_millis(RECONNECT_INTERVAL).take(MAX_RECONNECT_ATTEMPTS); let channel = Retry::retry( interval, - || async { connect_with_uds(socket_path.clone()).await }, + || async { + match connect_with_uds(socket_path.clone()).await { + Ok(channel) => Ok(channel), + Err(e) => { + warn!(?e, "Failed to connect to UDS socket"); + Err(Error::Connection(format!("Failed to connect: {:?}", e))) + } + } + }, |_: &Error| true, ) .await?; diff --git a/rust/numaflow-core/src/shared/server_info.rs b/rust/numaflow-core/src/shared/server_info.rs index 40ec6b37d6..ee3b1c8d6a 100644 --- a/rust/numaflow-core/src/shared/server_info.rs +++ b/rust/numaflow-core/src/shared/server_info.rs @@ -97,14 +97,17 @@ pub(crate) async fn sdk_server_info( // Read the server info file let server_info = read_server_info(&file_path, cln_token).await?; + // Get the container type from the server info file + let container_type = get_container_type(&file_path).unwrap_or(ContainerType::Unknown); + // Log the server info - info!("Server info file: {:?}", server_info); + info!(?container_type, ?server_info, "Server info file"); // Extract relevant fields from server info let sdk_version = &server_info.version; let min_numaflow_version = &server_info.minimum_numaflow_version; let sdk_language = &server_info.language; - let container_type = get_container_type(&file_path).unwrap_or(ContainerType::Unknown); + // Get version information let version_info = version::get_version_info(); let numaflow_version = &version_info.version; diff --git a/rust/numaflow-core/src/sink.rs b/rust/numaflow-core/src/sink.rs index 474f91e77f..0b30f4c30a 100644 --- a/rust/numaflow-core/src/sink.rs +++ b/rust/numaflow-core/src/sink.rs @@ -15,8 +15,13 @@ use tracing::{debug, error, info, warn}; use user_defined::UserDefinedSink; use crate::config::components::sink::{OnFailureStrategy, RetryConfig}; +use crate::config::is_mono_vertex; use crate::error::Error; use crate::message::{Message, ResponseFromSink, ResponseStatusFromSink}; +use crate::metrics::{ + monovertex_metrics, mvtx_forward_metric_labels, pipeline_forward_metric_labels, + pipeline_metrics, +}; use crate::tracker::TrackerHandle; use crate::Result; @@ -73,6 +78,12 @@ where } } } + + async fn run(mut self) { + while let Some(msg) = self.actor_messages.recv().await { + self.handle_message(msg).await; + } + } } pub(crate) enum SinkClientType { @@ -137,28 +148,22 @@ impl SinkWriterBuilder { SinkClientType::Log => { let log_sink = log::LogSink; tokio::spawn(async { - let mut actor = SinkActor::new(receiver, log_sink); - while let Some(msg) = actor.actor_messages.recv().await { - actor.handle_message(msg).await; - } + let actor = SinkActor::new(receiver, log_sink); + actor.run().await; }); } SinkClientType::Blackhole => { let blackhole_sink = blackhole::BlackholeSink; tokio::spawn(async { - let mut actor = SinkActor::new(receiver, blackhole_sink); - while let Some(msg) = actor.actor_messages.recv().await { - actor.handle_message(msg).await; - } + let actor = SinkActor::new(receiver, blackhole_sink); + actor.run().await; }); } SinkClientType::UserDefined(sink_client) => { let sink = UserDefinedSink::new(sink_client).await?; tokio::spawn(async { - let mut actor = SinkActor::new(receiver, sink); - while let Some(msg) = actor.actor_messages.recv().await { - actor.handle_message(msg).await; - } + let actor = SinkActor::new(receiver, sink); + actor.run().await; }); } }; @@ -169,28 +174,22 @@ impl SinkWriterBuilder { SinkClientType::Log => { let log_sink = log::LogSink; tokio::spawn(async { - let mut actor = SinkActor::new(fb_receiver, log_sink); - while let Some(msg) = actor.actor_messages.recv().await { - actor.handle_message(msg).await; - } + let actor = SinkActor::new(fb_receiver, log_sink); + actor.run().await; }); } SinkClientType::Blackhole => { let blackhole_sink = blackhole::BlackholeSink; tokio::spawn(async { - let mut actor = SinkActor::new(fb_receiver, blackhole_sink); - while let Some(msg) = actor.actor_messages.recv().await { - actor.handle_message(msg).await; - } + let actor = SinkActor::new(fb_receiver, blackhole_sink); + actor.run().await; }); } SinkClientType::UserDefined(sink_client) => { let sink = UserDefinedSink::new(sink_client).await?; tokio::spawn(async { - let mut actor = SinkActor::new(fb_receiver, sink); - while let Some(msg) = actor.actor_messages.recv().await { - actor.handle_message(msg).await; - } + let actor = SinkActor::new(fb_receiver, sink); + actor.run().await; }); } }; @@ -275,13 +274,15 @@ impl SinkWriter { .map(|msg| msg.id.offset.clone()) .collect::>(); + let total_msgs = batch.len(); // filter out the messages which needs to be dropped let batch = batch .into_iter() .filter(|msg| !msg.dropped()) .collect::>(); - let n = batch.len(); + let sink_start = time::Instant::now(); + let total_valid_msgs = batch.len(); match this.write(batch, cancellation_token.clone()).await { Ok(_) => { for offset in offsets { @@ -298,7 +299,31 @@ impl SinkWriter { } } - processed_msgs_count += n; + // publish sink metrics + if is_mono_vertex() { + monovertex_metrics() + .sink + .time + .get_or_create(mvtx_forward_metric_labels()) + .observe(sink_start.elapsed().as_micros() as f64); + monovertex_metrics() + .dropped_total + .get_or_create(mvtx_forward_metric_labels()) + .inc_by((total_msgs - total_valid_msgs) as u64); + } else { + pipeline_metrics() + .forwarder + .write_time + .get_or_create(pipeline_forward_metric_labels("Sink", None)) + .observe(sink_start.elapsed().as_micros() as f64); + pipeline_metrics() + .forwarder + .dropped_total + .get_or_create(pipeline_forward_metric_labels("Sink", None)) + .inc_by((total_msgs - total_valid_msgs) as u64); + } + + processed_msgs_count += total_msgs; if last_logged_at.elapsed().as_millis() >= 1000 { info!( "Processed {} messages at {:?}", @@ -326,6 +351,7 @@ impl SinkWriter { return Ok(()); } + let total_msgs = messages.len(); let mut attempts = 0; let mut error_map = HashMap::new(); let mut fallback_msgs = Vec::new(); @@ -388,12 +414,32 @@ impl SinkWriter { } } + let fb_msgs_total = fallback_msgs.len(); // If there are fallback messages, write them to the fallback sink if !fallback_msgs.is_empty() { self.handle_fallback_messages(fallback_msgs, retry_config) .await?; } + if is_mono_vertex() { + monovertex_metrics() + .sink + .write_total + .get_or_create(mvtx_forward_metric_labels()) + .inc_by((total_msgs - fb_msgs_total) as u64); + monovertex_metrics() + .fb_sink + .write_total + .get_or_create(mvtx_forward_metric_labels()) + .inc_by(fb_msgs_total as u64); + } else { + pipeline_metrics() + .forwarder + .write_total + .get_or_create(pipeline_forward_metric_labels("Sink", None)) + .inc_by(total_msgs as u64); + } + Ok(()) } @@ -605,9 +651,10 @@ impl Drop for SinkWriter { #[cfg(test)] mod tests { + use std::sync::Arc; + use chrono::Utc; use numaflow::sink; - use std::sync::Arc; use tokio::time::Duration; use tokio_util::sync::CancellationToken; diff --git a/rust/numaflow-core/src/sink/blackhole.rs b/rust/numaflow-core/src/sink/blackhole.rs index dd537d18b1..eb2f331360 100644 --- a/rust/numaflow-core/src/sink/blackhole.rs +++ b/rust/numaflow-core/src/sink/blackhole.rs @@ -19,9 +19,10 @@ impl Sink for BlackholeSink { #[cfg(test)] mod tests { - use chrono::Utc; use std::sync::Arc; + use chrono::Utc; + use super::BlackholeSink; use crate::message::IntOffset; use crate::message::{Message, MessageID, Offset, ResponseFromSink, ResponseStatusFromSink}; diff --git a/rust/numaflow-core/src/sink/log.rs b/rust/numaflow-core/src/sink/log.rs index a82670e8d8..9ae426f1f2 100644 --- a/rust/numaflow-core/src/sink/log.rs +++ b/rust/numaflow-core/src/sink/log.rs @@ -35,9 +35,10 @@ impl Sink for LogSink { #[cfg(test)] mod tests { - use chrono::Utc; use std::sync::Arc; + use chrono::Utc; + use super::LogSink; use crate::message::IntOffset; use crate::message::{Message, MessageID, Offset, ResponseFromSink, ResponseStatusFromSink}; diff --git a/rust/numaflow-core/src/sink/user_defined.rs b/rust/numaflow-core/src/sink/user_defined.rs index a1817b1ae0..efb9c1178d 100644 --- a/rust/numaflow-core/src/sink/user_defined.rs +++ b/rust/numaflow-core/src/sink/user_defined.rs @@ -1,7 +1,3 @@ -use crate::message::{Message, ResponseFromSink}; -use crate::sink::Sink; -use crate::Error; -use crate::Result; use numaflow_pb::clients::sink::sink_client::SinkClient; use numaflow_pb::clients::sink::{Handshake, SinkRequest, SinkResponse, TransmissionStatus}; use tokio::sync::mpsc; @@ -10,6 +6,11 @@ use tonic::transport::Channel; use tonic::{Request, Streaming}; use tracing::error; +use crate::message::{Message, ResponseFromSink}; +use crate::sink::Sink; +use crate::Error; +use crate::Result; + const DEFAULT_CHANNEL_SIZE: usize = 1000; /// User-Defined Sink code writes messages to a custom [SinkWriter]. @@ -118,9 +119,10 @@ impl Sink for UserDefinedSink { #[cfg(test)] mod tests { + use std::sync::Arc; + use chrono::offset::Utc; use numaflow::sink; - use std::sync::Arc; use tokio::sync::mpsc; use tracing::info; diff --git a/rust/numaflow-core/src/source.rs b/rust/numaflow-core/src/source.rs index 3f2816514f..66361d84ac 100644 --- a/rust/numaflow-core/src/source.rs +++ b/rust/numaflow-core/src/source.rs @@ -129,6 +129,12 @@ where } } } + + async fn run(mut self) { + while let Some(msg) = self.receiver.recv().await { + self.handle_message(msg).await; + } + } } /// Source is used to read, ack, and get the pending messages count from the source. @@ -150,31 +156,25 @@ impl Source { match src_type { SourceType::UserDefinedSource(reader, acker, lag_reader) => { tokio::spawn(async move { - let mut actor = SourceActor::new(receiver, reader, acker, lag_reader); - while let Some(msg) = actor.receiver.recv().await { - actor.handle_message(msg).await; - } + let actor = SourceActor::new(receiver, reader, acker, lag_reader); + actor.run().await; }); } SourceType::Generator(reader, acker, lag_reader) => { tokio::spawn(async move { - let mut actor = SourceActor::new(receiver, reader, acker, lag_reader); - while let Some(msg) = actor.receiver.recv().await { - actor.handle_message(msg).await; - } + let actor = SourceActor::new(receiver, reader, acker, lag_reader); + actor.run().await; }); } SourceType::Pulsar(pulsar_source) => { tokio::spawn(async move { - let mut actor = SourceActor::new( + let actor = SourceActor::new( receiver, pulsar_source.clone(), pulsar_source.clone(), pulsar_source, ); - while let Some(msg) = actor.receiver.recv().await { - actor.handle_message(msg).await; - } + actor.run().await; }); } }; diff --git a/rust/numaflow-core/src/source/generator.rs b/rust/numaflow-core/src/source/generator.rs index fdc5d590a5..855030f73b 100644 --- a/rust/numaflow-core/src/source/generator.rs +++ b/rust/numaflow-core/src/source/generator.rs @@ -1,8 +1,9 @@ +use tokio_stream::StreamExt; + use crate::config::components::source::GeneratorConfig; use crate::message::{Message, Offset}; use crate::reader; use crate::source; -use tokio_stream::StreamExt; /// Stream Generator returns a set of messages for every `.next` call. It will throttle itself if /// the call exceeds the RPU. It will return a max (batch size, RPU) till the quota for that unit of diff --git a/rust/numaflow-core/src/tracker.rs b/rust/numaflow-core/src/tracker.rs index a1dd32662b..a4ef30e24c 100644 --- a/rust/numaflow-core/src/tracker.rs +++ b/rust/numaflow-core/src/tracker.rs @@ -8,14 +8,16 @@ //! //! In the future Watermark will also be propagated based on this. -use crate::error::Error; -use crate::message::ReadAck; -use crate::Result; -use bytes::Bytes; use std::collections::HashMap; + +use bytes::Bytes; use tokio::sync::{mpsc, oneshot}; use tracing::warn; +use crate::error::Error; +use crate::message::ReadAck; +use crate::Result; + /// TrackerEntry represents the state of a tracked message. #[derive(Debug)] struct TrackerEntry { @@ -246,10 +248,11 @@ impl TrackerHandle { #[cfg(test)] mod tests { - use super::*; use tokio::sync::oneshot; use tokio::time::{timeout, Duration}; + use super::*; + #[tokio::test] async fn test_insert_update_delete() { let handle = TrackerHandle::new(); diff --git a/rust/numaflow-core/src/transformer.rs b/rust/numaflow-core/src/transformer.rs index d6d63bdfea..0b26a7e76a 100644 --- a/rust/numaflow-core/src/transformer.rs +++ b/rust/numaflow-core/src/transformer.rs @@ -1,5 +1,6 @@ -use numaflow_pb::clients::sourcetransformer::source_transform_client::SourceTransformClient; use std::sync::Arc; + +use numaflow_pb::clients::sourcetransformer::source_transform_client::SourceTransformClient; use tokio::sync::{mpsc, oneshot, OwnedSemaphorePermit, Semaphore}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; @@ -7,7 +8,9 @@ use tokio_stream::StreamExt; use tonic::transport::Channel; use tracing::error; +use crate::error::Error; use crate::message::Message; +use crate::metrics::{monovertex_metrics, mvtx_forward_metric_labels}; use crate::tracker::TrackerHandle; use crate::transformer::user_defined::UserDefinedTransformer; use crate::Result; @@ -104,6 +107,7 @@ impl Transformer { // invoke transformer and then wait for the one-shot tokio::spawn(async move { + let start_time = tokio::time::Instant::now(); let _permit = permit; let (sender, receiver) = oneshot::channel(); @@ -141,6 +145,11 @@ impl Transformer { .expect("failed to discard tracker"); } } + monovertex_metrics() + .transformer + .time + .get_or_create(mvtx_forward_metric_labels()) + .observe(start_time.elapsed().as_micros() as f64); }); Ok(()) @@ -156,14 +165,16 @@ impl Transformer { let transform_handle = self.sender.clone(); let tracker_handle = self.tracker_handle.clone(); - // FIXME: batch_size should not be used, introduce a new config called udf concurrenc + // FIXME: batch_size should not be used, introduce a new config called udf concurrency let semaphore = Arc::new(Semaphore::new(self.concurrency)); let handle = tokio::spawn(async move { let mut input_stream = input_stream; while let Some(read_msg) = input_stream.next().await { - let permit = Arc::clone(&semaphore).acquire_owned().await.unwrap(); + let permit = Arc::clone(&semaphore).acquire_owned().await.map_err(|e| { + Error::Transformer(format!("failed to acquire semaphore: {}", e)) + })?; Self::transform( transform_handle.clone(), diff --git a/rust/numaflow/src/main.rs b/rust/numaflow/src/main.rs index e25fbf9c66..60e26ef850 100644 --- a/rust/numaflow/src/main.rs +++ b/rust/numaflow/src/main.rs @@ -1,13 +1,14 @@ +use std::collections::HashMap; use std::env; +use std::error::Error; +use std::sync::Arc; -use tracing::{error, info}; +use tracing::error; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; #[tokio::main] -async fn main() { - let args: Vec = env::args().collect(); - +async fn main() -> Result<(), Box> { // Set up the tracing subscriber. RUST_LOG can be used to set the log level. // The default log level is `info`. The `axum::rejection=trace` enables showing // rejections from built-in extractors at `TRACE` level. @@ -20,21 +21,31 @@ async fn main() { ) .with(tracing_subscriber::fmt::layer().with_ansi(false)) .init(); + if let Err(e) = run().await { + error!("{e:?}"); + return Err(e); + } + Ok(()) +} +async fn run() -> Result<(), Box> { + let args: Vec = env::args().collect(); // Based on the argument, run the appropriate component. if args.contains(&"--serving".to_string()) { - if let Err(e) = serving::serve().await { - error!("Error running serving: {}", e); - } + let env_vars: HashMap = env::vars().collect(); + let settings: serving::Settings = env_vars.try_into()?; + let settings = Arc::new(settings); + serving::serve(settings) + .await + .map_err(|e| format!("Error running serving: {e:?}"))?; } else if args.contains(&"--servesink".to_string()) { - if let Err(e) = servesink::servesink().await { - info!("Error running servesink: {}", e); - } + servesink::servesink() + .await + .map_err(|e| format!("Error running servesink: {e:?}"))?; } else if args.contains(&"--rust".to_string()) { - if let Err(e) = numaflow_core::run().await { - error!("Error running rust binary: {}", e); - } - } else { - error!("Invalid argument. Use --serving, --servesink, or --rust."); + numaflow_core::run() + .await + .map_err(|e| format!("Error running rust binary: {e:?}"))? } + Err("Invalid argument. Use --serving, --servesink, or --rust".into()) } diff --git a/rust/serving/Cargo.toml b/rust/serving/Cargo.toml index d62a1d2d8f..de2f8bb820 100644 --- a/rust/serving/Cargo.toml +++ b/rust/serving/Cargo.toml @@ -27,7 +27,6 @@ tower = "0.4.13" tower-http = { version = "0.5.2", features = ["trace", "timeout"] } uuid = { version = "1.10.0", features = ["v4"] } redis = { version = "0.26.0", features = ["tokio-comp", "aio", "connection-manager"] } -config = "0.14.0" trait-variant = "0.1.2" chrono = { version = "0.4", features = ["serde"] } base64 = "0.22.1" diff --git a/rust/serving/config/default.toml b/rust/serving/config/default.toml deleted file mode 100644 index 448672abc1..0000000000 --- a/rust/serving/config/default.toml +++ /dev/null @@ -1,17 +0,0 @@ -tid_header = "ID" -app_listen_port = 3000 -metrics_server_listen_port = 3001 -upstream_addr = "localhost:8888" -drain_timeout_secs = 10 -host_ip = "localhost" - -[jetstream] -stream = "default" -url = "localhost:4222" - -[redis] -addr = "redis://127.0.0.1/" -max_tasks = 50 -retries = 5 -retries_duration_millis = 100 -ttl_secs = 1 diff --git a/rust/serving/config/jetstream.conf b/rust/serving/config/jetstream.conf deleted file mode 100644 index e09998c0ac..0000000000 --- a/rust/serving/config/jetstream.conf +++ /dev/null @@ -1,4 +0,0 @@ -jetstream: { - max_mem_store: 1MiB, - max_file_store: 1GiB -} \ No newline at end of file diff --git a/rust/serving/config/pipeline_spec.json b/rust/serving/config/pipeline_spec.json deleted file mode 100644 index 1698329e86..0000000000 --- a/rust/serving/config/pipeline_spec.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "vertices": [ - { - "name": "in" - }, { - "name": "cat" - }, { - "name": "out" - } - ], - "edges": [ - { - "from": "in", - "to": "cat" - }, - { - "from": "cat", - "to": "out" - } - ] -} \ No newline at end of file diff --git a/rust/serving/src/app.rs b/rust/serving/src/app.rs index d1d29c4c21..56d4a33cb3 100644 --- a/rust/serving/src/app.rs +++ b/rust/serving/src/app.rs @@ -1,5 +1,5 @@ -use std::env; use std::net::SocketAddr; +use std::sync::Arc; use std::time::Duration; use async_nats::jetstream; @@ -17,7 +17,7 @@ use tokio::signal; use tower::ServiceBuilder; use tower_http::timeout::TimeoutLayer; use tower_http::trace::{DefaultOnResponse, TraceLayer}; -use tracing::{debug, info, info_span, Level}; +use tracing::{info, info_span, Level}; use uuid::Uuid; use self::{ @@ -26,9 +26,11 @@ use self::{ }; use crate::app::callback::store::Store; use crate::app::tracker::MessageGraph; -use crate::pipeline::min_pipeline_spec; -use crate::Error::{InitError, MetricsServer}; -use crate::{app::callback::state::State as CallbackState, config, metrics::capture_metrics}; +use crate::config::JetStreamConfig; +use crate::pipeline::PipelineDCG; +use crate::Error::InitError; +use crate::Settings; +use crate::{app::callback::state::State as CallbackState, metrics::capture_metrics}; /// manage callbacks pub(crate) mod callback; @@ -41,10 +43,6 @@ mod message_path; // TODO: merge message_path and tracker mod response; mod tracker; -const ENV_NUMAFLOW_SERVING_JETSTREAM_USER: &str = "NUMAFLOW_ISBSVC_JETSTREAM_USER"; -const ENV_NUMAFLOW_SERVING_JETSTREAM_PASSWORD: &str = "NUMAFLOW_ISBSVC_JETSTREAM_PASSWORD"; -const ENV_NUMAFLOW_SERVING_AUTH_TOKEN: &str = "NUMAFLOW_SERVING_AUTH_TOKEN"; - /// Everything for numaserve starts here. The routing, middlewares, proxying, etc. // TODO // - [ ] implement an proxy and pass in UUID in the header if not present @@ -52,19 +50,23 @@ const ENV_NUMAFLOW_SERVING_AUTH_TOKEN: &str = "NUMAFLOW_SERVING_AUTH_TOKEN"; /// Start the main application Router and the axum server. pub(crate) async fn start_main_server( - addr: SocketAddr, + settings: Arc, tls_config: RustlsConfig, + pipeline_spec: PipelineDCG, ) -> crate::Result<()> { - debug!(?addr, "App server started"); + let app_addr: SocketAddr = format!("0.0.0.0:{}", &settings.app_listen_port) + .parse() + .map_err(|e| InitError(format!("{e:?}")))?; + let tid_header = settings.tid_header.clone(); let layers = ServiceBuilder::new() // Add tracing to all requests .layer( TraceLayer::new_for_http() - .make_span_with(|req: &Request| { + .make_span_with(move |req: &Request| { let tid = req .headers() - .get(&config().tid_header) + .get(&tid_header) .and_then(|v| v.to_str().ok()) .map(|v| v.to_string()) .unwrap_or_else(|| Uuid::new_v4().to_string()); @@ -83,13 +85,16 @@ pub(crate) async fn start_main_server( .layer( // Graceful shutdown will wait for outstanding requests to complete. Add a timeout so // requests don't hang forever. - TimeoutLayer::new(Duration::from_secs(config().drain_timeout_secs)), + TimeoutLayer::new(Duration::from_secs(settings.drain_timeout_secs)), ) // Add auth middleware to all user facing routes - .layer(middleware::from_fn(auth_middleware)); + .layer(middleware::from_fn_with_state( + settings.api_auth_token.clone(), + auth_middleware, + )); // Create the message graph from the pipeline spec and the redis store - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).map_err(|e| { + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).map_err(|e| { InitError(format!( "Creating message graph from pipeline spec: {:?}", e @@ -97,11 +102,8 @@ pub(crate) async fn start_main_server( })?; // Create a redis store to store the callbacks and the custom responses - let redis_store = callback::store::redisstore::RedisConnection::new( - &config().redis.addr, - config().redis.max_tasks, - ) - .await?; + let redis_store = + callback::store::redisstore::RedisConnection::new(settings.redis.clone()).await?; let state = CallbackState::new(msg_graph, redis_store).await?; let handle = Handle::new(); @@ -109,14 +111,17 @@ pub(crate) async fn start_main_server( tokio::spawn(graceful_shutdown(handle.clone())); // Create a Jetstream context - let js_context = create_js_context().await?; + let js_context = create_js_context(&settings.jetstream).await?; + + let router = setup_app(settings, js_context, state).await?.layer(layers); + + info!(?app_addr, "Starting application server"); - let router = setup_app(js_context, state).await?.layer(layers); - axum_server::bind_rustls(addr, tls_config) + axum_server::bind_rustls(app_addr, tls_config) .handle(handle) .serve(router.into_make_service()) .await - .map_err(|e| MetricsServer(format!("Starting web server for metrics: {}", e)))?; + .map_err(|e| InitError(format!("Starting web server for metrics: {}", e)))?; Ok(()) } @@ -149,19 +154,16 @@ async fn graceful_shutdown(handle: Handle) { handle.graceful_shutdown(Some(Duration::from_secs(30))); } -async fn create_js_context() -> crate::Result { - // Check for user and password in the Jetstream configuration - let js_config = &config().jetstream; - +async fn create_js_context(js_config: &JetStreamConfig) -> crate::Result { // Connect to Jetstream with user and password if they are set - let js_client = match ( - env::var(ENV_NUMAFLOW_SERVING_JETSTREAM_USER), - env::var(ENV_NUMAFLOW_SERVING_JETSTREAM_PASSWORD), - ) { - (Ok(user), Ok(password)) => { + let js_client = match js_config.auth.as_ref() { + Some(auth) => { async_nats::connect_with_options( &js_config.url, - async_nats::ConnectOptions::with_user_and_password(user, password), + async_nats::ConnectOptions::with_user_and_password( + auth.username.clone(), + auth.password.clone(), + ), ) .await } @@ -170,8 +172,7 @@ async fn create_js_context() -> crate::Result { .map_err(|e| { InitError(format!( "Connecting to jetstream server {}: {}", - &config().jetstream.url, - e + &js_config.url, e )) })?; Ok(jetstream::new(js_client)) @@ -185,7 +186,11 @@ const PUBLISH_ENDPOINTS: [&str; 3] = [ // auth middleware to do token based authentication for all user facing routes // if auth is enabled. -async fn auth_middleware(request: axum::extract::Request, next: Next) -> Response { +async fn auth_middleware( + State(api_auth_token): State>, + request: axum::extract::Request, + next: Next, +) -> Response { let path = request.uri().path(); // we only need to check for the presence of the auth token in the request headers for the publish endpoints @@ -193,8 +198,8 @@ async fn auth_middleware(request: axum::extract::Request, next: Next) -> Respons return next.run(request).await; } - match env::var(ENV_NUMAFLOW_SERVING_AUTH_TOKEN) { - Ok(token) => { + match api_auth_token { + Some(token) => { // Check for the presence of the auth token in the request headers let auth_token = match request.headers().get("Authorization") { Some(token) => token, @@ -216,22 +221,35 @@ async fn auth_middleware(request: axum::extract::Request, next: Next) -> Respons next.run(request).await } } - Err(_) => { + None => { // If the auth token is not set, we don't need to check for the presence of the auth token in the request headers next.run(request).await } } } +#[derive(Clone)] +pub(crate) struct AppState { + pub(crate) settings: Arc, + pub(crate) callback_state: CallbackState, + pub(crate) context: Context, +} + async fn setup_app( + settings: Arc, context: Context, - state: CallbackState, + callback_state: CallbackState, ) -> crate::Result { + let app_state = AppState { + settings, + callback_state: callback_state.clone(), + context: context.clone(), + }; let parent = Router::new() .route("/health", get(health_check)) .route("/livez", get(livez)) // Liveliness check .route("/readyz", get(readyz)) - .with_state((state.clone(), context.clone())); // Readiness check + .with_state(app_state.clone()); // Readiness check // a pool based client implementation for direct proxy, this client is cloneable. let client: direct_proxy::Client = @@ -240,8 +258,11 @@ async fn setup_app( // let's nest each endpoint let app = parent - .nest("/v1/direct", direct_proxy(client)) - .nest("/v1/process", routes(context, state).await?); + .nest( + "/v1/direct", + direct_proxy(client, app_state.settings.upstream_addr.clone()), + ) + .nest("/v1/process", routes(app_state).await?); Ok(app) } @@ -250,16 +271,20 @@ async fn health_check() -> impl IntoResponse { "ok" } -async fn livez( - State((_state, _context)): State<(CallbackState, Context)>, -) -> impl IntoResponse { +async fn livez() -> impl IntoResponse { StatusCode::NO_CONTENT } async fn readyz( - State((mut state, context)): State<(CallbackState, Context)>, + State(app): State>, ) -> impl IntoResponse { - if state.ready().await && context.get_stream(&config().jetstream.stream).await.is_ok() { + if app.callback_state.clone().ready().await + && app + .context + .get_stream(&app.settings.jetstream.stream) + .await + .is_ok() + { StatusCode::NO_CONTENT } else { StatusCode::INTERNAL_SERVER_ERROR @@ -267,11 +292,14 @@ async fn readyz( } async fn routes( - context: Context, - state: CallbackState, + app_state: AppState, ) -> crate::Result { - let jetstream_proxy = jetstream_proxy(context, state.clone()).await?; - let callback_router = callback_handler(state.clone()); + let state = app_state.callback_state.clone(); + let jetstream_proxy = jetstream_proxy(app_state.clone()).await?; + let callback_router = callback_handler( + app_state.settings.tid_header.clone(), + app_state.callback_state.clone(), + ); let message_path_handler = get_message_path(state); Ok(jetstream_proxy .merge(callback_router) @@ -280,8 +308,6 @@ async fn routes( #[cfg(test)] mod tests { - use std::net::SocketAddr; - use async_nats::jetstream::stream; use axum::http::StatusCode; use tokio::time::{sleep, Duration}; @@ -291,6 +317,8 @@ mod tests { use crate::app::callback::store::memstore::InMemoryStore; use crate::config::generate_certs; + const PIPELINE_SPEC_ENCODED: &str = "eyJ2ZXJ0aWNlcyI6W3sibmFtZSI6ImluIiwic291cmNlIjp7InNlcnZpbmciOnsiYXV0aCI6bnVsbCwic2VydmljZSI6dHJ1ZSwibXNnSURIZWFkZXJLZXkiOiJYLU51bWFmbG93LUlkIiwic3RvcmUiOnsidXJsIjoicmVkaXM6Ly9yZWRpczo2Mzc5In19fSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIiLCJlbnYiOlt7Im5hbWUiOiJSVVNUX0xPRyIsInZhbHVlIjoiZGVidWcifV19LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InBsYW5uZXIiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJwbGFubmVyIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InRpZ2VyIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsidGlnZXIiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZG9nIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZG9nIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6ImVsZXBoYW50IiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZWxlcGhhbnQiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiYXNjaWlhcnQiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJhc2NpaWFydCJdLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJidWlsdGluIjpudWxsLCJncm91cEJ5IjpudWxsfSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwic2NhbGUiOnsibWluIjoxfSwidXBkYXRlU3RyYXRlZ3kiOnsidHlwZSI6IlJvbGxpbmdVcGRhdGUiLCJyb2xsaW5nVXBkYXRlIjp7Im1heFVuYXZhaWxhYmxlIjoiMjUlIn19fSx7Im5hbWUiOiJzZXJ2ZS1zaW5rIiwic2luayI6eyJ1ZHNpbmsiOnsiY29udGFpbmVyIjp7ImltYWdlIjoic2VydmVzaW5rOjAuMSIsImVudiI6W3sibmFtZSI6Ik5VTUFGTE9XX0NBTExCQUNLX1VSTF9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctQ2FsbGJhY2stVXJsIn0seyJuYW1lIjoiTlVNQUZMT1dfTVNHX0lEX0hFQURFUl9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctSWQifV0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn19LCJyZXRyeVN0cmF0ZWd5Ijp7fX0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZXJyb3Itc2luayIsInNpbmsiOnsidWRzaW5rIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6InNlcnZlc2luazowLjEiLCJlbnYiOlt7Im5hbWUiOiJOVU1BRkxPV19DQUxMQkFDS19VUkxfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUNhbGxiYWNrLVVybCJ9LHsibmFtZSI6Ik5VTUFGTE9XX01TR19JRF9IRUFERVJfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUlkIn1dLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9fSwicmV0cnlTdHJhdGVneSI6e319LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19XSwiZWRnZXMiOlt7ImZyb20iOiJpbiIsInRvIjoicGxhbm5lciIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImFzY2lpYXJ0IiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiYXNjaWlhcnQiXX19fSx7ImZyb20iOiJwbGFubmVyIiwidG8iOiJ0aWdlciIsImNvbmRpdGlvbnMiOnsidGFncyI6eyJvcGVyYXRvciI6Im9yIiwidmFsdWVzIjpbInRpZ2VyIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZG9nIiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiZG9nIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZWxlcGhhbnQiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlbGVwaGFudCJdfX19LHsiZnJvbSI6InRpZ2VyIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZG9nIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZWxlcGhhbnQiLCJ0byI6InNlcnZlLXNpbmsiLCJjb25kaXRpb25zIjpudWxsfSx7ImZyb20iOiJhc2NpaWFydCIsInRvIjoic2VydmUtc2luayIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImVycm9yLXNpbmsiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlcnJvciJdfX19XSwibGlmZWN5Y2xlIjp7fSwid2F0ZXJtYXJrIjp7fX0="; + type Result = core::result::Result; type Error = Box; @@ -302,9 +330,14 @@ mod tests { .await .unwrap(); - let addr = SocketAddr::from(([127, 0, 0, 1], 0)); + let settings = Arc::new(Settings { + app_listen_port: 0, + ..Settings::default() + }); + let server = tokio::spawn(async move { - let result = start_main_server(addr, tls_config).await; + let pipeline_spec = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let result = start_main_server(settings, tls_config, pipeline_spec).await; assert!(result.is_ok()) }); @@ -319,9 +352,10 @@ mod tests { #[cfg(feature = "all-tests")] #[tokio::test] async fn test_setup_app() -> Result<()> { - let client = async_nats::connect(&config().jetstream.url).await?; + let settings = Arc::new(Settings::default()); + let client = async_nats::connect(&settings.jetstream.url).await?; let context = jetstream::new(client); - let stream_name = &config().jetstream.stream; + let stream_name = &settings.jetstream.stream; let stream = context .get_or_create_stream(stream::Config { @@ -334,11 +368,12 @@ mod tests { assert!(stream.is_ok()); let mem_store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec())?; + let pipeline_spec = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec)?; let callback_state = CallbackState::new(msg_graph, mem_store).await?; - let result = setup_app(context, callback_state).await; + let result = setup_app(settings, context, callback_state).await; assert!(result.is_ok()); Ok(()) } @@ -346,9 +381,10 @@ mod tests { #[cfg(feature = "all-tests")] #[tokio::test] async fn test_livez() -> Result<()> { - let client = async_nats::connect(&config().jetstream.url).await?; + let settings = Arc::new(Settings::default()); + let client = async_nats::connect(&settings.jetstream.url).await?; let context = jetstream::new(client); - let stream_name = &config().jetstream.stream; + let stream_name = &settings.jetstream.stream; let stream = context .get_or_create_stream(stream::Config { @@ -361,11 +397,12 @@ mod tests { assert!(stream.is_ok()); let mem_store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec())?; + let pipeline_spec = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec)?; let callback_state = CallbackState::new(msg_graph, mem_store).await?; - let result = setup_app(context, callback_state).await; + let result = setup_app(settings, context, callback_state).await; let request = Request::builder().uri("/livez").body(Body::empty())?; @@ -377,9 +414,10 @@ mod tests { #[cfg(feature = "all-tests")] #[tokio::test] async fn test_readyz() -> Result<()> { - let client = async_nats::connect(&config().jetstream.url).await?; + let settings = Arc::new(Settings::default()); + let client = async_nats::connect(&settings.jetstream.url).await?; let context = jetstream::new(client); - let stream_name = &config().jetstream.stream; + let stream_name = &settings.jetstream.stream; let stream = context .get_or_create_stream(stream::Config { @@ -392,11 +430,12 @@ mod tests { assert!(stream.is_ok()); let mem_store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec())?; + let pipeline_spec = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec)?; let callback_state = CallbackState::new(msg_graph, mem_store).await?; - let result = setup_app(context, callback_state).await; + let result = setup_app(settings, context, callback_state).await; let request = Request::builder().uri("/readyz").body(Body::empty())?; @@ -415,9 +454,10 @@ mod tests { #[cfg(feature = "all-tests")] #[tokio::test] async fn test_auth_middleware() -> Result<()> { - let client = async_nats::connect(&config().jetstream.url).await?; + let settings = Arc::new(Settings::default()); + let client = async_nats::connect(&settings.jetstream.url).await?; let context = jetstream::new(client); - let stream_name = &config().jetstream.stream; + let stream_name = &settings.jetstream.stream; let stream = context .get_or_create_stream(stream::Config { @@ -430,17 +470,23 @@ mod tests { assert!(stream.is_ok()); let mem_store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec())?; + let pipeline_spec = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec)?; let callback_state = CallbackState::new(msg_graph, mem_store).await?; + let app_state = AppState { + settings, + callback_state, + context, + }; + let app = Router::new() - .nest( - "/v1/process", - routes(context, callback_state).await.unwrap(), - ) - .layer(middleware::from_fn(auth_middleware)); + .nest("/v1/process", routes(app_state).await.unwrap()) + .layer(middleware::from_fn_with_state( + Some("test_token".to_owned()), + auth_middleware, + )); - env::set_var(ENV_NUMAFLOW_SERVING_AUTH_TOKEN, "test_token"); let res = app .oneshot( axum::extract::Request::builder() @@ -451,7 +497,6 @@ mod tests { .await?; assert_eq!(res.status(), StatusCode::UNAUTHORIZED); - env::remove_var(ENV_NUMAFLOW_SERVING_AUTH_TOKEN); Ok(()) } } diff --git a/rust/serving/src/app/callback.rs b/rust/serving/src/app/callback.rs index 2fe7a2f6fe..b4d43868ee 100644 --- a/rust/serving/src/app/callback.rs +++ b/rust/serving/src/app/callback.rs @@ -1,14 +1,14 @@ use axum::{body::Bytes, extract::State, http::HeaderMap, routing, Json, Router}; use serde::{Deserialize, Serialize}; -use state::State as CallbackState; use tracing::error; use self::store::Store; use crate::app::response::ApiError; -use crate::config; /// in-memory state store including connection tracking pub(crate) mod state; +use state::State as CallbackState; + /// store for storing the state pub(crate) mod store; @@ -21,38 +21,58 @@ pub(crate) struct CallbackRequest { pub(crate) tags: Option>, } +#[derive(Clone)] +struct CallbackAppState { + tid_header: String, + callback_state: CallbackState, +} + pub fn callback_handler( - callback_store: CallbackState, + tid_header: String, + callback_state: CallbackState, ) -> Router { + let app_state = CallbackAppState { + tid_header, + callback_state, + }; Router::new() .route("/callback", routing::post(callback)) .route("/callback_save", routing::post(callback_save)) - .with_state(callback_store) + .with_state(app_state) } async fn callback_save( - State(mut proxy_state): State>, + State(app_state): State>, headers: HeaderMap, body: Bytes, ) -> Result<(), ApiError> { let id = headers - .get(&config().tid_header) + .get(&app_state.tid_header) .map(|id| String::from_utf8_lossy(id.as_bytes()).to_string()) .ok_or_else(|| ApiError::BadRequest("Missing id header".to_string()))?; - proxy_state.save_response(id, body).await.map_err(|e| { - error!(error=?e, "Saving body from callback save request"); - ApiError::InternalServerError("Failed to save body from callback save request".to_string()) - })?; + app_state + .callback_state + .clone() + .save_response(id, body) + .await + .map_err(|e| { + error!(error=?e, "Saving body from callback save request"); + ApiError::InternalServerError( + "Failed to save body from callback save request".to_string(), + ) + })?; Ok(()) } async fn callback( - State(mut proxy_state): State>, + State(app_state): State>, Json(payload): Json>, ) -> Result<(), ApiError> { - proxy_state + app_state + .callback_state + .clone() .insert_callback_requests(payload) .await .map_err(|e| { @@ -72,16 +92,20 @@ mod tests { use tower::ServiceExt; use super::*; + use crate::app::callback::state::State as CallbackState; use crate::app::callback::store::memstore::InMemoryStore; use crate::app::tracker::MessageGraph; - use crate::pipeline::min_pipeline_spec; + use crate::pipeline::PipelineDCG; + + const PIPELINE_SPEC_ENCODED: &str = "eyJ2ZXJ0aWNlcyI6W3sibmFtZSI6ImluIiwic291cmNlIjp7InNlcnZpbmciOnsiYXV0aCI6bnVsbCwic2VydmljZSI6dHJ1ZSwibXNnSURIZWFkZXJLZXkiOiJYLU51bWFmbG93LUlkIiwic3RvcmUiOnsidXJsIjoicmVkaXM6Ly9yZWRpczo2Mzc5In19fSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIiLCJlbnYiOlt7Im5hbWUiOiJSVVNUX0xPRyIsInZhbHVlIjoiZGVidWcifV19LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InBsYW5uZXIiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJwbGFubmVyIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InRpZ2VyIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsidGlnZXIiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZG9nIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZG9nIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6ImVsZXBoYW50IiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZWxlcGhhbnQiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiYXNjaWlhcnQiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJhc2NpaWFydCJdLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJidWlsdGluIjpudWxsLCJncm91cEJ5IjpudWxsfSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwic2NhbGUiOnsibWluIjoxfSwidXBkYXRlU3RyYXRlZ3kiOnsidHlwZSI6IlJvbGxpbmdVcGRhdGUiLCJyb2xsaW5nVXBkYXRlIjp7Im1heFVuYXZhaWxhYmxlIjoiMjUlIn19fSx7Im5hbWUiOiJzZXJ2ZS1zaW5rIiwic2luayI6eyJ1ZHNpbmsiOnsiY29udGFpbmVyIjp7ImltYWdlIjoic2VydmVzaW5rOjAuMSIsImVudiI6W3sibmFtZSI6Ik5VTUFGTE9XX0NBTExCQUNLX1VSTF9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctQ2FsbGJhY2stVXJsIn0seyJuYW1lIjoiTlVNQUZMT1dfTVNHX0lEX0hFQURFUl9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctSWQifV0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn19LCJyZXRyeVN0cmF0ZWd5Ijp7fX0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZXJyb3Itc2luayIsInNpbmsiOnsidWRzaW5rIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6InNlcnZlc2luazowLjEiLCJlbnYiOlt7Im5hbWUiOiJOVU1BRkxPV19DQUxMQkFDS19VUkxfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUNhbGxiYWNrLVVybCJ9LHsibmFtZSI6Ik5VTUFGTE9XX01TR19JRF9IRUFERVJfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUlkIn1dLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9fSwicmV0cnlTdHJhdGVneSI6e319LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19XSwiZWRnZXMiOlt7ImZyb20iOiJpbiIsInRvIjoicGxhbm5lciIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImFzY2lpYXJ0IiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiYXNjaWlhcnQiXX19fSx7ImZyb20iOiJwbGFubmVyIiwidG8iOiJ0aWdlciIsImNvbmRpdGlvbnMiOnsidGFncyI6eyJvcGVyYXRvciI6Im9yIiwidmFsdWVzIjpbInRpZ2VyIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZG9nIiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiZG9nIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZWxlcGhhbnQiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlbGVwaGFudCJdfX19LHsiZnJvbSI6InRpZ2VyIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZG9nIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZWxlcGhhbnQiLCJ0byI6InNlcnZlLXNpbmsiLCJjb25kaXRpb25zIjpudWxsfSx7ImZyb20iOiJhc2NpaWFydCIsInRvIjoic2VydmUtc2luayIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImVycm9yLXNpbmsiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlcnJvciJdfX19XSwibGlmZWN5Y2xlIjp7fSwid2F0ZXJtYXJrIjp7fX0="; #[tokio::test] async fn test_callback_failure() { let store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let state = CallbackState::new(msg_graph, store).await.unwrap(); - let app = callback_handler(state); + let app = callback_handler("ID".to_owned(), state); let payload = vec![CallbackRequest { id: "test_id".to_string(), @@ -106,7 +130,8 @@ mod tests { #[tokio::test] async fn test_callback_success() { let store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let mut state = CallbackState::new(msg_graph, store).await.unwrap(); let x = state.register("test_id".to_string()); @@ -115,7 +140,7 @@ mod tests { let _ = x.await.unwrap(); }); - let app = callback_handler(state); + let app = callback_handler("ID".to_owned(), state); let payload = vec![ CallbackRequest { @@ -157,9 +182,10 @@ mod tests { #[tokio::test] async fn test_callback_save() { let store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let state = CallbackState::new(msg_graph, store).await.unwrap(); - let app = callback_handler(state); + let app = callback_handler("ID".to_owned(), state); let res = Request::builder() .method("POST") @@ -176,9 +202,10 @@ mod tests { #[tokio::test] async fn test_without_id_header() { let store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let state = CallbackState::new(msg_graph, store).await.unwrap(); - let app = callback_handler(state); + let app = callback_handler("ID".to_owned(), state); let res = Request::builder() .method("POST") diff --git a/rust/serving/src/app/callback/state.rs b/rust/serving/src/app/callback/state.rs index db145f5beb..293478ead2 100644 --- a/rust/serving/src/app/callback/state.rs +++ b/rust/serving/src/app/callback/state.rs @@ -236,11 +236,14 @@ mod tests { use super::*; use crate::app::callback::store::memstore::InMemoryStore; - use crate::pipeline::min_pipeline_spec; + use crate::pipeline::PipelineDCG; + + const PIPELINE_SPEC_ENCODED: &str = "eyJ2ZXJ0aWNlcyI6W3sibmFtZSI6ImluIiwic291cmNlIjp7InNlcnZpbmciOnsiYXV0aCI6bnVsbCwic2VydmljZSI6dHJ1ZSwibXNnSURIZWFkZXJLZXkiOiJYLU51bWFmbG93LUlkIiwic3RvcmUiOnsidXJsIjoicmVkaXM6Ly9yZWRpczo2Mzc5In19fSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIiLCJlbnYiOlt7Im5hbWUiOiJSVVNUX0xPRyIsInZhbHVlIjoiZGVidWcifV19LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InBsYW5uZXIiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJwbGFubmVyIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InRpZ2VyIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsidGlnZXIiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZG9nIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZG9nIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6ImVsZXBoYW50IiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZWxlcGhhbnQiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiYXNjaWlhcnQiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJhc2NpaWFydCJdLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJidWlsdGluIjpudWxsLCJncm91cEJ5IjpudWxsfSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwic2NhbGUiOnsibWluIjoxfSwidXBkYXRlU3RyYXRlZ3kiOnsidHlwZSI6IlJvbGxpbmdVcGRhdGUiLCJyb2xsaW5nVXBkYXRlIjp7Im1heFVuYXZhaWxhYmxlIjoiMjUlIn19fSx7Im5hbWUiOiJzZXJ2ZS1zaW5rIiwic2luayI6eyJ1ZHNpbmsiOnsiY29udGFpbmVyIjp7ImltYWdlIjoic2VydmVzaW5rOjAuMSIsImVudiI6W3sibmFtZSI6Ik5VTUFGTE9XX0NBTExCQUNLX1VSTF9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctQ2FsbGJhY2stVXJsIn0seyJuYW1lIjoiTlVNQUZMT1dfTVNHX0lEX0hFQURFUl9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctSWQifV0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn19LCJyZXRyeVN0cmF0ZWd5Ijp7fX0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZXJyb3Itc2luayIsInNpbmsiOnsidWRzaW5rIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6InNlcnZlc2luazowLjEiLCJlbnYiOlt7Im5hbWUiOiJOVU1BRkxPV19DQUxMQkFDS19VUkxfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUNhbGxiYWNrLVVybCJ9LHsibmFtZSI6Ik5VTUFGTE9XX01TR19JRF9IRUFERVJfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUlkIn1dLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9fSwicmV0cnlTdHJhdGVneSI6e319LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19XSwiZWRnZXMiOlt7ImZyb20iOiJpbiIsInRvIjoicGxhbm5lciIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImFzY2lpYXJ0IiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiYXNjaWlhcnQiXX19fSx7ImZyb20iOiJwbGFubmVyIiwidG8iOiJ0aWdlciIsImNvbmRpdGlvbnMiOnsidGFncyI6eyJvcGVyYXRvciI6Im9yIiwidmFsdWVzIjpbInRpZ2VyIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZG9nIiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiZG9nIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZWxlcGhhbnQiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlbGVwaGFudCJdfX19LHsiZnJvbSI6InRpZ2VyIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZG9nIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZWxlcGhhbnQiLCJ0byI6InNlcnZlLXNpbmsiLCJjb25kaXRpb25zIjpudWxsfSx7ImZyb20iOiJhc2NpaWFydCIsInRvIjoic2VydmUtc2luayIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImVycm9yLXNpbmsiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlcnJvciJdfX19XSwibGlmZWN5Y2xlIjp7fSwid2F0ZXJtYXJrIjp7fX0="; #[tokio::test] async fn test_state() { - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let store = InMemoryStore::new(); let mut state = State::new(msg_graph, store).await.unwrap(); @@ -277,16 +280,37 @@ mod tests { }, CallbackRequest { id: id.clone(), - vertex: "cat".to_string(), + vertex: "planner".to_string(), cb_time: 12345, from_vertex: "in".to_string(), + tags: Some(vec!["tiger".to_owned(), "asciiart".to_owned()]), + }, + CallbackRequest { + id: id.clone(), + vertex: "tiger".to_string(), + cb_time: 12345, + from_vertex: "planner".to_string(), + tags: None, + }, + CallbackRequest { + id: id.clone(), + vertex: "asciiart".to_string(), + cb_time: 12345, + from_vertex: "planner".to_string(), + tags: None, + }, + CallbackRequest { + id: id.clone(), + vertex: "serve-sink".to_string(), + cb_time: 12345, + from_vertex: "tiger".to_string(), tags: None, }, CallbackRequest { id: id.clone(), - vertex: "out".to_string(), + vertex: "serve-sink".to_string(), cb_time: 12345, - from_vertex: "cat".to_string(), + from_vertex: "asciiart".to_string(), tags: None, }, ]; @@ -300,7 +324,8 @@ mod tests { #[tokio::test] async fn test_retrieve_saved_no_entry() { - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let store = InMemoryStore::new(); let mut state = State::new(msg_graph, store).await.unwrap(); @@ -315,7 +340,8 @@ mod tests { #[tokio::test] async fn test_insert_callback_requests_invalid_id() { - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let store = InMemoryStore::new(); let mut state = State::new(msg_graph, store).await.unwrap(); @@ -336,7 +362,8 @@ mod tests { #[tokio::test] async fn test_retrieve_subgraph_from_storage_no_entry() { - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let store = InMemoryStore::new(); let mut state = State::new(msg_graph, store).await.unwrap(); diff --git a/rust/serving/src/app/callback/store/redisstore.rs b/rust/serving/src/app/callback/store/redisstore.rs index 6e5decf880..4439e7ce8e 100644 --- a/rust/serving/src/app/callback/store/redisstore.rs +++ b/rust/serving/src/app/callback/store/redisstore.rs @@ -8,9 +8,10 @@ use tokio::sync::Semaphore; use super::PayloadToSave; use crate::app::callback::CallbackRequest; +use crate::config::RedisConfig; use crate::consts::SAVED; +use crate::Error; use crate::Error::Connection; -use crate::{config, Error}; const LPUSH: &str = "LPUSH"; const LRANGE: &str = "LRANGE"; @@ -19,14 +20,14 @@ const EXPIRE: &str = "EXPIRE"; // Handle to the Redis actor. #[derive(Clone)] pub(crate) struct RedisConnection { - max_tasks: usize, conn_manager: ConnectionManager, + config: RedisConfig, } impl RedisConnection { /// Creates a new RedisConnection with concurrent operations on Redis set by max_tasks. - pub(crate) async fn new(addr: &str, max_tasks: usize) -> crate::Result { - let client = redis::Client::open(addr) + pub(crate) async fn new(config: RedisConfig) -> crate::Result { + let client = redis::Client::open(config.addr.as_str()) .map_err(|e| Connection(format!("Creating Redis client: {e:?}")))?; let conn = client .get_connection_manager() @@ -34,37 +35,13 @@ impl RedisConnection { .map_err(|e| Connection(format!("Connecting to Redis server: {e:?}")))?; Ok(Self { conn_manager: conn, - max_tasks, + config, }) } - async fn handle_write_requests( - conn_manager: &mut ConnectionManager, - msg: PayloadToSave, - ) -> crate::Result<()> { - match msg { - PayloadToSave::Callback { key, value } => { - // Convert the CallbackRequest to a byte array - let value = serde_json::to_vec(&*value) - .map_err(|e| Error::StoreWrite(format!("Serializing payload - {}", e)))?; - - Self::write_to_redis(conn_manager, &key, &value).await - } - - // Write the byte array to Redis - PayloadToSave::DatumFromPipeline { key, value } => { - // we have to differentiate between the saved responses and the callback requests - // saved responses are stored in "id_SAVED", callback requests are stored in "id" - let key = format!("{}_{}", key, SAVED); - let value: Vec = value.into(); - - Self::write_to_redis(conn_manager, &key, &value).await - } - } - } - async fn execute_redis_cmd( conn_manager: &mut ConnectionManager, + ttl_secs: Option, key: &str, val: &Vec, ) -> Result<(), RedisError> { @@ -72,7 +49,7 @@ impl RedisConnection { pipe.cmd(LPUSH).arg(key).arg(val); // if the ttl is configured, add the EXPIRE command to the pipeline - if let Some(ttl) = config().redis.ttl_secs { + if let Some(ttl) = ttl_secs { pipe.cmd(EXPIRE).arg(key).arg(ttl); } @@ -81,19 +58,21 @@ impl RedisConnection { } // write to Redis with retries - async fn write_to_redis( - conn_manager: &mut ConnectionManager, - key: &str, - value: &Vec, - ) -> crate::Result<()> { - let interval = fixed::Interval::from_millis(config().redis.retries_duration_millis.into()) - .take(config().redis.retries); + async fn write_to_redis(&self, key: &str, value: &Vec) -> crate::Result<()> { + let interval = fixed::Interval::from_millis(self.config.retries_duration_millis.into()) + .take(self.config.retries); Retry::retry( interval, || async { // https://hackmd.io/@compiler-errors/async-closures - Self::execute_redis_cmd(&mut conn_manager.clone(), key, value).await + Self::execute_redis_cmd( + &mut self.conn_manager.clone(), + self.config.ttl_secs, + key, + value, + ) + .await }, |e: &RedisError| !e.is_unrecoverable_error(), ) @@ -102,6 +81,31 @@ impl RedisConnection { } } +async fn handle_write_requests( + redis_conn: RedisConnection, + msg: PayloadToSave, +) -> crate::Result<()> { + match msg { + PayloadToSave::Callback { key, value } => { + // Convert the CallbackRequest to a byte array + let value = serde_json::to_vec(&*value) + .map_err(|e| Error::StoreWrite(format!("Serializing payload - {}", e)))?; + + redis_conn.write_to_redis(&key, &value).await + } + + // Write the byte array to Redis + PayloadToSave::DatumFromPipeline { key, value } => { + // we have to differentiate between the saved responses and the callback requests + // saved responses are stored in "id_SAVED", callback requests are stored in "id" + let key = format!("{}_{}", key, SAVED); + let value: Vec = value.into(); + + redis_conn.write_to_redis(&key, &value).await + } + } +} + // It is possible to move the methods defined here to be methods on the Redis actor and communicate through channels. // With that, all public APIs defined on RedisConnection can be on &self (immutable). impl super::Store for RedisConnection { @@ -110,13 +114,13 @@ impl super::Store for RedisConnection { let mut tasks = vec![]; // This is put in place not to overload Redis and also way some kind of // flow control. - let sem = Arc::new(Semaphore::new(self.max_tasks)); + let sem = Arc::new(Semaphore::new(self.config.max_tasks)); for msg in messages { let permit = Arc::clone(&sem).acquire_owned().await; - let mut _conn_mgr = self.conn_manager.clone(); + let redis_conn = self.clone(); let task = tokio::spawn(async move { let _permit = permit; - Self::handle_write_requests(&mut _conn_mgr, msg).await + handle_write_requests(redis_conn, msg).await }); tasks.push(task); } @@ -205,12 +209,16 @@ mod tests { #[tokio::test] async fn test_redis_store() { - let redis_connection = RedisConnection::new("no_such_redis://127.0.0.1:6379", 10).await; + let redis_config = RedisConfig { + addr: "no_such_redis://127.0.0.1:6379".to_owned(), + max_tasks: 10, + ..Default::default() + }; + let redis_connection = RedisConnection::new(redis_config).await; assert!(redis_connection.is_err()); // Test Redis connection - let redis_connection = - RedisConnection::new(format!("redis://127.0.0.1:{}", "6379").as_str(), 10).await; + let redis_connection = RedisConnection::new(RedisConfig::default()).await; assert!(redis_connection.is_ok()); let key = uuid::Uuid::new_v4().to_string(); @@ -273,7 +281,11 @@ mod tests { #[tokio::test] async fn test_redis_ttl() { - let redis_connection = RedisConnection::new("redis://127.0.0.1:6379", 10) + let redis_config = RedisConfig { + max_tasks: 10, + ..Default::default() + }; + let redis_connection = RedisConnection::new(redis_config) .await .expect("Failed to connect to Redis"); @@ -287,14 +299,12 @@ mod tests { }); // Save with TTL of 1 second + redis_connection + .write_to_redis(&key, &serde_json::to_vec(&*value).unwrap()) + .await + .expect("Failed to write to Redis"); + let mut conn_manager = redis_connection.conn_manager.clone(); - RedisConnection::write_to_redis( - &mut conn_manager, - &key, - &serde_json::to_vec(&*value).unwrap(), - ) - .await - .expect("Failed to write to Redis"); let exists: bool = conn_manager .exists(&key) diff --git a/rust/serving/src/app/direct_proxy.rs b/rust/serving/src/app/direct_proxy.rs index 1f80ff5e7f..9f08321e23 100644 --- a/rust/serving/src/app/direct_proxy.rs +++ b/rust/serving/src/app/direct_proxy.rs @@ -11,7 +11,6 @@ use hyper_util::client::legacy::connect::HttpConnector; use tracing::error; use crate::app::response::ApiError; -use crate::config; pub(crate) type Client = hyper_util::client::legacy::Client; @@ -24,11 +23,15 @@ pub(crate) type Client = hyper_util::client::legacy::Client #[derive(Clone, Debug)] struct ProxyState { client: Client, + upstream_addr: String, } /// Router for direct proxy. -pub(crate) fn direct_proxy(client: Client) -> Router { - let proxy_state = ProxyState { client }; +pub(crate) fn direct_proxy(client: Client, upstream_addr: String) -> Router { + let proxy_state = ProxyState { + client, + upstream_addr, + }; Router::new() // https://docs.rs/axum/latest/axum/struct.Router.html#wildcards @@ -44,7 +47,7 @@ async fn proxy( // This handler is registered with wildcard capture /*upstream. So the path here will never be empty. let path_query = request.uri().path_and_query().unwrap(); - let upstream_uri = format!("http://{}{}", &config().upstream_addr, path_query); + let upstream_uri = format!("http://{}{}", &proxy_state.upstream_addr, path_query); *request.uri_mut() = Uri::try_from(&upstream_uri) .inspect_err(|e| error!(?e, upstream_uri, "Parsing URI for upstream")) .map_err(|e| ApiError::BadRequest(e.to_string()))?; @@ -69,10 +72,8 @@ mod tests { use tower::ServiceExt; use crate::app::direct_proxy::direct_proxy; - use crate::config; - async fn start_server() { - let addr = config().upstream_addr.to_string(); + async fn start_server(addr: String) { let listener = TcpListener::bind(&addr).await.unwrap(); tokio::spawn(async move { loop { @@ -98,11 +99,12 @@ mod tests { #[tokio::test] async fn test_direct_proxy() { - start_server().await; + let upstream_addr = "localhost:4321".to_owned(); + start_server(upstream_addr.clone()).await; let client = hyper_util::client::legacy::Client::<(), ()>::builder(TokioExecutor::new()) .build(HttpConnector::new()); - let app = direct_proxy(client); + let app = direct_proxy(client, upstream_addr.clone()); // Test valid request let res = Request::builder() diff --git a/rust/serving/src/app/jetstream_proxy.rs b/rust/serving/src/app/jetstream_proxy.rs index 8123197857..af7d3917ff 100644 --- a/rust/serving/src/app/jetstream_proxy.rs +++ b/rust/serving/src/app/jetstream_proxy.rs @@ -1,4 +1,4 @@ -use std::borrow::Borrow; +use std::{borrow::Borrow, sync::Arc}; use async_nats::{jetstream::Context, HeaderMap as JSHeaderMap}; use axum::{ @@ -12,10 +12,9 @@ use axum::{ use tracing::error; use uuid::Uuid; -use super::callback::{state::State as CallbackState, store::Store}; +use super::{callback::store::Store, AppState}; use crate::app::callback::state; use crate::app::response::{ApiError, ServeResponse}; -use crate::config; // TODO: // - [ ] better health check @@ -33,34 +32,31 @@ use crate::config; // "from_vertex": "a" // } -const ID_HEADER_KEY: &str = "X-Numaflow-Id"; const CALLBACK_URL_KEY: &str = "X-Numaflow-Callback-Url"; - const NUMAFLOW_RESP_ARRAY_LEN: &str = "Numaflow-Array-Len"; const NUMAFLOW_RESP_ARRAY_IDX_LEN: &str = "Numaflow-Array-Index-Len"; -#[derive(Clone)] struct ProxyState { + tid_header: String, context: Context, callback: state::State, - stream: &'static str, + stream: String, callback_url: String, } pub(crate) async fn jetstream_proxy( - context: Context, - callback_store: CallbackState, + state: AppState, ) -> crate::Result { - let proxy_state = ProxyState { - context, - callback: callback_store, - stream: &config().jetstream.stream, + let proxy_state = Arc::new(ProxyState { + tid_header: state.settings.tid_header.clone(), + context: state.context.clone(), + callback: state.callback_state.clone(), + stream: state.settings.jetstream.stream.clone(), callback_url: format!( "https://{}:{}/v1/process/callback", - config().host_ip, - config().app_listen_port + state.settings.host_ip, state.settings.app_listen_port ), - }; + }); let router = Router::new() .route("/async", post(async_publish)) @@ -71,27 +67,28 @@ pub(crate) async fn jetstream_proxy( } async fn sync_publish_serve( - State(mut proxy_state): State>, + State(proxy_state): State>>, headers: HeaderMap, body: Bytes, ) -> impl IntoResponse { - let id = extract_id_from_headers(&headers); + let id = extract_id_from_headers(&proxy_state.tid_header, &headers); // Register the ID in the callback proxy state - let notify = proxy_state.callback.register(id.clone()); + let notify = proxy_state.callback.clone().register(id.clone()); if let Err(e) = publish_to_jetstream( - proxy_state.stream, + proxy_state.stream.clone(), &proxy_state.callback_url, headers, body, - proxy_state.context, - id.clone(), + proxy_state.context.clone(), + proxy_state.tid_header.as_str(), + id.as_str(), ) .await { // Deregister the ID in the callback proxy state if writing to Jetstream fails - let _ = proxy_state.callback.deregister(&id).await; + let _ = proxy_state.callback.clone().deregister(&id).await; error!(error = ?e, "Publishing message to Jetstream for sync serve request"); return Err(ApiError::BadGateway( "Failed to write message to Jetstream".to_string(), @@ -106,7 +103,7 @@ async fn sync_publish_serve( )); } - let result = match proxy_state.callback.retrieve_saved(&id).await { + let result = match proxy_state.callback.clone().retrieve_saved(&id).await { Ok(result) => result, Err(e) => { error!(error = ?e, "Failed to retrieve from redis"); @@ -140,27 +137,28 @@ async fn sync_publish_serve( } async fn sync_publish( - State(mut proxy_state): State>, + State(proxy_state): State>>, headers: HeaderMap, body: Bytes, ) -> Result, ApiError> { - let id = extract_id_from_headers(&headers); + let id = extract_id_from_headers(&proxy_state.tid_header, &headers); // Register the ID in the callback proxy state - let notify = proxy_state.callback.register(id.clone()); + let notify = proxy_state.callback.clone().register(id.clone()); if let Err(e) = publish_to_jetstream( - proxy_state.stream, + proxy_state.stream.clone(), &proxy_state.callback_url, headers, body, - proxy_state.context, - id.clone(), + proxy_state.context.clone(), + &proxy_state.tid_header, + id.as_str(), ) .await { // Deregister the ID in the callback proxy state if writing to Jetstream fails - let _ = proxy_state.callback.deregister(&id).await; + let _ = proxy_state.callback.clone().deregister(&id).await; error!(error = ?e, "Publishing message to Jetstream for sync request"); return Err(ApiError::BadGateway( "Failed to write message to Jetstream".to_string(), @@ -189,19 +187,19 @@ async fn sync_publish( } async fn async_publish( - State(proxy_state): State>, + State(proxy_state): State>>, headers: HeaderMap, body: Bytes, ) -> Result, ApiError> { - let id = extract_id_from_headers(&headers); - + let id = extract_id_from_headers(&proxy_state.tid_header, &headers); let result = publish_to_jetstream( - proxy_state.stream, + proxy_state.stream.clone(), &proxy_state.callback_url, headers, body, - proxy_state.context, - id.clone(), + proxy_state.context.clone(), + &proxy_state.tid_header, + id.as_str(), ) .await; @@ -222,12 +220,13 @@ async fn async_publish( /// Write to JetStream and return the metadata. It is responsible for getting the ID from the header. async fn publish_to_jetstream( - stream: &'static str, + stream: String, callback_url: &str, headers: HeaderMap, body: Bytes, js_context: Context, - id: String, // Added ID as a parameter + id_header: &str, + id_header_value: &str, ) -> Result<(), async_nats::Error> { let mut js_headers = JSHeaderMap::new(); @@ -236,24 +235,22 @@ async fn publish_to_jetstream( js_headers.append(k.as_ref(), String::from_utf8_lossy(v.as_bytes()).borrow()) } - js_headers.append(ID_HEADER_KEY, id.as_str()); // Use the passed ID + js_headers.append(id_header, id_header_value); // Use the passed ID js_headers.append(CALLBACK_URL_KEY, callback_url); js_context .publish_with_headers(stream, js_headers, body) .await - .inspect_err(|e| error!(stream, error=?e, "Publishing message to stream"))? + .map_err(|e| format!("Publishing message to stream: {e:?}"))? .await - .inspect_err( - |e| error!(stream, error=?e, "Waiting for acknowledgement of published message"), - )?; + .map_err(|e| format!("Waiting for acknowledgement of published message: {e:?}"))?; Ok(()) } // extracts the ID from the headers, if not found, generates a new UUID -fn extract_id_from_headers(headers: &HeaderMap) -> String { - headers.get(&config().tid_header).map_or_else( +fn extract_id_from_headers(tid_header: &str, headers: &HeaderMap) -> String { + headers.get(tid_header).map_or_else( || Uuid::new_v4().to_string(), |v| String::from_utf8_lossy(v.as_bytes()).to_string(), ) @@ -273,12 +270,15 @@ mod tests { use tower::ServiceExt; use super::*; + use crate::app::callback::state::State as CallbackState; use crate::app::callback::store::memstore::InMemoryStore; use crate::app::callback::store::PayloadToSave; use crate::app::callback::CallbackRequest; use crate::app::tracker::MessageGraph; - use crate::pipeline::min_pipeline_spec; - use crate::Error; + use crate::pipeline::PipelineDCG; + use crate::{Error, Settings}; + + const PIPELINE_SPEC_ENCODED: &str = "eyJ2ZXJ0aWNlcyI6W3sibmFtZSI6ImluIiwic291cmNlIjp7InNlcnZpbmciOnsiYXV0aCI6bnVsbCwic2VydmljZSI6dHJ1ZSwibXNnSURIZWFkZXJLZXkiOiJYLU51bWFmbG93LUlkIiwic3RvcmUiOnsidXJsIjoicmVkaXM6Ly9yZWRpczo2Mzc5In19fSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIiLCJlbnYiOlt7Im5hbWUiOiJSVVNUX0xPRyIsInZhbHVlIjoiZGVidWcifV19LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InBsYW5uZXIiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJwbGFubmVyIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InRpZ2VyIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsidGlnZXIiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZG9nIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZG9nIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6ImVsZXBoYW50IiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZWxlcGhhbnQiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiYXNjaWlhcnQiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJhc2NpaWFydCJdLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJidWlsdGluIjpudWxsLCJncm91cEJ5IjpudWxsfSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwic2NhbGUiOnsibWluIjoxfSwidXBkYXRlU3RyYXRlZ3kiOnsidHlwZSI6IlJvbGxpbmdVcGRhdGUiLCJyb2xsaW5nVXBkYXRlIjp7Im1heFVuYXZhaWxhYmxlIjoiMjUlIn19fSx7Im5hbWUiOiJzZXJ2ZS1zaW5rIiwic2luayI6eyJ1ZHNpbmsiOnsiY29udGFpbmVyIjp7ImltYWdlIjoic2VydmVzaW5rOjAuMSIsImVudiI6W3sibmFtZSI6Ik5VTUFGTE9XX0NBTExCQUNLX1VSTF9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctQ2FsbGJhY2stVXJsIn0seyJuYW1lIjoiTlVNQUZMT1dfTVNHX0lEX0hFQURFUl9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctSWQifV0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn19LCJyZXRyeVN0cmF0ZWd5Ijp7fX0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZXJyb3Itc2luayIsInNpbmsiOnsidWRzaW5rIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6InNlcnZlc2luazowLjEiLCJlbnYiOlt7Im5hbWUiOiJOVU1BRkxPV19DQUxMQkFDS19VUkxfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUNhbGxiYWNrLVVybCJ9LHsibmFtZSI6Ik5VTUFGTE9XX01TR19JRF9IRUFERVJfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUlkIn1dLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9fSwicmV0cnlTdHJhdGVneSI6e319LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19XSwiZWRnZXMiOlt7ImZyb20iOiJpbiIsInRvIjoicGxhbm5lciIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImFzY2lpYXJ0IiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiYXNjaWlhcnQiXX19fSx7ImZyb20iOiJwbGFubmVyIiwidG8iOiJ0aWdlciIsImNvbmRpdGlvbnMiOnsidGFncyI6eyJvcGVyYXRvciI6Im9yIiwidmFsdWVzIjpbInRpZ2VyIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZG9nIiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiZG9nIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZWxlcGhhbnQiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlbGVwaGFudCJdfX19LHsiZnJvbSI6InRpZ2VyIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZG9nIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZWxlcGhhbnQiLCJ0byI6InNlcnZlLXNpbmsiLCJjb25kaXRpb25zIjpudWxsfSx7ImZyb20iOiJhc2NpaWFydCIsInRvIjoic2VydmUtc2luayIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImVycm9yLXNpbmsiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlcnJvciJdfX19XSwibGlmZWN5Y2xlIjp7fSwid2F0ZXJtYXJrIjp7fX0="; #[derive(Clone)] struct MockStore; @@ -303,7 +303,9 @@ mod tests { #[tokio::test] async fn test_async_publish() -> Result<(), Box> { - let client = async_nats::connect(&config().jetstream.url) + let settings = Settings::default(); + let settings = Arc::new(settings); + let client = async_nats::connect(&settings.jetstream.url) .await .map_err(|e| format!("Connecting to Jetstream: {:?}", e))?; @@ -318,14 +320,20 @@ mod tests { ..Default::default() }) .await - .map_err(|e| format!("creating stream {}: {}", &config().jetstream.url, e))?; + .map_err(|e| format!("creating stream {}: {}", &settings.jetstream.url, e))?; let mock_store = MockStore {}; - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()) + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec) .map_err(|e| format!("Failed to create message graph from pipeline spec: {:?}", e))?; let callback_state = CallbackState::new(msg_graph, mock_store).await?; - let app = jetstream_proxy(context, callback_state).await?; + let app_state = AppState { + callback_state, + context, + settings, + }; + let app = jetstream_proxy(app_state).await?; let res = Request::builder() .method("POST") .uri("/async") @@ -384,7 +392,8 @@ mod tests { #[tokio::test] async fn test_sync_publish() { - let client = async_nats::connect(&config().jetstream.url).await.unwrap(); + let settings = Settings::default(); + let client = async_nats::connect(&settings.jetstream.url).await.unwrap(); let context = jetstream::new(client); let id = "foobar"; let stream_name = "sync_pub"; @@ -396,16 +405,21 @@ mod tests { ..Default::default() }) .await - .map_err(|e| format!("creating stream {}: {}", &config().jetstream.url, e)); + .map_err(|e| format!("creating stream {}: {}", &settings.jetstream.url, e)); let mem_store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let mut callback_state = CallbackState::new(msg_graph, mem_store).await.unwrap(); - let app = jetstream_proxy(context, callback_state.clone()) - .await - .unwrap(); + let settings = Arc::new(settings); + let app_state = AppState { + settings, + callback_state: callback_state.clone(), + context, + }; + let app = jetstream_proxy(app_state).await.unwrap(); tokio::spawn(async move { let cbs = create_default_callbacks(id); @@ -448,7 +462,8 @@ mod tests { #[tokio::test] async fn test_sync_publish_serve() { - let client = async_nats::connect(&config().jetstream.url).await.unwrap(); + let settings = Arc::new(Settings::default()); + let client = async_nats::connect(&settings.jetstream.url).await.unwrap(); let context = jetstream::new(client); let id = "foobar"; let stream_name = "sync_serve_pub"; @@ -460,16 +475,21 @@ mod tests { ..Default::default() }) .await - .map_err(|e| format!("creating stream {}: {}", &config().jetstream.url, e)); + .map_err(|e| format!("creating stream {}: {}", &settings.jetstream.url, e)); let mem_store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let mut callback_state = CallbackState::new(msg_graph, mem_store).await.unwrap(); - let app = jetstream_proxy(context, callback_state.clone()) - .await - .unwrap(); + let app_state = AppState { + settings, + callback_state: callback_state.clone(), + context, + }; + + let app = jetstream_proxy(app_state).await.unwrap(); // pipeline is in -> cat -> out, so we will have 3 callback requests let cbs = create_default_callbacks(id); diff --git a/rust/serving/src/app/message_path.rs b/rust/serving/src/app/message_path.rs index 933c58a815..20e5701864 100644 --- a/rust/serving/src/app/message_path.rs +++ b/rust/serving/src/app/message_path.rs @@ -46,12 +46,15 @@ mod tests { use super::*; use crate::app::callback::store::memstore::InMemoryStore; use crate::app::tracker::MessageGraph; - use crate::pipeline::min_pipeline_spec; + use crate::pipeline::PipelineDCG; + + const PIPELINE_SPEC_ENCODED: &str = "eyJ2ZXJ0aWNlcyI6W3sibmFtZSI6ImluIiwic291cmNlIjp7InNlcnZpbmciOnsiYXV0aCI6bnVsbCwic2VydmljZSI6dHJ1ZSwibXNnSURIZWFkZXJLZXkiOiJYLU51bWFmbG93LUlkIiwic3RvcmUiOnsidXJsIjoicmVkaXM6Ly9yZWRpczo2Mzc5In19fSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIiLCJlbnYiOlt7Im5hbWUiOiJSVVNUX0xPRyIsInZhbHVlIjoiZGVidWcifV19LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InBsYW5uZXIiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJwbGFubmVyIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InRpZ2VyIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsidGlnZXIiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZG9nIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZG9nIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6ImVsZXBoYW50IiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZWxlcGhhbnQiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiYXNjaWlhcnQiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJhc2NpaWFydCJdLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJidWlsdGluIjpudWxsLCJncm91cEJ5IjpudWxsfSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwic2NhbGUiOnsibWluIjoxfSwidXBkYXRlU3RyYXRlZ3kiOnsidHlwZSI6IlJvbGxpbmdVcGRhdGUiLCJyb2xsaW5nVXBkYXRlIjp7Im1heFVuYXZhaWxhYmxlIjoiMjUlIn19fSx7Im5hbWUiOiJzZXJ2ZS1zaW5rIiwic2luayI6eyJ1ZHNpbmsiOnsiY29udGFpbmVyIjp7ImltYWdlIjoic2VydmVzaW5rOjAuMSIsImVudiI6W3sibmFtZSI6Ik5VTUFGTE9XX0NBTExCQUNLX1VSTF9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctQ2FsbGJhY2stVXJsIn0seyJuYW1lIjoiTlVNQUZMT1dfTVNHX0lEX0hFQURFUl9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctSWQifV0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn19LCJyZXRyeVN0cmF0ZWd5Ijp7fX0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZXJyb3Itc2luayIsInNpbmsiOnsidWRzaW5rIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6InNlcnZlc2luazowLjEiLCJlbnYiOlt7Im5hbWUiOiJOVU1BRkxPV19DQUxMQkFDS19VUkxfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUNhbGxiYWNrLVVybCJ9LHsibmFtZSI6Ik5VTUFGTE9XX01TR19JRF9IRUFERVJfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUlkIn1dLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9fSwicmV0cnlTdHJhdGVneSI6e319LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19XSwiZWRnZXMiOlt7ImZyb20iOiJpbiIsInRvIjoicGxhbm5lciIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImFzY2lpYXJ0IiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiYXNjaWlhcnQiXX19fSx7ImZyb20iOiJwbGFubmVyIiwidG8iOiJ0aWdlciIsImNvbmRpdGlvbnMiOnsidGFncyI6eyJvcGVyYXRvciI6Im9yIiwidmFsdWVzIjpbInRpZ2VyIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZG9nIiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiZG9nIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZWxlcGhhbnQiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlbGVwaGFudCJdfX19LHsiZnJvbSI6InRpZ2VyIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZG9nIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZWxlcGhhbnQiLCJ0byI6InNlcnZlLXNpbmsiLCJjb25kaXRpb25zIjpudWxsfSx7ImZyb20iOiJhc2NpaWFydCIsInRvIjoic2VydmUtc2luayIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImVycm9yLXNpbmsiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlcnJvciJdfX19XSwibGlmZWN5Y2xlIjp7fSwid2F0ZXJtYXJrIjp7fX0="; #[tokio::test] async fn test_message_path_not_present() { let store = InMemoryStore::new(); - let msg_graph = MessageGraph::from_pipeline(min_pipeline_spec()).unwrap(); + let pipeline_spec: PipelineDCG = PIPELINE_SPEC_ENCODED.parse().unwrap(); + let msg_graph = MessageGraph::from_pipeline(&pipeline_spec).unwrap(); let state = CallbackState::new(msg_graph, store).await.unwrap(); let app = get_message_path(state); diff --git a/rust/serving/src/config.rs b/rust/serving/src/config.rs index 82e663c8f5..7ba3778d00 100644 --- a/rust/serving/src/config.rs +++ b/rust/serving/src/config.rs @@ -1,40 +1,23 @@ +use std::collections::HashMap; use std::fmt::Debug; -use std::path::Path; -use std::{env, sync::OnceLock}; use async_nats::rustls; use base64::prelude::BASE64_STANDARD; use base64::Engine; -use config::Config; use rcgen::{generate_simple_self_signed, Certificate, CertifiedKey, KeyPair}; use serde::{Deserialize, Serialize}; -use tracing::info; use crate::Error::ParseConfig; -use crate::{Error, Result}; -const ENV_PREFIX: &str = "NUMAFLOW_SERVING"; const ENV_NUMAFLOW_SERVING_SOURCE_OBJECT: &str = "NUMAFLOW_SERVING_SOURCE_OBJECT"; const ENV_NUMAFLOW_SERVING_JETSTREAM_URL: &str = "NUMAFLOW_ISBSVC_JETSTREAM_URL"; const ENV_NUMAFLOW_SERVING_JETSTREAM_STREAM: &str = "NUMAFLOW_SERVING_JETSTREAM_STREAM"; const ENV_NUMAFLOW_SERVING_STORE_TTL: &str = "NUMAFLOW_SERVING_STORE_TTL"; - -pub fn config() -> &'static Settings { - static CONF: OnceLock = OnceLock::new(); - CONF.get_or_init(|| { - let config_dir = env::var("CONFIG_PATH").unwrap_or_else(|_| { - info!("Config directory is not specified, using default config directory: './config'"); - String::from("config") - }); - - match Settings::load(config_dir) { - Ok(v) => v, - Err(e) => { - panic!("Failed to load configuration: {:?}", e); - } - } - }) -} +const ENV_NUMAFLOW_SERVING_HOST_IP: &str = "NUMAFLOW_SERVING_HOST_IP"; +const ENV_NUMAFLOW_SERVING_APP_PORT: &str = "NUMAFLOW_SERVING_APP_LISTEN_PORT"; +const ENV_NUMAFLOW_SERVING_JETSTREAM_USER: &str = "NUMAFLOW_ISBSVC_JETSTREAM_USER"; +const ENV_NUMAFLOW_SERVING_JETSTREAM_PASSWORD: &str = "NUMAFLOW_ISBSVC_JETSTREAM_PASSWORD"; +const ENV_NUMAFLOW_SERVING_AUTH_TOKEN: &str = "NUMAFLOW_SERVING_AUTH_TOKEN"; pub fn generate_certs() -> std::result::Result<(Certificate, KeyPair), String> { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -43,13 +26,47 @@ pub fn generate_certs() -> std::result::Result<(Certificate, KeyPair), String> { Ok((cert, key_pair)) } -#[derive(Debug, Deserialize)] +#[derive(Deserialize, Clone, PartialEq)] +pub struct BasicAuth { + pub username: String, + pub password: String, +} + +impl Debug for BasicAuth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let passwd_printable = if self.password.len() > 4 { + let passwd: String = self + .password + .chars() + .skip(self.password.len() - 2) + .take(2) + .collect(); + format!("***{}", passwd) + } else { + "*****".to_owned() + }; + write!(f, "{}:{}", self.username, passwd_printable) + } +} + +#[derive(Debug, Deserialize, Clone, PartialEq)] pub struct JetStreamConfig { pub stream: String, pub url: String, + pub auth: Option, +} + +impl Default for JetStreamConfig { + fn default() -> Self { + Self { + stream: "default".to_owned(), + url: "localhost:4222".to_owned(), + auth: None, + } + } } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone, PartialEq)] pub struct RedisConfig { pub addr: String, pub max_tasks: usize, @@ -58,7 +75,20 @@ pub struct RedisConfig { pub ttl_secs: Option, } -#[derive(Debug, Deserialize)] +impl Default for RedisConfig { + fn default() -> Self { + Self { + addr: "redis://127.0.0.1:6379".to_owned(), + max_tasks: 50, + retries: 5, + retries_duration_millis: 100, + // TODO: we might need an option type here. Zero value of u32 can be used instead of None + ttl_secs: Some(1), + } + } +} + +#[derive(Debug, Deserialize, Clone, PartialEq)] pub struct Settings { pub tid_header: String, pub app_listen_port: u16, @@ -69,6 +99,23 @@ pub struct Settings { pub redis: RedisConfig, /// The IP address of the numaserve pod. This will be used to construct the value for X-Numaflow-Callback-Url header pub host_ip: String, + pub api_auth_token: Option, +} + +impl Default for Settings { + fn default() -> Self { + Self { + tid_header: "ID".to_owned(), + app_listen_port: 3000, + metrics_server_listen_port: 3001, + upstream_addr: "localhost:8888".to_owned(), + drain_timeout_secs: 10, + jetstream: JetStreamConfig::default(), + redis: RedisConfig::default(), + host_ip: "127.0.0.1".to_owned(), + api_auth_token: None, + } + } } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -84,92 +131,104 @@ pub struct CallbackStorageConfig { pub url: String, } -impl Settings { - fn load>(config_dir: P) -> Result { - let config_dir = config_dir.as_ref(); - if !config_dir.is_dir() { - return Err(Error::Other(format!( - "Path {} is not a directory", - config_dir.to_string_lossy() - ))); +/// This implementation is to load settings from env variables +impl TryFrom> for Settings { + type Error = crate::Error; + fn try_from(env_vars: HashMap) -> std::result::Result { + let host_ip = env_vars + .get(ENV_NUMAFLOW_SERVING_HOST_IP) + .ok_or_else(|| { + ParseConfig(format!( + "Environment variable {ENV_NUMAFLOW_SERVING_HOST_IP} is not set" + )) + })? + .to_owned(); + + let mut settings = Settings { + host_ip, + ..Default::default() + }; + + if let Some(jetstream_url) = env_vars.get(ENV_NUMAFLOW_SERVING_JETSTREAM_URL) { + settings.jetstream.url = jetstream_url.to_owned(); } - let settings = Config::builder() - .add_source(config::File::from(config_dir.join("default.toml"))) - .add_source( - config::Environment::with_prefix(ENV_PREFIX) - .prefix_separator("_") - .separator("."), - ) - .build() - .map_err(|e| ParseConfig(format!("generating runtime configuration: {e:?}")))?; - - let mut settings = settings - .try_deserialize::() - .map_err(|e| ParseConfig(format!("parsing runtime configuration: {e:?}")))?; - - // Update JetStreamConfig from environment variables - if let Ok(url) = env::var(ENV_NUMAFLOW_SERVING_JETSTREAM_URL) { - settings.jetstream.url = url; + if let Some(jetstream_stream) = env_vars.get(ENV_NUMAFLOW_SERVING_JETSTREAM_STREAM) { + settings.jetstream.stream = jetstream_stream.to_owned(); } - if let Ok(stream) = env::var(ENV_NUMAFLOW_SERVING_JETSTREAM_STREAM) { - settings.jetstream.stream = stream; + + if let Some(api_auth_token) = env_vars.get(ENV_NUMAFLOW_SERVING_AUTH_TOKEN) { + settings.api_auth_token = Some(api_auth_token.to_owned()); } - let source_spec_encoded = env::var(ENV_NUMAFLOW_SERVING_SOURCE_OBJECT); - - match source_spec_encoded { - Ok(source_spec_encoded) => { - let source_spec_decoded = BASE64_STANDARD - .decode(source_spec_encoded.as_bytes()) - .map_err(|e| ParseConfig(format!("decoding NUMAFLOW_SERVING_SOURCE: {e:?}")))?; - - let source_spec = serde_json::from_slice::(&source_spec_decoded) - .map_err(|e| ParseConfig(format!("parsing NUMAFLOW_SERVING_SOURCE: {e:?}")))?; - - // Update tid_header from source_spec - if let Some(msg_id_header_key) = source_spec.msg_id_header_key { - settings.tid_header = msg_id_header_key; - } - - // Update redis.addr from source_spec, currently we only support redis as callback storage - settings.redis.addr = source_spec.callback_storage.url; - - // Update redis.ttl_secs from environment variable - settings.redis.ttl_secs = match env::var(ENV_NUMAFLOW_SERVING_STORE_TTL) { - Ok(ttl_secs) => Some(ttl_secs.parse().map_err(|e| { - ParseConfig(format!( - "parsing NUMAFLOW_SERVING_STORE_TTL: expected u32, got {:?}", - e - )) - })?), - Err(_) => None, - }; - - Ok(settings) - } - Err(_) => Ok(settings), + if let Some(app_port) = env_vars.get(ENV_NUMAFLOW_SERVING_APP_PORT) { + settings.app_listen_port = app_port.parse().map_err(|e| { + ParseConfig(format!( + "Parsing {ENV_NUMAFLOW_SERVING_APP_PORT}(set to '{app_port}'): {e:?}" + )) + })?; } + + // If username is set, the password also must be set + if let Some(username) = env_vars.get(ENV_NUMAFLOW_SERVING_JETSTREAM_USER) { + let Some(password) = env_vars.get(ENV_NUMAFLOW_SERVING_JETSTREAM_PASSWORD) else { + return Err(ParseConfig(format!("Env variable {ENV_NUMAFLOW_SERVING_JETSTREAM_USER} is set, but {ENV_NUMAFLOW_SERVING_JETSTREAM_PASSWORD} is not set"))); + }; + settings.jetstream.auth = Some(BasicAuth { + username: username.to_owned(), + password: password.to_owned(), + }); + } + + // Update redis.ttl_secs from environment variable + if let Some(ttl_secs) = env_vars.get(ENV_NUMAFLOW_SERVING_STORE_TTL) { + let ttl_secs: u32 = ttl_secs.parse().map_err(|e| { + ParseConfig(format!("parsing {ENV_NUMAFLOW_SERVING_STORE_TTL}: {e:?}")) + })?; + settings.redis.ttl_secs = Some(ttl_secs); + } + + let Some(source_spec_encoded) = env_vars.get(ENV_NUMAFLOW_SERVING_SOURCE_OBJECT) else { + return Ok(settings); + }; + + let source_spec_decoded = BASE64_STANDARD + .decode(source_spec_encoded.as_bytes()) + .map_err(|e| ParseConfig(format!("decoding NUMAFLOW_SERVING_SOURCE: {e:?}")))?; + + let source_spec = serde_json::from_slice::(&source_spec_decoded) + .map_err(|e| ParseConfig(format!("parsing NUMAFLOW_SERVING_SOURCE: {e:?}")))?; + + // Update tid_header from source_spec + if let Some(msg_id_header_key) = source_spec.msg_id_header_key { + settings.tid_header = msg_id_header_key; + } + + // Update redis.addr from source_spec, currently we only support redis as callback storage + settings.redis.addr = source_spec.callback_storage.url; + + Ok(settings) } } #[cfg(test)] mod tests { - use std::env; - use super::*; #[test] - fn test_config() { - // Set up the environment variable for the config directory - env::set_var("RUN_ENV", "Development"); - env::set_var("APP_HOST_IP", "10.244.0.6"); - env::set_var("CONFIG_PATH", "config"); + fn test_basic_auth_debug_print() { + let auth = BasicAuth { + username: "js-auth-user".into(), + password: "js-auth-password".into(), + }; + let auth_debug = format!("{auth:?}"); + assert_eq!(auth_debug, "js-auth-user:***rd"); + } - // Call the config method - let settings = config(); + #[test] + fn test_default_config() { + let settings = Settings::default(); - // Assert that the settings are as expected assert_eq!(settings.tid_header, "ID"); assert_eq!(settings.app_listen_port, 3000); assert_eq!(settings.metrics_server_listen_port, 3001); @@ -177,9 +236,66 @@ mod tests { assert_eq!(settings.drain_timeout_secs, 10); assert_eq!(settings.jetstream.stream, "default"); assert_eq!(settings.jetstream.url, "localhost:4222"); - assert_eq!(settings.redis.addr, "redis://127.0.0.1/"); + assert_eq!(settings.redis.addr, "redis://127.0.0.1:6379"); assert_eq!(settings.redis.max_tasks, 50); assert_eq!(settings.redis.retries, 5); assert_eq!(settings.redis.retries_duration_millis, 100); } + + #[test] + fn test_config_parse() { + // Set up the environment variables + let env_vars = [ + ( + ENV_NUMAFLOW_SERVING_JETSTREAM_URL, + "nats://isbsvc-default-js-svc.default.svc:4222", + ), + ( + ENV_NUMAFLOW_SERVING_JETSTREAM_STREAM, + "ascii-art-pipeline-in-serving-source", + ), + (ENV_NUMAFLOW_SERVING_JETSTREAM_USER, "js-auth-user"), + (ENV_NUMAFLOW_SERVING_JETSTREAM_PASSWORD, "js-user-password"), + (ENV_NUMAFLOW_SERVING_HOST_IP, "10.2.3.5"), + (ENV_NUMAFLOW_SERVING_AUTH_TOKEN, "api-auth-token"), + (ENV_NUMAFLOW_SERVING_APP_PORT, "8443"), + (ENV_NUMAFLOW_SERVING_STORE_TTL, "86400"), + (ENV_NUMAFLOW_SERVING_SOURCE_OBJECT, "eyJhdXRoIjpudWxsLCJzZXJ2aWNlIjp0cnVlLCJtc2dJREhlYWRlcktleSI6IlgtTnVtYWZsb3ctSWQiLCJzdG9yZSI6eyJ1cmwiOiJyZWRpczovL3JlZGlzOjYzNzkifX0=") + ]; + + // Call the config method + let settings: Settings = env_vars + .into_iter() + .map(|(key, val)| (key.to_owned(), val.to_owned())) + .collect::>() + .try_into() + .unwrap(); + + let expected_config = Settings { + tid_header: "X-Numaflow-Id".into(), + app_listen_port: 8443, + metrics_server_listen_port: 3001, + upstream_addr: "localhost:8888".into(), + drain_timeout_secs: 10, + jetstream: JetStreamConfig { + stream: "ascii-art-pipeline-in-serving-source".into(), + url: "nats://isbsvc-default-js-svc.default.svc:4222".into(), + auth: Some(BasicAuth { + username: "js-auth-user".into(), + password: "js-user-password".into(), + }), + }, + redis: RedisConfig { + addr: "redis://redis:6379".into(), + max_tasks: 50, + retries: 5, + retries_duration_millis: 100, + ttl_secs: Some(86400), + }, + host_ip: "10.2.3.5".into(), + api_auth_token: Some("api-auth-token".into()), + }; + + assert_eq!(settings, expected_config); + } } diff --git a/rust/serving/src/lib.rs b/rust/serving/src/lib.rs index 09e2dfcaa5..796313bdb2 100644 --- a/rust/serving/src/lib.rs +++ b/rust/serving/src/lib.rs @@ -1,42 +1,61 @@ +use std::env; use std::net::SocketAddr; +use std::sync::Arc; use axum_server::tls_rustls::RustlsConfig; use tracing::info; pub use self::error::{Error, Result}; +use self::pipeline::PipelineDCG; use crate::app::start_main_server; -use crate::config::{config, generate_certs}; +use crate::config::generate_certs; use crate::metrics::start_https_metrics_server; -use crate::pipeline::min_pipeline_spec; mod app; + mod config; +pub use config::Settings; + mod consts; mod error; mod metrics; mod pipeline; -pub async fn serve() -> std::result::Result<(), Box> -{ +const ENV_MIN_PIPELINE_SPEC: &str = "NUMAFLOW_SERVING_MIN_PIPELINE_SPEC"; + +pub async fn serve( + settings: Arc, +) -> std::result::Result<(), Box> { let (cert, key) = generate_certs()?; let tls_config = RustlsConfig::from_pem(cert.pem().into(), key.serialize_pem().into()) .await .map_err(|e| format!("Failed to create tls config {:?}", e))?; - info!(config = ?config(), pipeline_spec = ? min_pipeline_spec(), "Starting server with config and pipeline spec"); + // TODO: Move all env variables into one place. Some env variables are loaded when Settings is initialized + let pipeline_spec: PipelineDCG = env::var(ENV_MIN_PIPELINE_SPEC) + .map_err(|_| { + format!("Pipeline spec is not set using environment variable {ENV_MIN_PIPELINE_SPEC}") + })? + .parse() + .map_err(|e| { + format!( + "Parsing pipeline spec: {}: error={e:?}", + env::var(ENV_MIN_PIPELINE_SPEC).unwrap() + ) + })?; + + info!(config = ?settings, ?pipeline_spec, "Starting server with config and pipeline spec"); // Start the metrics server, which serves the prometheus metrics. let metrics_addr: SocketAddr = - format!("0.0.0.0:{}", &config().metrics_server_listen_port).parse()?; + format!("0.0.0.0:{}", &settings.metrics_server_listen_port).parse()?; let metrics_server_handle = tokio::spawn(start_https_metrics_server(metrics_addr, tls_config.clone())); - let app_addr: SocketAddr = format!("0.0.0.0:{}", &config().app_listen_port).parse()?; - // Start the main server, which serves the application. - let app_server_handle = tokio::spawn(start_main_server(app_addr, tls_config)); + let app_server_handle = tokio::spawn(start_main_server(settings, tls_config, pipeline_spec)); // TODO: is try_join the best? we need to short-circuit at the first failure tokio::try_join!(flatten(app_server_handle), flatten(metrics_server_handle))?; diff --git a/rust/serving/src/metrics.rs b/rust/serving/src/metrics.rs index 830a37c0c5..4c64760d4d 100644 --- a/rust/serving/src/metrics.rs +++ b/rust/serving/src/metrics.rs @@ -97,6 +97,7 @@ pub(crate) async fn start_https_metrics_server( ) -> crate::Result<()> { let metrics_app = Router::new().route("/metrics", get(metrics_handler)); + tracing::info!(?addr, "Starting metrics server"); axum_server::bind_rustls(addr, tls_config) .serve(metrics_app.into_make_service()) .await diff --git a/rust/serving/src/pipeline.rs b/rust/serving/src/pipeline.rs index 042e5923b4..d782e3d73a 100644 --- a/rust/serving/src/pipeline.rs +++ b/rust/serving/src/pipeline.rs @@ -1,5 +1,4 @@ -use std::env; -use std::sync::OnceLock; +use std::str::FromStr; use base64::prelude::BASE64_STANDARD; use base64::Engine; @@ -8,16 +7,6 @@ use serde::{Deserialize, Serialize}; use crate::Error::ParseConfig; -const ENV_MIN_PIPELINE_SPEC: &str = "NUMAFLOW_SERVING_MIN_PIPELINE_SPEC"; - -pub fn min_pipeline_spec() -> &'static PipelineDCG { - static PIPELINE: OnceLock = OnceLock::new(); - PIPELINE.get_or_init(|| match PipelineDCG::load() { - Ok(pipeline) => pipeline, - Err(e) => panic!("Failed to load minimum pipeline spec: {:?}", e), - }) -} - // OperatorType is an enum that contains the types of operators // that can be used in the conditions for the edge. #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] @@ -53,6 +42,7 @@ impl From for OperatorType { } // Tag is a struct that contains the information about the tags for the edge +#[cfg_attr(test, derive(PartialEq))] #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Tag { pub operator: Option, @@ -60,6 +50,7 @@ pub struct Tag { } // Conditions is a struct that contains the information about the conditions for the edge +#[cfg_attr(test, derive(PartialEq))] #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Conditions { pub tags: Option, @@ -87,27 +78,17 @@ pub struct Vertex { pub name: String, } -impl PipelineDCG { - pub fn load() -> crate::Result { - let full_pipeline_spec = match env::var(ENV_MIN_PIPELINE_SPEC) { - Ok(env_value) => { - // If the environment variable is set, decode and parse the pipeline - let decoded = BASE64_STANDARD - .decode(env_value.as_bytes()) - .map_err(|e| ParseConfig(format!("decoding pipeline from env: {e:?}")))?; - - serde_json::from_slice::(&decoded) - .map_err(|e| ParseConfig(format!("parsing pipeline from env: {e:?}")))? - } - Err(_) => { - // If the environment variable is not set, read the pipeline from a file - let file_path = "./config/pipeline_spec.json"; - let file_contents = std::fs::read_to_string(file_path) - .map_err(|e| ParseConfig(format!("reading pipeline file: {e:?}")))?; - serde_json::from_str::(&file_contents) - .map_err(|e| ParseConfig(format!("parsing pipeline file: {e:?}")))? - } - }; +impl FromStr for PipelineDCG { + type Err = crate::Error; + + fn from_str(pipeline_spec_encoded: &str) -> Result { + let full_pipeline_spec_decoded = BASE64_STANDARD + .decode(pipeline_spec_encoded) + .map_err(|e| ParseConfig(format!("Decoding pipeline from env: {e:?}")))?; + + let full_pipeline_spec = + serde_json::from_slice::(&full_pipeline_spec_decoded) + .map_err(|e| ParseConfig(format!("parsing pipeline from env: {e:?}")))?; let vertices: Vec = full_pipeline_spec .vertices @@ -148,17 +129,29 @@ mod tests { #[test] fn test_pipeline_load() { - let pipeline = min_pipeline_spec(); - assert_eq!(pipeline.vertices.len(), 3); - assert_eq!(pipeline.edges.len(), 2); + let pipeline: PipelineDCG = "eyJ2ZXJ0aWNlcyI6W3sibmFtZSI6ImluIiwic291cmNlIjp7InNlcnZpbmciOnsiYXV0aCI6bnVsbCwic2VydmljZSI6dHJ1ZSwibXNnSURIZWFkZXJLZXkiOiJYLU51bWFmbG93LUlkIiwic3RvcmUiOnsidXJsIjoicmVkaXM6Ly9yZWRpczo2Mzc5In19fSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIiLCJlbnYiOlt7Im5hbWUiOiJSVVNUX0xPRyIsInZhbHVlIjoiZGVidWcifV19LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InBsYW5uZXIiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJwbGFubmVyIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6InRpZ2VyIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsidGlnZXIiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZG9nIiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZG9nIl0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sImJ1aWx0aW4iOm51bGwsImdyb3VwQnkiOm51bGx9LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19LHsibmFtZSI6ImVsZXBoYW50IiwidWRmIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6ImFzY2lpOjAuMSIsImFyZ3MiOlsiZWxlcGhhbnQiXSwicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwiYnVpbHRpbiI6bnVsbCwiZ3JvdXBCeSI6bnVsbH0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiYXNjaWlhcnQiLCJ1ZGYiOnsiY29udGFpbmVyIjp7ImltYWdlIjoiYXNjaWk6MC4xIiwiYXJncyI6WyJhc2NpaWFydCJdLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJidWlsdGluIjpudWxsLCJncm91cEJ5IjpudWxsfSwiY29udGFpbmVyVGVtcGxhdGUiOnsicmVzb3VyY2VzIjp7fSwiaW1hZ2VQdWxsUG9saWN5IjoiTmV2ZXIifSwic2NhbGUiOnsibWluIjoxfSwidXBkYXRlU3RyYXRlZ3kiOnsidHlwZSI6IlJvbGxpbmdVcGRhdGUiLCJyb2xsaW5nVXBkYXRlIjp7Im1heFVuYXZhaWxhYmxlIjoiMjUlIn19fSx7Im5hbWUiOiJzZXJ2ZS1zaW5rIiwic2luayI6eyJ1ZHNpbmsiOnsiY29udGFpbmVyIjp7ImltYWdlIjoic2VydmVzaW5rOjAuMSIsImVudiI6W3sibmFtZSI6Ik5VTUFGTE9XX0NBTExCQUNLX1VSTF9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctQ2FsbGJhY2stVXJsIn0seyJuYW1lIjoiTlVNQUZMT1dfTVNHX0lEX0hFQURFUl9LRVkiLCJ2YWx1ZSI6IlgtTnVtYWZsb3ctSWQifV0sInJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn19LCJyZXRyeVN0cmF0ZWd5Ijp7fX0sImNvbnRhaW5lclRlbXBsYXRlIjp7InJlc291cmNlcyI6e30sImltYWdlUHVsbFBvbGljeSI6Ik5ldmVyIn0sInNjYWxlIjp7Im1pbiI6MX0sInVwZGF0ZVN0cmF0ZWd5Ijp7InR5cGUiOiJSb2xsaW5nVXBkYXRlIiwicm9sbGluZ1VwZGF0ZSI6eyJtYXhVbmF2YWlsYWJsZSI6IjI1JSJ9fX0seyJuYW1lIjoiZXJyb3Itc2luayIsInNpbmsiOnsidWRzaW5rIjp7ImNvbnRhaW5lciI6eyJpbWFnZSI6InNlcnZlc2luazowLjEiLCJlbnYiOlt7Im5hbWUiOiJOVU1BRkxPV19DQUxMQkFDS19VUkxfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUNhbGxiYWNrLVVybCJ9LHsibmFtZSI6Ik5VTUFGTE9XX01TR19JRF9IRUFERVJfS0VZIiwidmFsdWUiOiJYLU51bWFmbG93LUlkIn1dLCJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9fSwicmV0cnlTdHJhdGVneSI6e319LCJjb250YWluZXJUZW1wbGF0ZSI6eyJyZXNvdXJjZXMiOnt9LCJpbWFnZVB1bGxQb2xpY3kiOiJOZXZlciJ9LCJzY2FsZSI6eyJtaW4iOjF9LCJ1cGRhdGVTdHJhdGVneSI6eyJ0eXBlIjoiUm9sbGluZ1VwZGF0ZSIsInJvbGxpbmdVcGRhdGUiOnsibWF4VW5hdmFpbGFibGUiOiIyNSUifX19XSwiZWRnZXMiOlt7ImZyb20iOiJpbiIsInRvIjoicGxhbm5lciIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImFzY2lpYXJ0IiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiYXNjaWlhcnQiXX19fSx7ImZyb20iOiJwbGFubmVyIiwidG8iOiJ0aWdlciIsImNvbmRpdGlvbnMiOnsidGFncyI6eyJvcGVyYXRvciI6Im9yIiwidmFsdWVzIjpbInRpZ2VyIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZG9nIiwiY29uZGl0aW9ucyI6eyJ0YWdzIjp7Im9wZXJhdG9yIjoib3IiLCJ2YWx1ZXMiOlsiZG9nIl19fX0seyJmcm9tIjoicGxhbm5lciIsInRvIjoiZWxlcGhhbnQiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlbGVwaGFudCJdfX19LHsiZnJvbSI6InRpZ2VyIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZG9nIiwidG8iOiJzZXJ2ZS1zaW5rIiwiY29uZGl0aW9ucyI6bnVsbH0seyJmcm9tIjoiZWxlcGhhbnQiLCJ0byI6InNlcnZlLXNpbmsiLCJjb25kaXRpb25zIjpudWxsfSx7ImZyb20iOiJhc2NpaWFydCIsInRvIjoic2VydmUtc2luayIsImNvbmRpdGlvbnMiOm51bGx9LHsiZnJvbSI6InBsYW5uZXIiLCJ0byI6ImVycm9yLXNpbmsiLCJjb25kaXRpb25zIjp7InRhZ3MiOnsib3BlcmF0b3IiOiJvciIsInZhbHVlcyI6WyJlcnJvciJdfX19XSwibGlmZWN5Y2xlIjp7fSwid2F0ZXJtYXJrIjp7fX0=".parse().unwrap(); + + assert_eq!(pipeline.vertices.len(), 8); + assert_eq!(pipeline.edges.len(), 10); assert_eq!(pipeline.vertices[0].name, "in"); assert_eq!(pipeline.edges[0].from, "in"); - assert_eq!(pipeline.edges[0].to, "cat"); + assert_eq!(pipeline.edges[0].to, "planner"); assert!(pipeline.edges[0].conditions.is_none()); - assert_eq!(pipeline.vertices[1].name, "cat"); - assert_eq!(pipeline.vertices[2].name, "out"); - assert_eq!(pipeline.edges[1].from, "cat"); - assert_eq!(pipeline.edges[1].to, "out"); + assert_eq!(pipeline.vertices[1].name, "planner"); + assert_eq!(pipeline.edges[1].from, "planner"); + assert_eq!(pipeline.edges[1].to, "asciiart"); + assert_eq!( + pipeline.edges[1].conditions, + Some(Conditions { + tags: Some(Tag { + operator: Some(OperatorType::Or), + values: vec!["asciiart".to_owned()] + }) + }) + ); + + assert_eq!(pipeline.vertices[2].name, "tiger"); + assert_eq!(pipeline.vertices[3].name, "dog"); } } diff --git a/server/apis/v1/handler.go b/server/apis/v1/handler.go index 9e588daed8..17f0322c2a 100644 --- a/server/apis/v1/handler.go +++ b/server/apis/v1/handler.go @@ -1133,6 +1133,27 @@ func (h *handler) GetMonoVertex(c *gin.Context) { c.JSON(http.StatusOK, NewNumaflowAPIResponse(nil, monoVertexResp)) } +// DeleteMonoVertex is used to delete a mono vertex +func (h *handler) DeleteMonoVertex(c *gin.Context) { + ns, monoVertex := c.Param("namespace"), c.Param("mono-vertex") + + // Check if the mono vertex exists + _, err := h.numaflowClient.MonoVertices(ns).Get(c, monoVertex, metav1.GetOptions{}) + if err != nil { + h.respondWithError(c, fmt.Sprintf("Failed to fetch mono vertex %q in namespace %q, %s", monoVertex, ns, err.Error())) + return + } + + // Delete the mono vertex + err = h.numaflowClient.MonoVertices(ns).Delete(c, monoVertex, metav1.DeleteOptions{}) + if err != nil { + h.respondWithError(c, fmt.Sprintf("Failed to delete mono vertex %q in namespace %q, %s", monoVertex, ns, err.Error())) + return + } + + c.JSON(http.StatusOK, NewNumaflowAPIResponse(nil, nil)) +} + // CreateMonoVertex is used to create a mono vertex func (h *handler) CreateMonoVertex(c *gin.Context) { if h.opts.readonly { diff --git a/server/apis/v1/promql_service_test.go b/server/apis/v1/promql_service_test.go index 3d923bb850..ed50a4ec68 100644 --- a/server/apis/v1/promql_service_test.go +++ b/server/apis/v1/promql_service_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" @@ -38,13 +40,46 @@ func (m *MockPrometheusAPI) QueryRange(ctx context.Context, query string, r v1.R return mockResponse, nil, nil } +func compareFilters(query1, query2 string) bool { + //Extract the filter portions of the queries + filters1 := extractfilters(query1) + filters2 := extractfilters(query2) + return reflect.DeepEqual(filters1, filters2) +} + // comparePrometheusQueries compares two Prometheus queries, ignoring the order of filters within the curly braces func comparePrometheusQueries(query1, query2 string) bool { - // Extract the filter portions of the queries + //Extract the filter portions of the queries filters1 := extractfilters(query1) filters2 := extractfilters(query2) - // Compare the filter portions using reflect.DeepEqual, which ignores order - return reflect.DeepEqual(filters1, filters2) + //Compare the filter portions using reflect.DeepEqual, which ignores order + if !reflect.DeepEqual(filters1, filters2) { + return false // Filters don't match + } + + //Remove filter portions from the queries + query1 = removeFilters(query1) + query2 = removeFilters(query2) + + //Normalize the remaining parts of the queries + query1 = normalizeQuery(query1) + query2 = normalizeQuery(query2) + + //Compare the normalized queries + return cmp.Equal(query1, query2, cmpopts.IgnoreUnexported(struct{}{})) + +} + +func normalizeQuery(query string) string { + // Remove extra whitespace and normalize case + query = strings.TrimSpace(strings.ToLower(query)) + return query +} + +// remove filters within {} +func removeFilters(query string) string { + re := regexp.MustCompile(`\{(.*?)\}`) + return re.ReplaceAllString(query, "") } // extractfilters extracts the key-value pairs within the curly braces @@ -97,7 +132,7 @@ func Test_PopulateReqMap(t *testing.T) { assert.Equal(t, actualMap["$quantile"], expectedMap["$quantile"]) assert.Equal(t, actualMap["$duration"], expectedMap["$duration"]) assert.Equal(t, actualMap["$dimension"], expectedMap["$dimension"]) - if !comparePrometheusQueries(expectedMap["$filters"], actualMap["$filters"]) { + if !compareFilters(expectedMap["$filters"], actualMap["$filters"]) { t.Errorf("filters do not match") } }) @@ -121,7 +156,7 @@ func Test_PopulateReqMap(t *testing.T) { assert.Equal(t, actualMap["$duration"], expectedMap["$duration"]) assert.Equal(t, actualMap["$dimension"], expectedMap["$dimension"]) - if !comparePrometheusQueries(expectedMap["$filters"], actualMap["$filters"]) { + if !compareFilters(expectedMap["$filters"], actualMap["$filters"]) { t.Errorf("filters do not match") } }) @@ -160,7 +195,7 @@ func Test_PromQueryBuilder(t *testing.T) { "pod": "test-pod", }, }, - expectedQuery: `histogram_quantile(0.90, sum by(test_dimension,le) (rate(test_bucket{namespace= "test_namespace", mvtx_name= "test-mono-vertex", pod= "test-pod"}[5m])))`, + expectedQuery: `histogram_quantile(0.90, sum by(test_dimension,le) (rate(test_metric{namespace= "test_namespace", mvtx_name= "test-mono-vertex", pod= "test-pod"}[5m])))`, }, { name: "Missing placeholder in req", @@ -274,7 +309,7 @@ func Test_PromQueryBuilder(t *testing.T) { }) } - // tests for gauge metrics + // tests for mono-vertex gauge metrics var gauge_service = &PromQlService{ PlaceHolders: map[string]map[string][]string{ "monovtx_pending": { @@ -283,7 +318,7 @@ func Test_PromQueryBuilder(t *testing.T) { }, Expression: map[string]map[string]string{ "monovtx_pending": { - "mono-vertex": "$metric_name{$filters}", + "mono-vertex": "sum($metric_name{$filters}) by ($dimension, period)", }, }, } @@ -305,7 +340,7 @@ func Test_PromQueryBuilder(t *testing.T) { "period": "5m", }, }, - expectedQuery: `monovtx_pending{namespace= "test_namespace", mvtx_name= "test_mvtx", period= "5m"}`, + expectedQuery: `sum(monovtx_pending{namespace= "test_namespace", mvtx_name= "test_mvtx", period= "5m"}) by (mvtx_name, period)`, }, { name: "Missing metric name in service config", @@ -337,6 +372,72 @@ func Test_PromQueryBuilder(t *testing.T) { } }) } + + // tests for pipeline gauge metrics + var pl_gauge_service = &PromQlService{ + PlaceHolders: map[string]map[string][]string{ + "vertex_pending_messages": { + "vertex": {"$dimension", "$metric_name", "$filters"}, + }, + }, + Expression: map[string]map[string]string{ + "vertex_pending_messages": { + "vertex": "sum($metric_name{$filters}) by ($dimension, period)", + }, + }, + } + + pl_gauge_metrics_tests := []struct { + name string + requestBody MetricsRequestBody + expectedQuery string + expectError bool + }{ + { + name: "Successful pipeline gauge metrics template substitution", + requestBody: MetricsRequestBody{ + MetricName: "vertex_pending_messages", + Dimension: "vertex", + Filters: map[string]string{ + "namespace": "test_namespace", + "pipeline": "test_pipeline", + "vertex": "test_vertex", + "period": "5m", + }, + }, + expectedQuery: `sum(vertex_pending_messages{namespace= "test_namespace", pipeline= "test_pipeline", vertex= "test_vertex", period= "5m"}) by (vertex, period)`, + }, + { + name: "Missing metric name in service config", + requestBody: MetricsRequestBody{ + MetricName: "non_existent_metric", + Dimension: "mono-vertex", + Filters: map[string]string{ + "namespace": "test_namespace", + "pipeline": "test_pipeline", + "vertex": "test_vertex", + "period": "5m", + }, + }, + expectError: true, + }, + } + + for _, tt := range pl_gauge_metrics_tests { + t.Run(tt.name, func(t *testing.T) { + actualQuery, err := pl_gauge_service.BuildQuery(tt.requestBody) + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + if !comparePrometheusQueries(tt.expectedQuery, actualQuery) { + t.Errorf("Prometheus queries do not match.\nExpected: %s\nGot: %s", tt.expectedQuery, actualQuery) + } else { + t.Log("Prometheus queries match!") + } + } + }) + } } func Test_QueryPrometheus(t *testing.T) { @@ -386,14 +487,37 @@ func Test_QueryPrometheus(t *testing.T) { assert.Equal(t, 1, matrix.Len()) }) - t.Run("Successful gauge query", func(t *testing.T) { + t.Run("Successful mono-vertex gauge query", func(t *testing.T) { + mockAPI := &MockPrometheusAPI{} + promQlService := &PromQlService{ + PrometheusClient: &Prometheus{ + Api: mockAPI, + }, + } + query := `sum(monovtx_pending{namespace="default", mvtx_name="test-mvtx", period="5m"}) by (mvtx_name, period)` + startTime := time.Now().Add(-30 * time.Minute) + endTime := time.Now() + + ctx := context.Background() + result, err := promQlService.QueryPrometheus(ctx, query, startTime, endTime) + + assert.NoError(t, err) + assert.NotNil(t, result) + + // for query range , response should be a matrix + matrix, ok := result.(model.Matrix) + assert.True(t, ok) + assert.Equal(t, 1, matrix.Len()) + }) + + t.Run("Successful pipeline gauge query", func(t *testing.T) { mockAPI := &MockPrometheusAPI{} promQlService := &PromQlService{ PrometheusClient: &Prometheus{ Api: mockAPI, }, } - query := `monovtx_pending{namespace="default", mvtx_name="test-mvtx", pending="5m"}` + query := `sum(vertex_pending_messages{namespace="default", pipeline="test-pipeline", vertex="test-vertex", period="5m"}) by (vertex, period)` startTime := time.Now().Add(-30 * time.Minute) endTime := time.Now() diff --git a/server/routes/routes.go b/server/routes/routes.go index 1872e90181..500a32a587 100644 --- a/server/routes/routes.go +++ b/server/routes/routes.go @@ -171,6 +171,8 @@ func v1Routes(ctx context.Context, r gin.IRouter, dexObj *v1.DexObject, localUse r.GET("/namespaces/:namespace/mono-vertices", handler.ListMonoVertices) // Get the mono vertex information. r.GET("/namespaces/:namespace/mono-vertices/:mono-vertex", handler.GetMonoVertex) + // Delete a mono-vertex. + r.DELETE("/namespaces/:namespace/mono-vertices/:mono-vertex", handler.DeleteMonoVertex) // Get all the pods of a mono vertex. r.GET("/namespaces/:namespace/mono-vertices/:mono-vertex/pods", handler.ListMonoVertexPods) // Create a mono vertex. diff --git a/test/api-e2e/api_test.go b/test/api-e2e/api_test.go index b23735e699..e1800ea678 100644 --- a/test/api-e2e/api_test.go +++ b/test/api-e2e/api_test.go @@ -35,6 +35,10 @@ type APISuite struct { E2ESuite } +func TestAPISuite(t *testing.T) { + suite.Run(t, new(APISuite)) +} + func (s *APISuite) TestGetSysInfo() { defer s.Given().When().UXServerPodPortForward(8043, 8443).TerminateAllPodPortForwards() @@ -209,9 +213,17 @@ func (s *APISuite) TestAPIsForIsbAndPipelineAndMonoVertex() { Expect(). Status(200).Body().Raw() assert.Contains(s.T(), listMonoVertexBody, testMonoVertex1Name) + + // deletes a mono-vertex + deleteMonoVertex := HTTPExpect(s.T(), "https://localhost:8145").DELETE(fmt.Sprintf("/api/v1/namespaces/%s/mono-vertices/%s", Namespace, testMonoVertex1Name)). + Expect(). + Status(200).Body().Raw() + var deleteMonoVertexSuccessExpect = `"data":null` + assert.Contains(s.T(), deleteMonoVertex, deleteMonoVertexSuccessExpect) + } -func (s *APISuite) TestAPIsForMetricsAndWatermarkAndPods() { +func (s *APISuite) TestAPIsForMetricsAndWatermarkAndPodsForPipeline() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -275,8 +287,68 @@ func (s *APISuite) TestAPIsForMetricsAndWatermarkAndPods() { Expect(). Status(200).Body().Raw() assert.Contains(s.T(), getVerticesPodsBody, `simple-pipeline-input-0`) + + // Call the DiscoverMetrics API for the vertex object + discoverMetricsBodyForVertex := HTTPExpect(s.T(), "https://localhost:8146").GET("/api/v1/metrics-discovery/object/vertex"). + Expect(). + Status(200).Body().Raw() + + // Check that the response contains expected metrics for vertex object + assert.Contains(s.T(), discoverMetricsBodyForVertex, "forwarder_data_read_total") + + // Call the API to get input vertex pods info + getVertexPodsInfoBody := HTTPExpect(s.T(), "https://localhost:8146"). + GET(fmt.Sprintf("/api/v1/namespaces/%s/pipelines/%s/vertices/%s/pods-info", Namespace, pipelineName, "input")). + Expect(). + Status(200).Body().Raw() + + // Check that the response contains expected pod details + assert.Contains(s.T(), getVertexPodsInfoBody, `"name":`) // Check for pod name + assert.Contains(s.T(), getVertexPodsInfoBody, `"status":`) // Check for pod status + assert.Contains(s.T(), getVertexPodsInfoBody, `"totalCPU":`) // Check for pod's cpu usage + assert.Contains(s.T(), getVertexPodsInfoBody, `"totalMemory":`) // Check for pod's memory usage + assert.Contains(s.T(), getVertexPodsInfoBody, `"containerDetailsMap":`) // Check for pod's containers } -func TestAPISuite(t *testing.T) { - suite.Run(t, new(APISuite)) +func (s *APISuite) TestMetricsAPIsForMonoVertex() { + _, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + w := s.Given().MonoVertex("@testdata/mono-vertex.yaml"). + When(). + CreateMonoVertexAndWait() + defer w.DeleteMonoVertexAndWait() + + monoVertexName := "mono-vertex" + + defer w.UXServerPodPortForward(8149, 8443).TerminateAllPodPortForwards() + + w.Expect().MonoVertexPodsRunning() + // Expect the messages to reach the sink. + w.Expect().RedisSinkContains("mono-vertex", "199") + w.Expect().RedisSinkContains("mono-vertex", "200") + + // Call the API to get mono vertex pods info + getMonoVertexPodsInfoBody := HTTPExpect(s.T(), "https://localhost:8149"). + GET(fmt.Sprintf("/api/v1/namespaces/%s/mono-vertices/%s/pods-info", Namespace, monoVertexName)). + Expect(). + Status(200).Body().Raw() + + // Check that the response contains expected pod details + assert.Contains(s.T(), getMonoVertexPodsInfoBody, `"name":`) // Check for pod name + assert.Contains(s.T(), getMonoVertexPodsInfoBody, `"status":`) // Check for pod status + assert.Contains(s.T(), getMonoVertexPodsInfoBody, `"totalCPU":`) // Check for pod's cpu usage + assert.Contains(s.T(), getMonoVertexPodsInfoBody, `"totalMemory":`) // Check for pod's memory usage + assert.Contains(s.T(), getMonoVertexPodsInfoBody, `"containerDetailsMap":`) // Check for pod's containers + + // Call the DiscoverMetrics API for mono-vertex + discoverMetricsBodyForMonoVertex := HTTPExpect(s.T(), "https://localhost:8149").GET("/api/v1/metrics-discovery/object/mono-vertex"). + Expect(). + Status(200).Body().Raw() + + // Check that the response contains expected metrics for mono-vertex + assert.Contains(s.T(), discoverMetricsBodyForMonoVertex, "monovtx_processing_time_bucket") + assert.Contains(s.T(), discoverMetricsBodyForMonoVertex, "monovtx_sink_time_bucket") + assert.Contains(s.T(), discoverMetricsBodyForMonoVertex, "monovtx_read_total") + assert.Contains(s.T(), discoverMetricsBodyForMonoVertex, "monovtx_pending") } diff --git a/test/api-e2e/testdata.go b/test/api-e2e/testdata.go index 411a7b586a..bdee0da422 100644 --- a/test/api-e2e/testdata.go +++ b/test/api-e2e/testdata.go @@ -168,7 +168,7 @@ var ( "source": { "udsource": { "container": { - "image": "quay.io/numaio/numaflow-java/source-simple-source:stable" + "image": "quay.io/numaio/numaflow-rs/simple-source:stable" } }, "transformer": { @@ -180,7 +180,7 @@ var ( "sink": { "udsink": { "container": { - "image": "quay.io/numaio/numaflow-java/simple-sink:stable" + "image": "quay.io/numaio/numaflow-rs/sink-log:stable" } } } diff --git a/test/api-e2e/testdata/mono-vertex.yaml b/test/api-e2e/testdata/mono-vertex.yaml new file mode 100644 index 0000000000..c90ae8bd9e --- /dev/null +++ b/test/api-e2e/testdata/mono-vertex.yaml @@ -0,0 +1,26 @@ +apiVersion: numaflow.numaproj.io/v1alpha1 +kind: MonoVertex +metadata: + name: mono-vertex +spec: + scale: + min: 1 + source: + udsource: + container: + image: quay.io/numaio/numaflow-go/source-simple-source:stable + imagePullPolicy: Always + transformer: + container: + image: quay.io/numaio/numaflow-go/mapt-assign-event-time:stable + imagePullPolicy: Always + sink: + udsink: + container: + # A redis sink for e2e testing, see https://github.com/numaproj/numaflow-go/tree/main/pkg/sinker/examples/redis_sink + image: quay.io/numaio/numaflow-go/redis-sink:stable + imagePullPolicy: Always + env: + - name: SINK_HASH_KEY + # Use the name of the mono vertex as the key + value: "mono-vertex" \ No newline at end of file diff --git a/ui/src/components/pages/Pipeline/partials/Graph/partials/NodeInfo/partials/Pods/partials/PodDetails/partials/Metrics/index.tsx b/ui/src/components/pages/Pipeline/partials/Graph/partials/NodeInfo/partials/Pods/partials/PodDetails/partials/Metrics/index.tsx index a32fec5322..71f5ee862d 100644 --- a/ui/src/components/pages/Pipeline/partials/Graph/partials/NodeInfo/partials/Pods/partials/PodDetails/partials/Metrics/index.tsx +++ b/ui/src/components/pages/Pipeline/partials/Graph/partials/NodeInfo/partials/Pods/partials/PodDetails/partials/Metrics/index.tsx @@ -65,6 +65,11 @@ export function Metrics({ namespaceId, pipelineId, type, vertexId }: MetricsProp return ( {discoveredMetrics?.data?.map((metric: any) => { + if ( + type === "source" && + metric?.metric_name === "vertex_pending_messages" + ) + return null; const panelId = `${metric?.metric_name}-panel`; return ( { switch (metricName) { case "monovtx_pending": - return "period"; + case "vertex_pending_messages": + return dimension === "pod" ? ["pod", "period"] : ["period"]; } switch (dimension) { case "mono-vertex": - return "mvtx_name"; + return ["mvtx_name"]; default: - return dimension; + return [dimension]; } }, []); @@ -160,7 +161,18 @@ const LineChartComponent = ({ metricsReq?.metric_name ); chartData?.forEach((item) => { - const labelVal = item?.metric?.[label]; + let labelVal = ""; + label?.forEach((eachLabel: string) => { + if (item?.metric?.[eachLabel] !== undefined) { + labelVal += (labelVal ? "-" : "") + item.metric[eachLabel]; + } + }); + + // Remove initial hyphen if labelVal is not empty + if (labelVal.startsWith("-") && labelVal.length > 1) { + labelVal = labelVal.substring(1); + } + labels.push(labelVal); item?.values?.forEach(([timestamp, value]: [number, string]) => { const date = new Date(timestamp * 1000); diff --git a/ui/src/components/pages/Pipeline/partials/Graph/partials/NodeInfo/partials/Pods/partials/PodDetails/partials/Metrics/utils/constants.ts b/ui/src/components/pages/Pipeline/partials/Graph/partials/NodeInfo/partials/Pods/partials/PodDetails/partials/Metrics/utils/constants.ts index da9f955be9..46d6b61752 100644 --- a/ui/src/components/pages/Pipeline/partials/Graph/partials/NodeInfo/partials/Pods/partials/PodDetails/partials/Metrics/utils/constants.ts +++ b/ui/src/components/pages/Pipeline/partials/Graph/partials/NodeInfo/partials/Pods/partials/PodDetails/partials/Metrics/utils/constants.ts @@ -39,8 +39,8 @@ export const metricNameMap: { [p: string]: string } = { monovtx_sink_time_bucket: "Mono Vertex Sink Write Time Latency", forwarder_data_read_total: - "Vertex Read Processing Rate", - monovtx_read_total: - "Mono Vertex Read Processing Rate", - monovtx_pending: "Mono Vertex Pending", + "Vertex Read Processing Rate (messages per second)", + monovtx_read_total: "Mono Vertex Read Processing Rate (messages per second)", + monovtx_pending: "Mono Vertex Pending Messages", + vertex_pending_messages: "Vertex Pending Messages", }; diff --git a/ui/yarn.lock b/ui/yarn.lock index e507771e3f..5779331c0a 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -8333,9 +8333,9 @@ mz@^2.7.0: thenify-all "^1.0.0" nanoid@^3.3.7: - version "3.3.7" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" - integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== + version "3.3.8" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" + integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== natural-compare-lite@^1.4.0: version "1.4.0"