Health checks

components/operators/openshift-gitops/instance/components/health-check-openshift-ai/kustomization.yaml

@@ -11,13 +11,6 @@ patches:
   - path: patch-notebook-health-check.yaml
     target:
       kind: ArgoCD
-  # These have not yet been implimented/tested
-  # - path: patch-inferencegraph-health-check.yaml
-  #   target:
-  #     kind: ArgoCD
   - path: patch-inferenceservice-health-check.yaml
     target:
       kind: ArgoCD
-  # - path: patch-servingruntime-health-check.yaml
-  #   target:
-  #     kind: ArgoCD

components/operators/openshift-gitops/instance/components/health-check-openshift-ai/patch-inferenceservice-health-check.yaml

@@ -4,27 +4,33 @@
     group: serving.kserve.io
     kind: InferenceService
     check: |
-      health_status = {}
+      local health_status = {}
+
       health_status.status = "Progressing"
       health_status.message = "Waiting for InferenceService to report status..."
 
       if obj.status ~= nil then
-        progressing = false
 
-        if obj.modelStatus ~= nil then
-          if obj.modelStatus.transitionStatus == "InProgress" then
-            progressing = true
+        local progressing = false
+        local degraded = false
+        local status_false = 0
+        local status_unknown = 0
+        local msg = ""
+
+        if obj.status.modelStatus ~= nil then
+          if obj.status.modelStatus.transitionStatus ~= "UpToDate" then
+            if obj.status.modelStatus.transitionStatus == "InProgress" then
+              progressing = true
+            else
+              degraded = true
+            end
+            msg = msg .. "0: transitionStatus | " .. obj.status.modelStatus.transitionStatus .. "\n"
           end
         end
-
-        if obj.status.conditions ~= nil then
-          msg = ""
 
+        if obj.status.conditions ~= nil then
           for i, condition in pairs(obj.status.conditions) do
 
-            status_false = 0
-            status_unknown = 0
-
             if condition.status == "Unknown" then
               status_unknown = status_unknown + 1
             elseif condition.status == "False" then
@@ -44,9 +50,10 @@
 
           end
 
-          if progressing == false and status_unknown == 0 and status_false == 0 then
+          if progressing == false and degraded == false and status_unknown == 0 and status_false == 0 then
             health_status.status = "Healthy"
-          elseif progressing == true then
+            msg = "InferenceService is healthy."
+          elseif degraded == false and status_unknown > 0 then
             health_status.status = "Progressing"
           else
             health_status.status = "Degraded"