Adding namespace check by default

cerberus/kubernetes/client.py

@@ -78,6 +78,33 @@ def list_namespaces():
     return namespaces
 
 
+# Monitor the status of all specified namespaces
+# and set the status to true or false
+def monitor_namespaces_status(watch_namespaces, watch_terminating_namespaces, iteration, iter_track_time):
+    namespaces = []
+    none_terminating = True
+    if watch_terminating_namespaces:
+        watch_nodes_start_time = time.time()
+        try:
+            ret = cli.list_namespace(pretty=True)
+        except ApiException as e:
+            logging.error("Exception when calling CoreV1Api->list_namespace: %s\n" % e)
+            sys.exit(1)
+        for namespace in ret.items:
+            if namespace.metadata.name in watch_namespaces:
+                if namespace.status.phase != "Active":
+                    namespaces.append(namespace.metadata.name)
+                    none_terminating = False
+        iter_track_time["watch_terminating_namespaces"] = time.time() - watch_nodes_start_time
+        logging.info("Iteration %s: No Terminating Namespaces status: %s" % (iteration, str(none_terminating)))
+    else:
+        logging.info(
+            "Cerberus is not monitoring namespaces, so setting the status "
+            "to True and assuming that the namespaces are Active"
+        )
+    return namespaces
+
+
 # Get node status
 def get_node_info(node):
     try:

config/config.yaml

@@ -5,6 +5,7 @@ cerberus:
     watch_nodes: True                                    # Set to True for the cerberus to monitor the cluster nodes
     watch_cluster_operators: True                        # Set to True for cerberus to monitor cluster operators
     watch_url_routes:                                    # Route url's you want to monitor, this is a double array with the url and optional authorization parameter
+    watch_terminating_namespaces: True                   # Set to True to monitor if any namespaces in the 'watch_namespaces' list start terminating
     watch_master_schedulable:                            # When enabled checks for the schedulable master nodes with given label.
         enabled: True
         label: node-role.kubernetes.io/master

docs/config.md

@@ -6,6 +6,7 @@ Cerberus Config Components Explained
 * [Watch Routes](#watch-routes)
 * [Watch Master Schedulable Status](#watch-master-schedulable-status)
 * [Watch Namespaces](#watch-namespaces)
+* [Watch Terminating Namespaces](#watch-terminating-namespaces)
 * [Publish Status](#publish-status)
 * [Inpsect Components](#inspect-components)
 * [Custom Checks](#custom-checks)
@@ -20,6 +21,7 @@ cerberus:
     port: 8081                                           # http server port where cerberus status is published
     watch_nodes: True                                    # Set to True for the cerberus to monitor the cluster nodes
     watch_cluster_operators: True                        # Set to True for cerberus to monitor cluster operators
+    watch_terminating_namespaces: True                   # Set to True to monitor if any namespaces (set below under 'watch_namespaces' start terminating
     watch_url_routes:
     # Route url's you want to monitor, this is a double array with the url and optional authorization parameter
     watch_master_schedulable:                            # When enabled checks for the schedulable master nodes with given label.
@@ -111,6 +113,10 @@ For example, `^openshift-.*$` can be used to watch all namespaces that start wit
 Or you can use `^.*$` to watch all namespaces in your cluster
 
 
+#### Watch Terminating Namespaces
+When `watch_terminating_namespaces` is set to True, this will monitor the status of all the namespaces defind under watch namespaces and report a failure if any are terminating.
+If set to False will not query or report the status of the terminating namespaces
+
 #### Publish Status
 Parameter to set if you want to publish the go/no-go signal to the http server
 

start_cerberus.py

@@ -79,6 +79,7 @@ def main(cfg):
         watch_nodes = config["cerberus"].get("watch_nodes", False)
         watch_cluster_operators = config["cerberus"].get("watch_cluster_operators", False)
         watch_namespaces = config["cerberus"].get("watch_namespaces", [])
+        watch_terminating_namespaces = config["cerberus"].get("watch_terminating_namespaces", True)
         watch_url_routes = config["cerberus"].get("watch_url_routes", [])
         watch_master_schedulable = config["cerberus"].get("watch_master_schedulable", {})
         cerberus_publish_status = config["cerberus"].get("cerberus_publish_status", False)
@@ -241,6 +242,7 @@ def main(cfg):
                     (watch_nodes_status, failed_nodes),
                     (watch_cluster_operators_status, failed_operators),
                     (failed_routes),
+                    (terminating_namespaces),
                 ) = pool.map(
                     smap,
                     [
@@ -257,6 +259,13 @@ def main(cfg):
                             iter_track_time,
                         ),
                         functools.partial(kubecli.process_routes, watch_url_routes, iter_track_time),
+                        functools.partial(
+                            kubecli.monitor_namespaces_status,
+                            watch_namespaces,
+                            watch_terminating_namespaces,
+                            iteration,
+                            iter_track_time,
+                        ),
                     ],
                 )
 
@@ -330,6 +339,11 @@ def main(cfg):
                         dbcli.insert(datetime.now(), time.time(), 1, "pod crash", failures, component)
                     logging.info("")
 
+                watch_teminating_ns = True
+                if terminating_namespaces:
+                    watch_teminating_ns = False
+                    logging.info("Iteration %s: Terminating namespaces %s" % (iteration, str(terminating_namespaces)))
+
                 # Logging the failed checking of routes
                 watch_routes_status = True
                 if failed_routes:
@@ -347,6 +361,7 @@ def main(cfg):
                     and watch_cluster_operators_status
                     and server_status
                     and watch_routes_status
+                    and watch_teminating_ns
                 )
 
                 if distribution == "openshift":