Taking out start and end time for critical alerts (#572)

* taking out start and end time" Signed-off-by: Paige Rubendall <[email protected]> * adding only break when alert fires Signed-off-by: Paige Rubendall <[email protected]> * fail at end if alert had fired Signed-off-by: Paige Rubendall <[email protected]> * adding new krkn-lib function with no range Signed-off-by: Paige Rubendall <[email protected]> * updating requirements to new krkn-lib Signed-off-by: Paige Rubendall <[email protected]> --------- Signed-off-by: Paige Rubendall <[email protected]>
krkn-chaos · Feb 19, 2024 · c440dc4 · c440dc4
1 parent b174c51
commit c440dc4
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 7 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -15,7 +15,7 @@ google-api-python-client==2.116.0
 ibm_cloud_sdk_core==3.18.0
 ibm_vpc==0.20.0
 jinja2==3.1.3
-krkn-lib==1.4.11
+krkn-lib==1.4.12
 lxml==5.1.0
 kubernetes==26.1.0
 oauth2client==4.1.3

diff --git a/run_kraken.py b/run_kraken.py
@@ -215,6 +215,7 @@ def main(cfg):
 
         # Capture the start time
         start_time = int(time.time())
+        critical_alerts_count = 0
 
         chaos_telemetry = ChaosRunTelemetry()
         chaos_telemetry.run_uuid = run_uuid
@@ -352,17 +353,14 @@ def main(cfg):
                             ##PROM
                             query = r"""ALERTS{severity="critical"}"""
                             end_time = datetime.datetime.now()
-                            critical_alerts = prometheus.process_prom_query_in_range(
-                                query,
-                                start_time = datetime.datetime.fromtimestamp(start_time),
-                                end_time = end_time
-
+                            critical_alerts = prometheus.process_query(
+                                query
                             )
                             critical_alerts_count = len(critical_alerts)
                             if critical_alerts_count > 0:
                                 logging.error("Critical alerts are firing: %s", critical_alerts)
                                 logging.error("Please check, exiting")
-                                sys.exit(1)
+                                break
                             else:
                                 logging.info("No critical alerts are firing!!")
 
@@ -441,6 +439,10 @@ def main(cfg):
                 logging.error("Alert profile is not defined")
                 sys.exit(1)
 
+        if critical_alerts_count > 0:
+            logging.error("Critical alerts are firing, please check; exiting")
+            sys.exit(1)
+
         if failed_post_scenarios:
             logging.error(
                 "Post scenarios are still failing at the end of all iterations"