diff --git a/requirements.txt b/requirements.txt index cbbb2275..24ec00e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ google-api-python-client==2.116.0 ibm_cloud_sdk_core==3.18.0 ibm_vpc==0.20.0 jinja2==3.1.3 -krkn-lib==1.4.11 +krkn-lib==1.4.12 lxml==5.1.0 kubernetes==26.1.0 oauth2client==4.1.3 diff --git a/run_kraken.py b/run_kraken.py index 370b2c23..e1a95466 100644 --- a/run_kraken.py +++ b/run_kraken.py @@ -215,6 +215,7 @@ def main(cfg): # Capture the start time start_time = int(time.time()) + critical_alerts_count = 0 chaos_telemetry = ChaosRunTelemetry() chaos_telemetry.run_uuid = run_uuid @@ -352,17 +353,14 @@ def main(cfg): ##PROM query = r"""ALERTS{severity="critical"}""" end_time = datetime.datetime.now() - critical_alerts = prometheus.process_prom_query_in_range( - query, - start_time = datetime.datetime.fromtimestamp(start_time), - end_time = end_time - + critical_alerts = prometheus.process_query( + query ) critical_alerts_count = len(critical_alerts) if critical_alerts_count > 0: logging.error("Critical alerts are firing: %s", critical_alerts) logging.error("Please check, exiting") - sys.exit(1) + break else: logging.info("No critical alerts are firing!!") @@ -441,6 +439,10 @@ def main(cfg): logging.error("Alert profile is not defined") sys.exit(1) + if critical_alerts_count > 0: + logging.error("Critical alerts are firing, please check; exiting") + sys.exit(1) + if failed_post_scenarios: logging.error( "Post scenarios are still failing at the end of all iterations"