diff --git a/controller/collect-data.go b/controller/collect-data.go index 588f29e6..671017dd 100644 --- a/controller/collect-data.go +++ b/controller/collect-data.go @@ -16,10 +16,23 @@ import ( clientTypes "k8s.io/apimachinery/pkg/types" ) +//go:generate mockgen -destination=mocks/mock_collect-data.go -package=mocks github.com/litmuschaos/chaos-exporter/controller ResultCollector + +// ResultCollector interface for the both functions GetResultList and getExperimentMetricsFromResult +type ResultCollector interface { + GetResultList(clients clients.ClientSets, chaosNamespace string, monitoringEnabled *MonitoringEnabled) (litmuschaosv1alpha1.ChaosResultList, error) + GetExperimentMetricsFromResult(chaosResult *litmuschaosv1alpha1.ChaosResult, clients clients.ClientSets) (bool, error) + SetResultDetails() + GetResultDetails() ChaosResultDetails +} +type ResultDetails struct { + resultDetails ChaosResultDetails +} + // GetResultList return the result list correspond to the monitoring enabled chaosengine -func GetResultList(clients clients.ClientSets, chaosNamespace string, monitoringEnabled *MonitoringEnabled) (litmuschaosv1alpha1.ChaosResultList, error) { +func (r *ResultDetails) GetResultList(clients clients.ClientSets, chaosNamespace string, monitoringEnabled *MonitoringEnabled) (litmuschaosv1alpha1.ChaosResultList, error) { - chaosResultList, err := clients.LitmusClient.ChaosResults(chaosNamespace).List(context.Background(), metav1.ListOptions{}) + chaosResultList, err := clients.LitmusClient.LitmuschaosV1alpha1().ChaosResults(chaosNamespace).List(context.Background(), metav1.ListOptions{}) if err != nil { return litmuschaosv1alpha1.ChaosResultList{}, err } @@ -41,15 +54,14 @@ func GetResultList(clients clients.ClientSets, chaosNamespace string, monitoring return *chaosResultList, nil } -// getExperimentMetricsFromResult derive all the metrics data from the chaosresult and set into resultDetails struct -func (resultDetails *ChaosResultDetails) getExperimentMetricsFromResult(chaosResult *litmuschaosv1alpha1.ChaosResult, clients clients.ClientSets) (bool, error) { +// GetExperimentMetricsFromResult derive all the metrics data from the chaosresult and set into resultDetails struct +func (r *ResultDetails) GetExperimentMetricsFromResult(chaosResult *litmuschaosv1alpha1.ChaosResult, clients clients.ClientSets) (bool, error) { verdict := strings.ToLower(string(chaosResult.Status.ExperimentStatus.Verdict)) probeSuccesPercentage, err := getProbeSuccessPercentage(chaosResult) if err != nil { return false, err } - - engine, err := clients.LitmusClient.ChaosEngines(chaosResult.Namespace).Get(context.Background(), chaosResult.Spec.EngineName, metav1.GetOptions{}) + engine, err := clients.LitmusClient.LitmuschaosV1alpha1().ChaosEngines(chaosResult.Namespace).Get(context.Background(), chaosResult.Spec.EngineName, metav1.GetOptions{}) if err != nil { // k8serrors.IsNotFound(err) checking k8s resource is found or not, // It will skip this result if k8s resource is not found. @@ -64,9 +76,8 @@ func (resultDetails *ChaosResultDetails) getExperimentMetricsFromResult(chaosRes if err != nil { return false, err } - // setting all the values inside resultdetails struct - resultDetails.setName(chaosResult.Name). + r.resultDetails.setName(chaosResult.Name). setUID(chaosResult.UID). setNamespace(chaosResult.Namespace). setProbeSuccessPercentage(probeSuccesPercentage). @@ -89,8 +100,8 @@ func (resultDetails *ChaosResultDetails) getExperimentMetricsFromResult(chaosRes // experiment's final verdict[passed,failed,stopped] is already exported/overridden // and 'litmuschaos_experiment_verdict' metric was reset to 0 if engine.Status.EngineStatus == v1alpha1.EngineStatusCompleted { - result, ok := matchVerdict[string(resultDetails.UID)] - if !ok || (ok && result.Verdict == resultDetails.Verdict && result.VerdictReset) { + result, ok := matchVerdict[string(r.resultDetails.UID)] + if !ok || (ok && result.Verdict == r.resultDetails.Verdict && result.VerdictReset) { return true, nil } } @@ -98,10 +109,19 @@ func (resultDetails *ChaosResultDetails) getExperimentMetricsFromResult(chaosRes return false, nil } +func (r *ResultDetails) SetResultDetails() { + r.resultDetails.PassedExperiments = 0 + r.resultDetails.AwaitedExperiments = 0 + r.resultDetails.FailedExperiments = 0 +} + +func (r *ResultDetails) GetResultDetails() ChaosResultDetails { + return r.resultDetails +} + // initialiseResult create the new instance of the ChaosResultDetails struct func initialiseResult() *ChaosResultDetails { return &ChaosResultDetails{} - } // setName sets name inside resultDetails struct diff --git a/controller/collect-data_test.go b/controller/collect-data_test.go new file mode 100644 index 00000000..94553475 --- /dev/null +++ b/controller/collect-data_test.go @@ -0,0 +1,187 @@ +package controller_test + +import ( + "context" + "github.com/litmuschaos/chaos-exporter/controller" + "github.com/litmuschaos/chaos-exporter/pkg/clients" + "github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1" + litmusFakeClientSet "github.com/litmuschaos/chaos-operator/pkg/client/clientset/versioned/fake" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "testing" +) + +func TestGetResultList(t *testing.T) { + FakeChaosNameSpace := "Fake Namespace" + FakeEngineName := "Fake Engine" + + tests := []struct { + name string + execFunc func(client clients.ClientSets, chaosResult *v1alpha1.ChaosResult) + chaosResult *v1alpha1.ChaosResult + monitoring *controller.MonitoringEnabled + isErr bool + }{ + { + name: "success:chaos result found", + execFunc: func(client clients.ClientSets, chaosResult *v1alpha1.ChaosResult) { + _, err := client.LitmusClient.LitmuschaosV1alpha1().ChaosResults(chaosResult.Namespace).Create(context.Background(), chaosResult, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("chaosresult not created") + } + }, + + chaosResult: &v1alpha1.ChaosResult{ + ObjectMeta: metav1.ObjectMeta{ + Name: FakeEngineName, + Namespace: FakeChaosNameSpace, + }, + Spec: v1alpha1.ChaosResultSpec{ + ExperimentName: "exp-1", + EngineName: FakeEngineName, + }, + }, + + isErr: false, + monitoring: &controller.MonitoringEnabled{ + IsChaosResultsAvailable: true, + }, + }, + { + name: "success:empty chaosResult", + chaosResult: &v1alpha1.ChaosResult{}, + execFunc: func(client clients.ClientSets, chaosResult *v1alpha1.ChaosResult) {}, + isErr: false, + monitoring: &controller.MonitoringEnabled{ + IsChaosResultsAvailable: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := CreateFakeClient(t) + tt.execFunc(client, tt.chaosResult) + resultDetails := &controller.ResultDetails{} + _, err := resultDetails.GetResultList(client, FakeChaosNameSpace, tt.monitoring) + if tt.isErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestGetExperimentMetricsFromResult(t *testing.T) { + FakeEngineName := "Fake Engine" + FakeNamespace := "Fake Namespace" + fakeServiceAcc := "Fake Service Account" + fakeAppLabel := "Fake Label" + fakeAppKind := "Fake Kind" + + tests := map[string]struct { + chaosengine *v1alpha1.ChaosEngine + chaosresult *v1alpha1.ChaosResult + expectedVerdict bool + isErr bool + verdict bool + execFunc func(client clients.ClientSets, engine *v1alpha1.ChaosEngine, result *v1alpha1.ChaosResult) + }{ + "success": { + chaosengine: &v1alpha1.ChaosEngine{ + ObjectMeta: metav1.ObjectMeta{ + Name: FakeEngineName, + Namespace: FakeNamespace, + }, + Spec: v1alpha1.ChaosEngineSpec{ + ChaosServiceAccount: fakeServiceAcc, + Appinfo: v1alpha1.ApplicationParams{ + Appns: FakeNamespace, + Applabel: fakeAppLabel, + AppKind: fakeAppKind, + }, + Experiments: []v1alpha1.ExperimentList{ + { + Name: "Fake-Exp-Name", + }, + }, + }, + Status: v1alpha1.ChaosEngineStatus{ + EngineStatus: v1alpha1.EngineStatusCompleted, + Experiments: []v1alpha1.ExperimentStatuses{ + { + Name: "Fake-Exp-Name", + Status: v1alpha1.ExperimentStatusRunning, + }, + }, + }, + }, + chaosresult: &v1alpha1.ChaosResult{ + ObjectMeta: metav1.ObjectMeta{ + Name: FakeEngineName + "-" + "Fake-Exp-Name", + Namespace: FakeNamespace, + UID: "Fake-UID", + }, + Spec: v1alpha1.ChaosResultSpec{ + EngineName: FakeEngineName, + ExperimentName: "Fake-Exp-Name", + }, + Status: v1alpha1.ChaosResultStatus{ + ExperimentStatus: v1alpha1.TestStatus{ + Phase: "Completed", + Verdict: "Pass", + }, + History: &v1alpha1.HistoryDetails{}, + }, + }, + + execFunc: func(client clients.ClientSets, engine *v1alpha1.ChaosEngine, result *v1alpha1.ChaosResult) { + _, err := client.LitmusClient.LitmuschaosV1alpha1().ChaosEngines(engine.Namespace).Create(context.Background(), engine, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("engine not created for test, err: %v", err) + } + + _, err = client.LitmusClient.LitmuschaosV1alpha1().ChaosResults(result.Namespace).Create(context.Background(), result, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("chaosresult not created fortest, err: %v", err) + } + }, + isErr: false, + verdict: true, + }, + "failure: No Chaos Engine": { + chaosresult: &v1alpha1.ChaosResult{}, + isErr: false, + verdict: true, + execFunc: func(client clients.ClientSets, engine *v1alpha1.ChaosEngine, result *v1alpha1.ChaosResult) { + }, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + client := CreateFakeClient(t) + resultDetails := controller.ResultDetails{} + tt.execFunc(client, tt.chaosengine, tt.chaosresult) + verdict, err := resultDetails.GetExperimentMetricsFromResult(tt.chaosresult, client) + assert.Equal(t, tt.verdict, verdict) + if tt.isErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func CreateFakeClient(t *testing.T) clients.ClientSets { + cs := clients.ClientSets{} + cs.KubeClient = fake.NewSimpleClientset([]runtime.Object{}...) + cs.LitmusClient = litmusFakeClientSet.NewSimpleClientset([]runtime.Object{}...) + return cs +} diff --git a/controller/controller.go b/controller/controller.go index 0c581034..88666ad9 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -31,10 +31,13 @@ func Exporter(clients clients.ClientSets) { // Register the fixed (count) chaos metrics log.Info("Registering Fixed Metrics") - gaugeMetrics := GaugeMetrics{} + r := MetricesCollecter{ + ResultCollector: &ResultDetails{}, + } + //gaugeMetrics := GaugeMetrics{} overallChaosResults := litmuschaosv1alpha1.ChaosResultList{} - gaugeMetrics.InitializeGaugeMetrics(). + r.GaugeMetrics.InitializeGaugeMetrics(). RegisterFixedMetrics() monitoringEnabled := MonitoringEnabled{ @@ -43,7 +46,7 @@ func Exporter(clients clients.ClientSets) { } for { - if err := gaugeMetrics.GetLitmusChaosMetrics(clients, &overallChaosResults, &monitoringEnabled); err != nil { + if err := r.GetLitmusChaosMetrics(clients, &overallChaosResults, &monitoringEnabled); err != nil { log.Errorf("err: %v", err) } time.Sleep(1000 * time.Millisecond) diff --git a/controller/handle-result-deletion_test.go b/controller/handle-result-deletion_test.go new file mode 100644 index 00000000..f5e05567 --- /dev/null +++ b/controller/handle-result-deletion_test.go @@ -0,0 +1,101 @@ +package controller + +import ( + "errors" + "github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func Test_unsetDeletedChaosResults(t *testing.T) { + + tests := []struct { + name string + execFunc func(details *ChaosResultDetails) + isErr bool + resultDetails *ChaosResultDetails + oldChaosResult *v1alpha1.ChaosResultList + newChaosResult *v1alpha1.ChaosResultList + }{ + { + name: "success: deleted chaosResult", + execFunc: func(details *ChaosResultDetails) { + details.setResultData() + }, + resultDetails: &ChaosResultDetails{ + UID: "FAKE-UID-OLD", + }, + oldChaosResult: &v1alpha1.ChaosResultList{ + Items: []v1alpha1.ChaosResult{ + { + ObjectMeta: metav1.ObjectMeta{ + UID: "FAKE-UID-OLD", + }, + }, + }, + }, + newChaosResult: &v1alpha1.ChaosResultList{ + Items: []v1alpha1.ChaosResult{ + { + ObjectMeta: metav1.ObjectMeta{ + UID: "FAKE-UID-NEW", + }, + }, + }, + }, + isErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.execFunc(tt.resultDetails) + + r := MetricesCollecter{} + r.GaugeMetrics.InitializeGaugeMetrics() + r.GaugeMetrics.unsetDeletedChaosResults(tt.oldChaosResult, tt.newChaosResult) + if len(resultStore) != 0 && tt.isErr { + require.Error(t, errors.New("not able to remove result from resultStore")) + } + }) + } + +} + +func Test_unsetOutdatedChaosResults(t *testing.T) { + + tests := []struct { + name string + execFunc func(details ChaosResultDetails) + isErr bool + + oldResultDetails ChaosResultDetails + newResultDetails ChaosResultDetails + }{ + { + name: "success: verdict changed", + execFunc: func(details ChaosResultDetails) { + r := &ResultData{} + matchVerdict[string(details.UID)] = r.setVerdict(details.Verdict) + }, + oldResultDetails: ChaosResultDetails{ + UID: "UID", + Verdict: "Awaited", + }, + newResultDetails: ChaosResultDetails{ + UID: "UID", + Verdict: "Pass", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.execFunc(tt.oldResultDetails) + + r := MetricesCollecter{} + r.GaugeMetrics.InitializeGaugeMetrics() + r.GaugeMetrics.unsetOutdatedMetrics(tt.newResultDetails) + }) + } + +} diff --git a/controller/mocks/mock_collect-data.go b/controller/mocks/mock_collect-data.go new file mode 100644 index 00000000..1ebf59ef --- /dev/null +++ b/controller/mocks/mock_collect-data.go @@ -0,0 +1,93 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/litmuschaos/chaos-exporter/controller (interfaces: ResultCollector) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + controller "github.com/litmuschaos/chaos-exporter/controller" + clients "github.com/litmuschaos/chaos-exporter/pkg/clients" + v1alpha1 "github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1" +) + +// MockResultCollector is a mock of ResultCollector interface. +type MockResultCollector struct { + ctrl *gomock.Controller + recorder *MockResultCollectorMockRecorder +} + +// MockResultCollectorMockRecorder is the mock recorder for MockResultCollector. +type MockResultCollectorMockRecorder struct { + mock *MockResultCollector +} + +// NewMockResultCollector creates a new mock instance. +func NewMockResultCollector(ctrl *gomock.Controller) *MockResultCollector { + mock := &MockResultCollector{ctrl: ctrl} + mock.recorder = &MockResultCollectorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockResultCollector) EXPECT() *MockResultCollectorMockRecorder { + return m.recorder +} + +// GetExperimentMetricsFromResult mocks base method. +func (m *MockResultCollector) GetExperimentMetricsFromResult(arg0 *v1alpha1.ChaosResult, arg1 clients.ClientSets) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetExperimentMetricsFromResult", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetExperimentMetricsFromResult indicates an expected call of GetExperimentMetricsFromResult. +func (mr *MockResultCollectorMockRecorder) GetExperimentMetricsFromResult(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExperimentMetricsFromResult", reflect.TypeOf((*MockResultCollector)(nil).GetExperimentMetricsFromResult), arg0, arg1) +} + +// GetResultDetails mocks base method. +func (m *MockResultCollector) GetResultDetails() controller.ChaosResultDetails { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetResultDetails") + ret0, _ := ret[0].(controller.ChaosResultDetails) + return ret0 +} + +// GetResultDetails indicates an expected call of GetResultDetails. +func (mr *MockResultCollectorMockRecorder) GetResultDetails() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResultDetails", reflect.TypeOf((*MockResultCollector)(nil).GetResultDetails)) +} + +// GetResultList mocks base method. +func (m *MockResultCollector) GetResultList(arg0 clients.ClientSets, arg1 string, arg2 *controller.MonitoringEnabled) (v1alpha1.ChaosResultList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetResultList", arg0, arg1, arg2) + ret0, _ := ret[0].(v1alpha1.ChaosResultList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetResultList indicates an expected call of GetResultList. +func (mr *MockResultCollectorMockRecorder) GetResultList(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResultList", reflect.TypeOf((*MockResultCollector)(nil).GetResultList), arg0, arg1, arg2) +} + +// SetResultDetails mocks base method. +func (m *MockResultCollector) SetResultDetails() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetResultDetails") +} + +// SetResultDetails indicates an expected call of SetResultDetails. +func (mr *MockResultCollectorMockRecorder) SetResultDetails() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetResultDetails", reflect.TypeOf((*MockResultCollector)(nil).SetResultDetails)) +} diff --git a/controller/scrap_test.go b/controller/scrap_test.go new file mode 100644 index 00000000..1a44c5ab --- /dev/null +++ b/controller/scrap_test.go @@ -0,0 +1,88 @@ +package controller_test + +import ( + "github.com/golang/mock/gomock" + "github.com/litmuschaos/chaos-exporter/controller" + "github.com/litmuschaos/chaos-exporter/controller/mocks" + v1alpha1 "github.com/litmuschaos/chaos-operator/api/litmuschaos/v1alpha1" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func TestGetLitmusChaosMetrics(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + mockCollectData := mocks.NewMockResultCollector(mockCtl) + + r := controller.MetricesCollecter{ + ResultCollector: mockCollectData, + } + + r.GaugeMetrics.InitializeGaugeMetrics().RegisterFixedMetrics() + + tests := []struct { + name string + execFunc func() + isErr bool + monitoring *controller.MonitoringEnabled + overallChaosResult *v1alpha1.ChaosResultList + }{ + { + name: "success", + execFunc: func() { + mockCollectData.EXPECT().GetResultList(gomock.Any(), gomock.Any(), gomock.Any()). + Return(v1alpha1.ChaosResultList{ + Items: []v1alpha1.ChaosResult{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "chaosresult-1", + }, + }, + }, + }, nil).Times(1) + mockCollectData.EXPECT().GetExperimentMetricsFromResult(gomock.Any(), gomock.Any()).Return(false, nil).Times(1) + mockCollectData.EXPECT().SetResultDetails() + mockCollectData.EXPECT().GetResultDetails().Return(controller.ChaosResultDetails{ + UID: "FAKE-UID", + }).Times(1) + }, + overallChaosResult: &v1alpha1.ChaosResultList{ + Items: []v1alpha1.ChaosResult{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "chaosresult-1", + }, + }, + }, + }, + monitoring: &controller.MonitoringEnabled{}, + isErr: false, + }, + { + name: "failure: no ChaosResultList found", + execFunc: func() { + mockCollectData.EXPECT().GetResultList(gomock.Any(), gomock.Any(), gomock.Any()). + Return(v1alpha1.ChaosResultList{}, errors.New("Fake Error")).Times(1) + }, + overallChaosResult: &v1alpha1.ChaosResultList{}, + monitoring: &controller.MonitoringEnabled{}, + isErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.execFunc() + + client := CreateFakeClient(t) + err := r.GetLitmusChaosMetrics(client, tt.overallChaosResult, tt.monitoring) + if tt.isErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } + +} diff --git a/controller/scrape.go b/controller/scrape.go index b1da15d1..502055a5 100644 --- a/controller/scrape.go +++ b/controller/scrape.go @@ -34,7 +34,7 @@ import ( var err error // GetLitmusChaosMetrics derive and send the chaos metrics -func (gaugeMetrics *GaugeMetrics) GetLitmusChaosMetrics(clients clients.ClientSets, overallChaosResults *litmuschaosv1alpha1.ChaosResultList, monitoringEnabled *MonitoringEnabled) error { +func (m *MetricesCollecter) GetLitmusChaosMetrics(clients clients.ClientSets, overallChaosResults *litmuschaosv1alpha1.ChaosResultList, monitoringEnabled *MonitoringEnabled) error { engineCount := 0 // initialising the parameters for the namespaced scope metrics @@ -53,13 +53,12 @@ func (gaugeMetrics *GaugeMetrics) GetLitmusChaosMetrics(clients clients.ClientSe } watchNamespace := os.Getenv("WATCH_NAMESPACE") // Getting list of all the chaosresults for the monitoring - resultList, err := GetResultList(clients, watchNamespace, monitoringEnabled) + resultList, err := m.ResultCollector.GetResultList(clients, watchNamespace, monitoringEnabled) if err != nil { return err } - // unset the metrics correspond to deleted chaosresults - gaugeMetrics.unsetDeletedChaosResults(overallChaosResults, &resultList) + m.GaugeMetrics.unsetDeletedChaosResults(overallChaosResults, &resultList) // updating the overall chaosresults items to latest overallChaosResults.Items = resultList.Items @@ -68,14 +67,10 @@ func (gaugeMetrics *GaugeMetrics) GetLitmusChaosMetrics(clients clients.ClientSe // otherwise it derive metrics for all chaosresults present inside cluster for _, chaosresult := range resultList.Items { - resultDetails := ChaosResultDetails{ - PassedExperiments: 0, - FailedExperiments: 0, - AwaitedExperiments: 0, - } - + m.ResultCollector.SetResultDetails() // deriving metrics data from the chaosresult - skip, err := resultDetails.getExperimentMetricsFromResult(&chaosresult, clients) + skip, err := m.ResultCollector.GetExperimentMetricsFromResult(&chaosresult, clients) + resultDetails := m.ResultCollector.GetResultDetails() if err != nil { return err } @@ -110,8 +105,8 @@ func (gaugeMetrics *GaugeMetrics) GetLitmusChaosMetrics(clients clients.ClientSe }) // setting chaosresult metrics for the given chaosresult - verdictValue := gaugeMetrics.unsetOutdatedMetrics(resultDetails) - gaugeMetrics.setResultChaosMetrics(resultDetails, verdictValue) + verdictValue := m.GaugeMetrics.unsetOutdatedMetrics(resultDetails) + m.GaugeMetrics.setResultChaosMetrics(resultDetails, verdictValue) // setting chaosresult aws metrics for the given chaosresult, which can be used for cloudwatch if awsConfig.Namespace != "" && awsConfig.ClusterName != "" && awsConfig.Service != "" { awsConfig.setAwsResultChaosMetrics(resultDetails) @@ -128,7 +123,7 @@ func (gaugeMetrics *GaugeMetrics) GetLitmusChaosMetrics(clients clients.ClientSe } //setting aggregate metrics from the all chaosresults - gaugeMetrics.setNamespacedChaosMetrics(namespacedScopeMetrics, watchNamespace) + m.GaugeMetrics.setNamespacedChaosMetrics(namespacedScopeMetrics, watchNamespace) //setting aggregate aws metrics from the all chaosresults, which can be used for cloudwatch if awsConfig.Namespace != "" && awsConfig.ClusterName != "" && awsConfig.Service != "" { awsConfig.setAwsNamespacedChaosMetrics(namespacedScopeMetrics) diff --git a/controller/types.go b/controller/types.go index 9c46f671..86a509ae 100644 --- a/controller/types.go +++ b/controller/types.go @@ -282,6 +282,11 @@ type GaugeMetrics struct { ClusterScopedExperimentsRunCount *prometheus.GaugeVec } +type MetricesCollecter struct { + ResultCollector ResultCollector + GaugeMetrics GaugeMetrics +} + // MonitoringEnabled contains existence/availability of chaosEngines and chaosResults type MonitoringEnabled struct { IsChaosResultsAvailable bool diff --git a/go.mod b/go.mod index 1cc1244b..6ea59d9d 100644 --- a/go.mod +++ b/go.mod @@ -4,15 +4,17 @@ go 1.20 require ( github.com/aws/aws-sdk-go v1.40.27 - github.com/litmuschaos/chaos-operator v0.0.0-20230602170015-d019f63af50f + github.com/golang/mock v1.5.0 + github.com/litmuschaos/chaos-operator v0.0.0-20230629040437-de73ffdd63da github.com/litmuschaos/litmus-go v0.0.0-20230605073551-d73728198577 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.15.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.1 github.com/sirupsen/logrus v1.9.3 - k8s.io/api v0.22.2 - k8s.io/apimachinery v0.22.1 + github.com/stretchr/testify v1.7.0 + k8s.io/api v0.26.0 + k8s.io/apimachinery v0.26.0 k8s.io/client-go v12.0.0+incompatible ) @@ -27,9 +29,10 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/evanphx/json-patch v4.11.0+incompatible // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/go-logr/logr v0.4.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.6 // indirect @@ -43,12 +46,13 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect - golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect + golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect golang.org/x/sys v0.5.0 // indirect golang.org/x/term v0.5.0 // indirect @@ -60,10 +64,11 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.9.0 // indirect - k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect + k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e // indirect + k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect sigs.k8s.io/controller-runtime v0.10.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index 96a812fe..c5e6c2f0 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -149,8 +150,10 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -179,6 +182,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -240,6 +244,7 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -310,8 +315,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/litmuschaos/chaos-operator v0.0.0-20230602170015-d019f63af50f h1:RJMARNqpMgUHDrI9Oy4pMr8RTAh/CkYha4Wn7xXHt9Q= -github.com/litmuschaos/chaos-operator v0.0.0-20230602170015-d019f63af50f/go.mod h1:uIIUKHTPSEwSC52esrSSxgEe3pLoLlfOGXRmhLcgKy4= +github.com/litmuschaos/chaos-operator v0.0.0-20230629040437-de73ffdd63da h1:SqhQjwgUpjCFvx5pUHYxJJ1d6SvTX1NLEV8UZUr3IUQ= +github.com/litmuschaos/chaos-operator v0.0.0-20230629040437-de73ffdd63da/go.mod h1:NqSQCsjBXi6QyTaAt28CTWmL+vFWkdmXAs36wPm8+Is= github.com/litmuschaos/litmus-go v0.0.0-20230605073551-d73728198577 h1:8+3Ucb0VEY9F441vlweT1UEb1/4wbYxMXxGLad0It9M= github.com/litmuschaos/litmus-go v0.0.0-20230605073551-d73728198577/go.mod h1:/88zb8T8g75lMSqt6LjXdCEYCxPAWVCbViymkPcWbjg= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -865,13 +870,15 @@ k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= @@ -880,8 +887,8 @@ sigs.k8s.io/controller-runtime v0.10.0 h1:HgyZmMpjUOrtkaFtCnfxsR1bGRuFoAczSNbn2M sigs.k8s.io/controller-runtime v0.10.0/go.mod h1:GCdh6kqV6IY4LK0JLwX0Zm6g233RtVGdb/f0+KSfprg= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/clients/clientset.go b/pkg/clients/clientset.go index a60aa8f4..d544f4a1 100644 --- a/pkg/clients/clientset.go +++ b/pkg/clients/clientset.go @@ -2,8 +2,7 @@ package clients import ( "flag" - - chaosClient "github.com/litmuschaos/chaos-operator/pkg/client/clientset/versioned/typed/litmuschaos/v1alpha1" + clientv1alpha1 "github.com/litmuschaos/chaos-operator/pkg/client/clientset/versioned" "github.com/pkg/errors" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -12,8 +11,8 @@ import ( // ClientSets is a collection of clientSets and kubeConfig needed type ClientSets struct { - KubeClient *kubernetes.Clientset - LitmusClient *chaosClient.LitmuschaosV1alpha1Client + KubeClient kubernetes.Interface + LitmusClient clientv1alpha1.Interface KubeConfig *rest.Config } @@ -57,8 +56,8 @@ func GenerateK8sClientSet(config *rest.Config) (*kubernetes.Clientset, error) { } // GenerateLitmusClientSet will generate a LitmusClient -func GenerateLitmusClientSet(config *rest.Config) (*chaosClient.LitmuschaosV1alpha1Client, error) { - litmusClientSet, err := chaosClient.NewForConfig(config) +func GenerateLitmusClientSet(config *rest.Config) (*clientv1alpha1.Clientset, error) { + litmusClientSet, err := clientv1alpha1.NewForConfig(config) if err != nil { return nil, errors.Wrapf(err, "Unable to create LitmusClientSet, err: %v", err) } diff --git a/tests/bdd/bdd_test.go b/tests/bdd/bdd_test.go index 8924405d..184b3adf 100644 --- a/tests/bdd/bdd_test.go +++ b/tests/bdd/bdd_test.go @@ -21,6 +21,7 @@ import ( "flag" "fmt" "io/ioutil" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "net/http" "os" "os/exec" @@ -31,16 +32,16 @@ import ( "github.com/litmuschaos/litmus-go/pkg/utils/retry" "github.com/pkg/errors" - "github.com/litmuschaos/chaos-exporter/pkg/clients" - "github.com/litmuschaos/chaos-exporter/pkg/log" - chaosClient "github.com/litmuschaos/chaos-operator/pkg/client/clientset/versioned/typed/litmuschaos/v1alpha1" + clientv1alpha1 "github.com/litmuschaos/chaos-operator/pkg/client/clientset/versioned" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" appv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/litmuschaos/chaos-exporter/pkg/clients" + "github.com/litmuschaos/chaos-exporter/pkg/log" + "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/tools/clientcmd" @@ -72,7 +73,7 @@ var _ = BeforeSuite(func() { client.KubeClient, err = kubernetes.NewForConfig(config) Expect(err).To(BeNil(), "failed to generate k8sClientSet") - client.LitmusClient, err = chaosClient.NewForConfig(config) + client.LitmusClient, err = clientv1alpha1.NewForConfig(config) Expect(err).To(BeNil(), "failed to generate litmusClientSet") By("Installing Litmus") @@ -203,7 +204,7 @@ var _ = BeforeSuite(func() { }, } - _, err = client.LitmusClient.ChaosEngines("litmus").Create(context.Background(), chaosEngine, metav1.CreateOptions{}) + _, err = client.LitmusClient.LitmuschaosV1alpha1().ChaosEngines("litmus").Create(context.Background(), chaosEngine, metav1.CreateOptions{}) Expect(err).To( BeNil(), "while building ChaosEngine engine-nginx in namespace litmus", @@ -262,7 +263,7 @@ var _ = Describe("BDD on chaos-exporter", func() { By("Waiting for verdict to be Pass in ChaosResult") deadline := time.Now().Add(3 * time.Minute) for { - chaosengine, err := client.LitmusClient.ChaosResults("litmus").Get(context.Background(), "engine-nginx-pod-delete", metav1.GetOptions{}) + chaosengine, err := client.LitmusClient.LitmuschaosV1alpha1().ChaosResults("litmus").Get(context.Background(), "engine-nginx-pod-delete", metav1.GetOptions{}) time.Sleep(1 * time.Second) if time.Now().After(deadline) { fmt.Printf(`Timeout while waiting for verdict in the chaos result to be "Pass" \n`)