From 2b857b1875c3dae65d5cb852e1759beb7581d421 Mon Sep 17 00:00:00 2001 From: Jianrong Zhang Date: Thu, 12 Sep 2024 22:22:45 -0400 Subject: [PATCH] [issue-368] knative integration with DataIndex and JobService: fix workflow deletion hanging issu --- Makefile | 10 +++-- controllers/knative/monitoring.go | 5 ++- controllers/profiles/common/monitoring.go | 26 ++++++------ .../profiles/common/object_creators.go | 41 ++++++++++++++++--- main.go | 2 + operator.yaml | 2 +- .../{prometheus.yaml => monitoring.yaml} | 12 ++++++ 7 files changed, 74 insertions(+), 24 deletions(-) rename test/testdata/{prometheus.yaml => monitoring.yaml} (83%) diff --git a/Makefile b/Makefile index 2e8a167de..14dd7968b 100644 --- a/Makefile +++ b/Makefile @@ -236,6 +236,7 @@ KIND_VERSION ?= v0.20.0 KNATIVE_VERSION ?= v1.13.2 TIMEOUT_SECS ?= 180s PROMETHEUS_VERSION ?= v0.70.0 +GRAFANA_VERSION ?= v5.13.0 KNATIVE_SERVING_PREFIX ?= "https://github.com/knative/serving/releases/download/knative-$(KNATIVE_VERSION)" KNATIVE_EVENTING_PREFIX ?= "https://github.com/knative/eventing/releases/download/knative-$(KNATIVE_VERSION)" @@ -367,12 +368,15 @@ deploy-knative: create-cluster kubectl wait --for=condition=Ready=True KnativeServing/knative-serving -n knative-serving --timeout=$(TIMEOUT_SECS) kubectl wait --for=condition=Ready=True KnativeEventing/knative-eventing -n knative-eventing --timeout=$(TIMEOUT_SECS) -.PHONY: deploy-prometheus -deploy-prometheus: create-cluster +.PHONY: deploy-monitoring +deploy-monitoring: create-cluster kubectl create -f https://github.com/prometheus-operator/prometheus-operator/releases/download/$(PROMETHEUS_VERSION)/bundle.yaml kubectl wait --for=condition=Available=True deploy/prometheus-operator -n default --timeout=$(TIMEOUT_SECS) - kubectl apply -f ./test/testdata/prometheus.yaml + kubectl create -f https://github.com/grafana/grafana-operator/releases/$(GRAFANA_VERSION)/download/kustomize-cluster_scoped.yaml + kubectl wait --for=condition=Available=True deploy/grafana-operator-controller-manager -n grafana --timeout=$(TIMEOUT_SECS) + kubectl apply -f ./test/testdata/monitoring.yaml kubectl wait --for=condition=Available=True prometheus/prometheus -n default --timeout=$(TIMEOUT_SECS) + kubectl wait --for=condition=Available=True deploy/grafana-deployment -n default --timeout=$(TIMEOUT_SECS) .PHONY: delete-cluster delete-cluster: install-kind diff --git a/controllers/knative/monitoring.go b/controllers/knative/monitoring.go index 7e5c3f819..b35dd2815 100644 --- a/controllers/knative/monitoring.go +++ b/controllers/knative/monitoring.go @@ -29,7 +29,8 @@ type MonitoringAvailability struct { } const ( - prometheusGroup = "prometheuses.monitoring.coreos.com" + prometheusGroup = "monitoring.coreos.com" + grafanaGroup = "grafana.integreatly.org" ) func GetMonitoringAvailability(cfg *rest.Config) (*MonitoringAvailability, error) { @@ -45,7 +46,7 @@ func GetMonitoringAvailability(cfg *rest.Config) (*MonitoringAvailability, error if group.Name == prometheusGroup { result.Prometheus = true } - if group.Name == knativeEventingGroup { + if group.Name == grafanaGroup { result.Grafana = true } } diff --git a/controllers/profiles/common/monitoring.go b/controllers/profiles/common/monitoring.go index 4e7d85914..f45c9fb85 100644 --- a/controllers/profiles/common/monitoring.go +++ b/controllers/profiles/common/monitoring.go @@ -34,6 +34,7 @@ type monitoringObjectManager struct { func NewMonitoringEventingHandler(support *StateSupport) MonitoringEventingHandler { return &monitoringObjectManager{ serviceMonitor: NewObjectEnsurer(support.C, ServiceMonitorCreator), + dataSource: NewObjectEnsurer(support.C, DataSourceCreator), StateSupport: support, } } @@ -60,17 +61,18 @@ func (k monitoringObjectManager) Ensure(ctx context.Context, workflow *operatora } else if serviceMonitor != nil { objs = append(objs, serviceMonitor) } - /* - triggers := k.trigger.Ensure(ctx, workflow) - for _, trigger := range triggers { - if trigger.Error != nil { - return objs, trigger.Error - } - objs = append(objs, trigger.Object) - } - } - */ - return objs, nil } - return nil, nil + + if !MonitoringAvail.Grafana { + klog.V(log.I).InfoS("Grafana is not installed") + } else { + // create grafana data source + dataSource, _, err := k.dataSource.Ensure(ctx, workflow) + if err != nil { + return objs, err + } else if dataSource != nil { + objs = append(objs, dataSource) + } + } + return objs, nil } diff --git a/controllers/profiles/common/object_creators.go b/controllers/profiles/common/object_creators.go index 7593f5d67..53b821377 100644 --- a/controllers/profiles/common/object_creators.go +++ b/controllers/profiles/common/object_creators.go @@ -29,7 +29,8 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/workflowdef" servingv1 "knative.dev/serving/pkg/apis/serving/v1" - monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + grafana "github.com/grafana/grafana-operator/v5/api/v1beta1" + prometheus "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" cncfmodel "github.com/serverlessworkflow/sdk-go/v2/model" "github.com/imdario/mergo" @@ -452,26 +453,54 @@ func ManagedPropsConfigMapCreator(workflow *operatorapi.SonataFlow, platform *op } // ServiceMonitorCreator is an ObjectsCreator for Service Monitor for the workflow service. -// It will create v1.SinkBinding based on events defined in workflow. func ServiceMonitorCreator(workflow *operatorapi.SonataFlow) (client.Object, error) { lbl := workflowproj.GetMergedLabels(workflow) // subject must be deployment to inject K_SINK, service won't work - serviceMonitor := &monv1.ServiceMonitor{ + serviceMonitor := &prometheus.ServiceMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: workflow.Name, + Namespace: workflow.Namespace, + Labels: lbl, + }, + Spec: prometheus.ServiceMonitorSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + workflowproj.LabelWorkflow: workflow.Name, + workflowproj.LabelWorkflowNamespace: workflow.Namespace, + }, + }, + Endpoints: []prometheus.Endpoint{ + prometheus.Endpoint{ + Port: k8sServicePortName, + Path: k8sServicePortPath, + }, + }, + }, + } + return serviceMonitor, nil +} + +// DataSourceCreator is an ObjectsCreator for the grafana data source for the workflow service. +func DataSourceCreator(workflow *operatorapi.SonataFlow) (client.Object, error) { + lbl := workflowproj.GetMergedLabels(workflow) + + // subject must be deployment to inject K_SINK, service won't work + serviceMonitor := &grafana.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ Name: workflow.Name, Namespace: workflow.Namespace, Labels: lbl, }, - Spec: monv1.ServiceMonitorSpec{ + Spec: prometheus.ServiceMonitorSpec{ Selector: metav1.LabelSelector{ MatchLabels: map[string]string{ workflowproj.LabelWorkflow: workflow.Name, workflowproj.LabelWorkflowNamespace: workflow.Namespace, }, }, - Endpoints: []monv1.Endpoint{ - monv1.Endpoint{ + Endpoints: []prometheus.Endpoint{ + prometheus.Endpoint{ Port: k8sServicePortName, Path: k8sServicePortPath, }, diff --git a/main.go b/main.go index dc290e0ab..a85f62cdc 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,7 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + monv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -63,6 +64,7 @@ func init() { utilruntime.Must(sourcesv1.AddToScheme(scheme)) utilruntime.Must(eventingv1.AddToScheme(scheme)) utilruntime.Must(servingv1.AddToScheme(scheme)) + utilruntime.Must(monv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/operator.yaml b/operator.yaml index 29c606177..40643949d 100644 --- a/operator.yaml +++ b/operator.yaml @@ -27409,7 +27409,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/apache/incubator-kie-sonataflow-operator:latest + image: quay.io/jianrzha/kogito-serverless-operator:0.0.104 livenessProbe: httpGet: path: /healthz diff --git a/test/testdata/prometheus.yaml b/test/testdata/monitoring.yaml similarity index 83% rename from test/testdata/prometheus.yaml rename to test/testdata/monitoring.yaml index 39b5b6645..fe239b33c 100644 --- a/test/testdata/prometheus.yaml +++ b/test/testdata/monitoring.yaml @@ -11,6 +11,18 @@ spec: requests: memory: 400Mi --- +apiVersion: grafana.integreatly.org/v1beta1 +kind: Grafana +metadata: + name: grafana + labels: + dashboards: "grafana" +spec: + config: + security: + admin_user: root + admin_password: secret +--- apiVersion: v1 kind: ServiceAccount metadata: