[issue-368] knative integration with DataIndex and JobService (#467)

.github/workflows/e2e.yml

@@ -16,6 +16,7 @@ env:
   GO_VERSION: 1.21
   PYTHON_VERSION: "3.10"
   KIND_VERSION: v0.20.0
+  KNATIVE_VERSION: v1.13.2
   OPERATOR_IMAGE_NAME: "127.0.0.1:5001/kogito-serverless-operator:0.0.1"
 
 jobs:
@@ -60,7 +61,10 @@ jobs:
           cekit --version
 
       - name: Setup Kind cluster
-        run: make KIND_VERSION=${{ env.KIND_VERSION }} create-cluster
+        run: make KIND_VERSION=${{ env.KIND_VERSION }} BUILDER=docker create-cluster
+
+      - name: Deploy Knative Eventing and Serving
+        run: make KNATIVE_VERSION=${{ env.KNATIVE_VERSION }} deploy-knative
 
       - name: Set OPERATOR_IMAGE_NAME to point to Kind's local registry
         run: echo "OPERATOR_IMAGE_NAME=${{ env.OPERATOR_IMAGE_NAME }}" >> $GITHUB_ENV

.gitignore

@@ -27,7 +27,9 @@ Dockerfile
 /.idea/
 /.vscode/
 /target/
+/__debug*
+database/
 
 database/index.db
 e2e-test-report*.xml
-*.tar
+*.tar

Makefile

@@ -233,7 +233,11 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
 KUSTOMIZE_VERSION ?= v4.5.2
 CONTROLLER_TOOLS_VERSION ?= v0.9.2
 KIND_VERSION ?= v0.20.0
+KNATIVE_VERSION ?= v1.13.2
+TIMEOUT_SECS ?= 180s
 
+KNATIVE_SERVING_PREFIX ?= "https://github.com/knative/serving/releases/download/knative-$(KNATIVE_VERSION)"
+KNATIVE_EVENTING_PREFIX ?= "https://github.com/knative/eventing/releases/download/knative-$(KNATIVE_VERSION)"
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -352,8 +356,16 @@ install-kind:
 
 .PHONY: create-cluster
 create-cluster: install-kind
-	kind get clusters | grep kind >/dev/null || ./hack/ci/create-kind-cluster-with-registry.sh
-
+	kind get clusters | grep kind >/dev/null || ./hack/ci/create-kind-cluster-with-registry.sh $(BUILDER)
+
+.PHONY: deploy-knative
+deploy-knative: create-cluster
+	kubectl apply -f https://github.com/knative/operator/releases/download/knative-$(KNATIVE_VERSION)/operator.yaml
+	kubectl wait  --for=condition=Available=True deploy/knative-operator -n default --timeout=$(TIMEOUT_SECS)
+	kubectl apply -f ./test/testdata/knative_serving_eventing.yaml
+	kubectl wait  --for=condition=Ready=True KnativeServing/knative-serving -n knative-serving --timeout=$(TIMEOUT_SECS)
+	kubectl wait  --for=condition=Ready=True KnativeEventing/knative-eventing -n knative-eventing --timeout=$(TIMEOUT_SECS)
+
 .PHONY: delete-cluster
 delete-cluster: install-kind
-	kind delete cluster && docker rm -f kind-registry
+	kind delete cluster && $(BUILDER) rm -f kind-registry

api/v1alpha08/sonataflow_types.go

@@ -161,6 +161,18 @@ type SonataFlowSpec struct {
 	// Sink describes the sinkBinding details of this SonataFlow instance.
 	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="sink"
 	Sink *duckv1.Destination `json:"sink,omitempty"`
+	// Sources describes the list of sources used to create triggers for events consumed by this SonataFlow instance.
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="sources"
+	Sources []SonataFlowSourceSpec `json:"sources,omitempty"`
+}
+
+// SonataFlowSourceSpec defines the desired state of a source used for trigger creation
+// +k8s:openapi-gen=true
+type SonataFlowSourceSpec struct {
+	// Defines the eventType to filter the events
+	EventType string `json:"eventType"`
+	// Defines the broker used
+	duckv1.Destination `json:",inline"`
 }
 
 // SonataFlowStatus defines the observed state of SonataFlow
@@ -185,6 +197,19 @@ type SonataFlowStatus struct {
 	// Platform displays which platform is being used by this workflow
 	//+operator-sdk:csv:customresourcedefinitions:type=status,displayName="platform"
 	Platform *SonataFlowPlatformRef `json:"platform,omitempty"`
+	// Triggers list of triggers created for the SonataFlow
+	//+operator-sdk:csv:customresourcedefinitions:type=status,displayName="triggers"
+	Triggers []SonataFlowTriggerRef `json:"triggers,omitempty"`
+}
+
+// SonataFlowTriggerRef defines a trigger created for the SonataFlow.
+type SonataFlowTriggerRef struct {
+	// Name of the Trigger
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Trigger_Name"
+	Name string `json:"name"`
+	// Namespace of the Trigger
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Trigger_NS"
+	Namespace string `json:"namespace"`
 }
 
 func (s *SonataFlowStatus) GetTopLevelConditionType() api.ConditionType {

api/v1alpha08/sonataflowplatform_services_types.go

@@ -14,14 +14,41 @@
 
 package v1alpha08
 
+import (
+	duckv1 "knative.dev/pkg/apis/duck/v1"
+)
+
 // ServicesPlatformSpec describes the desired service configuration for workflows without the `sonataflow.org/profile: dev` annotation.
 type ServicesPlatformSpec struct {
 	// Deploys the Data Index service for use by workflows without the `sonataflow.org/profile: dev` annotation.
 	// +optional
-	DataIndex *ServiceSpec `json:"dataIndex,omitempty"`
+	DataIndex *DataIndexServiceSpec `json:"dataIndex,omitempty"`
 	// Deploys the Job service for use by workflows without the `sonataflow.org/profile: dev` annotation.
 	// +optional
-	JobService *ServiceSpec `json:"jobService,omitempty"`
+	JobService *JobServiceServiceSpec `json:"jobService,omitempty"`
+}
+
+// DataIndexServiceSpec defines the desired state of Dataindex service
+// +k8s:openapi-gen=true
+type DataIndexServiceSpec struct {
+	// Defines the common spec of a platform service
+	ServiceSpec `json:",inline"`
+	// Defines the source where the Dataindex receives events from
+	// +optional
+	Source *duckv1.Destination `json:"source,omitempty"`
+}
+
+// JobServiceServiceSpec defines the desired state of Jobservice service
+// +k8s:openapi-gen=true
+type JobServiceServiceSpec struct {
+	// Defines the common spec of a platform service
+	ServiceSpec `json:",inline"`
+	// Defines the sink where the Jobservice sends events to
+	// +optional
+	Sink *duckv1.Destination `json:"sink,omitempty"`
+	// Defines the source where the Jobservice receives events from
+	// +optional
+	Source *duckv1.Destination `json:"source,omitempty"`
 }
 
 // ServiceSpec defines the desired state of a platform service

api/v1alpha08/sonataflowplatform_types.go

@@ -21,6 +21,7 @@ package v1alpha08
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	duckv1 "knative.dev/pkg/apis/duck/v1"
 
 	"github.com/apache/incubator-kie-kogito-serverless-operator/api"
 )
@@ -47,6 +48,9 @@ type SonataFlowPlatformSpec struct {
 	// +optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Services"
 	Services *ServicesPlatformSpec `json:"services,omitempty"`
+	// Eventing describes the information required for Knative Eventing integration in the platform.
+	// +optional
+	Eventing *PlatformEventingSpec `json:"eventing,omitempty"`
 	// Persistence defines the platform persistence configuration. When this field is set,
 	// the configuration is used as the persistence for platform services and SonataFlow instances
 	// that don't provide one of their own.
@@ -61,6 +65,15 @@ type SonataFlowPlatformSpec struct {
 	Properties *PropertyPlatformSpec `json:"properties,omitempty"`
 }
 
+// PlatformEventingSpec specifies the Knative Eventing integration details in the platform.
+// +k8s:openapi-gen=true
+type PlatformEventingSpec struct {
+	// Broker to communicate with workflow deployment.  It can be the default broker when the workflow, Dataindex, or Jobservice does not have a sink or source specified.
+	// +optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="broker"
+	Broker *duckv1.Destination `json:"broker,omitempty"`
+}
+
 // PlatformCluster is the kind of orchestration cluster the platform is installed into
 // +kubebuilder:validation:Enum=kubernetes;openshift
 type PlatformCluster string
@@ -95,6 +108,19 @@ type SonataFlowPlatformStatus struct {
 	// ClusterPlatformRef information related to the (optional) active SonataFlowClusterPlatform
 	//+operator-sdk:csv:customresourcedefinitions:type=status,displayName="clusterPlatformRef"
 	ClusterPlatformRef *SonataFlowClusterPlatformRefStatus `json:"clusterPlatformRef,omitempty"`
+	// Triggers list of triggers created for the SonataFlowPlatform
+	//+operator-sdk:csv:customresourcedefinitions:type=status,displayName="triggers"
+	Triggers []SonataFlowPlatformTriggerRef `json:"triggers,omitempty"`
+}
+
+// SonataFlowPlatformTriggerRef defines a trigger created for the SonataFlowPlatform.
+type SonataFlowPlatformTriggerRef struct {
+	// Name of the Trigger
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Trigger_Name"
+	Name string `json:"name"`
+	// Namespace of the Trigger
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Trigger_NS"
+	Namespace string `json:"namespace"`
 }
 
 // SonataFlowClusterPlatformRefStatus information related to the (optional) active SonataFlowClusterPlatform