From 2350a9556b2c4ec9f13e76838ec5aca442f8f7a9 Mon Sep 17 00:00:00 2001 From: Jianrong Zhang Date: Tue, 13 Aug 2024 12:18:28 -0400 Subject: [PATCH] [issue-368] knative integration with DataIndex and JobService --- .github/workflows/e2e.yml | 6 +- .gitignore | 4 + Makefile | 20 +- api/v1alpha08/sonataflow_types.go | 25 ++ .../sonataflowplatform_services_types.go | 31 +- api/v1alpha08/sonataflowplatform_types.go | 26 ++ api/v1alpha08/zz_generated.deepcopy.go | 139 ++++++- ...taflow-operator.clusterserviceversion.yaml | 15 + .../sonataflow.org_sonataflowplatforms.yaml | 221 +++++++++++ .../manifests/sonataflow.org_sonataflows.yaml | 74 ++++ .../sonataflow.org_sonataflowplatforms.yaml | 221 +++++++++++ .../crd/bases/sonataflow.org_sonataflows.yaml | 74 ++++ ...taflow-operator.clusterserviceversion.yaml | 15 + controllers/knative/knative.go | 210 ++++++++++- controllers/platform/k8s.go | 114 +++++- controllers/platform/services/properties.go | 32 +- .../services/properties_services_test.go | 12 +- controllers/platform/services/services.go | 351 +++++++++++++++--- .../common/constants/platform_services.go | 36 +- .../profiles/common/constants/workflows.go | 2 +- controllers/profiles/common/ensurer.go | 71 +++- .../profiles/common/knative_eventing.go | 31 +- .../profiles/common/mutate_visitors.go | 36 +- .../profiles/common/object_creators.go | 151 ++++++-- .../profiles/common/object_creators_test.go | 208 +++++++++-- .../profiles/common/properties/knative.go | 17 +- .../profiles/common/properties/managed.go | 8 +- .../common/properties/managed_test.go | 43 ++- .../profiles/dev/object_creators_dev.go | 3 +- .../profiles/dev/object_creators_dev_test.go | 12 +- controllers/profiles/dev/profile_dev_test.go | 16 +- controllers/profiles/dev/states_dev.go | 8 +- controllers/profiles/factory/factory.go | 1 + .../profiles/gitops/profile_gitops_test.go | 18 +- .../profiles/preview/deployment_handler.go | 41 +- .../preview/deployment_handler_test.go | 5 + .../preview/object_creators_preview.go | 2 +- .../profiles/preview/profile_preview_test.go | 19 +- .../profiles/preview/states_preview.go | 13 +- controllers/sonataflow_controller.go | 54 ++- controllers/sonataflowplatform_controller.go | 43 ++- .../sonataflowplatform_controller_test.go | 280 ++++++++++++-- go.mod | 2 +- hack/ci/create-kind-cluster-with-registry.sh | 30 +- operator.yaml | 295 +++++++++++++++ test/e2e/clusterplatform_test.go | 8 +- test/e2e/helpers.go | 111 +++++- test/e2e/platform_test.go | 205 +++++++++- test/e2e/workflow_test.go | 53 ++- test/kubernetes_cli.go | 8 +- test/testdata/knative_default_broker.yaml | 65 ++++ test/testdata/knative_serving_eventing.yaml | 42 +++ .../ephemeral-with-workflow/00-broker.yaml | 20 + .../02-sonataflow_platform.yaml | 6 + .../kustomization.yaml | 1 + .../services/preview/ephemeral/00-broker.yaml | 20 + .../ephemeral/02-sonataflow_platform.yaml | 6 + .../preview/ephemeral/kustomization.yaml | 1 + .../platform-level-broker/01-postgres.yaml | 86 +++++ .../02-sonataflow_platform.yaml | 75 ++++ .../platform-level-broker/broker/broker.yaml | 22 ++ .../platform-level-broker/kustomization.yaml | 32 ++ ...4-sonataflow_callbackstatetimeouts.sw.yaml | 81 ++++ .../service-level-broker/01-postgres.yaml | 86 +++++ .../02-sonataflow_platform.yaml | 89 +++++ .../broker/00-broker.yaml | 47 +++ .../service-level-broker/kustomization.yaml | 32 ++ .../preview/postgreSQL/00-broker.yaml | 20 + .../postgreSQL/02-sonataflow_platform.yaml | 6 + .../preview/postgreSQL/kustomization.yaml | 1 + ...ow.org_v1alpha08_sonataflow_vet_event.yaml | 13 + ...alpha08_sonataflowplatform_withBroker.yaml | 48 +++ .../00-broker.yaml | 20 + .../02-sonataflow_platform.yaml | 6 + .../kustomization.yaml | 1 + test/yaml.go | 41 +- utils/kubernetes/volumes.go | 27 +- utils/kubernetes/volumes_test.go | 2 +- version/version.go | 2 +- workflowproj/operator.go | 16 +- workflowproj/workflowproj_test.go | 10 +- 81 files changed, 4030 insertions(+), 314 deletions(-) create mode 100644 test/testdata/knative_default_broker.yaml create mode 100644 test/testdata/knative_serving_eventing.yaml create mode 100644 test/testdata/platform/services/preview/ephemeral-with-workflow/00-broker.yaml create mode 100644 test/testdata/platform/services/preview/ephemeral/00-broker.yaml create mode 100644 test/testdata/platform/services/preview/knative/platform-level-broker/01-postgres.yaml create mode 100644 test/testdata/platform/services/preview/knative/platform-level-broker/02-sonataflow_platform.yaml create mode 100644 test/testdata/platform/services/preview/knative/platform-level-broker/broker/broker.yaml create mode 100644 test/testdata/platform/services/preview/knative/platform-level-broker/kustomization.yaml create mode 100644 test/testdata/platform/services/preview/knative/platform-level-broker/sonataflow/04-sonataflow_callbackstatetimeouts.sw.yaml create mode 100644 test/testdata/platform/services/preview/knative/service-level-broker/01-postgres.yaml create mode 100644 test/testdata/platform/services/preview/knative/service-level-broker/02-sonataflow_platform.yaml create mode 100644 test/testdata/platform/services/preview/knative/service-level-broker/broker/00-broker.yaml create mode 100644 test/testdata/platform/services/preview/knative/service-level-broker/kustomization.yaml create mode 100644 test/testdata/platform/services/preview/postgreSQL/00-broker.yaml create mode 100644 test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_withBroker.yaml create mode 100644 test/testdata/workflow/persistence/from_platform_with_di_and_js_services/00-broker.yaml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 491b9ca37..d126fecc6 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,6 +16,7 @@ env: GO_VERSION: 1.21 PYTHON_VERSION: "3.10" KIND_VERSION: v0.20.0 + KNATIVE_VERSION: v1.13.2 OPERATOR_IMAGE_NAME: "127.0.0.1:5001/kogito-serverless-operator:0.0.1" jobs: @@ -60,7 +61,10 @@ jobs: cekit --version - name: Setup Kind cluster - run: make KIND_VERSION=${{ env.KIND_VERSION }} create-cluster + run: make KIND_VERSION=${{ env.KIND_VERSION }} BUILDER=docker create-cluster + + - name: Deploy Knative Eventing and Serving + run: make KNATIVE_VERSION=${{ env.KNATIVE_VERSION }} deploy-knative - name: Set OPERATOR_IMAGE_NAME to point to Kind's local registry run: echo "OPERATOR_IMAGE_NAME=${{ env.OPERATOR_IMAGE_NAME }}" >> $GITHUB_ENV diff --git a/.gitignore b/.gitignore index d56122269..d4b7227d1 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,10 @@ Dockerfile /.idea/ /.vscode/ /target/ +/__debug* +database/ database/index.db e2e-test-report.xml + +bundle_tmp* \ No newline at end of file diff --git a/Makefile b/Makefile index f88a8759e..c2f6a30c9 100644 --- a/Makefile +++ b/Makefile @@ -233,7 +233,11 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest KUSTOMIZE_VERSION ?= v4.5.2 CONTROLLER_TOOLS_VERSION ?= v0.9.2 KIND_VERSION ?= v0.20.0 +KNATIVE_VERSION ?= v1.13.2 +TIMEOUT_SECS ?= 180s +KNATIVE_SERVING_PREFIX ?= "https://github.com/knative/serving/releases/download/knative-$(KNATIVE_VERSION)" +KNATIVE_EVENTING_PREFIX ?= "https://github.com/knative/eventing/releases/download/knative-$(KNATIVE_VERSION)" KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. @@ -338,7 +342,7 @@ generate-all: generate generate-deploy bundle addheaders vet fmt .PHONY: test-e2e # You will need to have a Minikube/Kind cluster up in running to run this target, and run container-builder before the test test-e2e: - go test ./test/e2e/* -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report.xml -timeout 60m + go test ./test/e2e/* -v -ginkgo.v -ginkgo.no-color -ginkgo.timeout=90m -ginkgo.junit-report=./e2e-test-report.xml -timeout 90m .PHONY: before-pr before-pr: test generate-all @@ -350,8 +354,16 @@ install-kind: .PHONY: create-cluster create-cluster: install-kind - kind get clusters | grep kind >/dev/null || ./hack/ci/create-kind-cluster-with-registry.sh - + kind get clusters | grep kind >/dev/null || ./hack/ci/create-kind-cluster-with-registry.sh $(BUILDER) + +.PHONY: deploy-knative +deploy-knative: create-cluster + kubectl apply -f https://github.com/knative/operator/releases/download/knative-$(KNATIVE_VERSION)/operator.yaml + kubectl wait --for=condition=Available=True deploy/knative-operator -n default --timeout=$(TIMEOUT_SECS) + kubectl apply -f ./test/testdata/knative_serving_eventing.yaml + kubectl wait --for=condition=Ready=True KnativeServing/knative-serving -n knative-serving --timeout=$(TIMEOUT_SECS) + kubectl wait --for=condition=Ready=True KnativeEventing/knative-eventing -n knative-eventing --timeout=$(TIMEOUT_SECS) + .PHONY: delete-cluster delete-cluster: install-kind - kind delete cluster && docker rm -f kind-registry + kind delete cluster && $(BUILDER) rm -f kind-registry diff --git a/api/v1alpha08/sonataflow_types.go b/api/v1alpha08/sonataflow_types.go index 66f8e3605..01452cc49 100644 --- a/api/v1alpha08/sonataflow_types.go +++ b/api/v1alpha08/sonataflow_types.go @@ -161,6 +161,18 @@ type SonataFlowSpec struct { // Sink describes the sinkBinding details of this SonataFlow instance. //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="sink" Sink *duckv1.Destination `json:"sink,omitempty"` + // Sources describes the list of sources used to create triggers for events consumed by this SonataFlow instance. + //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="sources" + Sources []SonataFlowSourceSpec `json:"sources,omitempty"` +} + +// SonataFlowSourceSpec defines the desired state of a source used for trigger creation +// +k8s:openapi-gen=true +type SonataFlowSourceSpec struct { + // Defines the eventType to filter the events + EventType string `json:"eventType"` + // Defines the broker used + duckv1.Destination `json:",inline"` } // SonataFlowStatus defines the observed state of SonataFlow @@ -185,6 +197,19 @@ type SonataFlowStatus struct { // Platform displays which platform is being used by this workflow //+operator-sdk:csv:customresourcedefinitions:type=status,displayName="platform" Platform *SonataFlowPlatformRef `json:"platform,omitempty"` + // Triggers list of triggers created for the SonataFlow + //+operator-sdk:csv:customresourcedefinitions:type=status,displayName="triggers" + Triggers []SonataFlowTriggerRef `json:"triggers,omitempty"` +} + +// SonataFlowTriggerRef defines a trigger created for the SonataFlow. +type SonataFlowTriggerRef struct { + // Name of the Trigger + //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Trigger_Name" + Name string `json:"name"` + // Namespace of the Trigger + //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Trigger_NS" + Namespace string `json:"namespace"` } func (s *SonataFlowStatus) GetTopLevelConditionType() api.ConditionType { diff --git a/api/v1alpha08/sonataflowplatform_services_types.go b/api/v1alpha08/sonataflowplatform_services_types.go index a16bc2782..229bb2faf 100644 --- a/api/v1alpha08/sonataflowplatform_services_types.go +++ b/api/v1alpha08/sonataflowplatform_services_types.go @@ -14,14 +14,41 @@ package v1alpha08 +import ( + duckv1 "knative.dev/pkg/apis/duck/v1" +) + // ServicesPlatformSpec describes the desired service configuration for workflows without the `sonataflow.org/profile: dev` annotation. type ServicesPlatformSpec struct { // Deploys the Data Index service for use by workflows without the `sonataflow.org/profile: dev` annotation. // +optional - DataIndex *ServiceSpec `json:"dataIndex,omitempty"` + DataIndex *DataIndexServiceSpec `json:"dataIndex,omitempty"` // Deploys the Job service for use by workflows without the `sonataflow.org/profile: dev` annotation. // +optional - JobService *ServiceSpec `json:"jobService,omitempty"` + JobService *JobServiceServiceSpec `json:"jobService,omitempty"` +} + +// DataIndexServiceSpec defines the desired state of Dataindex service +// +k8s:openapi-gen=true +type DataIndexServiceSpec struct { + // Defines the common spec of a platform service + ServiceSpec `json:",inline"` + // Defines the source where the Dataindex receives events from + // +optional + Source *duckv1.Destination `json:"source,omitempty"` +} + +// JobServiceServiceSpec defines the desired state of Jobservice service +// +k8s:openapi-gen=true +type JobServiceServiceSpec struct { + // Defines the common spec of a platform service + ServiceSpec `json:",inline"` + // Defines the sink where the Jobservice sends events to + // +optional + Sink *duckv1.Destination `json:"sink,omitempty"` + // Defines the source where the Jobservice receives events from + // +optional + Source *duckv1.Destination `json:"source,omitempty"` } // ServiceSpec defines the desired state of a platform service diff --git a/api/v1alpha08/sonataflowplatform_types.go b/api/v1alpha08/sonataflowplatform_types.go index b9f42c824..65dcf059f 100644 --- a/api/v1alpha08/sonataflowplatform_types.go +++ b/api/v1alpha08/sonataflowplatform_types.go @@ -21,6 +21,7 @@ package v1alpha08 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + duckv1 "knative.dev/pkg/apis/duck/v1" "github.com/apache/incubator-kie-kogito-serverless-operator/api" ) @@ -47,6 +48,9 @@ type SonataFlowPlatformSpec struct { // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Services" Services *ServicesPlatformSpec `json:"services,omitempty"` + // Eventing describes the information required for Knative Eventing integration in the platform. + // +optional + Eventing *PlatformEventingSpec `json:"eventing,omitempty"` // Persistence defines the platform persistence configuration. When this field is set, // the configuration is used as the persistence for platform services and SonataFlow instances // that don't provide one of their own. @@ -61,6 +65,15 @@ type SonataFlowPlatformSpec struct { Properties *PropertyPlatformSpec `json:"properties,omitempty"` } +// PlatformEventingSpec specifies the Knative Eventing integration details in the platform. +// +k8s:openapi-gen=true +type PlatformEventingSpec struct { + // Broker to communicate with workflow deployment. It can be the default broker when the workflow, Dataindex, or Jobservice does not have a sink or source specified. + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="broker" + Broker *duckv1.Destination `json:"broker,omitempty"` +} + // PlatformCluster is the kind of orchestration cluster the platform is installed into // +kubebuilder:validation:Enum=kubernetes;openshift type PlatformCluster string @@ -95,6 +108,19 @@ type SonataFlowPlatformStatus struct { // ClusterPlatformRef information related to the (optional) active SonataFlowClusterPlatform //+operator-sdk:csv:customresourcedefinitions:type=status,displayName="clusterPlatformRef" ClusterPlatformRef *SonataFlowClusterPlatformRefStatus `json:"clusterPlatformRef,omitempty"` + // Triggers list of triggers created for the SonataFlowPlatform + //+operator-sdk:csv:customresourcedefinitions:type=status,displayName="triggers" + Triggers []SonataFlowPlatformTriggerRef `json:"triggers,omitempty"` +} + +// SonataFlowPlatformTriggerRef defines a trigger created for the SonataFlowPlatform. +type SonataFlowPlatformTriggerRef struct { + // Name of the Trigger + //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Trigger_Name" + Name string `json:"name"` + // Namespace of the Trigger + //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Trigger_NS" + Namespace string `json:"namespace"` } // SonataFlowClusterPlatformRefStatus information related to the (optional) active SonataFlowClusterPlatform diff --git a/api/v1alpha08/zz_generated.deepcopy.go b/api/v1alpha08/zz_generated.deepcopy.go index b11518893..8924a44eb 100644 --- a/api/v1alpha08/zz_generated.deepcopy.go +++ b/api/v1alpha08/zz_generated.deepcopy.go @@ -213,6 +213,27 @@ func (in *ContainerSpec) DeepCopy() *ContainerSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataIndexServiceSpec) DeepCopyInto(out *DataIndexServiceSpec) { + *out = *in + in.ServiceSpec.DeepCopyInto(&out.ServiceSpec) + if in.Source != nil { + in, out := &in.Source, &out.Source + *out = new(duckv1.Destination) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataIndexServiceSpec. +func (in *DataIndexServiceSpec) DeepCopy() *DataIndexServiceSpec { + if in == nil { + return nil + } + out := new(DataIndexServiceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DevModePlatformSpec) DeepCopyInto(out *DevModePlatformSpec) { *out = *in @@ -342,6 +363,32 @@ func (in *FlowPodTemplateSpec) DeepCopy() *FlowPodTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JobServiceServiceSpec) DeepCopyInto(out *JobServiceServiceSpec) { + *out = *in + in.ServiceSpec.DeepCopyInto(&out.ServiceSpec) + if in.Sink != nil { + in, out := &in.Sink, &out.Sink + *out = new(duckv1.Destination) + (*in).DeepCopyInto(*out) + } + if in.Source != nil { + in, out := &in.Source, &out.Source + *out = new(duckv1.Destination) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobServiceServiceSpec. +func (in *JobServiceServiceSpec) DeepCopy() *JobServiceServiceSpec { + if in == nil { + return nil + } + out := new(JobServiceServiceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PersistenceOptionsSpec) DeepCopyInto(out *PersistenceOptionsSpec) { *out = *in @@ -383,6 +430,26 @@ func (in *PersistencePostgreSQL) DeepCopy() *PersistencePostgreSQL { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlatformEventingSpec) DeepCopyInto(out *PlatformEventingSpec) { + *out = *in + if in.Broker != nil { + in, out := &in.Broker, &out.Broker + *out = new(duckv1.Destination) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlatformEventingSpec. +func (in *PlatformEventingSpec) DeepCopy() *PlatformEventingSpec { + if in == nil { + return nil + } + out := new(PlatformEventingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PlatformPersistenceOptionsSpec) DeepCopyInto(out *PlatformPersistenceOptionsSpec) { *out = *in @@ -817,12 +884,12 @@ func (in *ServicesPlatformSpec) DeepCopyInto(out *ServicesPlatformSpec) { *out = *in if in.DataIndex != nil { in, out := &in.DataIndex, &out.DataIndex - *out = new(ServiceSpec) + *out = new(DataIndexServiceSpec) (*in).DeepCopyInto(*out) } if in.JobService != nil { in, out := &in.JobService, &out.JobService - *out = new(ServiceSpec) + *out = new(JobServiceServiceSpec) (*in).DeepCopyInto(*out) } } @@ -1208,6 +1275,11 @@ func (in *SonataFlowPlatformSpec) DeepCopyInto(out *SonataFlowPlatformSpec) { *out = new(ServicesPlatformSpec) (*in).DeepCopyInto(*out) } + if in.Eventing != nil { + in, out := &in.Eventing, &out.Eventing + *out = new(PlatformEventingSpec) + (*in).DeepCopyInto(*out) + } if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(PlatformPersistenceOptionsSpec) @@ -1246,6 +1318,11 @@ func (in *SonataFlowPlatformStatus) DeepCopyInto(out *SonataFlowPlatformStatus) *out = new(SonataFlowClusterPlatformRefStatus) (*in).DeepCopyInto(*out) } + if in.Triggers != nil { + in, out := &in.Triggers, &out.Triggers + *out = make([]SonataFlowPlatformTriggerRef, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SonataFlowPlatformStatus. @@ -1258,6 +1335,37 @@ func (in *SonataFlowPlatformStatus) DeepCopy() *SonataFlowPlatformStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SonataFlowPlatformTriggerRef) DeepCopyInto(out *SonataFlowPlatformTriggerRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SonataFlowPlatformTriggerRef. +func (in *SonataFlowPlatformTriggerRef) DeepCopy() *SonataFlowPlatformTriggerRef { + if in == nil { + return nil + } + out := new(SonataFlowPlatformTriggerRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SonataFlowSourceSpec) DeepCopyInto(out *SonataFlowSourceSpec) { + *out = *in + in.Destination.DeepCopyInto(&out.Destination) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SonataFlowSourceSpec. +func (in *SonataFlowSourceSpec) DeepCopy() *SonataFlowSourceSpec { + if in == nil { + return nil + } + out := new(SonataFlowSourceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SonataFlowSpec) DeepCopyInto(out *SonataFlowSpec) { *out = *in @@ -1274,6 +1382,13 @@ func (in *SonataFlowSpec) DeepCopyInto(out *SonataFlowSpec) { *out = new(duckv1.Destination) (*in).DeepCopyInto(*out) } + if in.Sources != nil { + in, out := &in.Sources, &out.Sources + *out = make([]SonataFlowSourceSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SonataFlowSpec. @@ -1307,6 +1422,11 @@ func (in *SonataFlowStatus) DeepCopyInto(out *SonataFlowStatus) { *out = new(SonataFlowPlatformRef) **out = **in } + if in.Triggers != nil { + in, out := &in.Triggers, &out.Triggers + *out = make([]SonataFlowTriggerRef, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SonataFlowStatus. @@ -1319,6 +1439,21 @@ func (in *SonataFlowStatus) DeepCopy() *SonataFlowStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SonataFlowTriggerRef) DeepCopyInto(out *SonataFlowTriggerRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SonataFlowTriggerRef. +func (in *SonataFlowTriggerRef) DeepCopy() *SonataFlowTriggerRef { + if in == nil { + return nil + } + out := new(SonataFlowTriggerRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkflowResources) DeepCopyInto(out *WorkflowResources) { *out = *in diff --git a/bundle/manifests/sonataflow-operator.clusterserviceversion.yaml b/bundle/manifests/sonataflow-operator.clusterserviceversion.yaml index e581164c6..5926784db 100644 --- a/bundle/manifests/sonataflow-operator.clusterserviceversion.yaml +++ b/bundle/manifests/sonataflow-operator.clusterserviceversion.yaml @@ -248,6 +248,11 @@ spec: no build required) displayName: DevMode path: devMode + - description: Broker to communicate with workflow deployment. It can be the + default broker when the workflow, Dataindex, or Jobservice does not have + a sink or source specified. + displayName: broker + path: eventing.broker - description: 'Services attributes for deploying supporting applications like Data Index & Job Service. Only workflows without the `sonataflow.org/profile: dev` annotation will be configured to use these service(s). Setting this @@ -275,6 +280,9 @@ spec: - description: Info generic information related to the build displayName: info path: info + - description: Triggers list of triggers created for the SonataFlowPlatform + displayName: triggers + path: triggers - description: Version the operator version controlling this Platform displayName: version path: version @@ -318,6 +326,10 @@ spec: - description: Sink describes the sinkBinding details of this SonataFlow instance. displayName: sink path: sink + - description: Sources describes the list of sources used to create triggers + for events consumed by this SonataFlow instance. + displayName: sources + path: sources statusDescriptors: - description: Address is used as a part of Addressable interface (status.address.url) for knative @@ -339,6 +351,9 @@ spec: workflow displayName: services path: services + - description: Triggers list of triggers created for the SonataFlow + displayName: triggers + path: triggers version: v1alpha08 description: |- SonataFlow Kubernetes Operator for deploying workflow applications diff --git a/bundle/manifests/sonataflow.org_sonataflowplatforms.yaml b/bundle/manifests/sonataflow.org_sonataflowplatforms.yaml index 09a2f2544..39dd5084e 100644 --- a/bundle/manifests/sonataflow.org_sonataflowplatforms.yaml +++ b/bundle/manifests/sonataflow.org_sonataflowplatforms.yaml @@ -420,6 +420,60 @@ spec: of the operator's default. type: string type: object + eventing: + description: Eventing describes the information required for Knative + Eventing integration in the platform. + properties: + broker: + description: Broker to communicate with workflow deployment. It + can be the default broker when the workflow, Dataindex, or Jobservice + does not have a sink or source specified. + properties: + CACerts: + description: CACerts are Certification Authority (CA) certificates + in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version of + the group. This can be used as an alternative to the + APIVersion, and then resolved using ResolveGroup. Note: + This API is EXPERIMENTAL and might break anytime. For + more details: https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the object + holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme and + non-empty host) pointing to the target or a relative URI. + Relative URIs will be resolved using the base URI retrieved + from Ref. + type: string + type: object + type: object persistence: description: Persistence defines the platform persistence configuration. When this field is set, the configuration is used as the persistence @@ -8427,6 +8481,56 @@ spec: type: object type: array type: object + source: + description: Defines the source where the Dataindex receives + events from + properties: + CACerts: + description: CACerts are Certification Authority (CA) + certificates in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address + Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version + of the group. This can be used as an alternative + to the APIVersion, and then resolved using ResolveGroup. + Note: This API is EXPERIMENTAL and might break anytime. + For more details: https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the + object holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme + and non-empty host) pointing to the target or a relative + URI. Relative URIs will be resolved using the base URI + retrieved from Ref. + type: string + type: object type: object jobService: description: 'Deploys the Job service for use by workflows without @@ -16299,6 +16403,106 @@ spec: type: object type: array type: object + sink: + description: Defines the sink where the Jobservice sends events + to + properties: + CACerts: + description: CACerts are Certification Authority (CA) + certificates in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address + Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version + of the group. This can be used as an alternative + to the APIVersion, and then resolved using ResolveGroup. + Note: This API is EXPERIMENTAL and might break anytime. + For more details: https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the + object holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme + and non-empty host) pointing to the target or a relative + URI. Relative URIs will be resolved using the base URI + retrieved from Ref. + type: string + type: object + source: + description: Defines the source where the Jobservice receives + events from + properties: + CACerts: + description: CACerts are Certification Authority (CA) + certificates in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address + Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version + of the group. This can be used as an alternative + to the APIVersion, and then resolved using ResolveGroup. + Note: This API is EXPERIMENTAL and might break anytime. + For more details: https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the + object holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme + and non-empty host) pointing to the target or a relative + URI. Relative URIs will be resolved using the base URI + retrieved from Ref. + type: string + type: object type: object type: object type: object @@ -16393,6 +16597,23 @@ spec: description: The generation observed by the deployment controller. format: int64 type: integer + triggers: + description: Triggers list of triggers created for the SonataFlowPlatform + items: + description: SonataFlowPlatformTriggerRef defines a trigger created + for the SonataFlowPlatform. + properties: + name: + description: Name of the Trigger + type: string + namespace: + description: Namespace of the Trigger + type: string + required: + - name + - namespace + type: object + type: array version: description: Version the operator version controlling this Platform type: string diff --git a/bundle/manifests/sonataflow.org_sonataflows.yaml b/bundle/manifests/sonataflow.org_sonataflows.yaml index 5ff75cae2..e0efa2b57 100644 --- a/bundle/manifests/sonataflow.org_sonataflows.yaml +++ b/bundle/manifests/sonataflow.org_sonataflows.yaml @@ -9402,6 +9402,63 @@ spec: will be resolved using the base URI retrieved from Ref. type: string type: object + sources: + description: Sources describes the list of sources used to create + triggers for events consumed by this SonataFlow instance. + items: + description: SonataFlowSourceSpec defines the desired state of a + source used for trigger creation + properties: + CACerts: + description: CACerts are Certification Authority (CA) certificates + in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + eventType: + description: Defines the eventType to filter the events + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version of the + group. This can be used as an alternative to the APIVersion, + and then resolved using ResolveGroup. Note: This API is + EXPERIMENTAL and might break anytime. For more details: + https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the object + holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme and + non-empty host) pointing to the target or a relative URI. + Relative URIs will be resolved using the base URI retrieved + from Ref. + type: string + required: + - eventType + type: object + type: array required: - flow type: object @@ -9500,6 +9557,23 @@ spec: type: string type: object type: object + triggers: + description: Triggers list of triggers created for the SonataFlow + items: + description: SonataFlowTriggerRef defines a trigger created for + the SonataFlow. + properties: + name: + description: Name of the Trigger + type: string + namespace: + description: Namespace of the Trigger + type: string + required: + - name + - namespace + type: object + type: array type: object type: object served: true diff --git a/config/crd/bases/sonataflow.org_sonataflowplatforms.yaml b/config/crd/bases/sonataflow.org_sonataflowplatforms.yaml index 648b1bcde..3146f621e 100644 --- a/config/crd/bases/sonataflow.org_sonataflowplatforms.yaml +++ b/config/crd/bases/sonataflow.org_sonataflowplatforms.yaml @@ -421,6 +421,60 @@ spec: of the operator's default. type: string type: object + eventing: + description: Eventing describes the information required for Knative + Eventing integration in the platform. + properties: + broker: + description: Broker to communicate with workflow deployment. It + can be the default broker when the workflow, Dataindex, or Jobservice + does not have a sink or source specified. + properties: + CACerts: + description: CACerts are Certification Authority (CA) certificates + in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version of + the group. This can be used as an alternative to the + APIVersion, and then resolved using ResolveGroup. Note: + This API is EXPERIMENTAL and might break anytime. For + more details: https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the object + holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme and + non-empty host) pointing to the target or a relative URI. + Relative URIs will be resolved using the base URI retrieved + from Ref. + type: string + type: object + type: object persistence: description: Persistence defines the platform persistence configuration. When this field is set, the configuration is used as the persistence @@ -8428,6 +8482,56 @@ spec: type: object type: array type: object + source: + description: Defines the source where the Dataindex receives + events from + properties: + CACerts: + description: CACerts are Certification Authority (CA) + certificates in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address + Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version + of the group. This can be used as an alternative + to the APIVersion, and then resolved using ResolveGroup. + Note: This API is EXPERIMENTAL and might break anytime. + For more details: https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the + object holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme + and non-empty host) pointing to the target or a relative + URI. Relative URIs will be resolved using the base URI + retrieved from Ref. + type: string + type: object type: object jobService: description: 'Deploys the Job service for use by workflows without @@ -16300,6 +16404,106 @@ spec: type: object type: array type: object + sink: + description: Defines the sink where the Jobservice sends events + to + properties: + CACerts: + description: CACerts are Certification Authority (CA) + certificates in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address + Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version + of the group. This can be used as an alternative + to the APIVersion, and then resolved using ResolveGroup. + Note: This API is EXPERIMENTAL and might break anytime. + For more details: https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the + object holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme + and non-empty host) pointing to the target or a relative + URI. Relative URIs will be resolved using the base URI + retrieved from Ref. + type: string + type: object + source: + description: Defines the source where the Jobservice receives + events from + properties: + CACerts: + description: CACerts are Certification Authority (CA) + certificates in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address + Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version + of the group. This can be used as an alternative + to the APIVersion, and then resolved using ResolveGroup. + Note: This API is EXPERIMENTAL and might break anytime. + For more details: https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the + object holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme + and non-empty host) pointing to the target or a relative + URI. Relative URIs will be resolved using the base URI + retrieved from Ref. + type: string + type: object type: object type: object type: object @@ -16394,6 +16598,23 @@ spec: description: The generation observed by the deployment controller. format: int64 type: integer + triggers: + description: Triggers list of triggers created for the SonataFlowPlatform + items: + description: SonataFlowPlatformTriggerRef defines a trigger created + for the SonataFlowPlatform. + properties: + name: + description: Name of the Trigger + type: string + namespace: + description: Namespace of the Trigger + type: string + required: + - name + - namespace + type: object + type: array version: description: Version the operator version controlling this Platform type: string diff --git a/config/crd/bases/sonataflow.org_sonataflows.yaml b/config/crd/bases/sonataflow.org_sonataflows.yaml index 7ee4675cf..c2965c608 100644 --- a/config/crd/bases/sonataflow.org_sonataflows.yaml +++ b/config/crd/bases/sonataflow.org_sonataflows.yaml @@ -9403,6 +9403,63 @@ spec: will be resolved using the base URI retrieved from Ref. type: string type: object + sources: + description: Sources describes the list of sources used to create + triggers for events consumed by this SonataFlow instance. + items: + description: SonataFlowSourceSpec defines the desired state of a + source used for trigger creation + properties: + CACerts: + description: CACerts are Certification Authority (CA) certificates + in PEM format according to https://www.rfc-editor.org/rfc/rfc7468. + If set, these CAs are appended to the set of CAs provided + by the Addressable target, if any. + type: string + eventType: + description: Defines the eventType to filter the events + type: string + ref: + description: Ref points to an Addressable. + properties: + address: + description: Address points to a specific Address Name. + type: string + apiVersion: + description: API version of the referent. + type: string + group: + description: 'Group of the API, without the version of the + group. This can be used as an alternative to the APIVersion, + and then resolved using ResolveGroup. Note: This API is + EXPERIMENTAL and might break anytime. For more details: + https://github.com/knative/eventing/issues/5086' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + This is optional field, it gets defaulted to the object + holding it if left out.' + type: string + required: + - kind + - name + type: object + uri: + description: URI can be an absolute URL(non-empty scheme and + non-empty host) pointing to the target or a relative URI. + Relative URIs will be resolved using the base URI retrieved + from Ref. + type: string + required: + - eventType + type: object + type: array required: - flow type: object @@ -9501,6 +9558,23 @@ spec: type: string type: object type: object + triggers: + description: Triggers list of triggers created for the SonataFlow + items: + description: SonataFlowTriggerRef defines a trigger created for + the SonataFlow. + properties: + name: + description: Name of the Trigger + type: string + namespace: + description: Namespace of the Trigger + type: string + required: + - name + - namespace + type: object + type: array type: object type: object served: true diff --git a/config/manifests/bases/sonataflow-operator.clusterserviceversion.yaml b/config/manifests/bases/sonataflow-operator.clusterserviceversion.yaml index 094fbfa89..f4316e016 100644 --- a/config/manifests/bases/sonataflow-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/sonataflow-operator.clusterserviceversion.yaml @@ -132,6 +132,11 @@ spec: no build required) displayName: DevMode path: devMode + - description: Broker to communicate with workflow deployment. It can be the + default broker when the workflow, Dataindex, or Jobservice does not have + a sink or source specified. + displayName: broker + path: eventing.broker - description: 'Services attributes for deploying supporting applications like Data Index & Job Service. Only workflows without the `sonataflow.org/profile: dev` annotation will be configured to use these service(s). Setting this @@ -159,6 +164,9 @@ spec: - description: Info generic information related to the build displayName: info path: info + - description: Triggers list of triggers created for the SonataFlowPlatform + displayName: triggers + path: triggers - description: Version the operator version controlling this Platform displayName: version path: version @@ -202,6 +210,10 @@ spec: - description: Sink describes the sinkBinding details of this SonataFlow instance. displayName: sink path: sink + - description: Sources describes the list of sources used to create triggers + for events consumed by this SonataFlow instance. + displayName: sources + path: sources statusDescriptors: - description: Address is used as a part of Addressable interface (status.address.url) for knative @@ -223,6 +235,9 @@ spec: workflow displayName: services path: services + - description: Triggers list of triggers created for the SonataFlow + displayName: triggers + path: triggers version: v1alpha08 description: |- SonataFlow Kubernetes Operator for deploying workflow applications diff --git a/controllers/knative/knative.go b/controllers/knative/knative.go index 929a96cc5..c99942cf8 100644 --- a/controllers/knative/knative.go +++ b/controllers/knative/knative.go @@ -20,20 +20,48 @@ package knative import ( + "context" + "fmt" + + operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/utils" + kubeutil "github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes" + "github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" + sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" clienteventingv1 "knative.dev/eventing/pkg/client/clientset/versioned/typed/eventing/v1" + "knative.dev/pkg/apis" + duckv1 "knative.dev/pkg/apis/duck/v1" clientservingv1 "knative.dev/serving/pkg/client/clientset/versioned/typed/serving/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var servingClient clientservingv1.ServingV1Interface var eventingClient clienteventingv1.EventingV1Interface +var discoveryClient discovery.DiscoveryInterface type Availability struct { Eventing bool Serving bool } +const ( + kSink = "K_SINK" + knativeBundleVolume = "kne-bundle-volume" + kCeOverRides = "K_CE_OVERRIDES" + knativeServingGroup = "serving.knative.dev" + knativeEventingGroup = "eventing.knative.dev" + knativeEventingAPIVersion = "eventing.knative.dev/v1" + knativeBrokerKind = "Broker" + knativeSinkProvided = "SinkProvided" +) + func GetKnativeServingClient(cfg *rest.Config) (clientservingv1.ServingV1Interface, error) { if servingClient == nil { if knServingClient, err := NewKnativeServingClient(cfg); err != nil { @@ -64,8 +92,23 @@ func NewKnativeEventingClient(cfg *rest.Config) (*clienteventingv1.EventingV1Cli return clienteventingv1.NewForConfig(cfg) } +func getDiscoveryClient(cfg *rest.Config) (discovery.DiscoveryInterface, error) { + if discoveryClient == nil { + if cli, err := discovery.NewDiscoveryClientForConfig(cfg); err != nil { + return nil, err + } else { + discoveryClient = cli + } + } + return discoveryClient, nil +} + +func SetDiscoveryClient(cli discovery.DiscoveryInterface) { + discoveryClient = cli +} + func GetKnativeAvailability(cfg *rest.Config) (*Availability, error) { - if cli, err := discovery.NewDiscoveryClientForConfig(cfg); err != nil { + if cli, err := getDiscoveryClient(cfg); err != nil { return nil, err } else { apiList, err := cli.ServerGroups() @@ -74,13 +117,174 @@ func GetKnativeAvailability(cfg *rest.Config) (*Availability, error) { } result := new(Availability) for _, group := range apiList.Groups { - if group.Name == "serving.knative.dev" { + if group.Name == knativeServingGroup { result.Serving = true } - if group.Name == "eventing.knative.dev" { + if group.Name == knativeEventingGroup { result.Eventing = true } } return result, nil } } + +// getRemotePlatform returns the remote platfrom referred by a SonataFlowClusterPlatform +func getRemotePlatform(pl *operatorapi.SonataFlowPlatform) (*operatorapi.SonataFlowPlatform, error) { + if pl.Status.ClusterPlatformRef != nil { + // Find the platform referred by the cluster platform + platform := &operatorapi.SonataFlowPlatform{} + if err := utils.GetClient().Get(context.TODO(), types.NamespacedName{Namespace: pl.Status.ClusterPlatformRef.PlatformRef.Namespace, Name: pl.Status.ClusterPlatformRef.PlatformRef.Name}, platform); err != nil { + return nil, fmt.Errorf("error reading the platform referred by the cluster platform") + } + return platform, nil + } + return nil, nil +} + +func getDestinationWithNamespace(dest *duckv1.Destination, namespace string) *duckv1.Destination { + if dest != nil && dest.Ref != nil && len(dest.Ref.Namespace) == 0 { + dest.Ref.Namespace = namespace + } + return dest +} + +func ValidateBroker(name, namespace string) error { + broker := &eventingv1.Broker{} + if err := utils.GetClient().Get(context.TODO(), types.NamespacedName{Name: name, Namespace: namespace}, broker); err != nil { + if errors.IsNotFound(err) { + return fmt.Errorf("broker %s in namespace %s does not exist", name, namespace) + } + return err + } + cond := broker.Status.GetCondition(apis.ConditionReady) + if cond != nil && cond.Status == corev1.ConditionTrue { + return nil + } + return fmt.Errorf("broker %s in namespace %s is not ready", name, namespace) +} + +func GetWorkflowSink(workflow *operatorapi.SonataFlow, pl *operatorapi.SonataFlowPlatform) (*duckv1.Destination, error) { + if workflow == nil { + return nil, nil + } + if workflow.Spec.Sink != nil { + return getDestinationWithNamespace(workflow.Spec.Sink, workflow.Namespace), nil + } + if pl != nil && pl.Spec.Eventing != nil && pl.Spec.Eventing.Broker != nil { + // no sink defined in the workflow, use the platform broker + return getDestinationWithNamespace(pl.Spec.Eventing.Broker, pl.Namespace), nil + } + // Find the remote platform referred by the cluster platform + platform, err := getRemotePlatform(pl) + if err != nil { + return nil, err + } + if platform != nil && platform.Spec.Eventing != nil && platform.Spec.Eventing.Broker != nil { + return getDestinationWithNamespace(platform.Spec.Eventing.Broker, platform.Namespace), nil + } + return nil, nil +} + +func IsKnativeBroker(kRef *duckv1.KReference) bool { + return kRef.APIVersion == knativeEventingAPIVersion && kRef.Kind == knativeBrokerKind +} + +func SaveKnativeData(dest *corev1.PodSpec, source *corev1.PodSpec) { + for _, volume := range source.Volumes { + if volume.Name == knativeBundleVolume { + kubeutil.AddOrReplaceVolume(dest, volume) + break + } + } + visitContainers(source, func(container *corev1.Container) { + visitContainers(dest, func(destContainer *corev1.Container) { + for _, mount := range container.VolumeMounts { + if mount.Name == knativeBundleVolume { + kubeutil.AddOrReplaceVolumeMount(destContainer, mount) + break + } + } + for _, env := range container.Env { + if env.Name == kSink || env.Name == kCeOverRides { + kubeutil.AddOrReplaceEnvVar(destContainer, env) + } + } + }) + }) +} + +func moveKnativeVolumeToEnd(vols []corev1.Volume) { + for i := 0; i < len(vols)-1; i++ { + if vols[i].Name == knativeBundleVolume { + vols[i], vols[i+1] = vols[i+1], vols[i] + } + } +} + +func moveKnativeVolumeMountToEnd(mounts []corev1.VolumeMount) { + for i := 0; i < len(mounts)-1; i++ { + if mounts[i].Name == knativeBundleVolume { + mounts[i], mounts[i+1] = mounts[i+1], mounts[i] + } + } +} + +// Knative Sinkbinding injects K_SINK env, a volume and volume mount. The volume and volume mount +// must be in the end of the array to avoid repeadly restarting of the workflow pod +func RestoreKnativeVolumeAndVolumeMount(podSpec *corev1.PodSpec) { + moveKnativeVolumeToEnd(podSpec.Volumes) + visitContainers(podSpec, func(container *corev1.Container) { + moveKnativeVolumeMountToEnd(container.VolumeMounts) + }) +} + +// containerVisitor is called with each container +type containerVisitor func(container *corev1.Container) + +// visitContainers invokes the visitor function for every container in the given pod template spec +func visitContainers(podSpec *corev1.PodSpec, visitor containerVisitor) { + for i := range podSpec.InitContainers { + visitor(&podSpec.InitContainers[i]) + } + for i := range podSpec.Containers { + visitor(&podSpec.Containers[i]) + } + for i := range podSpec.EphemeralContainers { + visitor((*corev1.Container)(&podSpec.EphemeralContainers[i].EphemeralContainerCommon)) + } +} + +// if a trigger is changed and it has namespace different from the platform is changed, reconcile the parent SonataFlowPlatform in the cluster. +func MapTriggerToPlatformRequests(ctx context.Context, object client.Object) []reconcile.Request { + if trigger, ok := object.(*eventingv1.Trigger); ok { + nameFound := "" + namespaceFound := "" + for k, v := range trigger.GetLabels() { + if k == workflowproj.LabelApp { + nameFound = v + } else if k == workflowproj.LabelAppNamespace { + namespaceFound = v + } + } + if len(nameFound) > 0 && len(namespaceFound) > 0 && namespaceFound != trigger.Namespace { + return []reconcile.Request{reconcile.Request{NamespacedName: types.NamespacedName{Name: nameFound, Namespace: namespaceFound}}} + } + } + return nil +} + +// Does the sinkbinding completed K_SINK injection? +func CheckKSinkInjected(name, namespace string) (bool, error) { + sb := &sourcesv1.SinkBinding{} + if err := utils.GetClient().Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("%s-sb", name), Namespace: namespace}, sb); err != nil { + if errors.IsNotFound(err) { + return false, nil // deployment hasn't been created yet + } + return false, err + } + cond := sb.Status.GetCondition(apis.ConditionType(knativeSinkProvided)) + if cond != nil && cond.Status == corev1.ConditionTrue { + return true, nil + } + return false, nil // K_SINK has not been injected yet +} diff --git a/controllers/platform/k8s.go b/controllers/platform/k8s.go index f7dc6d16a..60c622200 100644 --- a/controllers/platform/k8s.go +++ b/controllers/platform/k8s.go @@ -21,9 +21,11 @@ package platform import ( "context" + "fmt" operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" "github.com/apache/incubator-kie-kogito-serverless-operator/container-builder/client" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform/services" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/variables" @@ -31,10 +33,13 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/utils" kubeutil "github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes" "github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj" + "github.com/imdario/mergo" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" + sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -85,7 +90,10 @@ func createOrUpdateServiceComponents(ctx context.Context, client client.Client, if err := createOrUpdateDeployment(ctx, client, platform, psh); err != nil { return err } - return createOrUpdateService(ctx, client, platform, psh) + if err := createOrUpdateService(ctx, client, platform, psh); err != nil { + return err + } + return createOrUpdateKnativeResources(ctx, client, platform, psh) } func createOrUpdateDeployment(ctx context.Context, client client.Client, platform *operatorapi.SonataFlowPlatform, psh services.PlatformServiceHandler) error { @@ -137,6 +145,13 @@ func createOrUpdateDeployment(ctx context.Context, client client.Client, platfor serviceContainer.Name = psh.GetContainerName() replicas := psh.GetReplicaCount() + kSinkInjected, err := psh.CheckKSinkInjected() + if err != nil { + return nil + } + if !kSinkInjected { + replicas = 0 // Wait for K_SINK injection + } lbl, selectorLbl := getLabels(platform, psh) serviceDeploymentSpec := appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -182,15 +197,17 @@ func createOrUpdateDeployment(ctx context.Context, client client.Client, platfor // Create or Update the deployment if op, err := controllerutil.CreateOrUpdate(ctx, client, serviceDeployment, func() error { - serviceDeployment.Spec = serviceDeploymentSpec - + knative.SaveKnativeData(&serviceDeploymentSpec.Template.Spec, &serviceDeployment.Spec.Template.Spec) + err := mergo.Merge(&(serviceDeployment.Spec), serviceDeploymentSpec, mergo.WithOverride) + if err != nil { + return err + } return nil }); err != nil { return err } else { klog.V(log.I).InfoS("Deployment successfully reconciled", "operation", op) } - return nil } @@ -233,6 +250,8 @@ func createOrUpdateService(ctx context.Context, client client.Client, platform * func getLabels(platform *operatorapi.SonataFlowPlatform, psh services.PlatformServiceHandler) (map[string]string, map[string]string) { lbl := map[string]string{ + workflowproj.LabelApp: platform.Name, + workflowproj.LabelAppNamespace: platform.Namespace, workflowproj.LabelService: psh.GetServiceName(), workflowproj.LabelK8SName: psh.GetContainerName(), workflowproj.LabelK8SComponent: psh.GetServiceName(), @@ -251,6 +270,7 @@ func createOrUpdateConfigMap(ctx context.Context, client client.Client, platform return err } lbl, _ := getLabels(platform, psh) + dataStr := handler.Build() configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: psh.GetServiceCmName(), @@ -258,7 +278,7 @@ func createOrUpdateConfigMap(ctx context.Context, client client.Client, platform Labels: lbl, }, Data: map[string]string{ - workflowproj.ApplicationPropertiesFileName: handler.Build(), + workflowproj.ApplicationPropertiesFileName: dataStr, }, } if err := controllerutil.SetControllerReference(platform, configMap, client.Scheme()); err != nil { @@ -267,7 +287,7 @@ func createOrUpdateConfigMap(ctx context.Context, client client.Client, platform // Create or Update the service if op, err := controllerutil.CreateOrUpdate(ctx, client, configMap, func() error { - configMap.Data[workflowproj.ApplicationPropertiesFileName] = handler.WithUserProperties(configMap.Data[workflowproj.ApplicationPropertiesFileName]).Build() + configMap.Data[workflowproj.ApplicationPropertiesFileName] = handler.WithUserProperties(dataStr).Build() return nil }); err != nil { @@ -275,6 +295,88 @@ func createOrUpdateConfigMap(ctx context.Context, client client.Client, platform } else { klog.V(log.I).InfoS("ConfigMap successfully reconciled", "operation", op) } + return nil +} +func setSonataFlowPlatformFinalizer(ctx context.Context, c client.Client, platform *operatorapi.SonataFlowPlatform) error { + if !controllerutil.ContainsFinalizer(platform, constants.TriggerFinalizer) { + controllerutil.AddFinalizer(platform, constants.TriggerFinalizer) + return c.Update(ctx, platform) + } return nil } + +func createOrUpdateKnativeResources(ctx context.Context, client client.Client, platform *operatorapi.SonataFlowPlatform, psh services.PlatformServiceHandler) error { + lbl, _ := getLabels(platform, psh) + objs, err := psh.GenerateKnativeResources(platform, lbl) + if err != nil { + return err + } + // Create or update triggers + for _, obj := range objs { + if triggerDef, ok := obj.(*eventingv1.Trigger); ok { + if platform.Namespace == obj.GetNamespace() { + if err := controllerutil.SetControllerReference(platform, obj, client.Scheme()); err != nil { + return err + } + } else { + // This is for Knative trigger in a different namespace + // Set the finalizer for trigger cleanup when the platform is deleted + if err := setSonataFlowPlatformFinalizer(ctx, client, platform); err != nil { + return err + } + } + trigger := &eventingv1.Trigger{ + ObjectMeta: triggerDef.ObjectMeta, + } + _, err := controllerutil.CreateOrUpdate(ctx, client, trigger, func() error { + trigger.Spec = triggerDef.Spec + return nil + }) + if err != nil { + return err + } + addToSonataFlowPlatformTriggerList(platform, trigger) + } + } + + if err := client.Status().Update(ctx, platform); err != nil { + return err + } + + // Create or update sinkbindings + for _, obj := range objs { + if sbDef, ok := obj.(*sourcesv1.SinkBinding); ok { + if err := controllerutil.SetControllerReference(platform, obj, client.Scheme()); err != nil { + return err + } + sinkBinding := &sourcesv1.SinkBinding{ + ObjectMeta: sbDef.ObjectMeta, + } + _, err = controllerutil.CreateOrUpdate(ctx, client, sinkBinding, func() error { + sinkBinding.Spec = sbDef.Spec + return nil + }) + if err != nil { + return err + } + kSinkInjected, err := psh.CheckKSinkInjected() + if err != nil { + return err + } + if !kSinkInjected { + return fmt.Errorf("waiting for K_SINK injection for %s to complete", psh.GetServiceName()) + } + } + } + return nil +} + +func addToSonataFlowPlatformTriggerList(platform *operatorapi.SonataFlowPlatform, trigger *eventingv1.Trigger) { + for _, t := range platform.Status.Triggers { + if t.Name == trigger.Name && t.Namespace == trigger.Namespace { + return // trigger already exists + } + } + platform.Status.Triggers = append(platform.Status.Triggers, operatorapi.SonataFlowPlatformTriggerRef{Name: trigger.Name, Namespace: trigger.Namespace}) +} diff --git a/controllers/platform/services/properties.go b/controllers/platform/services/properties.go index c1deedcbb..3bbbbf13c 100644 --- a/controllers/platform/services/properties.go +++ b/controllers/platform/services/properties.go @@ -29,6 +29,7 @@ import ( "k8s.io/klog/v2" operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/workflowdef" @@ -161,6 +162,10 @@ func GenerateDataIndexWorkflowProperties(workflow *operatorapi.SonataFlow, platf props := properties.NewProperties() props.Set(constants.KogitoProcessDefinitionsEventsEnabled, "false") props.Set(constants.KogitoProcessInstancesEventsEnabled, "false") + sink, err := knative.GetWorkflowSink(workflow, platform) + if err != nil { + return nil, err + } di := NewDataIndexHandler(platform) if !profiles.IsDevProfile(workflow) && workflow != nil && workflow.Status.Services != nil && workflow.Status.Services.DataIndexRef != nil { serviceBaseUrl := workflow.Status.Services.DataIndexRef.Url @@ -170,8 +175,17 @@ func GenerateDataIndexWorkflowProperties(workflow *operatorapi.SonataFlow, platf props.Set(constants.KogitoProcessDefinitionsEventsErrorsEnabled, "true") props.Set(constants.KogitoDataIndexHealthCheckEnabled, "true") props.Set(constants.KogitoDataIndexURL, serviceBaseUrl) - props.Set(constants.KogitoProcessDefinitionsEventsURL, serviceBaseUrl+constants.KogitoProcessDefinitionsEventsPath) - props.Set(constants.KogitoProcessInstancesEventsURL, serviceBaseUrl+constants.KogitoProcessInstancesEventsPath) + if sink != nil { + props.Set(constants.KogitoProcessDefinitionsEventsConnector, constants.QuarkusHTTP) + props.Set(constants.KogitoProcessInstancesEventsConnector, constants.QuarkusHTTP) + props.Set(constants.KogitoProcessDefinitionsEventsURL, constants.KnativeInjectedEnvVar) + props.Set(constants.KogitoProcessInstancesEventsURL, constants.KnativeInjectedEnvVar) + props.Set(constants.KogitoProcessDefinitionsEventsMethod, constants.Post) + props.Set(constants.KogitoProcessInstancesEventsMethod, constants.Post) + } else { + props.Set(constants.KogitoProcessDefinitionsEventsURL, serviceBaseUrl+constants.KogitoProcessDefinitionsEventsPath) + props.Set(constants.KogitoProcessInstancesEventsURL, serviceBaseUrl+constants.KogitoProcessInstancesEventsPath) + } } } props.Sort() @@ -186,7 +200,11 @@ func GenerateDataIndexWorkflowProperties(workflow *operatorapi.SonataFlow, platf func GenerateJobServiceWorkflowProperties(workflow *operatorapi.SonataFlow, platform *operatorapi.SonataFlowPlatform) (*properties.Properties, error) { props := properties.NewProperties() props.Set(constants.JobServiceRequestEventsConnector, constants.QuarkusHTTP) - props.Set(constants.JobServiceRequestEventsURL, fmt.Sprintf("%s://localhost/v2/jobs/events", constants.JobServiceURLProtocol)) + props.Set(constants.JobServiceRequestEventsURL, fmt.Sprintf("%s://localhost%s", constants.DefaultHTTPProtocol, constants.JobServiceJobEventsPath)) + sink, err := knative.GetWorkflowSink(workflow, platform) + if err != nil { + return nil, err + } js := NewJobServiceHandler(platform) if !profiles.IsDevProfile(workflow) && workflow != nil && workflow.Status.Services != nil && workflow.Status.Services.JobServiceRef != nil { serviceBaseUrl := workflow.Status.Services.JobServiceRef.Url @@ -195,7 +213,13 @@ func GenerateJobServiceWorkflowProperties(workflow *operatorapi.SonataFlow, plat props.Set(constants.KogitoJobServiceHealthCheckEnabled, "true") } props.Set(constants.KogitoJobServiceURL, serviceBaseUrl) - props.Set(constants.JobServiceRequestEventsURL, serviceBaseUrl+constants.JobServiceJobEventsPath) + if sink != nil { + props.Set(constants.JobServiceRequestEventsURL, constants.KnativeInjectedEnvVar) + props.Set(constants.JobServiceRequestEventsConnector, constants.QuarkusHTTP) + props.Set(constants.JobServiceRequestEventsMethod, constants.Post) + } else { + props.Set(constants.JobServiceRequestEventsURL, serviceBaseUrl+constants.JobServiceJobEventsPath) + } } } props.Sort() diff --git a/controllers/platform/services/properties_services_test.go b/controllers/platform/services/properties_services_test.go index 46ea31a44..4e43898bd 100644 --- a/controllers/platform/services/properties_services_test.go +++ b/controllers/platform/services/properties_services_test.go @@ -177,7 +177,7 @@ func setJobServiceEnabledValue(v *bool) plfmOptionFn { p.Spec.Services = &operatorapi.ServicesPlatformSpec{} } if p.Spec.Services.JobService == nil { - p.Spec.Services.JobService = &operatorapi.ServiceSpec{} + p.Spec.Services.JobService = &operatorapi.JobServiceServiceSpec{} } p.Spec.Services.JobService.Enabled = v } @@ -189,7 +189,7 @@ func setDataIndexEnabledValue(v *bool) plfmOptionFn { p.Spec.Services = &operatorapi.ServicesPlatformSpec{} } if p.Spec.Services.DataIndex == nil { - p.Spec.Services.DataIndex = &operatorapi.ServiceSpec{} + p.Spec.Services.DataIndex = &operatorapi.DataIndexServiceSpec{} } p.Spec.Services.DataIndex.Enabled = v } @@ -201,7 +201,7 @@ func emptyDataIndexServiceSpec() plfmOptionFn { p.Spec.Services = &operatorapi.ServicesPlatformSpec{} } if p.Spec.Services.DataIndex == nil { - p.Spec.Services.DataIndex = &operatorapi.ServiceSpec{} + p.Spec.Services.DataIndex = &operatorapi.DataIndexServiceSpec{} } } } @@ -212,7 +212,7 @@ func emptyJobServiceSpec() plfmOptionFn { p.Spec.Services = &operatorapi.ServicesPlatformSpec{} } if p.Spec.Services.JobService == nil { - p.Spec.Services.JobService = &operatorapi.ServiceSpec{} + p.Spec.Services.JobService = &operatorapi.JobServiceServiceSpec{} } } } @@ -235,7 +235,7 @@ func setJobServiceJDBC(jdbc string) plfmOptionFn { p.Spec.Services = &operatorapi.ServicesPlatformSpec{} } if p.Spec.Services.JobService == nil { - p.Spec.Services.JobService = &operatorapi.ServiceSpec{} + p.Spec.Services.JobService = &operatorapi.JobServiceServiceSpec{} } if p.Spec.Services.JobService.Persistence == nil { p.Spec.Services.JobService.Persistence = &operatorapi.PersistenceOptionsSpec{} @@ -253,7 +253,7 @@ func setDataIndexJDBC(jdbc string) plfmOptionFn { p.Spec.Services = &operatorapi.ServicesPlatformSpec{} } if p.Spec.Services.DataIndex == nil { - p.Spec.Services.DataIndex = &operatorapi.ServiceSpec{} + p.Spec.Services.DataIndex = &operatorapi.DataIndexServiceSpec{} } if p.Spec.Services.DataIndex.Persistence == nil { p.Spec.Services.DataIndex.Persistence = &operatorapi.PersistenceOptionsSpec{} diff --git a/controllers/platform/services/services.go b/controllers/platform/services/services.go index 58e068daa..5df568373 100644 --- a/controllers/platform/services/services.go +++ b/controllers/platform/services/services.go @@ -23,18 +23,27 @@ import ( "fmt" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/cfg" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles" "github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes" + "github.com/imdario/mergo" + "github.com/magiconair/properties" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" + sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" + "knative.dev/pkg/apis" + duckv1 "knative.dev/pkg/apis/duck/v1" + "knative.dev/pkg/kmeta" + "knative.dev/pkg/tracker" + "sigs.k8s.io/controller-runtime/pkg/client" operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" - "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/persistence" - "github.com/magiconair/properties" - "github.com/apache/incubator-kie-kogito-serverless-operator/version" - "github.com/imdario/mergo" ) const ( @@ -73,6 +82,8 @@ type PlatformServiceHandler interface { MergePodSpec(podSpec corev1.PodSpec) (corev1.PodSpec, error) // GenerateServiceProperties returns a property object that contains the application properties required by the service deployment GenerateServiceProperties() (*properties.Properties, error) + // GenerateKnativeResources returns knative resources that bridge between workflow deploys and the service + GenerateKnativeResources(platform *operatorapi.SonataFlowPlatform, lbl map[string]string) ([]client.Object, error) // IsServiceSetInSpec returns true if the service is set in the spec. IsServiceSetInSpec() bool @@ -89,6 +100,11 @@ type PlatformServiceHandler interface { SetServiceUrlInPlatformStatus(clusterRefPlatform *operatorapi.SonataFlowPlatform) // SetServiceUrlInWorkflowStatus sets the service url in a workflow's status. SetServiceUrlInWorkflowStatus(workflow *operatorapi.SonataFlow) + + GetServiceSource() *duckv1.Destination + + // Check if K_SINK has injected for Job Service. No Op for Data Index + CheckKSinkInjected() (bool, error) } type DataIndexHandler struct { @@ -96,10 +112,10 @@ type DataIndexHandler struct { } func NewDataIndexHandler(platform *operatorapi.SonataFlowPlatform) PlatformServiceHandler { - return DataIndexHandler{platform: platform} + return &DataIndexHandler{platform: platform} } -func (d DataIndexHandler) GetContainerName() string { +func (d *DataIndexHandler) GetContainerName() string { return constants.DataIndexServiceName } @@ -114,7 +130,7 @@ func (d DataIndexHandler) GetServiceImageName(persistenceType constants.Persiste return fmt.Sprintf("%s-%s-%s:%s", constants.ImageNamePrefix, constants.DataIndexName, persistenceType.String(), version.GetServiceTagVersion()) } -func (d DataIndexHandler) GetServiceName() string { +func (d *DataIndexHandler) GetServiceName() string { return fmt.Sprintf("%s-%s", d.platform.Name, constants.DataIndexServiceName) } @@ -147,21 +163,21 @@ func (d DataIndexHandler) IsServiceSetInSpec() bool { return isDataIndexSet(d.platform) } -func (d DataIndexHandler) IsServiceEnabledInSpec() bool { +func (d *DataIndexHandler) IsServiceEnabledInSpec() bool { return isDataIndexEnabled(d.platform) } -func (d DataIndexHandler) isServiceEnabledInStatus() bool { +func (d *DataIndexHandler) isServiceEnabledInStatus() bool { return d.platform != nil && d.platform.Status.ClusterPlatformRef != nil && d.platform.Status.ClusterPlatformRef.Services != nil && d.platform.Status.ClusterPlatformRef.Services.DataIndexRef != nil && !isServicesSet(d.platform) } -func (d DataIndexHandler) IsServiceEnabled() bool { +func (d *DataIndexHandler) IsServiceEnabled() bool { return d.IsServiceEnabledInSpec() || d.isServiceEnabledInStatus() } -func (d DataIndexHandler) GetServiceBaseUrl() string { +func (d *DataIndexHandler) GetServiceBaseUrl() string { if d.IsServiceEnabledInSpec() { return d.GetLocalServiceBaseUrl() } @@ -171,11 +187,11 @@ func (d DataIndexHandler) GetServiceBaseUrl() string { return "" } -func (d DataIndexHandler) GetLocalServiceBaseUrl() string { - return GenerateServiceURL(constants.KogitoServiceURLProtocol, d.platform.Namespace, d.GetServiceName()) +func (d *DataIndexHandler) GetLocalServiceBaseUrl() string { + return GenerateServiceURL(constants.DefaultHTTPProtocol, d.platform.Namespace, d.GetServiceName()) } -func (d DataIndexHandler) GetEnvironmentVariables() []corev1.EnvVar { +func (d *DataIndexHandler) GetEnvironmentVariables() []corev1.EnvVar { return []corev1.EnvVar{ { Name: "KOGITO_DATA_INDEX_QUARKUS_PROFILE", @@ -192,7 +208,7 @@ func (d DataIndexHandler) GetEnvironmentVariables() []corev1.EnvVar { } } -func (d DataIndexHandler) GetPodResourceRequirements() corev1.ResourceRequirements { +func (d *DataIndexHandler) GetPodResourceRequirements() corev1.ResourceRequirements { return corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("100m"), @@ -205,20 +221,20 @@ func (d DataIndexHandler) GetPodResourceRequirements() corev1.ResourceRequiremen } } -func (d DataIndexHandler) MergePodSpec(podSpec corev1.PodSpec) (corev1.PodSpec, error) { +func (d *DataIndexHandler) MergePodSpec(podSpec corev1.PodSpec) (corev1.PodSpec, error) { c := podSpec.DeepCopy() err := mergo.Merge(c, d.platform.Spec.Services.DataIndex.PodTemplate.PodSpec.ToPodSpec(), mergo.WithOverride) return *c, err } // hasPostgreSQLConfigured returns true when either the SonataFlow Platform PostgreSQL CR's structure or the one in the Data Index service specification is not nil -func (d DataIndexHandler) hasPostgreSQLConfigured() bool { +func (d *DataIndexHandler) hasPostgreSQLConfigured() bool { return d.IsServiceSetInSpec() && ((d.platform.Spec.Services.DataIndex.Persistence != nil && d.platform.Spec.Services.DataIndex.Persistence.PostgreSQL != nil) || (d.platform.Spec.Persistence != nil && d.platform.Spec.Persistence.PostgreSQL != nil)) } -func (d DataIndexHandler) ConfigurePersistence(containerSpec *corev1.Container) *corev1.Container { +func (d *DataIndexHandler) ConfigurePersistence(containerSpec *corev1.Container) *corev1.Container { if d.hasPostgreSQLConfigured() { p := persistence.RetrievePostgreSQLConfiguration(d.platform.Spec.Services.DataIndex.Persistence, d.platform.Spec.Persistence, d.GetServiceName()) c := containerSpec.DeepCopy() @@ -235,33 +251,44 @@ func (d DataIndexHandler) MergeContainerSpec(containerSpec *corev1.Container) (* return mergeContainerSpec(containerSpec, &d.platform.Spec.Services.DataIndex.PodTemplate.Container) } -func (d DataIndexHandler) GetReplicaCount() int32 { +func (d *DataIndexHandler) GetReplicaCount() int32 { if d.platform.Spec.Services.DataIndex.PodTemplate.Replicas != nil { return *d.platform.Spec.Services.DataIndex.PodTemplate.Replicas } return 1 } -func (d DataIndexHandler) GetServiceCmName() string { +func (d *DataIndexHandler) GetServiceCmName() string { return fmt.Sprintf("%s-props", d.GetServiceName()) } -func (d DataIndexHandler) GenerateServiceProperties() (*properties.Properties, error) { +func (d *DataIndexHandler) GetServiceSource() *duckv1.Destination { + if d.platform.Spec.Services.DataIndex.Source != nil { + return d.platform.Spec.Services.DataIndex.Source + } + return GetPlatformBroker(d.platform) +} + +func (d *DataIndexHandler) GenerateServiceProperties() (*properties.Properties, error) { props := properties.NewProperties() props.Set(constants.KogitoServiceURLProperty, d.GetLocalServiceBaseUrl()) - props.Set(constants.DataIndexKafkaSmallRyeHealthProperty, "false") + props.Set(constants.DataIndexKafkaHealthCheck, "false") return props, nil } +func (d *DataIndexHandler) CheckKSinkInjected() (bool, error) { + return true, nil // No op +} + type JobServiceHandler struct { platform *operatorapi.SonataFlowPlatform } func NewJobServiceHandler(platform *operatorapi.SonataFlowPlatform) PlatformServiceHandler { - return JobServiceHandler{platform: platform} + return &JobServiceHandler{platform: platform} } -func (j JobServiceHandler) GetContainerName() string { +func (j *JobServiceHandler) GetContainerName() string { return constants.JobServiceName } @@ -276,11 +303,11 @@ func (j JobServiceHandler) GetServiceImageName(persistenceType constants.Persist return fmt.Sprintf("%s-%s-%s:%s", constants.ImageNamePrefix, constants.JobServiceName, persistenceType.String(), version.GetServiceTagVersion()) } -func (j JobServiceHandler) GetServiceName() string { +func (j *JobServiceHandler) GetServiceName() string { return fmt.Sprintf("%s-%s", j.platform.Name, constants.JobServiceName) } -func (j JobServiceHandler) GetServiceCmName() string { +func (j *JobServiceHandler) GetServiceCmName() string { return fmt.Sprintf("%s-props", j.GetServiceName()) } @@ -313,21 +340,21 @@ func (j JobServiceHandler) IsServiceSetInSpec() bool { return isJobServiceSet(j.platform) } -func (j JobServiceHandler) IsServiceEnabledInSpec() bool { +func (j *JobServiceHandler) IsServiceEnabledInSpec() bool { return isJobServiceEnabled(j.platform) } -func (j JobServiceHandler) isServiceEnabledInStatus() bool { +func (j *JobServiceHandler) isServiceEnabledInStatus() bool { return j.platform != nil && j.platform.Status.ClusterPlatformRef != nil && j.platform.Status.ClusterPlatformRef.Services != nil && j.platform.Status.ClusterPlatformRef.Services.JobServiceRef != nil && !isServicesSet(j.platform) } -func (j JobServiceHandler) IsServiceEnabled() bool { +func (j *JobServiceHandler) IsServiceEnabled() bool { return j.IsServiceEnabledInSpec() || j.isServiceEnabledInStatus() } -func (j JobServiceHandler) GetServiceBaseUrl() string { +func (j *JobServiceHandler) GetServiceBaseUrl() string { if j.IsServiceEnabledInSpec() { return j.GetLocalServiceBaseUrl() } @@ -337,11 +364,11 @@ func (j JobServiceHandler) GetServiceBaseUrl() string { return "" } -func (j JobServiceHandler) GetLocalServiceBaseUrl() string { - return GenerateServiceURL(constants.JobServiceURLProtocol, j.platform.Namespace, j.GetServiceName()) +func (j *JobServiceHandler) GetLocalServiceBaseUrl() string { + return GenerateServiceURL(constants.DefaultHTTPProtocol, j.platform.Namespace, j.GetServiceName()) } -func (j JobServiceHandler) GetEnvironmentVariables() []corev1.EnvVar { +func (j *JobServiceHandler) GetEnvironmentVariables() []corev1.EnvVar { return []corev1.EnvVar{ { Name: "QUARKUS_HTTP_CORS", @@ -354,7 +381,7 @@ func (j JobServiceHandler) GetEnvironmentVariables() []corev1.EnvVar { } } -func (j JobServiceHandler) GetPodResourceRequirements() corev1.ResourceRequirements { +func (j *JobServiceHandler) GetPodResourceRequirements() corev1.ResourceRequirements { return corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("250m"), @@ -367,7 +394,7 @@ func (j JobServiceHandler) GetPodResourceRequirements() corev1.ResourceRequireme } } -func (j JobServiceHandler) GetReplicaCount() int32 { +func (j *JobServiceHandler) GetReplicaCount() int32 { return 1 } @@ -376,13 +403,14 @@ func (j JobServiceHandler) MergeContainerSpec(containerSpec *corev1.Container) ( } // hasPostgreSQLConfigured returns true when either the SonataFlow Platform PostgreSQL CR's structure or the one in the Job service specification is not nil -func (j JobServiceHandler) hasPostgreSQLConfigured() bool { +func (j *JobServiceHandler) hasPostgreSQLConfigured() bool { return j.IsServiceSetInSpec() && ((j.platform.Spec.Services.JobService.Persistence != nil && j.platform.Spec.Services.JobService.Persistence.PostgreSQL != nil) || (j.platform.Spec.Persistence != nil && j.platform.Spec.Persistence.PostgreSQL != nil)) } -func (j JobServiceHandler) ConfigurePersistence(containerSpec *corev1.Container) *corev1.Container { +func (j *JobServiceHandler) ConfigurePersistence(containerSpec *corev1.Container) *corev1.Container { + if j.hasPostgreSQLConfigured() { c := containerSpec.DeepCopy() c.Image = j.GetServiceImageName(constants.PersistenceTypePostgreSQL) @@ -396,16 +424,22 @@ func (j JobServiceHandler) ConfigurePersistence(containerSpec *corev1.Container) return containerSpec } -func (j JobServiceHandler) MergePodSpec(podSpec corev1.PodSpec) (corev1.PodSpec, error) { +func (j *JobServiceHandler) MergePodSpec(podSpec corev1.PodSpec) (corev1.PodSpec, error) { c := podSpec.DeepCopy() err := mergo.Merge(c, j.platform.Spec.Services.JobService.PodTemplate.PodSpec.ToPodSpec(), mergo.WithOverride) return *c, err } -func (j JobServiceHandler) GenerateServiceProperties() (*properties.Properties, error) { +func (j *JobServiceHandler) GenerateServiceProperties() (*properties.Properties, error) { props := properties.NewProperties() - props.Set(constants.KogitoServiceURLProperty, GenerateServiceURL(constants.KogitoServiceURLProtocol, j.platform.Namespace, j.GetServiceName())) - props.Set(constants.JobServiceKafkaSmallRyeHealthProperty, "false") + + props.Set(constants.KogitoServiceURLProperty, GenerateServiceURL(constants.DefaultHTTPProtocol, j.platform.Namespace, j.GetServiceName())) + if j.GetServiceSource() == nil { + props.Set(constants.JobServiceKSinkInjectionHealthCheck, "false") + } else { + props.Set(constants.JobServiceKSinkInjectionHealthCheck, "true") + } + // add data source reactive URL if j.hasPostgreSQLConfigured() { p := persistence.RetrievePostgreSQLConfiguration(j.platform.Spec.Services.JobService.Persistence, j.platform.Spec.Persistence, j.GetServiceName()) @@ -417,9 +451,15 @@ func (j JobServiceHandler) GenerateServiceProperties() (*properties.Properties, } if isDataIndexEnabled(j.platform) { - di := NewDataIndexHandler(j.platform) props.Set(constants.JobServiceStatusChangeEvents, "true") - props.Set(constants.JobServiceStatusChangeEventsURL, di.GetLocalServiceBaseUrl()+"/jobs") + if j.GetServiceSource() == nil { + di := NewDataIndexHandler(j.platform) + props.Set(constants.JobServiceStatusChangeEventsURL, di.GetLocalServiceBaseUrl()+"/jobs") + } else { + props.Set(constants.JobServiceStatusChangeEventsURL, constants.KnativeInjectedEnvVar) + props.Set(constants.JobServiceStatusChangeEventsConnector, constants.QuarkusHTTP) + props.Set(constants.JobServiceStatusChangeEventsMethod, constants.Post) + } } props.Sort() return props, nil @@ -434,6 +474,20 @@ func SetServiceUrlsInWorkflowStatus(pl *operatorapi.SonataFlowPlatform, workflow tpsJS.SetServiceUrlInWorkflowStatus(workflow) } +func (j *JobServiceHandler) GetServiceSource() *duckv1.Destination { + if j.platform.Spec.Services.JobService.Source != nil { + return j.platform.Spec.Services.JobService.Source + } + return GetPlatformBroker(j.platform) +} + +func (j *JobServiceHandler) GetServiceSink() *duckv1.Destination { + if j.platform.Spec.Services.JobService.Sink != nil { + return j.platform.Spec.Services.JobService.Sink + } + return GetPlatformBroker(j.platform) +} + func isDataIndexEnabled(platform *operatorapi.SonataFlowPlatform) bool { return isDataIndexSet(platform) && platform.Spec.Services.DataIndex.Enabled != nil && *platform.Spec.Services.DataIndex.Enabled @@ -490,3 +544,214 @@ func mergeContainerPreservingEnvVars(dest *corev1.Container, source *corev1.Cont } return nil } + +// GetPlatformBroker gets the default broker for the platform. +func GetPlatformBroker(platform *operatorapi.SonataFlowPlatform) *duckv1.Destination { + if platform != nil && platform.Spec.Eventing != nil && platform.Spec.Eventing.Broker != nil { + return platform.Spec.Eventing.Broker + } + return nil +} + +func (d *DataIndexHandler) GetSourceBroker() *duckv1.Destination { + if d.platform != nil && d.platform.Spec.Services.DataIndex.Source != nil && d.platform.Spec.Services.DataIndex.Source.Ref != nil { + return d.platform.Spec.Services.DataIndex.Source + } + return GetPlatformBroker(d.platform) +} + +func (d *DataIndexHandler) newTrigger(labels map[string]string, brokerName, namespace, serviceName, tag, eventType, path string, platform *operatorapi.SonataFlowPlatform) *eventingv1.Trigger { + return &eventingv1.Trigger{ + ObjectMeta: metav1.ObjectMeta{ + Name: kmeta.ChildName(fmt.Sprintf("data-index-%s-", tag), string(platform.GetUID())), + Namespace: namespace, + Labels: labels, + }, + Spec: eventingv1.TriggerSpec{ + Broker: brokerName, + Filter: &eventingv1.TriggerFilter{ + Attributes: eventingv1.TriggerFilterAttributes{ + "type": eventType, + }, + }, + Subscriber: duckv1.Destination{ + Ref: &duckv1.KReference{ + Name: serviceName, + Namespace: platform.Namespace, + APIVersion: "v1", + Kind: "Service", + }, + URI: &apis.URL{ + Path: path, + }, + }, + }, + } +} +func (d *DataIndexHandler) GenerateKnativeResources(platform *operatorapi.SonataFlowPlatform, lbl map[string]string) ([]client.Object, error) { + broker := d.GetSourceBroker() + if broker == nil || len(broker.Ref.Name) == 0 { + return nil, nil // Nothing to do + } + brokerName := broker.Ref.Name + namespace := broker.Ref.Namespace + if len(namespace) == 0 { + namespace = platform.Namespace + } + if err := knative.ValidateBroker(brokerName, namespace); err != nil { + return nil, err + } + serviceName := d.GetServiceName() + return []client.Object{ + d.newTrigger(lbl, brokerName, namespace, serviceName, "process-error", "ProcessInstanceErrorDataEvent", constants.KogitoProcessInstancesEventsPath, platform), + d.newTrigger(lbl, brokerName, namespace, serviceName, "process-node", "ProcessInstanceNodeDataEvent", constants.KogitoProcessInstancesEventsPath, platform), + d.newTrigger(lbl, brokerName, namespace, serviceName, "process-sla", "ProcessInstanceSLADataEvent", constants.KogitoProcessInstancesEventsPath, platform), + d.newTrigger(lbl, brokerName, namespace, serviceName, "process-state", "ProcessInstanceStateDataEvent", constants.KogitoProcessInstancesEventsPath, platform), + d.newTrigger(lbl, brokerName, namespace, serviceName, "process-variable", "ProcessInstanceVariableDataEvent", constants.KogitoProcessInstancesEventsPath, platform), + d.newTrigger(lbl, brokerName, namespace, serviceName, "process-definition", "ProcessDefinitionEvent", constants.KogitoProcessDefinitionsEventsPath, platform), + d.newTrigger(lbl, brokerName, namespace, serviceName, "jobs", "JobEvent", constants.KogitoJobsPath, platform)}, nil +} + +func (d JobServiceHandler) GetSourceBroker() *duckv1.Destination { + if d.platform.Spec.Services.JobService.Source != nil && d.platform.Spec.Services.JobService.Source.Ref != nil { + return d.platform.Spec.Services.JobService.Source + } + return GetPlatformBroker(d.platform) +} + +func (d JobServiceHandler) GetSink() *duckv1.Destination { + if d.platform.Spec.Services.JobService.Sink != nil { + return d.platform.Spec.Services.JobService.Sink + } + return GetPlatformBroker(d.platform) +} + +func (j *JobServiceHandler) GenerateKnativeResources(platform *operatorapi.SonataFlowPlatform, lbl map[string]string) ([]client.Object, error) { + broker := j.GetSourceBroker() + sink := j.GetSink() + resultObjs := []client.Object{} + + if broker != nil && len(broker.Ref.Name) > 0 { + brokerName := broker.Ref.Name + namespace := broker.Ref.Namespace + if len(namespace) == 0 { + namespace = platform.Namespace + } + if err := knative.ValidateBroker(brokerName, namespace); err != nil { + return nil, err + } + jobCreateTrigger := &eventingv1.Trigger{ + ObjectMeta: metav1.ObjectMeta{ + Name: kmeta.ChildName("jobs-service-create-job-", string(platform.GetUID())), + Namespace: namespace, + Labels: lbl, + }, + Spec: eventingv1.TriggerSpec{ + Broker: brokerName, + Filter: &eventingv1.TriggerFilter{ + Attributes: eventingv1.TriggerFilterAttributes{ + "type": "job.create", + }, + }, + Subscriber: duckv1.Destination{ + Ref: &duckv1.KReference{ + Name: j.GetServiceName(), + Namespace: platform.Namespace, + APIVersion: "v1", + Kind: "Service", + }, + URI: &apis.URL{ + Path: constants.JobServiceJobEventsPath, + }, + }, + }, + } + resultObjs = append(resultObjs, jobCreateTrigger) + jobDeleteTrigger := &eventingv1.Trigger{ + ObjectMeta: metav1.ObjectMeta{ + Name: kmeta.ChildName("jobs-service-delete-job-", string(platform.GetUID())), + Namespace: namespace, + Labels: lbl, + }, + Spec: eventingv1.TriggerSpec{ + Broker: brokerName, + Filter: &eventingv1.TriggerFilter{ + Attributes: eventingv1.TriggerFilterAttributes{ + "type": "job.delete", + }, + }, + Subscriber: duckv1.Destination{ + Ref: &duckv1.KReference{ + Name: j.GetServiceName(), + Namespace: platform.Namespace, + APIVersion: "v1", + Kind: "Service", + }, + URI: &apis.URL{ + Path: constants.JobServiceJobEventsPath, + }, + }, + }, + } + resultObjs = append(resultObjs, jobDeleteTrigger) + } + if sink != nil { + sinkBinding := &sourcesv1.SinkBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-jobs-service-sb", platform.Name), + Namespace: platform.Namespace, + Labels: lbl, + }, + Spec: sourcesv1.SinkBindingSpec{ + SourceSpec: duckv1.SourceSpec{ + Sink: *sink, + }, + BindingSpec: duckv1.BindingSpec{ + Subject: tracker.Reference{ + Name: j.GetServiceName(), + Namespace: platform.Namespace, + APIVersion: "apps/v1", + Kind: "Deployment", + }, + }, + }, + } + resultObjs = append(resultObjs, sinkBinding) + } + return resultObjs, nil +} + +func (j *JobServiceHandler) CheckKSinkInjected() (bool, error) { + if j.GetSink() != nil { //job services has sink configured + return knative.CheckKSinkInjected(j.GetServiceName(), j.platform.Namespace) + } + return true, nil +} + +func IsDataIndexEnabled(plf *operatorapi.SonataFlowPlatform) bool { + if plf.Spec.Services != nil { + if plf.Spec.Services.DataIndex != nil { + return pointer.BoolDeref(plf.Spec.Services.DataIndex.Enabled, false) + } + return false + } + // Check if DataIndex is enabled in the platform status + if plf.Status.ClusterPlatformRef != nil && plf.Status.ClusterPlatformRef.Services != nil && plf.Status.ClusterPlatformRef.Services.DataIndexRef != nil && len(plf.Status.ClusterPlatformRef.Services.DataIndexRef.Url) > 0 { + return true + } + return false +} + +func IsJobServiceEnabled(plf *operatorapi.SonataFlowPlatform) bool { + if plf.Spec.Services != nil { + if plf.Spec.Services.JobService != nil { + return pointer.BoolDeref(plf.Spec.Services.JobService.Enabled, false) + } + return false + } + // Check if JobService is enabled in the platform status + if plf.Status.ClusterPlatformRef != nil && plf.Status.ClusterPlatformRef.Services != nil && plf.Status.ClusterPlatformRef.Services.JobServiceRef != nil && len(plf.Status.ClusterPlatformRef.Services.JobServiceRef.Url) > 0 { + return true + } + return false +} diff --git a/controllers/profiles/common/constants/platform_services.go b/controllers/profiles/common/constants/platform_services.go index 4065d9e92..0dc1fd087 100644 --- a/controllers/profiles/common/constants/platform_services.go +++ b/controllers/profiles/common/constants/platform_services.go @@ -20,40 +20,48 @@ package constants const ( - QuarkusHTTP = "quarkus-http" - + QuarkusHTTP = "quarkus-http" + Post = "POST" + DefaultHTTPProtocol = "http" ConfigMapWorkflowPropsVolumeName = "workflow-properties" - JobServiceRequestEventsURL = "mp.messaging.outgoing.kogito-job-service-job-request-events.url" - JobServiceRequestEventsConnector = "mp.messaging.outgoing.kogito-job-service-job-request-events.connector" - JobServiceStatusChangeEvents = "kogito.jobs-service.http.job-status-change-events" - JobServiceStatusChangeEventsURL = "mp.messaging.outgoing.kogito-job-service-job-status-events-http.url" - JobServiceURLProtocol = "http" - JobServiceDataSourceReactiveURL = "quarkus.datasource.reactive.url" - JobServiceJobEventsPath = "/v2/jobs/events" + JobServiceRequestEventsURL = "mp.messaging.outgoing.kogito-job-service-job-request-events.url" + JobServiceRequestEventsMethod = "mp.messaging.outgoing.kogito-job-service-job-request-events.method" + JobServiceRequestEventsConnector = "mp.messaging.outgoing.kogito-job-service-job-request-events.connector" + JobServiceStatusChangeEvents = "kogito.jobs-service.http.job-status-change-events" + JobServiceStatusChangeEventsURL = "mp.messaging.outgoing.kogito-job-service-job-status-events-http.url" + JobServiceStatusChangeEventsConnector = "mp.messaging.outgoing.kogito-job-service-job-status-events-http.connector" + JobServiceStatusChangeEventsMethod = "mp.messaging.outgoing.kogito-job-service-job-status-events-http.method" + JobServiceDataSourceReactiveURL = "quarkus.datasource.reactive.url" + JobServiceJobEventsPath = "/v2/jobs/events" + KogitoProcessInstancesEventsConnector = "mp.messaging.outgoing.kogito-processinstances-events.connector" + KogitoProcessInstancesEventsMethod = "mp.messaging.outgoing.kogito-processinstances-events.method" KogitoProcessInstancesEventsURL = "mp.messaging.outgoing.kogito-processinstances-events.url" KogitoProcessInstancesEventsEnabled = "kogito.events.processinstances.enabled" KogitoProcessInstancesEventsPath = "/processes" + KogitoProcessDefinitionsEventsConnector = "mp.messaging.outgoing.kogito-processdefinitions-events.connector" + KogitoProcessDefinitionsEventsMethod = "mp.messaging.outgoing.kogito-processdefinitions-events.method" KogitoProcessDefinitionsEventsURL = "mp.messaging.outgoing.kogito-processdefinitions-events.url" KogitoProcessDefinitionsEventsEnabled = "kogito.events.processdefinitions.enabled" KogitoProcessDefinitionsEventsErrorsEnabled = "kogito.events.processdefinitions.errors.propagate" KogitoProcessDefinitionsEventsPath = "/definitions" KogitoUserTasksEventsEnabled = "kogito.events.usertasks.enabled" + KogitoJobsPath = "/jobs" // KogitoDataIndexHealthCheckEnabled configures if a workflow must check for the data index availability as part // of its start health check. KogitoDataIndexHealthCheckEnabled = "kogito.data-index.health-enabled" // KogitoDataIndexURL configures the data index url, this value can be used internally by the workflow. KogitoDataIndexURL = "kogito.data-index.url" + // KogitoJobServiceHealthCheckEnabled configures if a workflow must check for the job service availability as part // of its start health check. KogitoJobServiceHealthCheckEnabled = "kogito.jobs-service.health-enabled" // KogitoJobServiceURL configures the jobs service, this value can be used internally by the workflow. - KogitoJobServiceURL = "kogito.jobs-service.url" - KogitoServiceURLProperty = "kogito.service.url" - KogitoServiceURLProtocol = "http" - DataIndexKafkaSmallRyeHealthProperty = `quarkus.smallrye-health.check."io.quarkus.kafka.client.health.KafkaHealthCheck".enabled` - JobServiceKafkaSmallRyeHealthProperty = `quarkus.smallrye-health.check."org.kie.kogito.jobs.service.messaging.http.health.knative.KSinkInjectionHealthCheck".enabled` + KogitoJobServiceURL = "kogito.jobs-service.url" + KogitoServiceURLProperty = "kogito.service.url" + DataIndexKafkaHealthCheck = `quarkus.smallrye-health.check."io.quarkus.kafka.client.health.KafkaHealthCheck".enabled` + JobServiceKSinkInjectionHealthCheck = `quarkus.smallrye-health.check."org.kie.kogito.jobs.service.messaging.http.health.knative.KSinkInjectionHealthCheck".enabled` DataIndexServiceName = "data-index-service" JobServiceName = "jobs-service" diff --git a/controllers/profiles/common/constants/workflows.go b/controllers/profiles/common/constants/workflows.go index 8087f963b..c771414a0 100644 --- a/controllers/profiles/common/constants/workflows.go +++ b/controllers/profiles/common/constants/workflows.go @@ -22,5 +22,5 @@ const ( KogitoIncomingEventsPath = "mp.messaging.incoming.kogito_incoming_stream.path" KnativeHealthEnabled = "org.kie.kogito.addons.knative.eventing.health-enabled" KnativeInjectedEnvVar = "${K_SINK}" - KnativeEventingBrokerDefault = "default" + TriggerFinalizer = "trigger-deletion" ) diff --git a/controllers/profiles/common/ensurer.go b/controllers/profiles/common/ensurer.go index ad4558d25..7de18dcc4 100644 --- a/controllers/profiles/common/ensurer.go +++ b/controllers/profiles/common/ensurer.go @@ -22,12 +22,13 @@ package common import ( "context" + operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/log" "k8s.io/klog/v2" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" ) var _ ObjectEnsurer = &defaultObjectEnsurer{} @@ -60,7 +61,7 @@ func NewObjectEnsurer(client client.Client, creator ObjectCreator) ObjectEnsurer } } -// NewObjectEnsurerWithPlatform see defaultObjectEnsurerWithPlatform +// NewObjectEnsurerWithPlatform see defaultObjectEnsurerWithPLatform func NewObjectEnsurerWithPlatform(client client.Client, creator ObjectCreatorWithPlatform) ObjectEnsurerWithPlatform { return &defaultObjectEnsurerWithPlatform{ c: client, @@ -97,6 +98,9 @@ func (d *defaultObjectEnsurerWithPlatform) Ensure(ctx context.Context, workflow if err != nil { return nil, result, err } + if object == nil { + return nil, result, nil + } if result, err = controllerutil.CreateOrPatch(ctx, d.c, object, func() error { for _, v := range visitors { @@ -137,6 +141,42 @@ type ObjectEnsurerResult struct { Error error } +// ObjectsEnsurer is an ensurer to apply multiple objects +type ObjectsEnsurerWithPlatform interface { + Ensure(ctx context.Context, workflow *operatorapi.SonataFlow, pl *operatorapi.SonataFlowPlatform, visitors ...MutateVisitor) []ObjectEnsurerResult +} + +func NewObjectsEnsurerWithPlatform(client client.Client, creator ObjectsCreatorWithPlatform) ObjectsEnsurerWithPlatform { + return &defaultObjectsEnsurerWithPlatform{ + c: client, + creator: creator, + } +} + +type defaultObjectsEnsurerWithPlatform struct { + ObjectsEnsurer + c client.Client + creator ObjectsCreatorWithPlatform +} + +func (d *defaultObjectsEnsurerWithPlatform) Ensure(ctx context.Context, workflow *operatorapi.SonataFlow, pl *operatorapi.SonataFlowPlatform, visitors ...MutateVisitor) []ObjectEnsurerResult { + result := controllerutil.OperationResultNone + + objects, err := d.creator(workflow, pl) + if err != nil { + return []ObjectEnsurerResult{{nil, result, err}} + } + var ensureResult []ObjectEnsurerResult + for _, object := range objects { + ensureObject, c, err := ensureObject(ctx, workflow, visitors, result, d.c, object) + ensureResult = append(ensureResult, ObjectEnsurerResult{ensureObject, c, err}) + if err != nil { + return ensureResult + } + } + return ensureResult +} + func NewObjectsEnsurer(client client.Client, creator ObjectsCreator) ObjectsEnsurer { return &defaultObjectsEnsurer{ c: client, @@ -168,6 +208,14 @@ func (d *defaultObjectsEnsurer) Ensure(ctx context.Context, workflow *operatorap return ensureResult } +func setWorkflowFinalizer(ctx context.Context, c client.Client, workflow *operatorapi.SonataFlow) error { + if !controllerutil.ContainsFinalizer(workflow, constants.TriggerFinalizer) { + controllerutil.AddFinalizer(workflow, constants.TriggerFinalizer) + return c.Update(ctx, workflow) + } + return nil +} + func ensureObject(ctx context.Context, workflow *operatorapi.SonataFlow, visitors []MutateVisitor, result controllerutil.OperationResult, c client.Client, object client.Object) (client.Object, controllerutil.OperationResult, error) { if result, err := controllerutil.CreateOrPatch(ctx, c, object, func() error { @@ -176,6 +224,14 @@ func ensureObject(ctx context.Context, workflow *operatorapi.SonataFlow, visitor return visitorErr } } + if trigger, ok := object.(*eventingv1.Trigger); ok { + addToSonataFlowTriggerList(workflow, trigger) + if workflow.Namespace != object.GetNamespace() { + // This is for Knative trigger in a different namespace + // Set the finalizer for trigger cleanup when the workflow is deleted + return setWorkflowFinalizer(ctx, c, workflow) + } + } return controllerutil.SetControllerReference(workflow, object, c.Scheme()) }); err != nil { return nil, result, err @@ -183,3 +239,12 @@ func ensureObject(ctx context.Context, workflow *operatorapi.SonataFlow, visitor klog.V(log.I).InfoS("Object operation finalized", "result", result, "kind", object.GetObjectKind().GroupVersionKind().String(), "name", object.GetName(), "namespace", object.GetNamespace()) return object, result, nil } + +func addToSonataFlowTriggerList(workflow *operatorapi.SonataFlow, trigger *eventingv1.Trigger) { + for _, t := range workflow.Status.Triggers { + if t.Name == trigger.Name && t.Namespace == trigger.Namespace { + return // trigger already exists + } + } + workflow.Status.Triggers = append(workflow.Status.Triggers, operatorapi.SonataFlowTriggerRef{Name: trigger.Name, Namespace: trigger.Namespace}) +} diff --git a/controllers/profiles/common/knative_eventing.go b/controllers/profiles/common/knative_eventing.go index 13f5bc49c..eb80884cc 100644 --- a/controllers/profiles/common/knative_eventing.go +++ b/controllers/profiles/common/knative_eventing.go @@ -17,9 +17,8 @@ package common import ( "context" - "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" - operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/log" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,15 +27,17 @@ import ( var _ KnativeEventingHandler = &knativeObjectManager{} type knativeObjectManager struct { - sinkBinding ObjectEnsurer - trigger ObjectsEnsurer + sinkBinding ObjectEnsurerWithPlatform + trigger ObjectsEnsurerWithPlatform + platform *operatorapi.SonataFlowPlatform *StateSupport } -func NewKnativeEventingHandler(support *StateSupport) KnativeEventingHandler { +func NewKnativeEventingHandler(support *StateSupport, pl *operatorapi.SonataFlowPlatform) KnativeEventingHandler { return &knativeObjectManager{ - sinkBinding: NewObjectEnsurer(support.C, SinkBindingCreator), - trigger: NewObjectsEnsurer(support.C, TriggersCreator), + sinkBinding: NewObjectEnsurerWithPlatform(support.C, SinkBindingCreator), + trigger: NewObjectsEnsurerWithPlatform(support.C, TriggersCreator), + platform: pl, StateSupport: support, } } @@ -48,23 +49,23 @@ type KnativeEventingHandler interface { func (k knativeObjectManager) Ensure(ctx context.Context, workflow *operatorapi.SonataFlow) ([]client.Object, error) { var objs []client.Object - if workflow.Spec.Flow.Events == nil { - // skip if no event is found - klog.V(log.I).InfoS("skip knative resource creation as no event is found") - } else if workflow.Spec.Sink == nil { - klog.V(log.I).InfoS("Spec.Sink is not provided") - } else if knativeAvail, err := knative.GetKnativeAvailability(k.Cfg); err != nil || knativeAvail == nil || !knativeAvail.Eventing { + knativeAvail, err := knative.GetKnativeAvailability(k.Cfg) + if err != nil { + klog.V(log.I).InfoS("Error checking Knative Eventing: %v", err) + return nil, err + } + if !knativeAvail.Eventing { klog.V(log.I).InfoS("Knative Eventing is not installed") } else { // create sinkBinding and trigger - sinkBinding, _, err := k.sinkBinding.Ensure(ctx, workflow) + sinkBinding, _, err := k.sinkBinding.Ensure(ctx, workflow, k.platform) if err != nil { return objs, err } else if sinkBinding != nil { objs = append(objs, sinkBinding) } - triggers := k.trigger.Ensure(ctx, workflow) + triggers := k.trigger.Ensure(ctx, workflow, k.platform) for _, trigger := range triggers { if trigger.Error != nil { return objs, trigger.Error diff --git a/controllers/profiles/common/mutate_visitors.go b/controllers/profiles/common/mutate_visitors.go index 097843856..88269379b 100644 --- a/controllers/profiles/common/mutate_visitors.go +++ b/controllers/profiles/common/mutate_visitors.go @@ -25,8 +25,12 @@ import ( "reflect" "slices" + operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/discovery" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/properties" + kubeutil "github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes" + "github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj" "github.com/imdario/mergo" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -35,10 +39,6 @@ import ( servingv1 "knative.dev/serving/pkg/apis/serving/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" - kubeutil "github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes" - "github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj" ) // ImageDeploymentMutateVisitor creates a visitor that mutates a vanilla Kubernetes Deployment to apply the given image in the DefaultContainerName container @@ -122,8 +122,11 @@ func EnsureDeployment(original *appsv1.Deployment, object *appsv1.Deployment) er object.Spec.Replicas = original.Spec.Replicas object.Spec.Selector = original.Spec.Selector object.Labels = original.GetLabels() + object.Finalizers = original.Finalizers // Clean up the volumes, they are inherited from original, additional are added by other visitors + // However, the knative data (voulmes, volumes mounts) must be preserved + knative.SaveKnativeData(&original.Spec.Template.Spec, &object.Spec.Template.Spec) object.Spec.Template.Spec.Volumes = nil for i := range object.Spec.Template.Spec.Containers { object.Spec.Template.Spec.Containers[i].VolumeMounts = nil @@ -154,6 +157,8 @@ func EnsureKService(original *servingv1.Service, object *servingv1.Service) erro object.Labels = original.GetLabels() // Clean up the volumes, they are inherited from original, additional are added by other visitors + // However, the knative data (voulmes, volumes mounts) must be preserved + knative.SaveKnativeData(&original.Spec.Template.Spec.PodSpec, &object.Spec.Template.Spec.PodSpec) object.Spec.Template.Spec.Volumes = nil for i := range object.Spec.Template.Spec.Containers { object.Spec.Template.Spec.Containers[i].VolumeMounts = nil @@ -216,8 +221,27 @@ func RolloutDeploymentIfCMChangedMutateVisitor(workflow *operatorapi.SonataFlow, return func(object client.Object) controllerutil.MutateFn { return func() error { deployment := object.(*appsv1.Deployment) - err := kubeutil.AnnotateDeploymentConfigChecksum(workflow, deployment, userPropsCM, managedPropsCM) - return err + return kubeutil.AnnotateDeploymentConfigChecksum(workflow, deployment, userPropsCM, managedPropsCM) + } + } +} + +func RestoreDeploymentVolumeAndVolumeMountMutateVisitor() MutateVisitor { + return func(object client.Object) controllerutil.MutateFn { + return func() error { + deployment := object.(*appsv1.Deployment) + knative.RestoreKnativeVolumeAndVolumeMount(&deployment.Spec.Template.Spec) + return nil + } + } +} + +func RestoreKServiceVolumeAndVolumeMountMutateVisitor() MutateVisitor { + return func(object client.Object) controllerutil.MutateFn { + return func() error { + service := object.(*servingv1.Service) + knative.RestoreKnativeVolumeAndVolumeMount(&service.Spec.Template.Spec.PodSpec) + return nil } } } diff --git a/controllers/profiles/common/object_creators.go b/controllers/profiles/common/object_creators.go index 8b72a2343..9a3eb544b 100644 --- a/controllers/profiles/common/object_creators.go +++ b/controllers/profiles/common/object_creators.go @@ -20,6 +20,7 @@ package common import ( + "context" "fmt" "strings" @@ -31,14 +32,19 @@ import ( "github.com/imdario/mergo" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" duckv1 "knative.dev/pkg/apis/duck/v1" + "knative.dev/pkg/kmeta" "knative.dev/pkg/tracker" "sigs.k8s.io/controller-runtime/pkg/client" operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform/services" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/persistence" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/properties" @@ -49,6 +55,15 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj" ) +const ( + knativeServingAPIVersion = "serving.knative.dev/v1" + knativeServiceKind = "Service" + deploymentAPIVersion = "apps/v1" + deploymentKind = "Deployment" + k8sServiceAPIVersion = "v1" + k8sServiceKind = "Service" +) + // ObjectCreator is the func that creates the initial reference object, if the object doesn't exist in the cluster, this one is created. // Can be used as a reference to keep the object immutable type ObjectCreator func(workflow *operatorapi.SonataFlow) (client.Object, error) @@ -60,6 +75,9 @@ type ObjectCreatorWithPlatform func(workflow *operatorapi.SonataFlow, platform * // ObjectsCreator creates multiple resources type ObjectsCreator func(workflow *operatorapi.SonataFlow) ([]client.Object, error) +// ObjectsCreatorWithPlatform creates multiple resources +type ObjectsCreatorWithPlatform func(workflow *operatorapi.SonataFlow, platform *operatorapi.SonataFlowPlatform) ([]client.Object, error) + const ( defaultHTTPServicePort = 80 @@ -70,6 +88,7 @@ const ( healthStartedFailureThreshold = 5 healthStartedPeriodSeconds = 15 healthStartedInitialDelaySeconds = 10 + healthSuccessThreshold = 1 ) // DeploymentCreator is an objectCreator for a base Kubernetes Deployments for profiles that need to deploy the workflow on a vanilla deployment. @@ -165,34 +184,44 @@ func defaultContainer(workflow *operatorapi.SonataFlow, plf *operatorapi.SonataF LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: constants.QuarkusHealthPathLive, - Port: variables.DefaultHTTPWorkflowPortIntStr, + Path: constants.QuarkusHealthPathLive, + Port: variables.DefaultHTTPWorkflowPortIntStr, + Scheme: corev1.URISchemeHTTP, }, }, - TimeoutSeconds: healthTimeoutSeconds, - PeriodSeconds: healthStartedPeriodSeconds, + InitialDelaySeconds: healthStartedInitialDelaySeconds, + TimeoutSeconds: healthTimeoutSeconds, + FailureThreshold: healthStartedFailureThreshold, + PeriodSeconds: healthStartedPeriodSeconds, + SuccessThreshold: healthSuccessThreshold, }, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: constants.QuarkusHealthPathReady, - Port: variables.DefaultHTTPWorkflowPortIntStr, + Path: constants.QuarkusHealthPathReady, + Port: variables.DefaultHTTPWorkflowPortIntStr, + Scheme: corev1.URISchemeHTTP, }, }, - TimeoutSeconds: healthTimeoutSeconds, - PeriodSeconds: healthStartedPeriodSeconds, + InitialDelaySeconds: healthStartedInitialDelaySeconds, + TimeoutSeconds: healthTimeoutSeconds, + FailureThreshold: healthStartedFailureThreshold, + PeriodSeconds: healthStartedPeriodSeconds, + SuccessThreshold: healthSuccessThreshold, }, StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: constants.QuarkusHealthPathStarted, - Port: variables.DefaultHTTPWorkflowPortIntStr, + Path: constants.QuarkusHealthPathStarted, + Port: variables.DefaultHTTPWorkflowPortIntStr, + Scheme: corev1.URISchemeHTTP, }, }, InitialDelaySeconds: healthStartedInitialDelaySeconds, TimeoutSeconds: healthTimeoutSeconds, FailureThreshold: healthStartedFailureThreshold, PeriodSeconds: healthStartedPeriodSeconds, + SuccessThreshold: healthSuccessThreshold, }, SecurityContext: kubeutil.SecurityDefaults(), } @@ -252,15 +281,30 @@ func ServiceCreator(workflow *operatorapi.SonataFlow) (client.Object, error) { // SinkBindingCreator is an ObjectsCreator for SinkBinding. // It will create v1.SinkBinding based on events defined in workflow. -func SinkBindingCreator(workflow *operatorapi.SonataFlow) (client.Object, error) { +func SinkBindingCreator(workflow *operatorapi.SonataFlow, plf *operatorapi.SonataFlowPlatform) (client.Object, error) { lbl := workflowproj.GetMergedLabels(workflow) - // skip if no produced event is found - if workflow.Spec.Sink == nil || !workflowdef.ContainsEventKind(workflow, cncfmodel.EventKindProduced) { - return nil, nil + sink, err := knative.GetWorkflowSink(workflow, plf) + if err != nil { + return nil, err } + dataIndexEnabled := services.IsDataIndexEnabled(plf) + jobServiceEnabled := services.IsJobServiceEnabled(plf) - sink := workflow.Spec.Sink + // skip if no produced event is found and there is no DataIndex/JobService enabled + if sink == nil { + if dataIndexEnabled || jobServiceEnabled || workflowdef.ContainsEventKind(workflow, cncfmodel.EventKindProduced) { + return nil, fmt.Errorf("a sink in the SonataFlow %s or broker in the SonataFlowPlatform %s should be configured when DataIndex or JobService is enabled", workflow.Name, plf.Name) + } + return nil, nil /*nothing to do*/ + } + + apiVersion := deploymentAPIVersion + kind := deploymentKind + if workflow.Spec.PodTemplate.DeploymentModel == operatorapi.KnativeDeploymentModel { + apiVersion = knativeServingAPIVersion // use knative serving API Version + kind = knativeServiceKind + } // subject must be deployment to inject K_SINK, service won't work sinkBinding := &sourcesv1.SinkBinding{ @@ -277,8 +321,8 @@ func SinkBindingCreator(workflow *operatorapi.SonataFlow) (client.Object, error) Subject: tracker.Reference{ Name: workflow.Name, Namespace: workflow.Namespace, - APIVersion: "apps/v1", - Kind: "Deployment", + APIVersion: apiVersion, + Kind: kind, }, }, }, @@ -286,12 +330,57 @@ func SinkBindingCreator(workflow *operatorapi.SonataFlow) (client.Object, error) return sinkBinding, nil } +func getBrokerRefFromPlatform(plf *operatorapi.SonataFlowPlatform) (*duckv1.KReference, error) { + // check the local platform + if plf.Spec.Eventing != nil && plf.Spec.Eventing.Broker != nil && plf.Spec.Eventing.Broker.Ref != nil { + ref := plf.Spec.Eventing.Broker.Ref.DeepCopy() + if len(ref.Namespace) == 0 { + ref.Namespace = plf.Namespace // default to the platform namespace + } + return ref, nil + } + // Check the cluster platform + if plf.Status.ClusterPlatformRef != nil && len(plf.Status.ClusterPlatformRef.PlatformRef.Name) > 0 { + platform := &operatorapi.SonataFlowPlatform{} + if err := utils.GetClient().Get(context.TODO(), types.NamespacedName{Namespace: plf.Status.ClusterPlatformRef.PlatformRef.Namespace, Name: plf.Status.ClusterPlatformRef.PlatformRef.Name}, platform); err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + return getBrokerRefFromPlatform(platform) + + } + return nil, nil +} + +func getBrokerRefForEventType(eventType string, workflow *operatorapi.SonataFlow, plf *operatorapi.SonataFlowPlatform) (*duckv1.KReference, error) { + // Check the workflow + for _, source := range workflow.Spec.Sources { + if source.EventType == eventType { + ref := source.Ref.DeepCopy() + if len(ref.Namespace) == 0 { + ref.Namespace = workflow.Namespace // default to the workflow namespace + } + return ref, nil + } + } + // get the broker from the local platform or cluster platform + return getBrokerRefFromPlatform(plf) +} + // TriggersCreator is an ObjectsCreator for Triggers. // It will create a list of eventingv1.Trigger based on events defined in workflow. -func TriggersCreator(workflow *operatorapi.SonataFlow) ([]client.Object, error) { +func TriggersCreator(workflow *operatorapi.SonataFlow, plf *operatorapi.SonataFlowPlatform) ([]client.Object, error) { var resultObjects []client.Object lbl := workflowproj.GetMergedLabels(workflow) + apiVersion := k8sServiceAPIVersion + kind := k8sServiceKind + if workflow.Spec.PodTemplate.DeploymentModel == operatorapi.KnativeDeploymentModel { + apiVersion = knativeServingAPIVersion // use knative serving API Version + kind = knativeServiceKind + } //consumed events := workflow.Spec.Flow.Events for _, event := range events { @@ -299,16 +388,29 @@ func TriggersCreator(workflow *operatorapi.SonataFlow) ([]client.Object, error) if event.Kind == cncfmodel.EventKindProduced { continue } - + brokerRef, err := getBrokerRefForEventType(event.Type, workflow, plf) + if err != nil { + return nil, err + } + if brokerRef == nil { + return nil, fmt.Errorf("no broker configured for eventType %s in SonataFlow %s", event.Type, workflow.Name) + } + if !knative.IsKnativeBroker(brokerRef) { + return nil, fmt.Errorf("no valid broker configured for eventType %s in SonataFlow %s", event.Type, workflow.Name) + } + if err := knative.ValidateBroker(brokerRef.Name, brokerRef.Namespace); err != nil { + return nil, err + } // construct eventingv1.Trigger + // The trigger must be created in the same namespace as the broker trigger := &eventingv1.Trigger{ ObjectMeta: metav1.ObjectMeta{ - Name: strings.ToLower(fmt.Sprintf("%s-%s-trigger", workflow.Name, event.Name)), - Namespace: workflow.Namespace, + Name: kmeta.ChildName(strings.ToLower(fmt.Sprintf("%s-%s-", workflow.Name, event.Name)), string(workflow.GetUID())), + Namespace: brokerRef.Namespace, Labels: lbl, }, Spec: eventingv1.TriggerSpec{ - Broker: constants.KnativeEventingBrokerDefault, + Broker: brokerRef.Name, Filter: &eventingv1.TriggerFilter{ Attributes: eventingv1.TriggerFilterAttributes{ "type": event.Type, @@ -318,8 +420,8 @@ func TriggersCreator(workflow *operatorapi.SonataFlow) ([]client.Object, error) Ref: &duckv1.KReference{ Name: workflow.Name, Namespace: workflow.Namespace, - APIVersion: "v1", - Kind: "Service", + APIVersion: apiVersion, + Kind: kind, }, }, }, @@ -344,6 +446,7 @@ func UserPropsConfigMapCreator(workflow *operatorapi.SonataFlow) (client.Object, // ManagedPropsConfigMapCreator creates an empty ConfigMap to hold the external application properties func ManagedPropsConfigMapCreator(workflow *operatorapi.SonataFlow, platform *operatorapi.SonataFlowPlatform) (client.Object, error) { + props, err := properties.ApplicationManagedProperties(workflow, platform) if err != nil { return nil, err diff --git a/controllers/profiles/common/object_creators_test.go b/controllers/profiles/common/object_creators_test.go index 074547e82..bf267f57f 100644 --- a/controllers/profiles/common/object_creators_test.go +++ b/controllers/profiles/common/object_creators_test.go @@ -23,13 +23,15 @@ import ( "context" "testing" - sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" - "github.com/magiconair/properties" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" + sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" + "knative.dev/pkg/kmeta" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/apache/incubator-kie-kogito-serverless-operator/utils" kubeutil "github.com/apache/incubator-kie-kogito-serverless-operator/utils/kubernetes" @@ -43,7 +45,7 @@ const platformName = "test-platform" func Test_ensureWorkflowPropertiesConfigMapMutator(t *testing.T) { workflow := test.GetBaseSonataFlowWithDevProfile(t.Name()) - platform := test.GetBasePlatform() + platform := test.GetBasePlatformInReadyPhase(workflow.Namespace) // can't be new managedProps, _ := ManagedPropsConfigMapCreator(workflow, platform) managedProps.SetUID("1") @@ -52,7 +54,7 @@ func Test_ensureWorkflowPropertiesConfigMapMutator(t *testing.T) { userProps, _ := UserPropsConfigMapCreator(workflow) userPropsCM := userProps.(*corev1.ConfigMap) - visitor := ManagedPropertiesMutateVisitor(context.TODO(), nil, workflow, nil, userPropsCM) + visitor := ManagedPropertiesMutateVisitor(context.TODO(), nil, workflow, platform, userPropsCM) mutateFn := visitor(managedProps) assert.NoError(t, mutateFn()) @@ -76,7 +78,8 @@ func Test_ensureWorkflowPropertiesConfigMapMutator(t *testing.T) { func Test_ensureWorkflowPropertiesConfigMapMutator_DollarReplacement(t *testing.T) { workflow := test.GetBaseSonataFlowWithDevProfile(t.Name()) - platform := test.GetBasePlatform() + platform := test.GetBasePlatformInReadyPhase(workflow.Namespace) + managedProps, _ := ManagedPropsConfigMapCreator(workflow, platform) managedProps.SetName(workflow.Name) managedProps.SetNamespace(workflow.Namespace) @@ -87,7 +90,7 @@ func Test_ensureWorkflowPropertiesConfigMapMutator_DollarReplacement(t *testing. userPropsCM := userProps.(*corev1.ConfigMap) userPropsCM.Data[workflowproj.ApplicationPropertiesFileName] = "mp.messaging.outgoing.kogito_outgoing_stream.url=${kubernetes:services.v1/event-listener}" - mutateVisitorFn := ManagedPropertiesMutateVisitor(context.TODO(), nil, workflow, nil, userPropsCM) + mutateVisitorFn := ManagedPropertiesMutateVisitor(context.TODO(), nil, workflow, platform, userPropsCM) err := mutateVisitorFn(managedPropsCM)() assert.NoError(t, err) @@ -147,7 +150,7 @@ func TestMergePodSpec(t *testing.T) { assert.Len(t, flowContainer.VolumeMounts, 1) } -func TestMergePodSpec_OverrideContainers(t *testing.T) { +func TestMergePodSpecOverrideContainers(t *testing.T) { workflow := test.GetBaseSonataFlow(t.Name()) workflow.Spec.PodTemplate = v1alpha08.FlowPodTemplateSpec{ PodSpec: v1alpha08.PodSpec{ @@ -179,11 +182,13 @@ func TestMergePodSpec_OverrideContainers(t *testing.T) { assert.Empty(t, flowContainer.Env) } -func Test_ensureWorkflowSinkBindingIsCreated(t *testing.T) { +func TestEnsureWorkflowSinkBindingWithWorkflowSinkIsCreated(t *testing.T) { workflow := test.GetVetEventSonataFlow(t.Name()) - + plf := test.GetBasePlatform() //On Kubernetes we want the service exposed in Dev with NodePort - sinkBinding, _ := SinkBindingCreator(workflow) + sinkBinding, err := SinkBindingCreator(workflow, plf) + assert.NoError(t, err) + assert.NotNil(t, sinkBinding) sinkBinding.SetUID("1") sinkBinding.SetResourceVersion("1") @@ -193,33 +198,180 @@ func Test_ensureWorkflowSinkBindingIsCreated(t *testing.T) { assert.NotNil(t, reflectSinkBinding.Spec) assert.NotEmpty(t, reflectSinkBinding.Spec.Sink) assert.Equal(t, reflectSinkBinding.Spec.Sink.Ref.Kind, "Broker") + assert.Equal(t, reflectSinkBinding.Spec.Sink.Ref.Name, "default") assert.NotNil(t, reflectSinkBinding.GetLabels()) assert.Equal(t, reflectSinkBinding.ObjectMeta.Labels, map[string]string{ - "sonataflow.org/workflow-app": "vet", - "app.kubernetes.io/name": "vet", - "app.kubernetes.io/component": "serverless-workflow", - "app.kubernetes.io/managed-by": "sonataflow-operator", - }) + "app": "vet", + "sonataflow.org/workflow-app": "vet", + "sonataflow.org/workflow-namespace": workflow.Namespace, + "app.kubernetes.io/name": "vet", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator"}) } -func Test_ensureWorkflowTriggersAreCreated(t *testing.T) { +func TestEnsureWorkflowSinkBindingWithPlatformBrokerIsCreated(t *testing.T) { workflow := test.GetVetEventSonataFlow(t.Name()) + workflow.Spec.Sink = nil + workflow.Spec.Sources = nil + plf := test.GetBasePlatformWithBroker() + sinkBinding, err := SinkBindingCreator(workflow, plf) + assert.NoError(t, err) + assert.NotNil(t, sinkBinding) + sinkBinding.SetUID("1") + sinkBinding.SetResourceVersion("1") - //On Kubernetes we want the service exposed in Dev with NodePort - triggers, _ := TriggersCreator(workflow) + reflectSinkBinding := sinkBinding.(*sourcesv1.SinkBinding) + + assert.NotNil(t, reflectSinkBinding) + assert.NotNil(t, reflectSinkBinding.Spec) + assert.NotEmpty(t, reflectSinkBinding.Spec.Sink) + assert.Equal(t, reflectSinkBinding.Spec.Sink.Ref.Kind, "Broker") + assert.Equal(t, reflectSinkBinding.Spec.Sink.Ref.Name, "default") + assert.NotNil(t, reflectSinkBinding.GetLabels()) + assert.Equal(t, reflectSinkBinding.ObjectMeta.Labels, map[string]string{"app": "vet", + "sonataflow.org/workflow-app": "vet", + "sonataflow.org/workflow-namespace": workflow.Namespace, + "app.kubernetes.io/name": "vet", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator"}) +} +func TestEnsureWorkflowSinkBindingWithoutBrokerAreNotCreated(t *testing.T) { + workflow := test.GetVetEventSonataFlow(t.Name()) + workflow.Spec.Sink = nil + workflow.Spec.Sources = nil + plf := test.GetBasePlatformWithBroker() + plf.Spec.Eventing = nil // No broker configured in the platform, but data index and jobs service are enabled + sinkBinding, err := SinkBindingCreator(workflow, plf) + assert.Error(t, err) + assert.Contains(t, err.Error(), "a sink in the SonataFlow vet or broker in the SonataFlowPlatform sonataflow-platform should be configured when DataIndex or JobService is enabled") + assert.Nil(t, sinkBinding) +} + +func getTrigger(name string, objs []client.Object) *eventingv1.Trigger { + for _, obj := range objs { + if trigger, ok := obj.(*eventingv1.Trigger); ok { + if trigger.Name == name { + return trigger + } + } + } + return nil +} + +func TestEnsureWorkflowTriggersWithPlatformBrokerAreCreated(t *testing.T) { + workflow := test.GetVetEventSonataFlow(t.Name()) + workflow.Spec.Sink = nil + workflow.Spec.Sources = nil + plf := test.GetBasePlatformWithBroker() + plf.Namespace = "platform-namespace" + plf.Spec.Eventing.Broker.Ref.Namespace = plf.Namespace + broker := test.GetDefaultBroker(plf.Namespace) + + // Create a fake client to mock API calls. + cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(workflow, broker).WithStatusSubresource(workflow, broker).Build() + utils.SetClient(cl) + + triggers, err := TriggersCreator(workflow, plf) + assert.NoError(t, err) assert.NotEmpty(t, triggers) assert.Len(t, triggers, 2) - for _, trigger := range triggers { - assert.Contains(t, []string{"vet-vetappointmentrequestreceived-trigger", "vet-vetappointmentinfo-trigger"}, trigger.GetName()) - assert.NotNil(t, trigger.GetLabels()) - assert.Equal(t, trigger.GetLabels(), map[string]string{ - "sonataflow.org/workflow-app": "vet", - "app.kubernetes.io/name": "vet", - "app.kubernetes.io/component": "serverless-workflow", - "app.kubernetes.io/managed-by": "sonataflow-operator", - }) - } + //Check the 1st trigger + name := kmeta.ChildName("vet-vetappointmentrequestreceived-", string(workflow.GetUID())) + trigger := getTrigger(name, triggers) + assert.NotNil(t, trigger) + assert.NotNil(t, trigger.GetLabels()) + assert.Equal(t, trigger.GetLabels(), map[string]string{"app": "vet", + "sonataflow.org/workflow-app": "vet", + "sonataflow.org/workflow-namespace": workflow.Namespace, + "app.kubernetes.io/name": "vet", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator"}) + assert.Equal(t, trigger.Namespace, plf.Namespace) //trigger should be in the platform namespace + assert.Equal(t, trigger.Spec.Broker, "default") + assert.NotNil(t, trigger.Spec.Filter) + assert.Len(t, trigger.Spec.Filter.Attributes, 1) + assert.Equal(t, trigger.Spec.Filter.Attributes["type"], "events.vet.appointments.request") + //Check the 2nd trigger + name = kmeta.ChildName("vet-vetappointmentinfo-", string(workflow.GetUID())) + trigger = getTrigger(name, triggers) + assert.NotNil(t, trigger) + assert.NotNil(t, trigger.GetLabels()) + assert.Equal(t, trigger.GetLabels(), map[string]string{"app": "vet", + "sonataflow.org/workflow-app": "vet", + "sonataflow.org/workflow-namespace": workflow.Namespace, + "app.kubernetes.io/name": "vet", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator"}) + assert.Equal(t, trigger.Namespace, plf.Namespace) //trigger should be in the platform namespace + assert.Equal(t, trigger.Spec.Broker, "default") + assert.NotNil(t, trigger.Spec.Filter) + assert.Len(t, trigger.Spec.Filter.Attributes, 1) + assert.Equal(t, trigger.Spec.Filter.Attributes["type"], "events.vet.appointments") +} + +func TestEnsureWorkflowTriggersWithWorkflowBrokerAreCreated(t *testing.T) { + workflow := test.GetVetEventSonataFlow(t.Name()) + workflow.Spec.Sources[0].Destination.Ref.Namespace = workflow.Namespace + workflow.Spec.Sources[1].Destination.Ref.Namespace = workflow.Namespace + plf := test.GetBasePlatform() // No broker defined in the platform + broker1 := test.GetDefaultBroker(workflow.Namespace) + broker1.Name = "broker-appointments-request" + broker2 := test.GetDefaultBroker(workflow.Namespace) + broker2.Name = "broker-appointments" + // Create a fake client to mock API calls. + cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(workflow, plf, broker1, broker2).WithStatusSubresource(workflow, plf, broker1, broker2).Build() + utils.SetClient(cl) + + triggers, err := TriggersCreator(workflow, plf) + assert.NoError(t, err) + assert.NotEmpty(t, triggers) + assert.Len(t, triggers, 2) + //Check the 1st trigger + name := kmeta.ChildName("vet-vetappointmentrequestreceived-", string(workflow.GetUID())) + + trigger := getTrigger(name, triggers) + assert.NotNil(t, trigger) + assert.NotNil(t, trigger.GetLabels()) + assert.Equal(t, trigger.GetLabels(), map[string]string{"app": "vet", + "sonataflow.org/workflow-app": "vet", + "sonataflow.org/workflow-namespace": workflow.Namespace, + "app.kubernetes.io/name": "vet", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator"}) + assert.Equal(t, trigger.Namespace, workflow.Namespace) //trigger should be in the workflow namespace + assert.Equal(t, trigger.Spec.Broker, "broker-appointments-request") + assert.NotNil(t, trigger.Spec.Filter) + assert.Len(t, trigger.Spec.Filter.Attributes, 1) + assert.Equal(t, trigger.Spec.Filter.Attributes["type"], "events.vet.appointments.request") + //Check the 2nd trigger + name = kmeta.ChildName("vet-vetappointmentinfo-", string(workflow.GetUID())) + trigger = getTrigger(name, triggers) + assert.NotNil(t, trigger) + assert.NotNil(t, trigger.GetLabels()) + assert.Equal(t, trigger.GetLabels(), map[string]string{"app": "vet", + "sonataflow.org/workflow-app": "vet", + "sonataflow.org/workflow-namespace": workflow.Namespace, + "app.kubernetes.io/name": "vet", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator"}) + assert.Equal(t, trigger.Namespace, workflow.Namespace) //trigger should be in the workflow namespace + assert.Equal(t, trigger.Spec.Broker, "broker-appointments") + assert.NotNil(t, trigger.Spec.Filter) + assert.Len(t, trigger.Spec.Filter.Attributes, 1) + assert.Equal(t, trigger.Spec.Filter.Attributes["type"], "events.vet.appointments") +} + +func TestEnsureWorkflowTriggersWithoutBrokerAreNotCreated(t *testing.T) { + workflow := test.GetVetEventSonataFlow(t.Name()) + workflow.Spec.Sink = nil + workflow.Spec.Sources = nil + plf := test.GetBasePlatform() + + triggers, err := TriggersCreator(workflow, plf) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no broker configured for eventType events.vet.appointments in SonataFlow vet") + assert.Nil(t, triggers) } func TestMergePodSpec_WithPostgreSQL_and_JDBC_URL_field(t *testing.T) { diff --git a/controllers/profiles/common/properties/knative.go b/controllers/profiles/common/properties/knative.go index 195dd21fa..ffcade76b 100644 --- a/controllers/profiles/common/properties/knative.go +++ b/controllers/profiles/common/properties/knative.go @@ -16,6 +16,7 @@ package properties import ( operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/workflowdef" "github.com/magiconair/properties" @@ -25,13 +26,17 @@ import ( // generateKnativeEventingWorkflowProperties returns the set of application properties required for the workflow to produce or consume // Knative Events. // Never nil. -func generateKnativeEventingWorkflowProperties(workflow *operatorapi.SonataFlow) (*properties.Properties, error) { +func generateKnativeEventingWorkflowProperties(workflow *operatorapi.SonataFlow, platform *operatorapi.SonataFlowPlatform) (*properties.Properties, error) { props := properties.NewProperties() - if workflow == nil || workflow.Spec.Sink == nil { + props.Set(constants.KnativeHealthEnabled, "false") + sink, err := knative.GetWorkflowSink(workflow, platform) + if err != nil { + return nil, err + } + if workflow == nil || sink == nil { props.Set(constants.KnativeHealthEnabled, "false") return props, nil } - // verify ${K_SINK} props.Set(constants.KnativeHealthEnabled, "true") if workflowdef.ContainsEventKind(workflow, cncfmodel.EventKindProduced) { props.Set(constants.KogitoOutgoingEventsConnector, constants.QuarkusHTTP) @@ -39,11 +44,7 @@ func generateKnativeEventingWorkflowProperties(workflow *operatorapi.SonataFlow) } if workflowdef.ContainsEventKind(workflow, cncfmodel.EventKindConsumed) { props.Set(constants.KogitoIncomingEventsConnector, constants.QuarkusHTTP) - var path = "/" - if workflow.Spec.Sink.URI != nil { - path = workflow.Spec.Sink.URI.Path - } - props.Set(constants.KogitoIncomingEventsPath, path) + props.Set(constants.KogitoIncomingEventsPath, "/") } return props, nil } diff --git a/controllers/profiles/common/properties/managed.go b/controllers/profiles/common/properties/managed.go index 98a73ad33..48208c582 100644 --- a/controllers/profiles/common/properties/managed.go +++ b/controllers/profiles/common/properties/managed.go @@ -110,9 +110,9 @@ func (a *managedPropertyHandler) Build() string { func (a *managedPropertyHandler) withKogitoServiceUrl() ManagedPropertyHandler { var kogitoServiceUrl string if len(a.workflow.Namespace) > 0 { - kogitoServiceUrl = fmt.Sprintf("%s://%s.%s", constants.KogitoServiceURLProtocol, a.workflow.Name, a.workflow.Namespace) + kogitoServiceUrl = fmt.Sprintf("%s://%s.%s", constants.DefaultHTTPProtocol, a.workflow.Name, a.workflow.Namespace) } else { - kogitoServiceUrl = fmt.Sprintf("%s://%s", constants.KogitoServiceURLProtocol, a.workflow.Name) + kogitoServiceUrl = fmt.Sprintf("%s://%s", constants.DefaultHTTPProtocol, a.workflow.Name) } return a.addDefaultManagedProperty(constants.KogitoServiceURLProperty, kogitoServiceUrl) } @@ -121,7 +121,7 @@ func (a *managedPropertyHandler) withKogitoServiceUrl() ManagedPropertyHandler { // See Service Discovery https://kubernetes.io/docs/concepts/services-networking/service/#dns func (a *managedPropertyHandler) withKafkaHealthCheckDisabled() ManagedPropertyHandler { a.addDefaultManagedProperty( - constants.DataIndexKafkaSmallRyeHealthProperty, + constants.DataIndexKafkaHealthCheck, "false", ) return a @@ -172,7 +172,7 @@ func NewManagedPropertyHandler(workflow *operatorapi.SonataFlow, platform *opera props.Merge(p) } - p, err := generateKnativeEventingWorkflowProperties(workflow) + p, err := generateKnativeEventingWorkflowProperties(workflow, platform) if err != nil { return nil, err } diff --git a/controllers/profiles/common/properties/managed_test.go b/controllers/profiles/common/properties/managed_test.go index d27d69e1d..6e9ee05e7 100644 --- a/controllers/profiles/common/properties/managed_test.go +++ b/controllers/profiles/common/properties/managed_test.go @@ -111,7 +111,8 @@ func (c *mockCatalogService) Query(ctx context.Context, uri discovery.ResourceUr func Test_appPropertyHandler_WithKogitoServiceUrl(t *testing.T) { workflow := test.GetBaseSonataFlow("default") - props, err := ApplicationManagedProperties(workflow, nil) + platform := test.GetBasePlatform() + props, err := ApplicationManagedProperties(workflow, platform) assert.NoError(t, err) assert.Contains(t, props, constants.KogitoServiceURLProperty) assert.Contains(t, props, "http://"+workflow.Name+"."+workflow.Namespace) @@ -121,11 +122,12 @@ func Test_appPropertyHandler_WithUserPropertiesWithNoUserOverrides(t *testing.T) //just add some user provided properties, no overrides. userProperties := "property1=value1\nproperty2=value2" workflow := test.GetBaseSonataFlow("default") - props, err := NewManagedPropertyHandler(workflow, nil) + platform := test.GetBasePlatform() + props, err := NewManagedPropertyHandler(workflow, platform) assert.NoError(t, err) generatedProps, propsErr := properties.LoadString(props.WithUserProperties(userProperties).Build()) assert.NoError(t, propsErr) - assert.Equal(t, 7, len(generatedProps.Keys())) + assert.Equal(t, 12, len(generatedProps.Keys())) assert.NotContains(t, "property1", generatedProps.Keys()) assert.NotContains(t, "property2", generatedProps.Keys()) assert.Equal(t, "http://greeting.default", generatedProps.GetString("kogito.service.url", "")) @@ -134,6 +136,11 @@ func Test_appPropertyHandler_WithUserPropertiesWithNoUserOverrides(t *testing.T) assert.Equal(t, "false", generatedProps.GetString("quarkus.devservices.enabled", "")) assert.Equal(t, "false", generatedProps.GetString("quarkus.kogito.devservices.enabled", "")) assert.Equal(t, "false", generatedProps.GetString(constants.KogitoUserTasksEventsEnabled, "")) + assert.Equal(t, "false", generatedProps.GetString(constants.KogitoProcessDefinitionsEventsEnabled, "")) + assert.Equal(t, "false", generatedProps.GetString(constants.KogitoProcessInstancesEventsEnabled, "")) + assert.Equal(t, "quarkus-http", generatedProps.GetString("mp.messaging.outgoing.kogito-job-service-job-request-events.connector", "")) + assert.Equal(t, "http://localhost/v2/jobs/events", generatedProps.GetString("mp.messaging.outgoing.kogito-job-service-job-request-events.url", "")) + assert.Equal(t, "false", generatedProps.GetString("org.kie.kogito.addons.knative.eventing.health-enabled", "")) } func Test_appPropertyHandler_WithUserPropertiesWithServiceDiscovery(t *testing.T) { @@ -149,7 +156,8 @@ func Test_appPropertyHandler_WithUserPropertiesWithServiceDiscovery(t *testing.T userProperties = userProperties + "broker2=${knative:brokers.v1.eventing.knative.dev/my-kn-broker2}\n" workflow := test.GetBaseSonataFlow(defaultNamespace) - props, err := NewManagedPropertyHandler(workflow, nil) + platform := test.GetBasePlatform() + props, err := NewManagedPropertyHandler(workflow, platform) assert.NoError(t, err) generatedProps, propsErr := properties.LoadString(props. WithUserProperties(userProperties). @@ -157,7 +165,7 @@ func Test_appPropertyHandler_WithUserPropertiesWithServiceDiscovery(t *testing.T Build()) generatedProps.DisableExpansion = true assert.NoError(t, propsErr) - assert.Equal(t, 21, len(generatedProps.Keys())) + assert.Equal(t, 26, len(generatedProps.Keys())) assert.NotContains(t, "property1", generatedProps.Keys()) assert.NotContains(t, "property2", generatedProps.Keys()) assertHasProperty(t, generatedProps, "service1", myService1Address) @@ -184,6 +192,13 @@ func Test_appPropertyHandler_WithUserPropertiesWithServiceDiscovery(t *testing.T assertHasProperty(t, generatedProps, "quarkus.devservices.enabled", "false") assertHasProperty(t, generatedProps, "quarkus.kogito.devservices.enabled", "false") assertHasProperty(t, generatedProps, constants.KogitoUserTasksEventsEnabled, "false") + + assertHasProperty(t, generatedProps, "org.kie.kogito.addons.knative.eventing.health-enabled", "false") + assertHasProperty(t, generatedProps, "kogito.events.processdefinitions.enabled", "false") + assertHasProperty(t, generatedProps, "kogito.events.processinstances.enabled", "false") + assertHasProperty(t, generatedProps, "kogito.events.usertasks.enabled", "false") + assertHasProperty(t, generatedProps, "mp.messaging.outgoing.kogito-job-service-job-request-events.connector", "quarkus-http") + assertHasProperty(t, generatedProps, "mp.messaging.outgoing.kogito-job-service-job-request-events.url", "http://localhost/v2/jobs/events") } func Test_appPropertyHandler_WithServicesWithUserOverrides(t *testing.T) { @@ -197,11 +212,15 @@ func Test_appPropertyHandler_WithServicesWithUserOverrides(t *testing.T) { platform.Namespace = ns platform.Spec = operatorapi.SonataFlowPlatformSpec{ Services: &operatorapi.ServicesPlatformSpec{ - DataIndex: &operatorapi.ServiceSpec{ - Enabled: &enabled, + DataIndex: &operatorapi.DataIndexServiceSpec{ + ServiceSpec: operatorapi.ServiceSpec{ + Enabled: &enabled, + }, }, - JobService: &operatorapi.ServiceSpec{ - Enabled: &enabled, + JobService: &operatorapi.JobServiceServiceSpec{ + ServiceSpec: operatorapi.ServiceSpec{ + Enabled: &enabled, + }, }, }, } @@ -636,7 +655,7 @@ func setJobServiceEnabledValue(v *bool) plfmOptionFn { p.Spec.Services = &operatorapi.ServicesPlatformSpec{} } if p.Spec.Services.JobService == nil { - p.Spec.Services.JobService = &operatorapi.ServiceSpec{} + p.Spec.Services.JobService = &operatorapi.JobServiceServiceSpec{} } p.Spec.Services.JobService.Enabled = v } @@ -648,7 +667,7 @@ func setDataIndexEnabledValue(v *bool) plfmOptionFn { p.Spec.Services = &operatorapi.ServicesPlatformSpec{} } if p.Spec.Services.DataIndex == nil { - p.Spec.Services.DataIndex = &operatorapi.ServiceSpec{} + p.Spec.Services.DataIndex = &operatorapi.DataIndexServiceSpec{} } p.Spec.Services.DataIndex.Enabled = v } @@ -672,7 +691,7 @@ func setJobServiceJDBC(jdbc string) plfmOptionFn { p.Spec.Services = &operatorapi.ServicesPlatformSpec{} } if p.Spec.Services.JobService == nil { - p.Spec.Services.JobService = &operatorapi.ServiceSpec{} + p.Spec.Services.JobService = &operatorapi.JobServiceServiceSpec{} } if p.Spec.Services.JobService.Persistence == nil { p.Spec.Services.JobService.Persistence = &operatorapi.PersistenceOptionsSpec{} diff --git a/controllers/profiles/dev/object_creators_dev.go b/controllers/profiles/dev/object_creators_dev.go index 0f6069f18..a9c920dbe 100644 --- a/controllers/profiles/dev/object_creators_dev.go +++ b/controllers/profiles/dev/object_creators_dev.go @@ -51,6 +51,7 @@ func serviceCreator(workflow *operatorapi.SonataFlow) (client.Object, error) { } func deploymentCreator(workflow *operatorapi.SonataFlow, plf *operatorapi.SonataFlowPlatform) (client.Object, error) { + obj, err := common.DeploymentCreator(workflow, plf) if err != nil { return nil, err @@ -154,7 +155,7 @@ func mountDevConfigMapsMutateVisitor(workflow *operatorapi.SonataFlow, flowDefCM if len(deployment.Spec.Template.Spec.Containers[flowContainerIdx].VolumeMounts) == 0 { deployment.Spec.Template.Spec.Containers[flowContainerIdx].VolumeMounts = make([]corev1.VolumeMount, 0, len(volumeMounts)) } - kubeutil.AddOrReplaceVolumeMount(flowContainerIdx, &deployment.Spec.Template.Spec, volumeMounts...) + kubeutil.AddOrReplaceVolumeMount(&deployment.Spec.Template.Spec.Containers[flowContainerIdx], volumeMounts...) return nil } diff --git a/controllers/profiles/dev/object_creators_dev_test.go b/controllers/profiles/dev/object_creators_dev_test.go index e9daec345..9a3f2b7b0 100644 --- a/controllers/profiles/dev/object_creators_dev_test.go +++ b/controllers/profiles/dev/object_creators_dev_test.go @@ -44,10 +44,12 @@ func Test_ensureWorkflowDevServiceIsExposed(t *testing.T) { assert.NotNil(t, reflectService.ObjectMeta) assert.NotNil(t, reflectService.ObjectMeta.Labels) assert.Equal(t, reflectService.ObjectMeta.Labels, map[string]string{ - "test": "test", - "sonataflow.org/workflow-app": "greeting", - "app.kubernetes.io/name": "greeting", - "app.kubernetes.io/component": "serverless-workflow", - "app.kubernetes.io/managed-by": "sonataflow-operator", + "app": "greeting", + "test": "test", + "sonataflow.org/workflow-app": "greeting", + "sonataflow.org/workflow-namespace": workflow.Namespace, + "app.kubernetes.io/name": "greeting", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator", }) } diff --git a/controllers/profiles/dev/profile_dev_test.go b/controllers/profiles/dev/profile_dev_test.go index c5671faff..b90001c41 100644 --- a/controllers/profiles/dev/profile_dev_test.go +++ b/controllers/profiles/dev/profile_dev_test.go @@ -51,6 +51,7 @@ import ( clientruntime "sigs.k8s.io/controller-runtime/pkg/client" "github.com/apache/incubator-kie-kogito-serverless-operator/api" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/test" ) @@ -59,6 +60,8 @@ func Test_OverrideStartupProbe(t *testing.T) { client := test.NewSonataFlowClientBuilder().WithRuntimeObjects(workflow).WithStatusSubresource(workflow).Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) + devReconciler := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()) result, err := devReconciler.Reconcile(context.TODO(), workflow) @@ -85,7 +88,7 @@ func Test_recoverFromFailureNoDeployment(t *testing.T) { workflow.Status.Manager().MarkFalse(api.RunningConditionType, api.DeploymentFailureReason, "") client := test.NewSonataFlowClientBuilder().WithRuntimeObjects(workflow).WithStatusSubresource(workflow).Build() - + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) reconciler := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()) // we are in failed state and have no objects @@ -126,6 +129,7 @@ func Test_newDevProfile(t *testing.T) { workflow := test.GetBaseSonataFlow(t.Name()) client := test.NewSonataFlowClientBuilder().WithRuntimeObjects(workflow).WithStatusSubresource(workflow).Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) devReconciler := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()) @@ -208,6 +212,8 @@ func Test_newDevProfile(t *testing.T) { func Test_devProfileImageDefaultsNoPlatform(t *testing.T) { workflow := test.GetBaseSonataFlowWithDevProfile(t.Name()) client := test.NewSonataFlowClientBuilder().WithRuntimeObjects(workflow).WithStatusSubresource(workflow).Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) + devReconciler := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()) result, err := devReconciler.Reconcile(context.TODO(), workflow) @@ -225,6 +231,8 @@ func Test_devProfileWithImageSnapshotOverrideWithPlatform(t *testing.T) { platform := test.GetBasePlatformWithDevBaseImageInReadyPhase(workflow.Namespace) client := test.NewSonataFlowClientBuilder().WithRuntimeObjects(workflow, platform).WithStatusSubresource(workflow, platform).Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) + devReconciler := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()) result, err := devReconciler.Reconcile(context.TODO(), workflow) @@ -242,6 +250,8 @@ func Test_devProfileWithWPlatformWithoutDevBaseImageAndWithBaseImage(t *testing. platform := test.GetBasePlatformWithBaseImageInReadyPhase(workflow.Namespace) client := test.NewSonataFlowClientBuilder().WithRuntimeObjects(workflow, platform).WithStatusSubresource(workflow, platform).Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) + devReconciler := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()) result, err := devReconciler.Reconcile(context.TODO(), workflow) @@ -259,6 +269,8 @@ func Test_devProfileWithPlatformWithoutDevBaseImageAndWithoutBaseImage(t *testin platform := test.GetBasePlatformInReadyPhase(workflow.Namespace) client := test.NewSonataFlowClientBuilder().WithRuntimeObjects(workflow, platform).WithStatusSubresource(workflow, platform).Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) + devReconciler := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()) result, err := devReconciler.Reconcile(context.TODO(), workflow) @@ -277,6 +289,7 @@ func Test_newDevProfileWithExternalConfigMaps(t *testing.T) { operatorapi.ConfigMapWorkflowResource{ConfigMap: corev1.LocalObjectReference{Name: configmapName}, WorkflowPath: "routes"}) client := test.NewSonataFlowClientBuilder().WithRuntimeObjects(workflow).WithStatusSubresource(workflow).Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) devReconciler := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()) @@ -391,6 +404,7 @@ func Test_VolumeWithCapitalizedPaths(t *testing.T) { workflow := test.GetSonataFlow(test.SonataFlowGreetingsWithStaticResourcesCR, t.Name()) client := test.NewSonataFlowClientBuilder().WithRuntimeObjects(workflow, configMap).WithStatusSubresource(workflow, configMap).Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) devReconciler := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()) diff --git a/controllers/profiles/dev/states_dev.go b/controllers/profiles/dev/states_dev.go index 17bd5a80a..dce0cd209 100644 --- a/controllers/profiles/dev/states_dev.go +++ b/controllers/profiles/dev/states_dev.go @@ -81,7 +81,7 @@ func (e *ensureRunningWorkflowState) Do(ctx context.Context, workflow *operatora if err != nil { return ctrl.Result{Requeue: false}, objs, err } - managedPropsCM, _, err := e.ensurers.managedPropsConfigMap.Ensure(ctx, workflow, pl, common.ManagedPropertiesMutateVisitor(ctx, e.StateSupport.Catalog, workflow, pl, userPropsCM.(*corev1.ConfigMap))) + managedPropsCM, _, err := e.ensurers.managedPropsConfigMap.Ensure(ctx, workflow, pl, common.ManagedPropertiesMutateVisitor(ctx, e.Catalog, workflow, pl, userPropsCM.(*corev1.ConfigMap))) if err != nil { return ctrl.Result{Requeue: false}, objs, err } @@ -117,12 +117,6 @@ func (e *ensureRunningWorkflowState) Do(ctx context.Context, workflow *operatora } objs = append(objs, route) - if knativeObjs, err := common.NewKnativeEventingHandler(e.StateSupport).Ensure(ctx, workflow); err != nil { - return ctrl.Result{RequeueAfter: constants.RequeueAfterFailure}, objs, err - } else { - objs = append(objs, knativeObjs...) - } - // First time reconciling this object, mark as wait for deployment if workflow.Status.GetTopLevelCondition().IsUnknown() { klog.V(log.I).InfoS("Workflow is in WaitingForDeployment Condition") diff --git a/controllers/profiles/factory/factory.go b/controllers/profiles/factory/factory.go index 511c6ca36..bd028b4c2 100644 --- a/controllers/profiles/factory/factory.go +++ b/controllers/profiles/factory/factory.go @@ -23,6 +23,7 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/gitops" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/preview" "github.com/apache/incubator-kie-kogito-serverless-operator/log" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" diff --git a/controllers/profiles/gitops/profile_gitops_test.go b/controllers/profiles/gitops/profile_gitops_test.go index 0a600ca73..c9cb5937c 100644 --- a/controllers/profiles/gitops/profile_gitops_test.go +++ b/controllers/profiles/gitops/profile_gitops_test.go @@ -20,6 +20,7 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/api" operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/test" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" @@ -38,6 +39,9 @@ func Test_Reconciler_ProdOps(t *testing.T) { client := test.NewSonataFlowClientBuilder(). WithRuntimeObjects(workflow). WithStatusSubresource(workflow, &operatorapi.SonataFlowBuild{}).Build() + + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) + result, err := NewProfileForOpsReconciler(client, &rest.Config{}, test.NewFakeRecorder()).Reconcile(context.TODO(), workflow) assert.NoError(t, err) @@ -65,11 +69,13 @@ func Test_Reconciler_ProdOps(t *testing.T) { assert.NotNil(t, deployment.ObjectMeta) assert.NotNil(t, deployment.ObjectMeta.Labels) assert.Equal(t, deployment.ObjectMeta.Labels, map[string]string{ - "test": "test", - "sonataflow.org/workflow-app": "simple", - "app.kubernetes.io/name": "simple", - "app.kubernetes.io/component": "serverless-workflow", - "app.kubernetes.io/managed-by": "sonataflow-operator", - "app.kubernetes.io/part-of": "sonataflow-platform", + "app": "simple", + "test": "test", + "sonataflow.org/workflow-app": "simple", + "sonataflow.org/workflow-namespace": workflow.Namespace, + "app.kubernetes.io/name": "simple", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator", + "app.kubernetes.io/part-of": "sonataflow-platform", }) } diff --git a/controllers/profiles/preview/deployment_handler.go b/controllers/profiles/preview/deployment_handler.go index dfbac2538..69148463a 100644 --- a/controllers/profiles/preview/deployment_handler.go +++ b/controllers/profiles/preview/deployment_handler.go @@ -16,6 +16,7 @@ package preview import ( "context" + "fmt" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" v1 "k8s.io/api/core/v1" @@ -26,6 +27,7 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/api" operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform/services" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/utils" @@ -53,6 +55,11 @@ func (d *DeploymentReconciler) reconcileWithImage(ctx context.Context, workflow return reconcile.Result{Requeue: false}, nil, err } + // Checks if the workflow has sink configured. + if requires, err := d.ensureKnativeSinkConfigured(workflow); requires || err != nil { + return reconcile.Result{Requeue: false}, nil, err + } + // Ensure objects result, objs, err := d.ensureObjects(ctx, workflow, image) if err != nil || result.Requeue { @@ -89,6 +96,32 @@ func (d *DeploymentReconciler) ensureKnativeServingRequired(workflow *operatorap return false, nil } +// if Knative Eventing is available, the workflow should have a sink configured, or the platform should have a broker defined +func (d *DeploymentReconciler) ensureKnativeSinkConfigured(workflow *operatorapi.SonataFlow) (bool, error) { + avail, err := knative.GetKnativeAvailability(d.Cfg) + if err != nil { + return true, err + } + if !avail.Eventing { + return false, nil + } + platform, err := platform.GetActivePlatform(context.TODO(), d.C, workflow.Namespace) + if err != nil { + return true, err + } + sink, err := knative.GetWorkflowSink(workflow, platform) + if err != nil { + return true, err + } + if sink == nil && (services.IsDataIndexEnabled(platform) || services.IsJobServiceEnabled(platform)) { + d.Recorder.Eventf(workflow, v1.EventTypeWarning, + "KnativeSinkNotConfigured", + "Failed to deploy workflow. No sink configured in the workflow or the platform when Job Service or Data Index Service is enabled.") + return true, fmt.Errorf("no sink configured in the workflow or the platform when Job Service or Data Index Service is enabled") + } + return false, nil +} + func (d *DeploymentReconciler) ensureObjects(ctx context.Context, workflow *operatorapi.SonataFlow, image string) (reconcile.Result, []client.Object, error) { pl, _ := platform.GetActivePlatform(ctx, d.C, workflow.Namespace) userPropsCM, _, err := d.ensurers.userPropsConfigMap.Ensure(ctx, workflow) @@ -121,7 +154,7 @@ func (d *DeploymentReconciler) ensureObjects(ctx context.Context, workflow *oper return reconcile.Result{}, nil, err } - eventingObjs, err := common.NewKnativeEventingHandler(d.StateSupport).Ensure(ctx, workflow) + eventingObjs, err := common.NewKnativeEventingHandler(d.StateSupport, pl).Ensure(ctx, workflow) if err != nil { return reconcile.Result{}, nil, err } @@ -149,7 +182,9 @@ func (d *DeploymentReconciler) deploymentModelMutateVisitors( if workflow.IsKnativeDeployment() { return []common.MutateVisitor{common.KServiceMutateVisitor(workflow, plf), common.ImageKServiceMutateVisitor(workflow, image), - mountConfigMapsMutateVisitor(workflow, userPropsCM, managedPropsCM)} + mountConfigMapsMutateVisitor(workflow, userPropsCM, managedPropsCM), + common.RestoreKServiceVolumeAndVolumeMountMutateVisitor(), + } } if utils.IsOpenShift() { @@ -157,11 +192,13 @@ func (d *DeploymentReconciler) deploymentModelMutateVisitors( mountConfigMapsMutateVisitor(workflow, userPropsCM, managedPropsCM), addOpenShiftImageTriggerDeploymentMutateVisitor(workflow, image), common.ImageDeploymentMutateVisitor(workflow, image), + common.RestoreDeploymentVolumeAndVolumeMountMutateVisitor(), common.RolloutDeploymentIfCMChangedMutateVisitor(workflow, userPropsCM, managedPropsCM), } } return []common.MutateVisitor{common.DeploymentMutateVisitor(workflow, plf), common.ImageDeploymentMutateVisitor(workflow, image), mountConfigMapsMutateVisitor(workflow, userPropsCM, managedPropsCM), + common.RestoreDeploymentVolumeAndVolumeMountMutateVisitor(), common.RolloutDeploymentIfCMChangedMutateVisitor(workflow, userPropsCM, managedPropsCM)} } diff --git a/controllers/profiles/preview/deployment_handler_test.go b/controllers/profiles/preview/deployment_handler_test.go index 5faf98bb9..123cdeb8b 100644 --- a/controllers/profiles/preview/deployment_handler_test.go +++ b/controllers/profiles/preview/deployment_handler_test.go @@ -20,6 +20,7 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/api/metadata" "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/test" "github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj" "github.com/magiconair/properties" @@ -44,6 +45,7 @@ func Test_CheckDeploymentModelIsKnative(t *testing.T) { WithStatusSubresource(workflow). Build() stateSupport := fakeReconcilerSupport(cli) + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) handler := NewDeploymentReconciler(stateSupport, NewObjectEnsurers(stateSupport)) result, objects, err := handler.ensureObjects(context.TODO(), workflow, "") @@ -70,6 +72,7 @@ func Test_CheckPodTemplateChangesReflectDeployment(t *testing.T) { WithStatusSubresource(workflow). Build() stateSupport := fakeReconcilerSupport(client) + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) handler := NewDeploymentReconciler(stateSupport, NewObjectEnsurers(stateSupport)) result, objects, err := handler.Reconcile(context.TODO(), workflow) @@ -105,6 +108,7 @@ func Test_CheckDeploymentRolloutAfterCMChange(t *testing.T) { WithStatusSubresource(workflow). Build() stateSupport := fakeReconcilerSupport(client) + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) handler := NewDeploymentReconciler(stateSupport, NewObjectEnsurers(stateSupport)) result, objects, err := handler.Reconcile(context.TODO(), workflow) @@ -167,6 +171,7 @@ func Test_CheckDeploymentUnchangedAfterCMChangeOtherKeys(t *testing.T) { WithStatusSubresource(workflow). Build() stateSupport := fakeReconcilerSupport(client) + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) handler := NewDeploymentReconciler(stateSupport, NewObjectEnsurers(stateSupport)) result, objects, err := handler.Reconcile(context.TODO(), workflow) diff --git a/controllers/profiles/preview/object_creators_preview.go b/controllers/profiles/preview/object_creators_preview.go index a48a4e5db..748d28136 100644 --- a/controllers/profiles/preview/object_creators_preview.go +++ b/controllers/profiles/preview/object_creators_preview.go @@ -95,7 +95,7 @@ func mountConfigMapsMutateVisitor(workflow *operatorapi.SonataFlow, userPropsCM kubeutil.VolumeProjectionAddConfigMap(defaultResourcesVolume.Projected, userPropsCM.Name, v1.KeyToPath{Key: workflowproj.ApplicationPropertiesFileName, Path: workflowproj.ApplicationPropertiesFileName}) kubeutil.VolumeProjectionAddConfigMap(defaultResourcesVolume.Projected, managedPropsCM.Name, v1.KeyToPath{Key: workflowproj.GetManagedPropertiesFileName(workflow), Path: workflowproj.GetManagedPropertiesFileName(workflow)}) kubeutil.AddOrReplaceVolume(podTemplateSpec, defaultResourcesVolume) - kubeutil.AddOrReplaceVolumeMount(idx, podTemplateSpec, + kubeutil.AddOrReplaceVolumeMount(&podTemplateSpec.Containers[idx], kubeutil.VolumeMount(constants.ConfigMapWorkflowPropsVolumeName, true, quarkusProdConfigMountPath)) return nil diff --git a/controllers/profiles/preview/profile_preview_test.go b/controllers/profiles/preview/profile_preview_test.go index ad1a1ed00..5341585cd 100644 --- a/controllers/profiles/preview/profile_preview_test.go +++ b/controllers/profiles/preview/profile_preview_test.go @@ -26,6 +26,7 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/api" operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common" "github.com/apache/incubator-kie-kogito-serverless-operator/test" "github.com/stretchr/testify/assert" @@ -49,6 +50,7 @@ func Test_Reconciler_ProdCustomPod(t *testing.T) { client := test.NewSonataFlowClientBuilder(). WithRuntimeObjects(workflow, build, platform). WithStatusSubresource(workflow, build, platform).Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) _, err := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()).Reconcile(context.TODO(), workflow) assert.NoError(t, err) @@ -64,11 +66,13 @@ func Test_Reconciler_ProdCustomPod(t *testing.T) { assert.NotNil(t, deployment.ObjectMeta) assert.NotNil(t, deployment.ObjectMeta.Labels) assert.Equal(t, deployment.ObjectMeta.Labels, map[string]string{ - "test": "test", - "sonataflow.org/workflow-app": "greeting", - "app.kubernetes.io/name": "greeting", - "app.kubernetes.io/component": "serverless-workflow", - "app.kubernetes.io/managed-by": "sonataflow-operator", + "app": "greeting", + "test": "test", + "sonataflow.org/workflow-app": "greeting", + "sonataflow.org/workflow-namespace": workflow.Namespace, + "app.kubernetes.io/name": "greeting", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator", }) } @@ -78,7 +82,7 @@ func Test_reconcilerProdBuildConditions(t *testing.T) { client := test.NewSonataFlowClientBuilder(). WithRuntimeObjects(workflow, platform). WithStatusSubresource(workflow, platform, &operatorapi.SonataFlowBuild{}).Build() - + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) result, err := NewProfileReconciler(client, &rest.Config{}, test.NewFakeRecorder()).Reconcile(context.TODO(), workflow) assert.NoError(t, err) @@ -140,6 +144,7 @@ func Test_deployWorkflowReconciliationHandler_handleObjects(t *testing.T) { WithRuntimeObjects(workflow, platform, build). WithStatusSubresource(workflow, platform, build). Build() + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) handler := &deployWithBuildWorkflowState{ StateSupport: fakeReconcilerSupport(client), ensurers: NewObjectEnsurers(&common.StateSupport{C: client}), @@ -168,7 +173,7 @@ func Test_GenerationAnnotationCheck(t *testing.T) { client := test.NewSonataFlowClientBuilder(). WithRuntimeObjects(workflow, platform). WithStatusSubresource(workflow, platform, &operatorapi.SonataFlowBuild{}).Build() - + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) handler := &deployWithBuildWorkflowState{ StateSupport: fakeReconcilerSupport(client), ensurers: NewObjectEnsurers(&common.StateSupport{C: client}), diff --git a/controllers/profiles/preview/states_preview.go b/controllers/profiles/preview/states_preview.go index 333c33976..6fadac9cb 100644 --- a/controllers/profiles/preview/states_preview.go +++ b/controllers/profiles/preview/states_preview.go @@ -21,6 +21,7 @@ package preview import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -70,7 +71,7 @@ func (h *newBuilderState) Do(ctx context.Context, workflow *operatorapi.SonataFl // available at build time. userPropsCM, _, err := h.ensurers.userPropsConfigMap.Ensure(ctx, workflow) if err != nil { - workflow.Status.Manager().MarkFalse(api.RunningConditionType, api.ExternalResourcesNotFoundReason, "Unable to retrieve the user properties config map") + workflow.Status.Manager().MarkFalse(api.RunningConditionType, api.ExternalResourcesNotFoundReason, fmt.Sprintf("Unable to retrieve the user properties config map: %v", err)) _, err = h.PerformStatusUpdate(ctx, workflow) return ctrl.Result{}, nil, err } @@ -78,7 +79,7 @@ func (h *newBuilderState) Do(ctx context.Context, workflow *operatorapi.SonataFl _, _, err = h.ensurers.managedPropsConfigMap.Ensure(ctx, workflow, pl, common.ManagedPropertiesMutateVisitor(ctx, h.StateSupport.Catalog, workflow, pl, userPropsCM.(*corev1.ConfigMap))) if err != nil { - workflow.Status.Manager().MarkFalse(api.RunningConditionType, api.ExternalResourcesNotFoundReason, "Unable to retrieve the managed properties config map") + workflow.Status.Manager().MarkFalse(api.RunningConditionType, api.ExternalResourcesNotFoundReason, fmt.Sprintf("Unable to retrieve the managed properties config map: %v", err)) _, err = h.PerformStatusUpdate(ctx, workflow) return ctrl.Result{}, nil, err } @@ -210,7 +211,13 @@ func (h *deployWithBuildWorkflowState) Do(ctx context.Context, workflow *operato } // didn't change, business as usual - return NewDeploymentReconciler(h.StateSupport, h.ensurers).reconcileWithImage(ctx, workflow, build.Status.ImageTag) + result, objs, err := NewDeploymentReconciler(h.StateSupport, h.ensurers).reconcileWithImage(ctx, workflow, build.Status.ImageTag) + if err != nil { + workflow.Status.Manager().MarkFalse(api.RunningConditionType, api.DeploymentFailureReason, fmt.Sprintf("Error in deploy the workflow:%s", err)) + _, err = h.PerformStatusUpdate(ctx, workflow) + return result, nil, err + } + return result, objs, err } func (h *deployWithBuildWorkflowState) PostReconcile(ctx context.Context, workflow *operatorapi.SonataFlow) error { diff --git a/controllers/sonataflow_controller.go b/controllers/sonataflow_controller.go index 7c1d28abb..3436cff0c 100644 --- a/controllers/sonataflow_controller.go +++ b/controllers/sonataflow_controller.go @@ -26,26 +26,32 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/api/metadata" "k8s.io/klog/v2" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" profiles "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/factory" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/rest" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" + sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" + servingv1 "knative.dev/serving/pkg/apis/serving/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/apache/incubator-kie-kogito-serverless-operator/api" - "github.com/apache/incubator-kie-kogito-serverless-operator/log" operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform" ) @@ -91,6 +97,15 @@ func (r *SonataFlowReconciler) Reconcile(ctx context.Context, req ctrl.Request) } r.setDefaults(workflow) + // If the workflow is being deleted, clean up the triggers on a different namespace + if workflow.DeletionTimestamp != nil && controllerutil.ContainsFinalizer(workflow, constants.TriggerFinalizer) { + err := r.cleanupTriggers(ctx, workflow) + if err != nil { + klog.V(log.E).ErrorS(err, "Failed to clean up triggers for workflow %s", workflow.Name) + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } // Only process resources assigned to the operator if !platform.IsOperatorHandlerConsideringLock(ctx, r.Client, req.Namespace, workflow) { @@ -112,6 +127,37 @@ func (r *SonataFlowReconciler) setDefaults(workflow *operatorapi.SonataFlow) { } } +func (r *SonataFlowReconciler) cleanupTriggers(ctx context.Context, workflow *operatorapi.SonataFlow) error { + plf, _ := platform.GetActivePlatform(ctx, r.Client, workflow.Namespace) + if plf == nil || len(plf.Status.Triggers) == 0 { + return nil + } + avail, err := knative.GetKnativeAvailability(r.Config) + if err != nil { + return err + } + if avail.Eventing { + for _, triggerRef := range workflow.Status.Triggers { + trigger := &eventingv1.Trigger{ + ObjectMeta: metav1.ObjectMeta{ + Name: triggerRef.Name, + Namespace: triggerRef.Namespace, + }, + } + if err := r.Client.Delete(ctx, trigger); err != nil { + return err + } + } + } + controllerutil.RemoveFinalizer(workflow, constants.TriggerFinalizer) + return r.Client.Update(ctx, workflow) +} + +// Delete implements a handler for the Delete event. +func (r *SonataFlowReconciler) Delete(e event.DeleteEvent) error { + return nil +} + func platformEnqueueRequestsFromMapFunc(c client.Client, p *operatorapi.SonataFlowPlatform) []reconcile.Request { var requests []reconcile.Request @@ -185,6 +231,9 @@ func (r *SonataFlowReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). Owns(&corev1.ConfigMap{}). + Owns(&servingv1.Service{}). + Owns(&eventingv1.Trigger{}). + Owns(&sourcesv1.SinkBinding{}). Owns(&operatorapi.SonataFlowBuild{}). Watches(&operatorapi.SonataFlowPlatform{}, handler.EnqueueRequestsFromMapFunc(func(c context.Context, a client.Object) []reconcile.Request { plat, ok := a.(*operatorapi.SonataFlowPlatform) @@ -202,5 +251,6 @@ func (r *SonataFlowReconciler) SetupWithManager(mgr ctrl.Manager) error { } return buildEnqueueRequestsFromMapFunc(mgr.GetClient(), build) })). + Watches(&eventingv1.Trigger{}, handler.EnqueueRequestsFromMapFunc(knative.MapTriggerToPlatformRequests)). Complete(r) } diff --git a/controllers/sonataflowplatform_controller.go b/controllers/sonataflowplatform_controller.go index e111a3212..a6887dac1 100644 --- a/controllers/sonataflowplatform_controller.go +++ b/controllers/sonataflowplatform_controller.go @@ -28,20 +28,26 @@ import ( operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" clientr "github.com/apache/incubator-kie-kogito-serverless-operator/container-builder/client" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/clusterplatform" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform/services" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/log" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" + sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" ctrlrun "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ctrl "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -118,6 +124,16 @@ func (r *SonataFlowPlatformReconciler) Reconcile(ctx context.Context, req reconc return reconcile.Result{}, err } + // If the platform is being deleted, clean up the triggers on a different namespace + if instance.DeletionTimestamp != nil && controllerutil.ContainsFinalizer(&instance, constants.TriggerFinalizer) { + err := r.cleanupTriggers(ctx, &instance) + if err != nil { + klog.V(log.E).ErrorS(err, "Failed to clean up triggers for platform %s in namespace %s", instance.Name, instance.Namespace) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + for _, a := range actions { cli, _ := clientr.FromCtrlClientSchemeAndConfig(r.Client, r.Scheme, r.Config) a.InjectClient(cli) @@ -140,12 +156,10 @@ func (r *SonataFlowPlatformReconciler) Reconcile(ctx context.Context, req reconc if target != nil { target.Status.ObservedGeneration = instance.Generation - if err := r.Client.Status().Patch(ctx, target, ctrl.MergeFrom(&instance)); err != nil { r.Recorder.Event(&instance, corev1.EventTypeNormal, "Status Updated", fmt.Sprintf("Updated platform condition %s", instance.Status.GetTopLevelCondition())) return reconcile.Result{}, err } - if err := r.Client.Update(ctx, target); err != nil { r.Recorder.Event(&instance, corev1.EventTypeNormal, "Spec Updated", fmt.Sprintf("Updated platform condition to %s", instance.Status.GetTopLevelCondition())) return reconcile.Result{}, err @@ -170,6 +184,28 @@ func (r *SonataFlowPlatformReconciler) Reconcile(ctx context.Context, req reconc } +func (r *SonataFlowPlatformReconciler) cleanupTriggers(ctx context.Context, platform *operatorapi.SonataFlowPlatform) error { + avail, err := knative.GetKnativeAvailability(r.Config) + if err != nil { + return err + } + if avail.Eventing { + for _, triggerRef := range platform.Status.Triggers { + trigger := &eventingv1.Trigger{ + ObjectMeta: metav1.ObjectMeta{ + Name: triggerRef.Name, + Namespace: triggerRef.Namespace, + }, + } + if err := r.Client.Delete(ctx, trigger); err != nil { + return err + } + } + } + controllerutil.RemoveFinalizer(platform, constants.TriggerFinalizer) + return r.Client.Update(ctx, platform) +} + // sonataFlowPlatformUpdateStatus If an active cluster platform exists, update platform.Status accordingly func (r *SonataFlowPlatformReconciler) updateSonataFlowPlatformStatus(ctx context.Context, req reconcile.Request, target *operatorapi.SonataFlowPlatform) error { // Fetch the active SonataFlowClusterPlatform instance @@ -224,8 +260,11 @@ func (r *SonataFlowPlatformReconciler) SetupWithManager(mgr ctrlrun.Manager) err Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). Owns(&corev1.ConfigMap{}). + Owns(&eventingv1.Trigger{}). + Owns(&sourcesv1.SinkBinding{}). Watches(&operatorapi.SonataFlowPlatform{}, handler.EnqueueRequestsFromMapFunc(r.mapPlatformToPlatformRequests)). Watches(&operatorapi.SonataFlowClusterPlatform{}, handler.EnqueueRequestsFromMapFunc(r.mapClusterPlatformToPlatformRequests)). + Watches(&eventingv1.Trigger{}, handler.EnqueueRequestsFromMapFunc(knative.MapTriggerToPlatformRequests)). Complete(r) } diff --git a/controllers/sonataflowplatform_controller_test.go b/controllers/sonataflowplatform_controller_test.go index 022c76063..9f3661a45 100644 --- a/controllers/sonataflowplatform_controller_test.go +++ b/controllers/sonataflowplatform_controller_test.go @@ -25,15 +25,22 @@ import ( "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/clusterplatform" + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/knative" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/platform/services" "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/test" + "github.com/apache/incubator-kie-kogito-serverless-operator/utils" "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" + sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" + duckv1 "knative.dev/pkg/apis/duck/v1" + "knative.dev/pkg/kmeta" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -56,6 +63,7 @@ func TestSonataFlowPlatformController(t *testing.T) { // Create a fake client to mock API calls. cl := test.NewSonataFlowClientBuilder().WithRuntimeObjects(ksp).WithStatusSubresource(ksp).Build() + utils.SetClient(cl) // Create a SonataFlowPlatformReconciler object with the scheme and fake client. r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} @@ -88,11 +96,12 @@ func TestSonataFlowPlatformController(t *testing.T) { // Create a SonataFlowPlatform object with metadata and spec. ksp := test.GetBasePlatformInReadyPhase(namespace) ksp.Spec.Services = &v1alpha08.ServicesPlatformSpec{ - DataIndex: &v1alpha08.ServiceSpec{}, + DataIndex: &v1alpha08.DataIndexServiceSpec{}, } // Create a fake client to mock API calls. cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(ksp).WithStatusSubresource(ksp).Build() + utils.SetClient(cl) // Create a SonataFlowPlatformReconciler object with the scheme and fake client. r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} @@ -167,20 +176,25 @@ func TestSonataFlowPlatformController(t *testing.T) { ksp := test.GetBasePlatformInReadyPhase(namespace) var replicas int32 = 2 ksp.Spec.Services = &v1alpha08.ServicesPlatformSpec{ - DataIndex: &v1alpha08.ServiceSpec{ - PodTemplate: v1alpha08.PodTemplateSpec{ - Replicas: &replicas, - Container: v1alpha08.ContainerSpec{ - Command: []string{"test:latest"}, + DataIndex: &v1alpha08.DataIndexServiceSpec{ + ServiceSpec: v1alpha08.ServiceSpec{ + PodTemplate: v1alpha08.PodTemplateSpec{ + Replicas: &replicas, + Container: v1alpha08.ContainerSpec{ + Command: []string{"test:latest"}, + }, }, }, + Source: nil, }, } - di := services.NewDataIndexHandler(ksp) - // Create a fake client to mock API calls. cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(ksp).WithStatusSubresource(ksp).Build() + utils.SetClient(cl) + + di := services.NewDataIndexHandler(ksp) + // Create a SonataFlowPlatformReconciler object with the scheme and fake client. r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} @@ -260,8 +274,8 @@ func TestSonataFlowPlatformController(t *testing.T) { // Check with persistence set ksp.Spec = v1alpha08.SonataFlowPlatformSpec{ Services: &v1alpha08.ServicesPlatformSpec{ - DataIndex: &v1alpha08.ServiceSpec{}, - JobService: &v1alpha08.ServiceSpec{}, + DataIndex: &v1alpha08.DataIndexServiceSpec{}, + JobService: &v1alpha08.JobServiceServiceSpec{}, }, Persistence: &v1alpha08.PlatformPersistenceOptionsSpec{ PostgreSQL: &v1alpha08.PlatformPersistencePostgreSQL{ @@ -277,6 +291,7 @@ func TestSonataFlowPlatformController(t *testing.T) { // Create a fake client to mock API calls. cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(ksp).WithStatusSubresource(ksp).Build() + utils.SetClient(cl) // Create a SonataFlowPlatformReconciler object with the scheme and fake client. r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} @@ -354,19 +369,23 @@ func TestSonataFlowPlatformController(t *testing.T) { urlJS := "jdbc:postgresql://localhost:5432/database?currentSchema=job-service" ksp.Spec = v1alpha08.SonataFlowPlatformSpec{ Services: &v1alpha08.ServicesPlatformSpec{ - DataIndex: &v1alpha08.ServiceSpec{ - Persistence: &v1alpha08.PersistenceOptionsSpec{ - PostgreSQL: &v1alpha08.PersistencePostgreSQL{ - SecretRef: v1alpha08.PostgreSQLSecretOptions{Name: "dataIndex"}, - JdbcUrl: urlDI, + DataIndex: &v1alpha08.DataIndexServiceSpec{ + ServiceSpec: v1alpha08.ServiceSpec{ + Persistence: &v1alpha08.PersistenceOptionsSpec{ + PostgreSQL: &v1alpha08.PersistencePostgreSQL{ + SecretRef: v1alpha08.PostgreSQLSecretOptions{Name: "dataIndex"}, + JdbcUrl: urlDI, + }, }, }, }, - JobService: &v1alpha08.ServiceSpec{ - Persistence: &v1alpha08.PersistenceOptionsSpec{ - PostgreSQL: &v1alpha08.PersistencePostgreSQL{ - SecretRef: v1alpha08.PostgreSQLSecretOptions{Name: "job"}, - JdbcUrl: urlJS, + JobService: &v1alpha08.JobServiceServiceSpec{ + ServiceSpec: v1alpha08.ServiceSpec{ + Persistence: &v1alpha08.PersistenceOptionsSpec{ + PostgreSQL: &v1alpha08.PersistencePostgreSQL{ + SecretRef: v1alpha08.PostgreSQLSecretOptions{Name: "job"}, + JdbcUrl: urlJS, + }, }, }, }, @@ -381,6 +400,7 @@ func TestSonataFlowPlatformController(t *testing.T) { // Create a fake client to mock API calls. cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(ksp).WithStatusSubresource(ksp).Build() + utils.SetClient(cl) // Create a SonataFlowPlatformReconciler object with the scheme and fake client. r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} @@ -467,11 +487,12 @@ func TestSonataFlowPlatformController(t *testing.T) { // Create a SonataFlowPlatform object with metadata and spec. ksp := test.GetBasePlatformInReadyPhase(namespace) ksp.Spec.Services = &v1alpha08.ServicesPlatformSpec{ - JobService: &v1alpha08.ServiceSpec{}, + JobService: &v1alpha08.JobServiceServiceSpec{}, } // Create a fake client to mock API calls. cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(ksp).WithStatusSubresource(ksp).Build() + utils.SetClient(cl) // Create a SonataFlowPlatformReconciler object with the scheme and fake client. r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} @@ -544,20 +565,22 @@ func TestSonataFlowPlatformController(t *testing.T) { ksp := test.GetBasePlatformInReadyPhase(namespace) var replicas int32 = 2 ksp.Spec.Services = &v1alpha08.ServicesPlatformSpec{ - JobService: &v1alpha08.ServiceSpec{ - PodTemplate: v1alpha08.PodTemplateSpec{ - Replicas: &replicas, - Container: v1alpha08.ContainerSpec{ - Command: []string{"test:latest"}, + JobService: &v1alpha08.JobServiceServiceSpec{ + ServiceSpec: v1alpha08.ServiceSpec{ + PodTemplate: v1alpha08.PodTemplateSpec{ + Replicas: &replicas, + Container: v1alpha08.ContainerSpec{ + Command: []string{"test:latest"}, + }, }, }, }, } - js := services.NewJobServiceHandler(ksp) - // Create a fake client to mock API calls. cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(ksp).WithStatusSubresource(ksp).Build() + utils.SetClient(cl) + js := services.NewJobServiceHandler(ksp) // Create a SonataFlowPlatformReconciler object with the scheme and fake client. r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} @@ -625,14 +648,15 @@ func TestSonataFlowPlatformController(t *testing.T) { // Create a SonataFlowPlatform object with metadata and spec. ksp := test.GetBasePlatformInReadyPhase(namespace) ksp.Spec.Services = &v1alpha08.ServicesPlatformSpec{ - DataIndex: &v1alpha08.ServiceSpec{}, - JobService: &v1alpha08.ServiceSpec{}, + DataIndex: &v1alpha08.DataIndexServiceSpec{}, + JobService: &v1alpha08.JobServiceServiceSpec{}, } - di := services.NewDataIndexHandler(ksp) - js := services.NewJobServiceHandler(ksp) // Create a fake client to mock API calls. cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(ksp).WithStatusSubresource(ksp).Build() + utils.SetClient(cl) + di := services.NewDataIndexHandler(ksp) + js := services.NewJobServiceHandler(ksp) // Create a SonataFlowPlatformReconciler object with the scheme and fake client. r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} @@ -693,14 +717,15 @@ func TestSonataFlowPlatformController(t *testing.T) { // Create a SonataFlowPlatform object with metadata and spec. ksp := test.GetBasePlatformInReadyPhase(namespace) ksp.Spec.Services = &v1alpha08.ServicesPlatformSpec{ - DataIndex: &v1alpha08.ServiceSpec{}, - JobService: &v1alpha08.ServiceSpec{}, + DataIndex: &v1alpha08.DataIndexServiceSpec{}, + JobService: &v1alpha08.JobServiceServiceSpec{}, } ksp2 := test.GetBasePlatformInReadyPhase(namespace) ksp2.Name = "ksp2" // Create a fake client to mock API calls. cl := test.NewSonataFlowClientBuilder().WithRuntimeObjects(kscp, ksp, ksp2).WithStatusSubresource(kscp, ksp, ksp2).Build() + utils.SetClient(cl) // Create a SonataFlowPlatformReconciler object with the scheme and fake client. r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} @@ -827,4 +852,191 @@ func TestSonataFlowPlatformController(t *testing.T) { assert.NotNil(t, ksp2.Status.ClusterPlatformRef) assert.Nil(t, ksp2.Status.ClusterPlatformRef.Services) }) + t.Run("verify that knative resources creation for job service and data index service with platform level broker is performed without error", func(t *testing.T) { + namespace := t.Name() + // Create a SonataFlowPlatform object with metadata and spec. + ksp := test.GetBasePlatformWithBrokerInReadyPhase(namespace) + broker := test.GetDefaultBroker(namespace) + brokerName := broker.Name + + // Create a fake client to mock API calls. + cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(ksp, broker).WithStatusSubresource(ksp, broker).Build() + utils.SetClient(cl) + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) + // Create a SonataFlowPlatformReconciler object with the scheme and fake client. + r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} + + // Mock request to simulate Reconcile() being called on an event for a + // watched resource . + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: ksp.Name, + Namespace: ksp.Namespace, + }, + } + _, err := r.Reconcile(context.TODO(), req) + if err != nil && err.Error() != "waiting for K_SINK injection for sonataflow-platform-jobs-service to complete" { + t.Fatalf("reconcile: (%v)", err) + } + + assert.NoError(t, cl.Get(context.TODO(), types.NamespacedName{Name: ksp.Name, Namespace: ksp.Namespace}, ksp)) + + // Perform some checks on the created CR + assert.Equal(t, "quay.io/kiegroup", ksp.Spec.Build.Config.Registry.Address) + assert.Equal(t, "regcred", ksp.Spec.Build.Config.Registry.Secret) + assert.Equal(t, v1alpha08.OperatorBuildStrategy, ksp.Spec.Build.Config.BuildStrategy) + assert.NotNil(t, ksp.Spec.Eventing) + assert.NotNil(t, ksp.Spec.Eventing.Broker) + assert.NotNil(t, ksp.Spec.Eventing.Broker.Ref) + assert.Equal(t, ksp.Spec.Eventing.Broker.Ref.Name, brokerName) + assert.NotNil(t, ksp.Spec.Services.DataIndex) + assert.NotNil(t, ksp.Spec.Services.DataIndex.Enabled) + assert.Equal(t, true, *ksp.Spec.Services.DataIndex.Enabled) + assert.NotNil(t, ksp.Spec.Services.JobService) + assert.NotNil(t, ksp.Spec.Services.JobService.Enabled) + assert.Equal(t, true, *ksp.Spec.Services.JobService.Enabled) + assert.Equal(t, v1alpha08.PlatformClusterKubernetes, ksp.Status.Cluster) + + assert.Equal(t, "", ksp.Status.GetTopLevelCondition().Reason) + + // Check Triggers + trigger := &eventingv1.Trigger{} + validateTrigger(t, cl, "jobs-service-create-job-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "jobs-service-delete-job-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "data-index-jobs-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "jobs-service-create-job-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "data-index-process-definition-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "data-index-process-error-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "data-index-process-node-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "data-index-process-sla-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "data-index-process-state-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "data-index-process-variable-", ksp.Namespace, ksp, trigger) + + // Check SinkBinding + sinkBinding := &sourcesv1.SinkBinding{} + assert.NoError(t, cl.Get(context.TODO(), types.NamespacedName{Name: "sonataflow-platform-jobs-service-sb", Namespace: ksp.Namespace}, sinkBinding)) + + }) + + t.Run("verify that knative resources creation for job service and data index service with services level brokers is performed without error", func(t *testing.T) { + namespace := t.Name() + // Create a SonataFlowPlatform object with metadata and spec. + ksp := test.GetBasePlatformWithBrokerInReadyPhase(namespace) + brokerName := "default" + brokerNameDataIndexSource := "broker-di-source" + brokerNameJobsServiceSource := "broker-jobs-source" + brokerNameJobsServiceSink := "broker-jobs-sink" + broker := test.GetDefaultBroker(namespace) + brokerDataIndexSource := test.GetDefaultBroker(namespace) + brokerDataIndexSource.Name = brokerNameDataIndexSource + brokerJobsServiceSource := test.GetDefaultBroker(namespace) + brokerJobsServiceSource.Name = brokerNameJobsServiceSource + brokerJobsServiceSink := test.GetDefaultBroker(namespace) + brokerJobsServiceSink.Name = brokerNameJobsServiceSink + + ksp.Spec.Services.DataIndex.Source = &duckv1.Destination{ + Ref: &duckv1.KReference{ + Name: brokerNameDataIndexSource, + Namespace: namespace, + APIVersion: "eventing.knative.dev/v1", + Kind: "Broker", + }, + } + ksp.Spec.Services.JobService.Sink = &duckv1.Destination{ + Ref: &duckv1.KReference{ + Name: brokerNameJobsServiceSink, + Namespace: namespace, + APIVersion: "eventing.knative.dev/v1", + Kind: "Broker", + }, + } + ksp.Spec.Services.JobService.Source = &duckv1.Destination{ + Ref: &duckv1.KReference{ + Name: brokerNameJobsServiceSource, + Namespace: namespace, + APIVersion: "eventing.knative.dev/v1", + Kind: "Broker", + }, + } + + // Create a fake client to mock API calls. + cl := test.NewKogitoClientBuilderWithOpenShift().WithRuntimeObjects(ksp, broker, brokerDataIndexSource, brokerJobsServiceSource, brokerJobsServiceSink).WithStatusSubresource(ksp, broker, brokerDataIndexSource, brokerJobsServiceSource, brokerJobsServiceSink).Build() + utils.SetClient(cl) + knative.SetDiscoveryClient(test.CreateFakeKnativeDiscoveryClient()) + // Create a SonataFlowPlatformReconciler object with the scheme and fake client. + r := &SonataFlowPlatformReconciler{cl, cl, cl.Scheme(), &rest.Config{}, &record.FakeRecorder{}} + + // Mock request to simulate Reconcile() being called on an event for a + // watched resource . + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: ksp.Name, + Namespace: ksp.Namespace, + }, + } + _, err := r.Reconcile(context.TODO(), req) + if err != nil && err.Error() != "waiting for K_SINK injection for sonataflow-platform-jobs-service to complete" { + t.Fatalf("reconcile: (%v)", err) + } + + assert.NoError(t, cl.Get(context.TODO(), types.NamespacedName{Name: ksp.Name, Namespace: ksp.Namespace}, ksp)) + + // Perform some checks on the created CR + assert.Equal(t, "quay.io/kiegroup", ksp.Spec.Build.Config.Registry.Address) + assert.Equal(t, "regcred", ksp.Spec.Build.Config.Registry.Secret) + assert.Equal(t, v1alpha08.OperatorBuildStrategy, ksp.Spec.Build.Config.BuildStrategy) + assert.NotNil(t, ksp.Spec.Eventing) + assert.NotNil(t, ksp.Spec.Eventing.Broker) + assert.NotNil(t, ksp.Spec.Eventing.Broker.Ref) + assert.Equal(t, ksp.Spec.Eventing.Broker.Ref.Name, brokerName) + assert.NotNil(t, ksp.Spec.Services.DataIndex) + assert.NotNil(t, ksp.Spec.Services.DataIndex.Enabled) + assert.Equal(t, true, *ksp.Spec.Services.DataIndex.Enabled) + assert.NotNil(t, ksp.Spec.Services.DataIndex.Source) + assert.NotNil(t, ksp.Spec.Services.DataIndex.Source.Ref) + assert.Equal(t, ksp.Spec.Services.DataIndex.Source.Ref.Name, brokerNameDataIndexSource) + assert.NotNil(t, ksp.Spec.Services.JobService) + assert.NotNil(t, ksp.Spec.Services.JobService.Enabled) + assert.Equal(t, true, *ksp.Spec.Services.JobService.Enabled) + assert.NotNil(t, ksp.Spec.Services.JobService.Source) + assert.NotNil(t, ksp.Spec.Services.JobService.Source.Ref) + assert.Equal(t, ksp.Spec.Services.JobService.Source.Ref.Name, brokerNameJobsServiceSource) + assert.NotNil(t, ksp.Spec.Services.JobService.Sink) + assert.NotNil(t, ksp.Spec.Services.JobService.Sink.Ref) + assert.Equal(t, ksp.Spec.Services.JobService.Sink.Ref.Name, brokerNameJobsServiceSink) + assert.Equal(t, v1alpha08.PlatformClusterKubernetes, ksp.Status.Cluster) + assert.Equal(t, "", ksp.Status.GetTopLevelCondition().Reason) + + // Check Triggers to have the service level source used + trigger := &eventingv1.Trigger{} + validateTrigger(t, cl, "jobs-service-create-job-", ksp.Namespace, ksp, trigger) + assert.Equal(t, trigger.Spec.Broker, brokerNameJobsServiceSource) + validateTrigger(t, cl, "jobs-service-delete-job-", ksp.Namespace, ksp, trigger) + assert.Equal(t, trigger.Spec.Broker, brokerNameJobsServiceSource) + validateTrigger(t, cl, "data-index-jobs-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "jobs-service-create-job-", ksp.Namespace, ksp, trigger) + validateTrigger(t, cl, "data-index-process-definition-", ksp.Namespace, ksp, trigger) + assert.Equal(t, trigger.Spec.Broker, brokerNameDataIndexSource) + validateTrigger(t, cl, "data-index-process-error-", ksp.Namespace, ksp, trigger) + assert.Equal(t, trigger.Spec.Broker, brokerNameDataIndexSource) + validateTrigger(t, cl, "data-index-process-node-", ksp.Namespace, ksp, trigger) + assert.Equal(t, trigger.Spec.Broker, brokerNameDataIndexSource) + validateTrigger(t, cl, "data-index-process-sla-", ksp.Namespace, ksp, trigger) + assert.Equal(t, trigger.Spec.Broker, brokerNameDataIndexSource) + validateTrigger(t, cl, "data-index-process-state-", ksp.Namespace, ksp, trigger) + assert.Equal(t, trigger.Spec.Broker, brokerNameDataIndexSource) + validateTrigger(t, cl, "data-index-process-variable-", ksp.Namespace, ksp, trigger) + assert.Equal(t, trigger.Spec.Broker, brokerNameDataIndexSource) + + // Check SinkBinding to have the sink level source used + sinkBinding := &sourcesv1.SinkBinding{} + assert.NoError(t, cl.Get(context.TODO(), types.NamespacedName{Name: "sonataflow-platform-jobs-service-sb", Namespace: ksp.Namespace}, sinkBinding)) + assert.NotNil(t, sinkBinding.Spec.Sink) + assert.NotNil(t, sinkBinding.Spec.Sink.Ref) + assert.Equal(t, sinkBinding.Spec.Sink.Ref.Name, brokerNameJobsServiceSink) + }) +} + +func validateTrigger(t *testing.T, cl client.WithWatch, prefix string, namespace string, ksp *v1alpha08.SonataFlowPlatform, trigger *eventingv1.Trigger) { + assert.NoError(t, cl.Get(context.TODO(), types.NamespacedName{Name: kmeta.ChildName(prefix, string(ksp.GetUID())), Namespace: namespace}, trigger)) } diff --git a/go.mod b/go.mod index 82e0303fd..bb49b029a 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/imdario/mergo v0.3.16 k8s.io/klog/v2 v2.100.1 + k8s.io/utils v0.0.0-20230711102312-30195339c3c7 knative.dev/eventing v0.26.0 ) @@ -128,7 +129,6 @@ require ( k8s.io/apiextensions-apiserver v0.27.6 // indirect k8s.io/component-base v0.27.6 // indirect k8s.io/kube-openapi v0.0.0-20230525220651-2546d827e515 // indirect - k8s.io/utils v0.0.0-20230711102312-30195339c3c7 // indirect knative.dev/networking v0.0.0-20231017124814-2a7676e912b7 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect diff --git a/hack/ci/create-kind-cluster-with-registry.sh b/hack/ci/create-kind-cluster-with-registry.sh index 2feb2c254..c8c871b22 100755 --- a/hack/ci/create-kind-cluster-with-registry.sh +++ b/hack/ci/create-kind-cluster-with-registry.sh @@ -16,6 +16,7 @@ set -o errexit +container_engine="$1" reg_name='kind-registry' reg_port='5001' @@ -31,6 +32,21 @@ reg_port='5001' cat < 1 { + GinkgoWriter.Println("multiple pods found") + return false // multiple pods found, wait for other pods to terminate + } + return true + }, 1*time.Minute, 5).Should(BeTrue()) +} + +func verifyTrigger(triggers []operatorapi.SonataFlowPlatformTriggerRef, namePrefix, path, ns, broker string) error { + GinkgoWriter.Println("Triggers from platform status:", triggers) + for _, ref := range triggers { + if strings.HasPrefix(ref.Name, namePrefix) && ref.Namespace == ns { + return verifyTriggerData(ref.Name, ns, path, broker) + } + } + return fmt.Errorf("failed to find trigger to verify with prefix: %v, namespace: %v", namePrefix, ns) +} + +func verifyTriggerData(name, ns, path, broker string) error { + cmd := exec.Command("kubectl", "get", "trigger", name, "-n", ns, "-ojsonpath={.spec.broker} {.status.subscriberUri} {.status.conditions[?(@.type=='Ready')].status}") + out, err := utils.Run(cmd) + if err != nil { + return err + } + data := strings.Fields(string(out)) + if len(data) == 3 && broker == data[0] && strings.HasSuffix(data[1], path) && data[2] == "True" { + return nil + } + return fmt.Errorf("failed to verify trigger %v, data=%s", name, string(out)) +} + +func verifySinkBinding(name, ns, broker string) error { + cmd := exec.Command("kubectl", "get", "sinkbinding", name, "-n", ns, "-ojsonpath={.status.sinkUri} {.status.conditions[?(@.type=='Ready')].status}") + out, err := utils.Run(cmd) + if err != nil { + return err + } + data := strings.Fields(string(out)) + if len(data) == 2 && strings.HasSuffix(data[0], broker) && data[1] == "True" { + return nil + } + return fmt.Errorf("failed to verify sinkbinding %v, data=%s", name, string(out)) +} diff --git a/test/e2e/platform_test.go b/test/e2e/platform_test.go index 511503be5..6074ef168 100644 --- a/test/e2e/platform_test.go +++ b/test/e2e/platform_test.go @@ -16,6 +16,7 @@ package e2e import ( "bytes" + "encoding/json" "fmt" "math/rand" "os/exec" @@ -24,6 +25,9 @@ import ( "time" "github.com/apache/incubator-kie-kogito-serverless-operator/api/metadata" + operatorapi "github.com/apache/incubator-kie-kogito-serverless-operator/api/v1alpha08" + + "github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/constants" "github.com/apache/incubator-kie-kogito-serverless-operator/test" "github.com/apache/incubator-kie-kogito-serverless-operator/test/utils" @@ -47,8 +51,9 @@ const ( var _ = Describe("Validate the persistence", Ordered, func() { var ( - projectDir string - targetNamespace string + projectDir string + targetNamespace string + targetNamespace2 string ) BeforeEach(func() { @@ -56,13 +61,25 @@ var _ = Describe("Validate the persistence", Ordered, func() { cmd := exec.Command("kubectl", "create", "namespace", targetNamespace) _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) + + targetNamespace2 = fmt.Sprintf("test-%d", rand.Intn(1024)+1) + cmd = exec.Command("kubectl", "create", "namespace", targetNamespace2) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { // Remove resources in test namespace with no failure - if !CurrentSpecReport().Failed() && len(targetNamespace) > 0 { - cmd := exec.Command("kubectl", "delete", "namespace", targetNamespace, "--wait") - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred()) + if !CurrentSpecReport().Failed() { + if len(targetNamespace) > 0 { + cmd := exec.Command("kubectl", "delete", "namespace", targetNamespace, "--wait") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + } + if len(targetNamespace2) > 0 { + cmd := exec.Command("kubectl", "delete", "namespace", targetNamespace2, "--wait") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + } } }) var _ = Context("with platform services", func() { @@ -83,11 +100,20 @@ var _ = Describe("Validate the persistence", Ordered, func() { Expect(err).NotTo(HaveOccurred()) By("Wait for SonataFlowPlatform CR to complete deployment") // wait for service deployments to be ready - EventuallyWithOffset(1, func() error { + EventuallyWithOffset(1, func() bool { cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "--for", "condition=Ready", "--timeout=5s") _, err = utils.Run(cmd) - return err - }, 20*time.Minute, 5).Should(Succeed()) + if err != nil { + return false + } + if profile == metadata.PreviewProfile.String() { + GinkgoWriter.Println("waitForPodRestartCompletion") + waitForPodRestartCompletion("app.kubernetes.io/name=jobs-service", targetNamespace) + GinkgoWriter.Println("waitForPodRestartCompletion done") + return true + } + return true + }, 30*time.Minute, 5).Should(BeTrue()) By("Evaluate status of service's health endpoint") cmd = exec.Command("kubectl", "get", "pod", "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}") output, err := utils.Run(cmd) @@ -156,4 +182,165 @@ var _ = Describe("Validate the persistence", Ordered, func() { Entry("and both Job Service and Data Index using the one defined in each service, discarding the one from the platform CR", test.GetSonataFlowE2EPlatformPersistenceSampleDataDirectory("overwritten_by_services")), ) + DescribeTable("when deploying a SonataFlowPlatform CR with brokers", func(testcaseDir string) { + By("Deploy the brokers") + cmd := exec.Command("kubectl", "create", "-n", targetNamespace, "-f", filepath.Join(projectDir, + testcaseDir, "broker")) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("Wait for the brokers to be ready") + EventuallyWithOffset(1, func() error { + cmd = exec.Command("kubectl", "wait", "broker", "-l", "test=test-e2e", "-n", targetNamespace, "--for", "condition=Ready=True", "--timeout=5s") + _, err = utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + + By("Deploy the CR") + var manifests []byte + EventuallyWithOffset(1, func() error { + var err error + cmd := exec.Command("kubectl", "kustomize", testcaseDir) + manifests, err = utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + cmd = exec.Command("kubectl", "create", "-n", targetNamespace, "-f", "-") + cmd.Stdin = bytes.NewBuffer(manifests) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + By("Wait for SonatatFlowPlatform CR to complete deployment") + // wait for service deployments to be ready + EventuallyWithOffset(1, func() error { + cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "--for", "condition=Ready", "--timeout=5s") + _, err = utils.Run(cmd) + return err + }, 10*time.Minute, 5).Should(Succeed()) + + GinkgoWriter.Println("waitForPodRestartCompletion") + waitForPodRestartCompletion("app.kubernetes.io/name=jobs-service", targetNamespace) + GinkgoWriter.Println("waitForPodRestartCompletion done") + + By("Evaluate status of all service's health endpoint") + cmd = exec.Command("kubectl", "get", "pod", "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}") + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + for _, pn := range strings.Split(string(output), " ") { + verifyHealthStatusInPod(pn, targetNamespace) + } + By("Evaluate triggers and sinkbindings") + cmd = exec.Command("kubectl", "get", "sonataflowplatform", "sonataflow-platform", "-n", targetNamespace, "-ojsonpath={.status.triggers}") + output, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + var triggers []operatorapi.SonataFlowPlatformTriggerRef + err = json.Unmarshal(output, &triggers) + Expect(err).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-error-", constants.KogitoProcessInstancesEventsPath, targetNamespace, "di-source")).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-node-", constants.KogitoProcessInstancesEventsPath, targetNamespace, "di-source")).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-sla-", constants.KogitoProcessInstancesEventsPath, targetNamespace, "di-source")).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-state-", constants.KogitoProcessInstancesEventsPath, targetNamespace, "di-source")).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-variable-", constants.KogitoProcessInstancesEventsPath, targetNamespace, "di-source")).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-definition-", constants.KogitoProcessDefinitionsEventsPath, targetNamespace, "di-source")).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-jobs-", constants.KogitoJobsPath, targetNamespace, "di-source")).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "jobs-service-create-job-", constants.JobServiceJobEventsPath, targetNamespace, "js-source")).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "jobs-service-delete-job-", constants.JobServiceJobEventsPath, targetNamespace, "js-source")).NotTo(HaveOccurred()) + Expect(verifySinkBinding("sonataflow-platform-jobs-service-sb", targetNamespace, "js-sink")).NotTo(HaveOccurred()) + }, + Entry("and both Job Service and Data Index have service level brokers", test.GetSonataFlowE2EPlatformServicesKnativeDirectory("service-level-broker")), + ) + + DescribeTable("when deploying a SonataFlowPlatform CR with platform broker", func(testcaseDir string, brokerInAnotherNamespace bool) { + By("Deploy the broker") + brokerName := "default" + brokerNamespace := targetNamespace + if brokerInAnotherNamespace { + brokerNamespace = targetNamespace2 + } + GinkgoWriter.Println(fmt.Sprintf("testcaseDir=%v, brokerNamespace = %s", testcaseDir, brokerNamespace)) + cmd := exec.Command("kubectl", "create", "-n", brokerNamespace, "-f", filepath.Join(projectDir, + testcaseDir, "broker")) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("Wait for the broker to be ready") + EventuallyWithOffset(1, func() error { + cmd = exec.Command("kubectl", "wait", "broker", brokerName, "-n", brokerNamespace, "--for", "condition=Ready=True", "--timeout=5s") + _, err = utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + + By("Deploy the SonataFlowPlatform CR") + var manifests []byte + EventuallyWithOffset(1, func() error { + var err error + cmd := exec.Command("kubectl", "kustomize", testcaseDir) + manifests, err = utils.Run(cmd) + return err + }, time.Minute, time.Second).Should(Succeed()) + manifestsUpdated := strings.ReplaceAll(string(manifests), "${BROKER_NAMESPACE}", brokerNamespace) + cmd = exec.Command("kubectl", "create", "-n", targetNamespace, "-f", "-") + cmd.Stdin = bytes.NewBuffer([]byte(manifestsUpdated)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("Wait for SonatatFlowPlatform CR to complete deployment") + // wait for service deployments to be ready + EventuallyWithOffset(1, func() error { + cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "--for", "condition=Ready", "--timeout=5s") + _, err = utils.Run(cmd) + return err + }, 10*time.Minute, 5).Should(Succeed()) + + GinkgoWriter.Println("waitForPodRestartCompletion") + waitForPodRestartCompletion("app.kubernetes.io/name=jobs-service", targetNamespace) + GinkgoWriter.Println("waitForPodRestartCompletion done") + + By("Evaluate status of all service's health endpoint") + cmd = exec.Command("kubectl", "get", "pod", "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}") + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + for _, pn := range strings.Split(string(output), " ") { + verifyHealthStatusInPod(pn, targetNamespace) + } + By("Evaluate triggers and sinkbindings for DI and JS") + cmd = exec.Command("kubectl", "get", "sonataflowplatform", "sonataflow-platform", "-n", targetNamespace, "-ojsonpath={.status.triggers}") + output, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + var triggers []operatorapi.SonataFlowPlatformTriggerRef + err = json.Unmarshal(output, &triggers) + Expect(err).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-error-", constants.KogitoProcessInstancesEventsPath, brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-node-", constants.KogitoProcessInstancesEventsPath, brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-sla-", constants.KogitoProcessInstancesEventsPath, brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-state-", constants.KogitoProcessInstancesEventsPath, brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-variable-", constants.KogitoProcessInstancesEventsPath, brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-process-definition-", constants.KogitoProcessDefinitionsEventsPath, brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "data-index-jobs-", constants.KogitoJobsPath, brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "jobs-service-create-job-", constants.JobServiceJobEventsPath, brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, "jobs-service-delete-job-", constants.JobServiceJobEventsPath, brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifySinkBinding("sonataflow-platform-jobs-service-sb", targetNamespace, brokerName)).NotTo(HaveOccurred()) + + By("Deploy the SonataFlow CR") + cmd = exec.Command("kubectl", "create", "-n", targetNamespace, "-f", filepath.Join(projectDir, + testcaseDir, "sonataflow")) + manifests, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + sfName := "callbackstatetimeouts" + By("Evaluate status of SonataFlow CR") + EventuallyWithOffset(1, func() bool { + return verifyWorkflowIsInRunningStateInNamespace(sfName, targetNamespace) + }, 10*time.Minute, 5).Should(BeTrue()) + + By("Evaluate triggers and sinkbindings for the workflow") + cmd = exec.Command("kubectl", "get", "sonataflow", sfName, "-n", targetNamespace, "-ojsonpath={.status.triggers}") + output, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + err = json.Unmarshal(output, &triggers) + Expect(err).NotTo(HaveOccurred()) + Expect(verifyTrigger(triggers, sfName, "", brokerNamespace, brokerName)).NotTo(HaveOccurred()) + Expect(verifySinkBinding(fmt.Sprintf("%s-sb", sfName), targetNamespace, brokerName)).NotTo(HaveOccurred()) + }, + Entry("and with broker and platform in the same namespace", test.GetSonataFlowE2EPlatformServicesKnativeDirectory("platform-level-broker"), false), + Entry("and with broker and platform in a separate namespace", test.GetSonataFlowE2EPlatformServicesKnativeDirectory("platform-level-broker"), true), + ) }) diff --git a/test/e2e/workflow_test.go b/test/e2e/workflow_test.go index ed8f95c4e..98471c8da 100644 --- a/test/e2e/workflow_test.go +++ b/test/e2e/workflow_test.go @@ -40,6 +40,10 @@ import ( . "github.com/onsi/gomega" ) +const ( + workflowAppLabel = "sonataflow.org/workflow-app" +) + var _ = Describe("SonataFlow Operator", Ordered, func() { var targetNamespace string @@ -52,9 +56,12 @@ var _ = Describe("SonataFlow Operator", Ordered, func() { AfterEach(func() { // Remove resources in test namespace if !CurrentSpecReport().Failed() && len(targetNamespace) > 0 { - cmd := exec.Command("kubectl", "delete", "namespace", targetNamespace, "--wait") + cmd := exec.Command("kubectl", "delete", "sonataflow", "--all", "-n", targetNamespace, "--wait") _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) + cmd = exec.Command("kubectl", "delete", "namespace", targetNamespace, "--wait") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) } }) @@ -161,14 +168,17 @@ var _ = Describe("Validate the persistence ", Ordered, func() { AfterEach(func() { // Remove platform CR if it exists if len(ns) > 0 { - cmd := exec.Command("kubectl", "delete", "namespace", ns, "--wait") + cmd := exec.Command("kubectl", "delete", "sonataflow", "--all", "-n", ns, "--wait") _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) + cmd = exec.Command("kubectl", "delete", "namespace", ns, "--wait") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) } }) - DescribeTable("when deploying a SonataFlow CR with PostgreSQL persistence", func(testcaseDir string, withPersistence bool) { + DescribeTable("when deploying a SonataFlow CR with PostgreSQL persistence", func(testcaseDir string, withPersistence bool, waitKSinkInjection bool) { By("Deploy the CR") var manifests []byte EventuallyWithOffset(1, func() error { @@ -183,15 +193,24 @@ var _ = Describe("Validate the persistence ", Ordered, func() { Expect(err).NotTo(HaveOccurred()) By("Wait for SonatatFlow CR to complete deployment") // wait for service deployments to be ready - EventuallyWithOffset(1, func() error { - cmd = exec.Command("kubectl", "wait", "pod", "-n", ns, "-l", "sonataflow.org/workflow-app", "--for", "condition=Ready", "--timeout=5s") + EventuallyWithOffset(1, func() bool { + cmd = exec.Command("kubectl", "wait", "pod", "-n", ns, "-l", workflowAppLabel, "--for", "condition=Ready", "--timeout=5s") out, err := utils.Run(cmd) + if err != nil { + return false + } GinkgoWriter.Printf("%s\n", string(out)) - return err - }, 12*time.Minute, 5).Should(Succeed()) + if !waitKSinkInjection { + return true + } + GinkgoWriter.Println("waitForPodRestartCompletion") + waitForPodRestartCompletion(workflowAppLabel, ns) + GinkgoWriter.Println("waitForPodRestartCompletion done") + return true + }, 25*time.Minute, 5).Should(BeTrue()) By("Evaluate status of the workflow's pod database connection health endpoint") - cmd = exec.Command("kubectl", "get", "pod", "-l", "sonataflow.org/workflow-app", "-n", ns, "-ojsonpath={.items[*].metadata.name}") + cmd = exec.Command("kubectl", "get", "pod", "-l", workflowAppLabel, "-n", ns, "-ojsonpath={.items[*].metadata.name}") output, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) EventuallyWithOffset(1, func() bool { @@ -226,7 +245,7 @@ var _ = Describe("Validate the persistence ", Ordered, func() { return false }, 1*time.Minute).Should(BeTrue()) // Persistence initialization checks - cmd = exec.Command("kubectl", "get", "pod", "-l", "sonataflow.org/workflow-app", "-n", ns, "-ojsonpath={.items[*].metadata.name}") + cmd = exec.Command("kubectl", "get", "pod", "-l", workflowAppLabel, "-n", ns, "-ojsonpath={.items[*].metadata.name}") output, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) podName := string(output) @@ -238,9 +257,9 @@ var _ = Describe("Validate the persistence ", Ordered, func() { By("Validate that the workflow persistence was properly initialized") Expect(logs).Should(ContainSubstring("Flyway Community Edition")) Expect(logs).Should(ContainSubstring("Database: jdbc:postgresql://postgres.%s:5432", ns)) - Expect(logs).Should(ContainSubstring("Creating schema \"callbackstatetimeouts\"")) - Expect(logs).Should(ContainSubstring("Migrating schema \"callbackstatetimeouts\" to version")) - Expect(logs).Should(MatchRegexp("Successfully applied \\d migrations to schema \"callbackstatetimeouts\"")) + result := verifySchemaMigration(logs, "callbackstatetimeouts") + GinkgoWriter.Println(fmt.Sprintf("verifySchemaMigration: %v", result)) + Expect(result).Should(BeTrue()) Expect(logs).Should(ContainSubstring("Profile prod activated")) } else { By("Validate that the workflow has no persistence") @@ -249,11 +268,11 @@ var _ = Describe("Validate the persistence ", Ordered, func() { Expect(logs).Should(ContainSubstring("Profile prod activated")) } }, - Entry("defined in the workflow from an existing kubernetes service as a reference", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("by_service"), true), - Entry("defined in the workflow and from the sonataflow platform", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("from_platform_overwritten_by_service"), true), - Entry("defined from the sonataflow platform as reference and with DI and JS", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("from_platform_with_di_and_js_services"), true), - Entry("defined from the sonataflow platform as reference and without DI and JS", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("from_platform_without_di_and_js_services"), true), - Entry("defined from the sonataflow platform as reference but not required by the workflow", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("from_platform_with_no_persistence_required"), false), + Entry("defined in the workflow from an existing kubernetes service as a reference", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("by_service"), true, false), + Entry("defined in the workflow and from the sonataflow platform", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("from_platform_overwritten_by_service"), true, false), + Entry("defined from the sonataflow platform as reference and with DI and JS", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("from_platform_with_di_and_js_services"), true, true), + Entry("defined from the sonataflow platform as reference and without DI and JS", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("from_platform_without_di_and_js_services"), true, false), + Entry("defined from the sonataflow platform as reference but not required by the workflow", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("from_platform_with_no_persistence_required"), false, false), ) }) diff --git a/test/kubernetes_cli.go b/test/kubernetes_cli.go index 5120a5caa..810dd1932 100644 --- a/test/kubernetes_cli.go +++ b/test/kubernetes_cli.go @@ -21,9 +21,14 @@ package test import ( "context" + "testing" "github.com/apache/incubator-kie-kogito-serverless-operator/utils" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" + sourcesv1 "knative.dev/eventing/pkg/apis/sources/v1" + servingv1 "knative.dev/serving/pkg/apis/serving/v1" + buildv1 "github.com/openshift/api/build/v1" imgv1 "github.com/openshift/api/image/v1" routev1 "github.com/openshift/api/route/v1" @@ -35,7 +40,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" - servingv1 "knative.dev/serving/pkg/apis/serving/v1" ctrl "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -97,6 +101,8 @@ func NewKogitoClientBuilderWithOpenShift() *SonataFlowClientBuilder { utilruntime.Must(buildv1.Install(s)) utilruntime.Must(imgv1.Install(s)) utilruntime.Must(operatorapi.AddToScheme(s)) + utilruntime.Must(eventingv1.AddToScheme(s)) + utilruntime.Must(sourcesv1.AddToScheme(s)) builder := fake.NewClientBuilder().WithScheme(s) return &SonataFlowClientBuilder{ innerBuilder: builder, diff --git a/test/testdata/knative_default_broker.yaml b/test/testdata/knative_default_broker.yaml new file mode 100644 index 000000000..58e3e025e --- /dev/null +++ b/test/testdata/knative_default_broker.yaml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + annotations: + eventing.knative.dev/broker.class: MTChannelBasedBroker + name: default +spec: + config: + apiVersion: v1 + kind: ConfigMap + name: config-br-default-channel + namespace: knative-eventing + delivery: + backoffDelay: PT0.2S + backoffPolicy: exponential + retry: 10 +status: + address: + name: http + url: http://broker-ingress.knative-eventing.svc.cluster.local/sonataflow-infra/default + annotations: + knative.dev/channelAPIVersion: messaging.knative.dev/v1 + knative.dev/channelAddress: http://default-kne-trigger-kn-channel.sonataflow-infra.svc.cluster.local + knative.dev/channelKind: InMemoryChannel + knative.dev/channelName: default-kne-trigger + conditions: + - lastTransitionTime: "2024-07-16T18:58:35Z" + status: "True" + type: Addressable + - lastTransitionTime: "2024-07-16T18:58:35Z" + message: No dead letter sink is configured. + reason: DeadLetterSinkNotConfigured + severity: Info + status: "True" + type: DeadLetterSinkResolved + - lastTransitionTime: "2024-07-16T18:58:35Z" + status: "True" + type: FilterReady + - lastTransitionTime: "2024-07-16T18:58:35Z" + status: "True" + type: IngressReady + - lastTransitionTime: "2024-07-16T18:58:35Z" + status: "True" + type: Ready + - lastTransitionTime: "2024-07-16T18:58:35Z" + status: "True" + type: TriggerChannelReady + observedGeneration: 1 \ No newline at end of file diff --git a/test/testdata/knative_serving_eventing.yaml b/test/testdata/knative_serving_eventing.yaml new file mode 100644 index 000000000..6f0d79efb --- /dev/null +++ b/test/testdata/knative_serving_eventing.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: knative-serving +--- +apiVersion: operator.knative.dev/v1beta1 +kind: KnativeServing +metadata: + name: knative-serving + namespace: knative-serving +spec: + ingress: + kourier: + enabled: true + config: + network: + ingress-class: "kourier.ingress.networking.knative.dev" +--- +apiVersion: v1 +kind: Namespace +metadata: + name: knative-eventing +--- +apiVersion: operator.knative.dev/v1beta1 +kind: KnativeEventing +metadata: + name: knative-eventing + namespace: knative-eventing \ No newline at end of file diff --git a/test/testdata/platform/services/preview/ephemeral-with-workflow/00-broker.yaml b/test/testdata/platform/services/preview/ephemeral-with-workflow/00-broker.yaml new file mode 100644 index 000000000..6152f24d7 --- /dev/null +++ b/test/testdata/platform/services/preview/ephemeral-with-workflow/00-broker.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: default +spec: {} + diff --git a/test/testdata/platform/services/preview/ephemeral-with-workflow/02-sonataflow_platform.yaml b/test/testdata/platform/services/preview/ephemeral-with-workflow/02-sonataflow_platform.yaml index 52b7d11f7..a27221171 100644 --- a/test/testdata/platform/services/preview/ephemeral-with-workflow/02-sonataflow_platform.yaml +++ b/test/testdata/platform/services/preview/ephemeral-with-workflow/02-sonataflow_platform.yaml @@ -17,6 +17,12 @@ kind: SonataFlowPlatform metadata: name: sonataflow-platform spec: + eventing: + broker: + ref: + apiVersion: eventing.knative.dev/v1 + kind: Broker + name: default build: config: strategyOptions: diff --git a/test/testdata/platform/services/preview/ephemeral-with-workflow/kustomization.yaml b/test/testdata/platform/services/preview/ephemeral-with-workflow/kustomization.yaml index 20286ada0..a3a9c95f9 100644 --- a/test/testdata/platform/services/preview/ephemeral-with-workflow/kustomization.yaml +++ b/test/testdata/platform/services/preview/ephemeral-with-workflow/kustomization.yaml @@ -13,6 +13,7 @@ # limitations under the License. resources: +- 00-broker.yaml - 02-sonataflow_platform.yaml - sonataflow/03-sonataflow_callbackstatetimeouts.sw.yaml diff --git a/test/testdata/platform/services/preview/ephemeral/00-broker.yaml b/test/testdata/platform/services/preview/ephemeral/00-broker.yaml new file mode 100644 index 000000000..6152f24d7 --- /dev/null +++ b/test/testdata/platform/services/preview/ephemeral/00-broker.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: default +spec: {} + diff --git a/test/testdata/platform/services/preview/ephemeral/02-sonataflow_platform.yaml b/test/testdata/platform/services/preview/ephemeral/02-sonataflow_platform.yaml index 52b7d11f7..a27221171 100644 --- a/test/testdata/platform/services/preview/ephemeral/02-sonataflow_platform.yaml +++ b/test/testdata/platform/services/preview/ephemeral/02-sonataflow_platform.yaml @@ -17,6 +17,12 @@ kind: SonataFlowPlatform metadata: name: sonataflow-platform spec: + eventing: + broker: + ref: + apiVersion: eventing.knative.dev/v1 + kind: Broker + name: default build: config: strategyOptions: diff --git a/test/testdata/platform/services/preview/ephemeral/kustomization.yaml b/test/testdata/platform/services/preview/ephemeral/kustomization.yaml index 5441bfce8..bb1fcd0e2 100644 --- a/test/testdata/platform/services/preview/ephemeral/kustomization.yaml +++ b/test/testdata/platform/services/preview/ephemeral/kustomization.yaml @@ -13,6 +13,7 @@ # limitations under the License. resources: +- 00-broker.yaml - 02-sonataflow_platform.yaml sortOptions: diff --git a/test/testdata/platform/services/preview/knative/platform-level-broker/01-postgres.yaml b/test/testdata/platform/services/preview/knative/platform-level-broker/01-postgres.yaml new file mode 100644 index 000000000..662de4c7b --- /dev/null +++ b/test/testdata/platform/services/preview/knative/platform-level-broker/01-postgres.yaml @@ -0,0 +1,86 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: postgres + name: postgres-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: postgres + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgres + template: + metadata: + labels: + app.kubernetes.io/name: postgres + spec: + containers: + - name: postgres + image: postgres:13.2-alpine + imagePullPolicy: 'IfNotPresent' + ports: + - containerPort: 5432 + volumeMounts: + - name: storage + mountPath: /var/lib/postgresql/data + envFrom: + - secretRef: + name: postgres-secrets + readinessProbe: + exec: + command: ["pg_isready"] + initialDelaySeconds: 15 + timeoutSeconds: 2 + livenessProbe: + exec: + command: ["pg_isready"] + initialDelaySeconds: 15 + timeoutSeconds: 2 + resources: + limits: + memory: "256Mi" + cpu: "500m" + volumes: + - name: storage + persistentVolumeClaim: + claimName: postgres-pvc +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: postgres + name: postgres +spec: + selector: + app.kubernetes.io/name: postgres + ports: + - port: 5432 diff --git a/test/testdata/platform/services/preview/knative/platform-level-broker/02-sonataflow_platform.yaml b/test/testdata/platform/services/preview/knative/platform-level-broker/02-sonataflow_platform.yaml new file mode 100644 index 000000000..07f37f770 --- /dev/null +++ b/test/testdata/platform/services/preview/knative/platform-level-broker/02-sonataflow_platform.yaml @@ -0,0 +1,75 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: sonataflow.org/v1alpha08 +kind: SonataFlowPlatform +metadata: + name: sonataflow-platform +spec: + eventing: + broker: + ref: + apiVersion: eventing.knative.dev/v1 + kind: Broker + name: default + namespace: ${BROKER_NAMESPACE} + build: + config: + strategyOptions: + KanikoBuildCacheEnabled: "true" + services: + dataIndex: + enabled: true + persistence: + postgresql: + jdbcUrl: jdbc:postgresql://postgres:5432/sonataflow?currentSchema=data-index-service + secretRef: + name: postgres-secrets + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + podTemplate: + initContainers: + - name: init-postgres + image: registry.access.redhat.com/ubi9/ubi-micro:latest + imagePullPolicy: IfNotPresent + command: [ 'sh', '-c', 'until (echo 1 > /dev/tcp/postgres.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local/5432) >/dev/null 2>&1; do echo "Waiting for postgres server"; sleep 3; done;' ] + container: + env: + - name: MY_CUSTOM_VARIABLE + value: "OKAY" + - name: QUARKUS_DATASOURCE_PASSWORD +# This value should not be used since it's already set by the operator. If used, the test will fail. + value: "SHOULD_NOT_BE_USED" + jobService: + enabled: true + persistence: + postgresql: + jdbcUrl: jdbc:postgresql://postgres:5432/sonataflow?currentSchema=jobs-service + secretRef: + name: postgres-secrets + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + podTemplate: + initContainers: + - name: init-postgres + image: registry.access.redhat.com/ubi9/ubi-micro:latest + imagePullPolicy: IfNotPresent + command: [ 'sh', '-c', 'until (echo 1 > /dev/tcp/postgres.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local/5432) >/dev/null 2>&1; do echo "Waiting for postgres server"; sleep 3; done;' ] + container: + env: + - name: MY_CUSTOM_VARIABLE + value: "OKAY" + - name: QUARKUS_DATASOURCE_PASSWORD +# This value should not be used since it's already set by the operator. If used, the test will fail. + value: "SHOULD_NOT_BE_USED" \ No newline at end of file diff --git a/test/testdata/platform/services/preview/knative/platform-level-broker/broker/broker.yaml b/test/testdata/platform/services/preview/knative/platform-level-broker/broker/broker.yaml new file mode 100644 index 000000000..219c2c1b1 --- /dev/null +++ b/test/testdata/platform/services/preview/knative/platform-level-broker/broker/broker.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: default +spec: {} + + + diff --git a/test/testdata/platform/services/preview/knative/platform-level-broker/kustomization.yaml b/test/testdata/platform/services/preview/knative/platform-level-broker/kustomization.yaml new file mode 100644 index 000000000..d3fd127c7 --- /dev/null +++ b/test/testdata/platform/services/preview/knative/platform-level-broker/kustomization.yaml @@ -0,0 +1,32 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: +- 01-postgres.yaml +- 02-sonataflow_platform.yaml + +generatorOptions: + disableNameSuffixHash: true + +secretGenerator: + - name: postgres-secrets + literals: + - POSTGRES_USER=sonataflow + - POSTGRES_PASSWORD=sonataflow + - POSTGRES_DATABASE=sonataflow + - PGDATA=/var/lib/pgsql/data/userdata + +sortOptions: + order: fifo + diff --git a/test/testdata/platform/services/preview/knative/platform-level-broker/sonataflow/04-sonataflow_callbackstatetimeouts.sw.yaml b/test/testdata/platform/services/preview/knative/platform-level-broker/sonataflow/04-sonataflow_callbackstatetimeouts.sw.yaml new file mode 100644 index 000000000..a76ac23fb --- /dev/null +++ b/test/testdata/platform/services/preview/knative/platform-level-broker/sonataflow/04-sonataflow_callbackstatetimeouts.sw.yaml @@ -0,0 +1,81 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: sonataflow.org/v1alpha08 +kind: SonataFlow +metadata: + name: callbackstatetimeouts + annotations: + sonataflow.org/description: Callback State Timeouts Example k8s + sonataflow.org/version: 0.0.1 + sonataflow.org/profile: preview +spec: + flow: + start: PrintStartMessage + events: + - name: callbackEvent + source: '' + type: callback_event_type + functions: + - name: systemOut + type: custom + operation: sysout + states: + - name: PrintStartMessage + type: operation + actions: + - name: printSystemOut + functionRef: + refName: systemOut + arguments: + message: "${\"callback-state-timeouts: \" + $WORKFLOW.instanceId + \" has started.\"}" + transition: CallbackState + - name: CallbackState + type: callback + action: + name: callbackAction + functionRef: + refName: systemOut + arguments: + message: "${\"callback-state-timeouts: \" + $WORKFLOW.instanceId + \" has executed the callbackFunction.\"}" + eventRef: callbackEvent + transition: CheckEventArrival + timeouts: + eventTimeout: PT30S + - name: CheckEventArrival + type: switch + dataConditions: + - condition: "${ .eventData != null }" + transition: EventArrived + defaultCondition: + transition: EventNotArrived + - name: EventArrived + type: inject + data: + exitMessage: "The callback event has arrived." + transition: PrintExitMessage + - name: EventNotArrived + type: inject + data: + exitMessage: "The callback event has not arrived, and the timeout has overdue." + transition: PrintExitMessage + - name: PrintExitMessage + type: operation + actions: + - name: printSystemOut + functionRef: + refName: systemOut + arguments: + message: "${\"callback-state-timeouts: \" + $WORKFLOW.instanceId + \" has finalized. \" + .exitMessage + \" eventData: \" + .eventData}" + end: true diff --git a/test/testdata/platform/services/preview/knative/service-level-broker/01-postgres.yaml b/test/testdata/platform/services/preview/knative/service-level-broker/01-postgres.yaml new file mode 100644 index 000000000..662de4c7b --- /dev/null +++ b/test/testdata/platform/services/preview/knative/service-level-broker/01-postgres.yaml @@ -0,0 +1,86 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: postgres + name: postgres-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: postgres + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgres + template: + metadata: + labels: + app.kubernetes.io/name: postgres + spec: + containers: + - name: postgres + image: postgres:13.2-alpine + imagePullPolicy: 'IfNotPresent' + ports: + - containerPort: 5432 + volumeMounts: + - name: storage + mountPath: /var/lib/postgresql/data + envFrom: + - secretRef: + name: postgres-secrets + readinessProbe: + exec: + command: ["pg_isready"] + initialDelaySeconds: 15 + timeoutSeconds: 2 + livenessProbe: + exec: + command: ["pg_isready"] + initialDelaySeconds: 15 + timeoutSeconds: 2 + resources: + limits: + memory: "256Mi" + cpu: "500m" + volumes: + - name: storage + persistentVolumeClaim: + claimName: postgres-pvc +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: postgres + name: postgres +spec: + selector: + app.kubernetes.io/name: postgres + ports: + - port: 5432 diff --git a/test/testdata/platform/services/preview/knative/service-level-broker/02-sonataflow_platform.yaml b/test/testdata/platform/services/preview/knative/service-level-broker/02-sonataflow_platform.yaml new file mode 100644 index 000000000..14793f28a --- /dev/null +++ b/test/testdata/platform/services/preview/knative/service-level-broker/02-sonataflow_platform.yaml @@ -0,0 +1,89 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: sonataflow.org/v1alpha08 +kind: SonataFlowPlatform +metadata: + name: sonataflow-platform +spec: + eventing: + broker: + ref: + apiVersion: eventing.knative.dev/v1 + kind: Broker + name: default + build: + config: + strategyOptions: + KanikoBuildCacheEnabled: "true" + services: + dataIndex: + enabled: true + source: + ref: + apiVersion: eventing.knative.dev/v1 + kind: Broker + name: di-source + persistence: + postgresql: + jdbcUrl: jdbc:postgresql://postgres:5432/sonataflow?currentSchema=data-index-service + secretRef: + name: postgres-secrets + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + podTemplate: + initContainers: + - name: init-postgres + image: registry.access.redhat.com/ubi9/ubi-micro:latest + imagePullPolicy: IfNotPresent + command: [ 'sh', '-c', 'until (echo 1 > /dev/tcp/postgres.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local/5432) >/dev/null 2>&1; do echo "Waiting for postgres server"; sleep 3; done;' ] + container: + env: + - name: MY_CUSTOM_VARIABLE + value: "OKAY" + - name: QUARKUS_DATASOURCE_PASSWORD +# This value should not be used since it's already set by the operator. If used, the test will fail. + value: "SHOULD_NOT_BE_USED" + jobService: + enabled: true + source: + ref: + apiVersion: eventing.knative.dev/v1 + kind: Broker + name: js-source + sink: + ref: + apiVersion: eventing.knative.dev/v1 + kind: Broker + name: js-sink + persistence: + postgresql: + jdbcUrl: jdbc:postgresql://postgres:5432/sonataflow?currentSchema=jobs-service + secretRef: + name: postgres-secrets + userKey: POSTGRES_USER + passwordKey: POSTGRES_PASSWORD + podTemplate: + initContainers: + - name: init-postgres + image: registry.access.redhat.com/ubi9/ubi-micro:latest + imagePullPolicy: IfNotPresent + command: [ 'sh', '-c', 'until (echo 1 > /dev/tcp/postgres.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local/5432) >/dev/null 2>&1; do echo "Waiting for postgres server"; sleep 3; done;' ] + container: + env: + - name: MY_CUSTOM_VARIABLE + value: "OKAY" + - name: QUARKUS_DATASOURCE_PASSWORD +# This value should not be used since it's already set by the operator. If used, the test will fail. + value: "SHOULD_NOT_BE_USED" \ No newline at end of file diff --git a/test/testdata/platform/services/preview/knative/service-level-broker/broker/00-broker.yaml b/test/testdata/platform/services/preview/knative/service-level-broker/broker/00-broker.yaml new file mode 100644 index 000000000..74fb9f790 --- /dev/null +++ b/test/testdata/platform/services/preview/knative/service-level-broker/broker/00-broker.yaml @@ -0,0 +1,47 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: default + labels: + test: test-e2e +spec: {} +--- +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: di-source + labels: + test: test-e2e +spec: {} +--- +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: js-sink + labels: + test: test-e2e +spec: {} +--- +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: js-source + labels: + test: test-e2e +spec: {} + + diff --git a/test/testdata/platform/services/preview/knative/service-level-broker/kustomization.yaml b/test/testdata/platform/services/preview/knative/service-level-broker/kustomization.yaml new file mode 100644 index 000000000..d3fd127c7 --- /dev/null +++ b/test/testdata/platform/services/preview/knative/service-level-broker/kustomization.yaml @@ -0,0 +1,32 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: +- 01-postgres.yaml +- 02-sonataflow_platform.yaml + +generatorOptions: + disableNameSuffixHash: true + +secretGenerator: + - name: postgres-secrets + literals: + - POSTGRES_USER=sonataflow + - POSTGRES_PASSWORD=sonataflow + - POSTGRES_DATABASE=sonataflow + - PGDATA=/var/lib/pgsql/data/userdata + +sortOptions: + order: fifo + diff --git a/test/testdata/platform/services/preview/postgreSQL/00-broker.yaml b/test/testdata/platform/services/preview/postgreSQL/00-broker.yaml new file mode 100644 index 000000000..6152f24d7 --- /dev/null +++ b/test/testdata/platform/services/preview/postgreSQL/00-broker.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: default +spec: {} + diff --git a/test/testdata/platform/services/preview/postgreSQL/02-sonataflow_platform.yaml b/test/testdata/platform/services/preview/postgreSQL/02-sonataflow_platform.yaml index 9514b81ac..d0f5f615f 100644 --- a/test/testdata/platform/services/preview/postgreSQL/02-sonataflow_platform.yaml +++ b/test/testdata/platform/services/preview/postgreSQL/02-sonataflow_platform.yaml @@ -17,6 +17,12 @@ kind: SonataFlowPlatform metadata: name: sonataflow-platform spec: + eventing: + broker: + ref: + apiVersion: eventing.knative.dev/v1 + kind: Broker + name: default build: config: strategyOptions: diff --git a/test/testdata/platform/services/preview/postgreSQL/kustomization.yaml b/test/testdata/platform/services/preview/postgreSQL/kustomization.yaml index d3fd127c7..cb33c50ae 100644 --- a/test/testdata/platform/services/preview/postgreSQL/kustomization.yaml +++ b/test/testdata/platform/services/preview/postgreSQL/kustomization.yaml @@ -13,6 +13,7 @@ # limitations under the License. resources: +- 00-broker.yaml - 01-postgres.yaml - 02-sonataflow_platform.yaml diff --git a/test/testdata/sonataflow.org_v1alpha08_sonataflow_vet_event.yaml b/test/testdata/sonataflow.org_v1alpha08_sonataflow_vet_event.yaml index 05e3a5b42..8636feb0e 100644 --- a/test/testdata/sonataflow.org_v1alpha08_sonataflow_vet_event.yaml +++ b/test/testdata/sonataflow.org_v1alpha08_sonataflow_vet_event.yaml @@ -29,6 +29,19 @@ spec: namespace: default apiVersion: eventing.knative.dev/v1 kind: Broker + sources: + - eventType: events.vet.appointments + ref: + name: broker-appointments + namespace: default + apiVersion: eventing.knative.dev/v1 + kind: Broker + - eventType: events.vet.appointments.request + ref: + name: broker-appointments-request + namespace: default + apiVersion: eventing.knative.dev/v1 + kind: Broker flow: events: - name: MakeVetAppointment diff --git a/test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_withBroker.yaml b/test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_withBroker.yaml new file mode 100644 index 000000000..d554d60b6 --- /dev/null +++ b/test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_withBroker.yaml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: sonataflow.org/v1alpha08 +kind: SonataFlowPlatform +metadata: + name: sonataflow-platform +spec: + properties: + flow: + - name: quarkus.log.level + value: INFO + build: + config: + registry: + address: quay.io/kiegroup + secret: regcred + eventing: + broker: + ref: + name: default + apiVersion: eventing.knative.dev/v1 + kind: Broker + services: + dataIndex: + enabled: true + podTemplate: + container: + resources: {} + jobService: + enabled: true + podTemplate: + container: + resources: {} diff --git a/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/00-broker.yaml b/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/00-broker.yaml new file mode 100644 index 000000000..6152f24d7 --- /dev/null +++ b/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/00-broker.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Apache Software Foundation (ASF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: eventing.knative.dev/v1 +kind: Broker +metadata: + name: default +spec: {} + diff --git a/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/02-sonataflow_platform.yaml b/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/02-sonataflow_platform.yaml index 2a7db22a6..27814312a 100644 --- a/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/02-sonataflow_platform.yaml +++ b/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/02-sonataflow_platform.yaml @@ -17,6 +17,12 @@ kind: SonataFlowPlatform metadata: name: sonataflow-platform spec: + eventing: + broker: + ref: + apiVersion: eventing.knative.dev/v1 + kind: Broker + name: default persistence: postgresql: secretRef: diff --git a/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/kustomization.yaml b/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/kustomization.yaml index 2aaac5b14..f494b37c0 100644 --- a/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/kustomization.yaml +++ b/test/testdata/workflow/persistence/from_platform_with_di_and_js_services/kustomization.yaml @@ -13,6 +13,7 @@ # limitations under the License. resources: +- 00-broker.yaml - 01-postgres.yaml - 02-sonataflow_platform.yaml - 03-configmap_callbackstatetimeouts-props.yaml diff --git a/test/yaml.go b/test/yaml.go index 368ac6b79..a0fbd8bb6 100644 --- a/test/yaml.go +++ b/test/yaml.go @@ -34,7 +34,11 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/discovery" + discfake "k8s.io/client-go/discovery/fake" + clienttesting "k8s.io/client-go/testing" "k8s.io/klog/v2" + eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -48,14 +52,15 @@ const ( SonataFlowGreetingsDataInputSchemaConfig = "v1_configmap_greetings_datainput.yaml" SonataFlowGreetingsStaticFilesConfig = "v1_configmap_greetings_staticfiles.yaml" sonataFlowPlatformYamlCR = "sonataflow.org_v1alpha08_sonataflowplatform.yaml" + sonataFlowPlatformWithBrokerYamlCR = "sonataflow.org_v1alpha08_sonataflowplatform_withBroker.yaml" sonataFlowPlatformWithCacheMinikubeYamlCR = "sonataflow.org_v1alpha08_sonataflowplatform_withCache_minikube.yaml" sonataFlowPlatformForOpenshift = "sonataflow.org_v1alpha08_sonataflowplatform_openshift.yaml" sonataFlowClusterPlatformYamlCR = "sonataflow.org_v1alpha08_sonataflowclusterplatform.yaml" sonataFlowBuilderConfig = "sonataflow-operator-builder-config_v1_configmap.yaml" sonataFlowBuildSucceed = "sonataflow.org_v1alpha08_sonataflowbuild.yaml" - - e2eSamples = "test/testdata/" - manifestsPath = "bundle/manifests/" + knativeDefaultBrokerCR = "knative_default_broker.yaml" + e2eSamples = "test/testdata/" + manifestsPath = "bundle/manifests/" ) var projectDir = "" @@ -256,6 +261,14 @@ func GetBasePlatform() *operatorapi.SonataFlowPlatform { return getSonataFlowPlatform(sonataFlowPlatformYamlCR) } +func GetBasePlatformWithBroker() *operatorapi.SonataFlowPlatform { + return getSonataFlowPlatform(sonataFlowPlatformWithBrokerYamlCR) +} + +func GetBasePlatformWithBrokerInReadyPhase(namespace string) *operatorapi.SonataFlowPlatform { + return GetSonataFlowPlatformInReadyPhase(sonataFlowPlatformWithBrokerYamlCR, namespace) +} + func GetPlatformMinikubeE2eTest() string { return e2eSamples + sonataFlowPlatformWithCacheMinikubeYamlCR } @@ -272,6 +285,10 @@ func GetSonataFlowE2EPlatformServicesDirectory() string { return filepath.Join(getTestDataDir(), "platform", "services") } +func GetSonataFlowE2EPlatformServicesKnativeDirectory(subdir string) string { + return filepath.Join(getTestDataDir(), "platform", "services", "preview", "knative", subdir) +} + func GetSonataFlowE2EPlatformNoServicesDirectory() string { return filepath.Join(getTestDataDir(), "platform", "noservices") } @@ -317,3 +334,21 @@ func getProjectDir() string { return projectDir } + +func CreateFakeKnativeDiscoveryClient() discovery.DiscoveryInterface { + return &discfake.FakeDiscovery{ + Fake: &clienttesting.Fake{ + Resources: []*metav1.APIResourceList{ + {GroupVersion: "serving.knative.dev/v1"}, + {GroupVersion: "eventing.knative.dev/v1"}, + }, + }, + } +} + +func GetDefaultBroker(namespace string) *eventingv1.Broker { + broker := &eventingv1.Broker{} + GetKubernetesResource(knativeDefaultBrokerCR, broker) + broker.Namespace = namespace + return broker +} diff --git a/utils/kubernetes/volumes.go b/utils/kubernetes/volumes.go index 71b56d620..e43553a0d 100644 --- a/utils/kubernetes/volumes.go +++ b/utils/kubernetes/volumes.go @@ -127,12 +127,11 @@ func AddOrReplaceVolume(podSpec *corev1.PodSpec, volumes ...corev1.Volume) { } // AddOrReplaceVolumeMount same as AddOrReplaceVolume, but with VolumeMounts in a specific container -func AddOrReplaceVolumeMount(containerIndex int, podSpec *corev1.PodSpec, mounts ...corev1.VolumeMount) { +func AddOrReplaceVolumeMount(container *corev1.Container, mounts ...corev1.VolumeMount) { // analogous to AddOrReplaceVolume function, the processing must be realized en order. // see: AddOrReplaceVolume mountsToAdd := make([]corev1.VolumeMount, 0) wasAdded := false - container := &podSpec.Containers[containerIndex] for _, mount := range mounts { wasAdded = false for i := 0; !wasAdded && i < len(container.VolumeMounts); i++ { @@ -147,7 +146,27 @@ func AddOrReplaceVolumeMount(containerIndex int, podSpec *corev1.PodSpec, mounts mountsToAdd = append(mountsToAdd, mount) } } - for _, mount := range mountsToAdd { - container.VolumeMounts = append(container.VolumeMounts, mount) + container.VolumeMounts = append(container.VolumeMounts, mountsToAdd...) +} + +// AddOrReplaceEnvVar adds or removes the given env variables to the PodSpec. +// If there's already an env variable with the same name, it's replaced. +func AddOrReplaceEnvVar(container *corev1.Container, envs ...corev1.EnvVar) { + envVarsToAdd := make([]corev1.EnvVar, 0) + wasAdded := false + for _, envVar := range envs { + wasAdded = false + for i := 0; !wasAdded && i < len(container.Env); i++ { + if envVar.Name == container.Env[i].Name { + // replace existing + container.Env[i] = envVar + wasAdded = true + } + } + if !wasAdded { + // remember to add it later in order + envVarsToAdd = append(envVarsToAdd, envVar) + } } + container.Env = append(container.Env, envVarsToAdd...) } diff --git a/utils/kubernetes/volumes_test.go b/utils/kubernetes/volumes_test.go index 6b296c671..a36afe378 100644 --- a/utils/kubernetes/volumes_test.go +++ b/utils/kubernetes/volumes_test.go @@ -122,7 +122,7 @@ func TestAddOrReplaceVolumeMount(t *testing.T) { {Name: "mount1", MountPath: "/dev"}, } - AddOrReplaceVolumeMount(0, &podSpec, mounts...) + AddOrReplaceVolumeMount(&podSpec.Containers[0], mounts...) assert.Len(t, podSpec.Containers[0].VolumeMounts, 2) assert.Equal(t, "/dev", podSpec.Containers[0].VolumeMounts[0].MountPath) assert.Equal(t, "/tmp/any/path", podSpec.Containers[0].VolumeMounts[1].MountPath) diff --git a/version/version.go b/version/version.go index eb0eb353d..bbc04934a 100644 --- a/version/version.go +++ b/version/version.go @@ -31,7 +31,7 @@ const ( // When released, this version should reflect the `major.minor` version in the registry. // For example, docker.io/apache/incubator-kie-sonataflow-operator:10.0 tagVersion = "main" - serviceTagVersion = "weekly-latest" + serviceTagVersion = "main-20240728" // weekly-latest ) // GetOperatorVersion gets the current binary version of the operator. Do not use it to compose image tags! diff --git a/workflowproj/operator.go b/workflowproj/operator.go index 4c7bed28f..3ea38d74a 100644 --- a/workflowproj/operator.go +++ b/workflowproj/operator.go @@ -35,6 +35,10 @@ const ( // ApplicationPropertiesFileName is the default application properties file name holding user properties ApplicationPropertiesFileName = "application.properties" workflowManagedConfigMapNameSuffix = "-managed-props" + // LabelApp key to use among object selectors, "app" is used among k8s applications to group objects in some UI consoles + LabelApp = "app" + // LabelAppNamespace namespace the k8s application is deployed + LabelAppNamespace = "app-namespace" // LabelService key to use among object selectors LabelService = metadata.Domain + "/service" // LabelWorkflow specialized label managed by the controller @@ -43,6 +47,8 @@ const ( LabelK8SComponent = "app.kubernetes.io/component" LabelK8SPartOF = "app.kubernetes.io/part-of" LabelK8SManagedBy = "app.kubernetes.io/managed-by" + // LabelWorkflowNamespace specialized label managed by the controller indicating the namespace of the workflow + LabelWorkflowNamespace = metadata.Domain + "/workflow-namespace" ) // SetTypeToObject sets the Kind and ApiVersion to a given object since the default constructor won't do it. @@ -87,10 +93,12 @@ func GetManagedPropertiesFileName(workflow *operatorapi.SonataFlow) string { // GetDefaultLabels gets the default labels based on the given workflow. func GetDefaultLabels(workflow *operatorapi.SonataFlow) map[string]string { labels := map[string]string{ - LabelWorkflow: workflow.Name, - LabelK8SName: workflow.Name, - LabelK8SComponent: "serverless-workflow", - LabelK8SManagedBy: "sonataflow-operator", + LabelWorkflow: workflow.Name, + LabelK8SName: workflow.Name, + LabelK8SComponent: "serverless-workflow", + LabelK8SManagedBy: "sonataflow-operator", + LabelApp: workflow.Name, + LabelWorkflowNamespace: workflow.Namespace, } if workflow.Status.Platform != nil { labels[LabelK8SPartOF] = workflow.Status.Platform.Name diff --git a/workflowproj/workflowproj_test.go b/workflowproj/workflowproj_test.go index 81dc4bb4b..9bd22d0ef 100644 --- a/workflowproj/workflowproj_test.go +++ b/workflowproj/workflowproj_test.go @@ -79,10 +79,12 @@ func Test_Handler_WorkflowMinimalAndPropsAndSpec(t *testing.T) { assert.NotNil(t, proj.Workflow) assert.NotNil(t, proj.Workflow.ObjectMeta) assert.Equal(t, proj.Workflow.ObjectMeta.Labels, map[string]string{ - "sonataflow.org/workflow-app": "hello", - "app.kubernetes.io/name": "hello", - "app.kubernetes.io/component": "serverless-workflow", - "app.kubernetes.io/managed-by": "sonataflow-operator", + "app": "hello", + "sonataflow.org/workflow-app": "hello", + "sonataflow.org/workflow-namespace": "default", + "app.kubernetes.io/name": "hello", + "app.kubernetes.io/component": "serverless-workflow", + "app.kubernetes.io/managed-by": "sonataflow-operator", }) assert.NotNil(t, proj.Properties) assert.NotEmpty(t, proj.Resources)