diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 101589cc..00f655f6 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,18 @@ metadata: creationTimestamp: null name: odh-model-controller-role rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - patch + - update + - watch + - delete - apiGroups: - "" resources: @@ -13,7 +25,6 @@ rules: - namespaces - pods - secrets - - serviceaccounts - services verbs: - create diff --git a/controllers/inferenceservice_controller.go b/controllers/inferenceservice_controller.go index d89e9cc0..69b6f0b1 100644 --- a/controllers/inferenceservice_controller.go +++ b/controllers/inferenceservice_controller.go @@ -115,31 +115,12 @@ func (r *OpenshiftInferenceServiceReconciler) SetupWithManager(mgr ctrl.Manager) Watches(&source.Kind{Type: &kservev1alpha1.ServingRuntime{}}, handler.EnqueueRequestsFromMapFunc(func(o client.Object) []reconcile.Request { r.log.Info("Reconcile event triggered by serving runtime: " + o.GetName()) - inferenceServicesList := &kservev1beta1.InferenceServiceList{} - opts := []client.ListOption{client.InNamespace(o.GetNamespace())} - - // Todo: Get only Inference Services that are deploying on the specific serving runtime - err := r.client.List(context.TODO(), inferenceServicesList, opts...) - if err != nil { - r.log.Info("Error getting list of inference services for namespace") - return []reconcile.Request{} - } - - if len(inferenceServicesList.Items) == 0 { - r.log.Info("No InferenceServices found for Serving Runtime: " + o.GetName()) - return []reconcile.Request{} - } - - reconcileRequests := make([]reconcile.Request, 0, len(inferenceServicesList.Items)) - for _, inferenceService := range inferenceServicesList.Items { - reconcileRequests = append(reconcileRequests, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: inferenceService.Name, - Namespace: inferenceService.Namespace, - }, - }) - } - return reconcileRequests + return r.getReconcileRequestsOnUpdateOfServingRuntime(o) + })). + Watches(&source.Kind{Type: &networkingv1.NetworkPolicy{}}, + handler.EnqueueRequestsFromMapFunc(func(o client.Object) []reconcile.Request { + r.log.Info("Reconcile event triggered by Network Policy: " + o.GetName()) + return r.getReconcileRequestsOnUpdateOfNetworkPolicy(o) })) err := builder.Complete(r) if err != nil { @@ -149,6 +130,61 @@ func (r *OpenshiftInferenceServiceReconciler) SetupWithManager(mgr ctrl.Manager) return nil } +func (r *OpenshiftInferenceServiceReconciler) getReconcileRequestsOnUpdateOfServingRuntime(o client.Object) []reconcile.Request { + inferenceServicesList := &kservev1beta1.InferenceServiceList{} + opts := []client.ListOption{client.InNamespace(o.GetNamespace())} + + // Todo: Get only Inference Services that are deploying on the specific serving runtime + err := r.client.List(context.TODO(), inferenceServicesList, opts...) + if err != nil { + r.log.Info("Error getting list of inference services for namespace") + return []reconcile.Request{} + } + + if len(inferenceServicesList.Items) == 0 { + r.log.Info("No InferenceServices found for Network Policy: " + o.GetName()) + return []reconcile.Request{} + } + + reconcileRequests := make([]reconcile.Request, 0, len(inferenceServicesList.Items)) + for _, inferenceService := range inferenceServicesList.Items { + reconcileRequests = append(reconcileRequests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: inferenceService.Name, + Namespace: inferenceService.Namespace, + }, + }) + } + return reconcileRequests +} + +func (r *OpenshiftInferenceServiceReconciler) getReconcileRequestsOnUpdateOfNetworkPolicy(o client.Object) []reconcile.Request { + inferenceServicesList := &kservev1beta1.InferenceServiceList{} + opts := []client.ListOption{client.InNamespace(o.GetNamespace())} + + err := r.client.List(context.TODO(), inferenceServicesList, opts...) + if err != nil { + r.log.Info("Error getting list of inference services for namespace") + return []reconcile.Request{} + } + + if len(inferenceServicesList.Items) == 0 { + r.log.Info("No InferenceServices found for Network Policy: " + o.GetName()) + return []reconcile.Request{} + } + + reconcileRequests := make([]reconcile.Request, 0, len(inferenceServicesList.Items)) + for _, inferenceService := range inferenceServicesList.Items { + reconcileRequests = append(reconcileRequests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: inferenceService.Name, + Namespace: inferenceService.Namespace, + }, + }) + } + return reconcileRequests +} + // general clean-up, mostly resources in different namespaces from kservev1beta1.InferenceService func (r *OpenshiftInferenceServiceReconciler) onDeletion(ctx context.Context, log logr.Logger, inferenceService *kservev1beta1.InferenceService) error { log.V(1).Info("Running cleanup logic") diff --git a/controllers/reconcilers/kserve_istio_podmonitor_reconciler.go b/controllers/reconcilers/kserve_istio_podmonitor_reconciler.go index 895d7e33..e7912a7a 100644 --- a/controllers/reconcilers/kserve_istio_podmonitor_reconciler.go +++ b/controllers/reconcilers/kserve_istio_podmonitor_reconciler.go @@ -83,6 +83,10 @@ func (r *KserveIstioPodMonitorReconciler) createDesiredResource(isvc *kservev1be Key: "istio-prometheus-ignore", Operator: metav1.LabelSelectorOpDoesNotExist, }, + { + Key: "modelmesh-service", + Operator: metav1.LabelSelectorOpDoesNotExist, + }, }, }, PodMetricsEndpoints: []v1.PodMetricsEndpoint{ diff --git a/controllers/reconcilers/mm_controller_networkpolicy_reconciler.go b/controllers/reconcilers/mm_controller_networkpolicy_reconciler.go new file mode 100644 index 00000000..95249a52 --- /dev/null +++ b/controllers/reconcilers/mm_controller_networkpolicy_reconciler.go @@ -0,0 +1,141 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconcilers + +import ( + "context" + "github.com/go-logr/logr" + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + "github.com/kserve/kserve/pkg/constants" + "github.com/opendatahub-io/odh-model-controller/controllers/comparators" + "github.com/opendatahub-io/odh-model-controller/controllers/processors" + "github.com/opendatahub-io/odh-model-controller/controllers/resources" + "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type MMControllerNetworkPolicyReconciler struct { + client client.Client + scheme *runtime.Scheme + networkPolicyHandler resources.NetworkPolicyHandler + deltaProcessor processors.DeltaProcessor +} + +func NewMMControllerNetworkPolicyReconciler(client client.Client, scheme *runtime.Scheme) *MMControllerNetworkPolicyReconciler { + return &MMControllerNetworkPolicyReconciler{ + client: client, + scheme: scheme, + networkPolicyHandler: resources.NewNetworkPolicyHandler(client), + deltaProcessor: processors.NewDeltaProcessor(), + } +} + +func (r *MMControllerNetworkPolicyReconciler) Reconcile(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) error { + + // Create Desired resource + desiredResource, err := r.createDesiredResource(isvc) + if err != nil { + return err + } + + // Get Existing resource + existingResource, err := r.getExistingResource(ctx, log, isvc) + if err != nil { + return err + } + + // Process Delta + if err = r.processDelta(ctx, log, desiredResource, existingResource); err != nil { + return err + } + return nil +} + +func (r *MMControllerNetworkPolicyReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) (*v1.NetworkPolicy, error) { + desiredNetworkPolicy := &v1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: getMMControllerNetworkPolicyName(), + Namespace: isvc.Namespace, + }, + Spec: v1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{}, + Ingress: []v1.NetworkPolicyIngressRule{ + { + From: []v1.NetworkPolicyPeer{ + { + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kubernetes.io/metadata.name": constants.KServeNamespace, + }, + }, + }, + }, + }, + }, + PolicyTypes: []v1.PolicyType{v1.PolicyTypeIngress}, + }, + } + return desiredNetworkPolicy, nil +} + +func (r *MMControllerNetworkPolicyReconciler) getExistingResource(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) (*v1.NetworkPolicy, error) { + return r.networkPolicyHandler.FetchNetworkPolicy(ctx, log, types.NamespacedName{Name: getMMControllerNetworkPolicyName(), Namespace: isvc.Namespace}) +} + +func (r *MMControllerNetworkPolicyReconciler) processDelta(ctx context.Context, log logr.Logger, desiredPod *v1.NetworkPolicy, existingPod *v1.NetworkPolicy) (err error) { + comparator := comparators.GetNetworkPolicyComparator() + delta := r.deltaProcessor.ComputeDelta(comparator, desiredPod, existingPod) + + if !delta.HasChanges() { + log.V(1).Info("No delta found") + return nil + } + + if delta.IsAdded() { + log.V(1).Info("Delta found", "create", desiredPod.GetName()) + if err = r.client.Create(ctx, desiredPod); err != nil { + return + } + } + if delta.IsUpdated() { + log.V(1).Info("Delta found", "update", existingPod.GetName()) + rp := existingPod.DeepCopy() + rp.Labels = desiredPod.Labels + rp.Spec = desiredPod.Spec + + if err = r.client.Update(ctx, rp); err != nil { + return + } + } + if delta.IsRemoved() { + log.V(1).Info("Delta found", "delete", existingPod.GetName()) + if err = r.client.Delete(ctx, existingPod); err != nil { + return + } + } + return nil +} + +func getMMControllerNetworkPolicyName() string { + return "allow-from-" + constants.KServeNamespace + "-ns" +} + +func (r *MMControllerNetworkPolicyReconciler) DeleteMMControllerNetworkPolicy(ctx context.Context, isvcNamespace string) error { + return r.networkPolicyHandler.DeleteNetworkPolicy(ctx, types.NamespacedName{Name: getMMControllerNetworkPolicyName(), Namespace: isvcNamespace}) +} diff --git a/controllers/reconcilers/mm_inferenceservice_reconciler.go b/controllers/reconcilers/mm_inferenceservice_reconciler.go index e46a7de8..464ca16a 100644 --- a/controllers/reconcilers/mm_inferenceservice_reconciler.go +++ b/controllers/reconcilers/mm_inferenceservice_reconciler.go @@ -29,6 +29,8 @@ type ModelMeshInferenceServiceReconciler struct { routeReconciler *ModelMeshRouteReconciler serviceAccountReconciler *ModelMeshServiceAccountReconciler clusterRoleBindingReconciler *ModelMeshClusterRoleBindingReconciler + controllerNetworkPolicy *MMControllerNetworkPolicyReconciler + routeNetworkPolicy *MMRouteNetworkPolicyReconciler } func NewModelMeshInferenceServiceReconciler(client client.Client, scheme *runtime.Scheme) *ModelMeshInferenceServiceReconciler { @@ -37,6 +39,8 @@ func NewModelMeshInferenceServiceReconciler(client client.Client, scheme *runtim routeReconciler: NewModelMeshRouteReconciler(client, scheme), serviceAccountReconciler: NewModelMeshServiceAccountReconciler(client, scheme), clusterRoleBindingReconciler: NewModelMeshClusterRoleBindingReconciler(client, scheme), + controllerNetworkPolicy: NewMMControllerNetworkPolicyReconciler(client, scheme), + routeNetworkPolicy: NewMMRouteNetworkPolicyReconciler(client, scheme), } } @@ -56,6 +60,17 @@ func (r *ModelMeshInferenceServiceReconciler) Reconcile(ctx context.Context, log if err := r.clusterRoleBindingReconciler.Reconcile(ctx, log, isvc); err != nil { return err } + + log.V(1).Info("Reconciling NetworkPolicy to allow traffic from Controller") + if err := r.controllerNetworkPolicy.Reconcile(ctx, log, isvc); err != nil { + return err + } + + log.V(1).Info("Reconciling NetworkPolicy to allow traffic from external routes") + if err := r.routeNetworkPolicy.Reconcile(ctx, log, isvc); err != nil { + return err + } + return nil } @@ -84,6 +99,16 @@ func (r *ModelMeshInferenceServiceReconciler) DeleteModelMeshResourcesIfNoMMIsvc if err := r.clusterRoleBindingReconciler.DeleteClusterRoleBinding(ctx, isvcNamespace); err != nil { return err } + + log.V(1).Info("Deleting Controller network policy object for target namespace") + if err := r.controllerNetworkPolicy.DeleteMMControllerNetworkPolicy(ctx, isvcNamespace); err != nil { + return err + } + + log.V(1).Info("Deleting Route network policy object for target namespace") + if err := r.routeNetworkPolicy.DeleteMMRouteNetworkPolicy(ctx, isvcNamespace); err != nil { + return err + } } return nil } diff --git a/controllers/reconcilers/mm_route_networkpolicy_reconciler.go b/controllers/reconcilers/mm_route_networkpolicy_reconciler.go new file mode 100644 index 00000000..317b32bf --- /dev/null +++ b/controllers/reconcilers/mm_route_networkpolicy_reconciler.go @@ -0,0 +1,144 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reconcilers + +import ( + "context" + "github.com/go-logr/logr" + kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1" + "github.com/opendatahub-io/odh-model-controller/controllers/comparators" + "github.com/opendatahub-io/odh-model-controller/controllers/processors" + "github.com/opendatahub-io/odh-model-controller/controllers/resources" + "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + modelMeshRouteNetworkPolicyName = "allow-modelmesh-route" +) + +type MMRouteNetworkPolicyReconciler struct { + client client.Client + scheme *runtime.Scheme + networkPolicyHandler resources.NetworkPolicyHandler + deltaProcessor processors.DeltaProcessor +} + +func NewMMRouteNetworkPolicyReconciler(client client.Client, scheme *runtime.Scheme) *MMRouteNetworkPolicyReconciler { + return &MMRouteNetworkPolicyReconciler{ + client: client, + scheme: scheme, + networkPolicyHandler: resources.NewNetworkPolicyHandler(client), + deltaProcessor: processors.NewDeltaProcessor(), + } +} + +func (r *MMRouteNetworkPolicyReconciler) Reconcile(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) error { + + // Create Desired resource + desiredResource, err := r.createDesiredResource(isvc) + if err != nil { + return err + } + + // Get Existing resource + existingResource, err := r.getExistingResource(ctx, log, isvc) + if err != nil { + return err + } + + // Process Delta + if err = r.processDelta(ctx, log, desiredResource, existingResource); err != nil { + return err + } + return nil +} + +func (r *MMRouteNetworkPolicyReconciler) createDesiredResource(isvc *kservev1beta1.InferenceService) (*v1.NetworkPolicy, error) { + desiredNetworkPolicy := &v1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: modelMeshRouteNetworkPolicyName, + Namespace: isvc.Namespace, + }, + Spec: v1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "modelmesh-service": "modelmesh-serving", + }, + }, + Ingress: []v1.NetworkPolicyIngressRule{ + { + From: []v1.NetworkPolicyPeer{ + { + NamespaceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "network.openshift.io/policy-group": "ingress", + }, + }, + }, + }, + }, + }, + PolicyTypes: []v1.PolicyType{v1.PolicyTypeIngress}, + }, + } + return desiredNetworkPolicy, nil +} + +func (r *MMRouteNetworkPolicyReconciler) getExistingResource(ctx context.Context, log logr.Logger, isvc *kservev1beta1.InferenceService) (*v1.NetworkPolicy, error) { + return r.networkPolicyHandler.FetchNetworkPolicy(ctx, log, types.NamespacedName{Name: modelMeshRouteNetworkPolicyName, Namespace: isvc.Namespace}) +} + +func (r *MMRouteNetworkPolicyReconciler) processDelta(ctx context.Context, log logr.Logger, desiredPod *v1.NetworkPolicy, existingPod *v1.NetworkPolicy) (err error) { + comparator := comparators.GetNetworkPolicyComparator() + delta := r.deltaProcessor.ComputeDelta(comparator, desiredPod, existingPod) + + if !delta.HasChanges() { + log.V(1).Info("No delta found") + return nil + } + + if delta.IsAdded() { + log.V(1).Info("Delta found", "create", desiredPod.GetName()) + if err = r.client.Create(ctx, desiredPod); err != nil { + return + } + } + if delta.IsUpdated() { + log.V(1).Info("Delta found", "update", existingPod.GetName()) + rp := existingPod.DeepCopy() + rp.Labels = desiredPod.Labels + rp.Spec = desiredPod.Spec + + if err = r.client.Update(ctx, rp); err != nil { + return + } + } + if delta.IsRemoved() { + log.V(1).Info("Delta found", "delete", existingPod.GetName()) + if err = r.client.Delete(ctx, existingPod); err != nil { + return + } + } + return nil +} + +func (r *MMRouteNetworkPolicyReconciler) DeleteMMRouteNetworkPolicy(ctx context.Context, isvcNamespace string) error { + return r.networkPolicyHandler.DeleteNetworkPolicy(ctx, types.NamespacedName{Name: modelMeshRouteNetworkPolicyName, Namespace: isvcNamespace}) +}