diff --git a/cli/cmd/resources/odiglet.go b/cli/cmd/resources/odiglet.go index b5cad80b6..d4c8919f8 100644 --- a/cli/cmd/resources/odiglet.go +++ b/cli/cmd/resources/odiglet.go @@ -96,6 +96,7 @@ func NewOdigletClusterRole(psp bool) *rbacv1.ClusterRole { "get", "list", "watch", + "patch", }, APIGroups: []string{""}, Resources: []string{ diff --git a/common/consts/consts.go b/common/consts/consts.go index a0287e309..6343c9856 100644 --- a/common/consts/consts.go +++ b/common/consts/consts.go @@ -26,6 +26,9 @@ const ( // Used to label instrumentation instances by the corresponding // instrumented app for better query performance. InstrumentedAppNameLabel = "instrumented-app" + // Used to indicate that the odiglet is installed on a node. + OdigletInstalledLabel = "odiglet-installed" + OdigletInstalledLabelValue = "true" ) var ( diff --git a/helm/odigos/templates/odiglet/clusterrole.yaml b/helm/odigos/templates/odiglet/clusterrole.yaml index 9cbb2d109..ae87cae41 100644 --- a/helm/odigos/templates/odiglet/clusterrole.yaml +++ b/helm/odigos/templates/odiglet/clusterrole.yaml @@ -8,13 +8,21 @@ rules: resources: - configmaps - namespaces - - nodes - pods - services verbs: - get - list - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - patch - apiGroups: - "" resources: diff --git a/instrumentor/controllers/instrumentationdevice/common.go b/instrumentor/controllers/instrumentationdevice/common.go index f76647777..29ff0e3e7 100644 --- a/instrumentor/controllers/instrumentationdevice/common.go +++ b/instrumentor/controllers/instrumentationdevice/common.go @@ -13,6 +13,7 @@ import ( "github.com/odigos-io/odigos/k8sutils/pkg/conditions" odigosk8sconsts "github.com/odigos-io/odigos/k8sutils/pkg/consts" "github.com/odigos-io/odigos/k8sutils/pkg/env" + "github.com/odigos-io/odigos/k8sutils/pkg/pod" "github.com/odigos-io/odigos/k8sutils/pkg/workload" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -128,6 +129,9 @@ func addInstrumentationDeviceToWorkload(ctx context.Context, kubeClient client.C } err, deviceApplied, tempDevicePartiallyApplied := instrumentation.ApplyInstrumentationDevicesToPodTemplate(podSpec, runtimeDetails, otelSdkToUse, obj, logger) + + pod.AddOdigletInstalledAffinity(podSpec, consts.OdigletInstalledLabel, consts.OdigletInstalledLabelValue) + if err != nil { return err } @@ -179,6 +183,8 @@ func removeInstrumentationDeviceFromWorkload(ctx context.Context, kubeClient cli // If instrumentation device is removed successfully, remove odigos.io/inject-instrumentation label to disable the webhook instrumentation.RemoveInjectInstrumentationLabel(podSpec) + pod.RemoveOdigletInstalledAffinity(podSpec, consts.OdigletInstalledLabel, consts.OdigletInstalledLabelValue) + instrumentation.RevertInstrumentationDevices(podSpec) err = instrumentation.RevertEnvOverwrites(workloadObj, podSpec) diff --git a/k8sutils/pkg/node/node.go b/k8sutils/pkg/node/node.go new file mode 100644 index 000000000..f5d4750d0 --- /dev/null +++ b/k8sutils/pkg/node/node.go @@ -0,0 +1,27 @@ +package node + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +func AddLabelToNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName string, labelKey string, labelValue string) error { + patch := []byte(`{"metadata": {"labels": {"` + labelKey + `": "` + labelValue + `"}}}`) + _, err := clientset.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return err + } + return nil +} + +func RemoveLabelFromNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName string, labelKey string) error { + patch := []byte(`{"metadata": {"labels": {"` + labelKey + `": null}}}`) + _, err := clientset.CoreV1().Nodes().Patch(ctx, nodeName, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + return err + } + return nil +} diff --git a/k8sutils/pkg/pod/pod.go b/k8sutils/pkg/pod/pod.go new file mode 100644 index 000000000..ec92f6045 --- /dev/null +++ b/k8sutils/pkg/pod/pod.go @@ -0,0 +1,112 @@ +package pod + +import ( + corev1 "k8s.io/api/core/v1" +) + +func AddOdigletInstalledAffinity(original *corev1.PodTemplateSpec, nodeLabelKey, nodeLabelValue string) { + // Ensure Affinity exists + if original.Spec.Affinity == nil { + original.Spec.Affinity = &corev1.Affinity{} + } + + // Ensure NodeAffinity exists + if original.Spec.Affinity.NodeAffinity == nil { + original.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{} + } + + // Ensure RequiredDuringSchedulingIgnoredDuringExecution exists + if original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{}, + } + } + + // Check if the term already exists + for _, term := range original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + for _, expr := range term.MatchExpressions { + if expr.Key == nodeLabelKey && expr.Operator == corev1.NodeSelectorOpIn { + for _, val := range expr.Values { + if val == nodeLabelValue { + // The term already exists, so return without adding a duplicate + return + } + } + } + } + } + + // Append the new NodeSelectorTerm if it doesn't exist + newTerm := corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: nodeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{nodeLabelValue}, + }, + }, + } + original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append( + original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, + newTerm, + ) +} + +// RemoveNodeAffinityFromPodTemplate removes a specific NodeAffinity rule from a PodTemplateSpec if it exists. +func RemoveOdigletInstalledAffinity(original *corev1.PodTemplateSpec, nodeLabelKey, nodeLabelValue string) { + if original.Spec.Affinity == nil || original.Spec.Affinity.NodeAffinity == nil { + // No affinity or node affinity present, nothing to remove + return + } + + if original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + // No required node affinity present, nothing to remove + return + } + + // Iterate over NodeSelectorTerms and remove terms that match the key and value + filteredTerms := []corev1.NodeSelectorTerm{} + for _, term := range original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + filteredExpressions := []corev1.NodeSelectorRequirement{} + for _, expr := range term.MatchExpressions { + // Only keep expressions that don't match the given key and value + if !(expr.Key == nodeLabelKey && expr.Operator == corev1.NodeSelectorOpIn && containsValue(expr.Values, nodeLabelValue)) { + filteredExpressions = append(filteredExpressions, expr) + } + } + + // Only add the term if it still has expressions after filtering + if len(filteredExpressions) > 0 { + term.MatchExpressions = filteredExpressions + filteredTerms = append(filteredTerms, term) + } + } + + // Update the NodeSelectorTerms with the filtered list or set to nil if empty + if len(filteredTerms) > 0 { + original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = filteredTerms + } else { + original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = nil + } + + // Clean up empty NodeAffinity if needed + if original.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil && + original.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution == nil { + original.Spec.Affinity.NodeAffinity = nil + } + + // Clean up empty Affinity if needed + if original.Spec.Affinity.NodeAffinity == nil && original.Spec.Affinity.PodAffinity == nil && original.Spec.Affinity.PodAntiAffinity == nil { + original.Spec.Affinity = nil + } +} + +// Helper function to check if a value is in a slice +func containsValue(values []string, value string) bool { + for _, v := range values { + if v == value { + return true + } + } + return false +} diff --git a/odiglet/cmd/main.go b/odiglet/cmd/main.go index a8e605b5a..f060a0a42 100644 --- a/odiglet/cmd/main.go +++ b/odiglet/cmd/main.go @@ -4,13 +4,15 @@ import ( "context" "os" + consts "github.com/odigos-io/odigos/common/consts" + detector "github.com/odigos-io/odigos/odiglet/pkg/detector" "github.com/odigos-io/odigos/odiglet/pkg/ebpf/sdks" "github.com/odigos-io/odigos/odiglet/pkg/instrumentation/fs" - detector "github.com/odigos-io/odigos/odiglet/pkg/detector" "github.com/kubevirt/device-plugin-manager/pkg/dpm" "github.com/odigos-io/odigos/common" k8senv "github.com/odigos-io/odigos/k8sutils/pkg/env" + k8snode "github.com/odigos-io/odigos/k8sutils/pkg/node" "github.com/odigos-io/odigos/odiglet/pkg/ebpf" "github.com/odigos-io/odigos/odiglet/pkg/env" "github.com/odigos-io/odigos/odiglet/pkg/instrumentation" @@ -109,6 +111,12 @@ func main() { os.Exit(-1) } + // add label of Odiglet Installed so the k8s-scheduler can schedule instrumented pods on this nodes + if err := k8snode.AddLabelToNode(ctx, clientset, env.Current.NodeName, consts.OdigletInstalledLabel, "true"); err != nil { + log.Logger.Error(err, "Failed to add label to the node") + os.Exit(-1) + } + <-ctx.Done() for _, director := range ebpfDirectors { director.Shutdown() @@ -118,6 +126,10 @@ func main() { log.Logger.Error(err, "Failed to stop runtime detector") os.Exit(-1) } + // Remove the label before exiting + if err := k8snode.RemoveLabelFromNode(ctx, clientset, env.Current.NodeName, consts.OdigletInstalledLabel); err != nil { + log.Logger.Error(err, "Failed to remove label from the node") + } log.Logger.V(0).Info("odiglet exiting") }