From c0bb3aac898d4ce9d92d52a18a034345889448c3 Mon Sep 17 00:00:00 2001 From: Cesar Celis Hernandez Date: Fri, 27 Oct 2023 12:16:02 -0400 Subject: [PATCH] Proposal for controlling minio via CRD in a declarative way --- .gitignore | 4 +- go.mod | 1 + go.sum | 5 ++ manifests/minio.min.io_tenants.yaml | 9 ++ pkg/apis/minio.min.io/v2/types.go | 14 ++- pkg/controller/main-controller.go | 13 ++- pkg/controller/operator.go | 129 ++++++++++++++++++++++++++++ pkg/controller/status.go | 27 ++++++ 8 files changed, 198 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 1e1c59cdfa8..17c4dbd5fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,6 @@ nancy examples/.DS_Store testing/openshift/bundle/* examples/**/obj/ -.idea \ No newline at end of file +.idea +operator.iml +go_build_github_com_minio_operator_cmd_operator diff --git a/go.mod b/go.mod index df4839cf46e..9b6efef8a27 100644 --- a/go.mod +++ b/go.mod @@ -133,6 +133,7 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/muesli/ansi v0.0.0-20221106050444-61f0cd9a192a // indirect diff --git a/go.sum b/go.sum index 7f641d04d38..85417df5f86 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= @@ -474,6 +476,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gopherjs/gopherjs v0.0.0-20220104163920-15ed2e8cf2bd/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= @@ -658,6 +661,8 @@ github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/manifests/minio.min.io_tenants.yaml b/manifests/minio.min.io_tenants.yaml index d18f067d261..ed545d9ab21 100644 --- a/manifests/minio.min.io_tenants.yaml +++ b/manifests/minio.min.io_tenants.yaml @@ -804,6 +804,13 @@ spec: type: string type: object type: array + ldapPolicyAttachToSingleUser: + type: object + properties: + user: + type: string + policy: + type: string certConfig: properties: commonName: @@ -4938,6 +4945,8 @@ spec: type: array provisionedBuckets: type: boolean + ldapPolicyAttachToSingleUser: + type: boolean provisionedUsers: type: boolean revision: diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index 8c6ed8c6ec2..242fda37458 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -72,6 +72,12 @@ type TenantDomains struct { Console string `json:"console,omitempty"` } +type LDAPPolicyAttach struct { + User string `json:"user,omitempty"` + Group string `json:"group,omitempty"` + Policy string `json:"policy,omitempty"` +} + // Features (`features`) - Object describing which MinIO features to enable/disable in the MinIO Tenant. + type Features struct { // *Optional* + @@ -361,6 +367,11 @@ type TenantSpec struct { // If provided, statefulset will add these volumes. You should set the rules for the corresponding volumes and volume mounts. We will not test this rule, k8s will show the result. // +optional AdditionalVolumeMounts []corev1.VolumeMount `json:"additionalVolumeMounts,omitempty"` + // *Optional* + + // + // If provided, Policy will be attached to LDAP User + // +optional + LDAPPolicyAttachToSingleUser *LDAPPolicyAttach `json:"ldapPolicyAttachToSingleUser,omitempty"` } // Logging describes Logging for MinIO tenants. @@ -545,7 +556,8 @@ type TenantStatus struct { // // Health Message regarding the State of the tenant // ProvisionedBuckets keeps track for telling if operator already created initial buckets for the tenant - ProvisionedBuckets bool `json:"provisionedBuckets,omitempty"` + ProvisionedBuckets bool `json:"provisionedBuckets,omitempty"` + LDAPPolicyAttachToSingleUser bool `json:"ldapPolicyAttachToSingleUser,omitempty"` } // CertificateConfig (`certConfig`) defines controlling attributes associated to any TLS certificate automatically generated by the Operator as part of tenant creation. These fields have no effect if `spec.autoCert: false`. diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go index bf6cbec1c34..702d56dfb30 100644 --- a/pkg/controller/main-controller.go +++ b/pkg/controller/main-controller.go @@ -19,6 +19,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/minio/operator/pkg/utils" "net/http" "os" "os/signal" @@ -26,8 +27,6 @@ import ( "syscall" "time" - "github.com/minio/operator/pkg/utils" - "github.com/minio/madmin-go/v2" "github.com/minio/operator/pkg/common" xcerts "github.com/minio/pkg/certs" @@ -1387,6 +1386,16 @@ func (c *Controller) syncHandler(key string) (Result, error) { } } + // Section to attach LDAP Policy to a user. + if tenant.Status.LDAPPolicyAttachToSingleUser { + klog.Info("A user already got the LDAP policy attached") + } else if tenant.Spec.LDAPPolicyAttachToSingleUser != nil { + error := c.ldapPolicyAttachToSingleUser(ctx, tenant) + if error != nil { + klog.Errorf("There was an error configuring LDAP User in the tenant: %s", error) + } + } + // Finally, we update the status block of the Tenant resource to reflect the // current state of the world tenant, err = c.updateTenantStatus(ctx, tenant, StatusInitialized, totalAvailableReplicas) diff --git a/pkg/controller/operator.go b/pkg/controller/operator.go index b33553b3eaf..4318427eb79 100644 --- a/pkg/controller/operator.go +++ b/pkg/controller/operator.go @@ -23,8 +23,13 @@ import ( "crypto/x509" "errors" "fmt" + corev1 "k8s.io/api/core/v1" + runetimetime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" "net" "net/http" + "strings" "time" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -318,6 +323,130 @@ func (c *Controller) createUsers(ctx context.Context, tenant *miniov2.Tenant, te return err } +func (c *Controller) verifyLDAPParams(ctx context.Context, tenant *miniov2.Tenant) (err error) { + // Verify there is a user + user := tenant.Spec.LDAPPolicyAttachToSingleUser.User + policy := tenant.Spec.LDAPPolicyAttachToSingleUser.Policy + if user != "" && policy != "" { + klog.Infof("Attach policy: %s to user: %user", policy, user) + } else { + klog.Errorf("User and Policy have to be provided") + return errors.New("User and Policy have to be provided") + } + return nil +} + +func (c *Controller) ldapPolicyAttachToSingleUser(ctx context.Context, tenant *miniov2.Tenant) (err error) { + result := c.verifyLDAPParams(ctx, tenant) + if result != nil { + klog.Errorf("LDAP Params did not pass validation") + return err + } + klog.Info("Configuring LDAP in the Tenant") + tenantName := tenant.Name + tenantNamespace := tenant.Namespace + pool0Name := tenant.Spec.Pools[0].Name + podName := tenantName + "-" + pool0Name + "-0" + request := c.kubeClientSet.CoreV1().RESTClient().Post().Resource("pods").Name(podName).Namespace( + tenantNamespace).SubResource("exec") + scheme := runetimetime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + klog.Error("There was an error trying to configure LDAP User") + return err + } + parameterCodec := runetimetime.NewParameterCodec(scheme) + LDAPPolicyAttachToSingleUser := tenant.Spec.LDAPPolicyAttachToSingleUser + User := LDAPPolicyAttachToSingleUser.User + Policy := LDAPPolicyAttachToSingleUser.Policy + MinIOServerEndpoint := tenant.MinIOServerEndpoint() + tenant.ConfigurationSecretName() + tenant.HasConfigurationSecret() + tenantSpecConfigName := tenant.Spec.Configuration.Name + secret, _ := c.kubeClientSet.CoreV1().Secrets(tenant.Namespace).Get(ctx, tenantSpecConfigName, metav1.GetOptions{}) + secretData := secret.Data + content := string(secretData["config.env"]) + contents := strings.Split(content, "\n") + rootAccessKey := "" + rootSecretKey := "" + for i := range contents { + content := contents[i] + if strings.Contains(content, "MINIO_ROOT_USER") { + valor := strings.Split(content, "=") + user := valor[1] + plainUser := user[1 : len(user)-1] + rootAccessKey = plainUser + } + if strings.Contains(content, "MINIO_ROOT_PASSWORD") { + valor := strings.Split(content, "=") + password := valor[1] + plainPassword := password[1 : len(password)-1] + rootSecretKey = plainPassword + } + } + // export set HOME=/tmp is to allow mc to work when not using root user in security context to avoid + // mc: Unable to save new mc config. mkdir /.mc: permission denied. + alias := "export set HOME=/tmp; mc alias set myminio " + MinIOServerEndpoint + " " + rootAccessKey + " " + rootSecretKey + ldapCommand := "mc idp ldap policy attach myminio " + Policy + " --user=" + "'" + User + "'" + klog.Info(ldapCommand) + finalCommand := alias + "; " + ldapCommand + cmds := []string{ + "sh", + "-c", + finalCommand, + } + request.VersionedParams(&corev1.PodExecOptions{ + Stdin: false, + Stdout: true, + Stderr: true, + TTY: false, + Container: "minio", + Command: cmds, + }, parameterCodec) + // Get a rest.Config from the kubeconfig file. This will be passed into all + // the client objects we create. + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + restConfig, err := kubeconfig.ClientConfig() + if err != nil { + klog.Errorf("Error attaching LDAP Policy to user %s", err) + return err + } + exec, err := remotecommand.NewSPDYExecutor(restConfig, "POST", request.URL()) + fmt.Println(exec) + if err != nil { + klog.Errorf("Error attaching LDAP Policy to user %s", err) + return err + } + var stdout, stderr bytes.Buffer + fmt.Println(stdout) + fmt.Println(stderr) + err = exec.Stream(remotecommand.StreamOptions{ + Stdin: nil, + Stdout: &stdout, + Stderr: &stderr, + Tty: false, + }) + if err != nil { + klog.Errorf("Error attaching LDAP Policy to user %s", err) + return err + } + actualOutput := stdout.String() + actualError := stderr.String() + klog.Infof("Output from pod: %v", actualOutput) + klog.Infof("Error from pod: %v", actualError) + // If expected output is contained by the actual output, it will be + // considered as a success. + if strings.Contains(actualError, "The specified policy change is already in effect") { + klog.Info("Command(s) succeeded, will not be executed again") + // By saving the command in the status, we guarantee that will not be re-executed. + if _, err = c.updateLDAPPolicyAttachToSingleUserStatus(ctx, tenant, true); err != nil { + klog.V(2).Infof(err.Error()) + } + } + return nil +} + func (c *Controller) createBuckets(ctx context.Context, tenant *miniov2.Tenant, tenantConfiguration map[string][]byte) (err error) { defer func() { if err == nil { diff --git a/pkg/controller/status.go b/pkg/controller/status.go index a3666dcb96e..7343c5ce75b 100644 --- a/pkg/controller/status.go +++ b/pkg/controller/status.go @@ -29,6 +29,33 @@ func (c *Controller) updateTenantStatus(ctx context.Context, tenant *miniov2.Ten return c.updateTenantStatusWithRetry(ctx, tenant, currentState, availableReplicas, true) } +func (c *Controller) updateLDAPPolicyAttachToSingleUserStatus(ctx context.Context, tenant *miniov2.Tenant, succeeded bool) (*miniov2.Tenant, error) { + return c.updateLDAPPolicyAttachToSingleUserWithRetry(ctx, tenant, succeeded, true) +} + +func (c *Controller) updateLDAPPolicyAttachToSingleUserWithRetry(ctx context.Context, tenant *miniov2.Tenant, succeeded bool, retry bool) ( + *miniov2.Tenant, error) { + tenantCopy := tenant.DeepCopy() + tenantCopy.Spec = miniov2.TenantSpec{} + tenantCopy.Status = *tenant.Status.DeepCopy() + tenantCopy.Status.LDAPPolicyAttachToSingleUser = succeeded + opts := metav1.UpdateOptions{} + t, err := c.minioClientSet.MinioV2().Tenants(tenant.Namespace).UpdateStatus(ctx, tenantCopy, opts) + t.EnsureDefaults() + if err != nil { + if k8serrors.IsConflict(err) && retry { + klog.Info("Hit conflict issue, getting latest version of tenant") + tenant, err = c.minioClientSet.MinioV2().Tenants(tenant.Namespace).Get(ctx, tenant.Name, metav1.GetOptions{}) + if err != nil { + return tenant, err + } + return c.updateLDAPPolicyAttachToSingleUserWithRetry(ctx, tenant, succeeded, false) + } + return t, err + } + return t, nil +} + func (c *Controller) updateTenantStatusWithRetry(ctx context.Context, tenant *miniov2.Tenant, currentState string, availableReplicas int32, retry bool) (*miniov2.Tenant, error) { // If we are updating the tenant with the same status as before we are going to skip it as to avoid a resource number // change and have the operator loop re-processing the tenant endlessly