diff --git a/UPGRADE.md b/UPGRADE.md index 11d78fe4a0b..c5ce0ec43f4 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -13,6 +13,14 @@ This release introduces a readiness probe to prevent kubernetes from routing tra > ⚠️ Upgrading to v6.0.0 will cause all pods to restart upon upgrade. +v6.0.1 +--- + +This release includes improvements to reduce the number of restarts to MinIO including when changes to environment +variables happen, the operator will perform an in-place update to the MinIO pods without restarting them. + +> ⚠️ Upgrading to v6.0.1 will cause all pods to restart upon upgrade due to new sidecar image. + v5.0.0 --- diff --git a/pkg/common/const.go b/pkg/common/const.go index 97481270cbd..d9866838174 100644 --- a/pkg/common/const.go +++ b/pkg/common/const.go @@ -18,9 +18,12 @@ package common // Constants for the webhook endpoints const ( - WebhookAPIVersion = "/webhook/v1" - UpgradeServerPort = "4221" - WebhookDefaultPort = "4222" - WebhookAPIBucketService = WebhookAPIVersion + "/bucketsrv" - WebhookAPIUpdate = WebhookAPIVersion + "/update" + WebhookAPIVersion = "/webhook/v1" + UpgradeServerPort = "4221" + WebhookDefaultPort = "4222" + WebhookAPIBucketService = WebhookAPIVersion + "/bucketsrv" + WebhookAPIUpdate = WebhookAPIVersion + "/update" + SidecarHTTPPort = "4224" + SidecarAPIVersion = "/sidecar/v1" + SidecarAPIConfigEndpoint = SidecarAPIVersion + "/config" ) diff --git a/pkg/configuration/tenant_configuration.go b/pkg/configuration/tenant_configuration.go new file mode 100644 index 00000000000..2900a6a19c6 --- /dev/null +++ b/pkg/configuration/tenant_configuration.go @@ -0,0 +1,243 @@ +// This file is part of MinIO Operator +// Copyright (c) 2023 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package configuration + +import ( + "fmt" + "sort" + "strings" + + miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" + "github.com/minio/operator/pkg/common" + "github.com/minio/operator/pkg/resources/statefulsets" + corev1 "k8s.io/api/core/v1" +) + +const ( + bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT" +) + +// GetFullTenantConfig returns the full configuration for the tenant considering the secret and the tenant spec +func GetFullTenantConfig(tenant *miniov2.Tenant, configSecret *corev1.Secret) (string, bool, bool) { + seededVars := parseConfEnvSecret(configSecret) + rootUserFound := false + rootPwdFound := false + for _, env := range seededVars { + if env.Name == "MINIO_ROOT_USER" { + rootUserFound = true + } + if env.Name == "MINIO_ACCESS_KEY" { + rootUserFound = true + } + if env.Name == "MINIO_ROOT_PASSWORD" { + rootPwdFound = true + } + if env.Name == "MINIO_SECRET_KEY" { + rootPwdFound = true + } + } + compiledConfig := buildTenantEnvs(tenant, seededVars) + configurationFileContent := envVarsToFileContent(compiledConfig) + return configurationFileContent, rootUserFound, rootPwdFound +} + +func parseConfEnvSecret(secret *corev1.Secret) map[string]corev1.EnvVar { + if secret == nil { + return nil + } + data := secret.Data["config.env"] + envMap := make(map[string]corev1.EnvVar) + + lines := strings.Split(string(data), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "export ") { + line = strings.TrimPrefix(line, "export ") + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + name := strings.TrimSpace(parts[0]) + value := strings.Trim(strings.TrimSpace(parts[1]), "\"") + envVar := corev1.EnvVar{ + Name: name, + Value: value, + } + envMap[name] = envVar + } + } + } + return envMap +} + +func buildTenantEnvs(tenant *miniov2.Tenant, cfgEnvExisting map[string]corev1.EnvVar) []corev1.EnvVar { + // Enable `mc admin update` style updates to MinIO binaries + // within the container, only operator is supposed to perform + // these operations. + envVarsMap := map[string]corev1.EnvVar{ + "MINIO_UPDATE": { + Name: "MINIO_UPDATE", + Value: "on", + }, + "MINIO_UPDATE_MINISIGN_PUBKEY": { + Name: "MINIO_UPDATE_MINISIGN_PUBKEY", + Value: "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav", + }, + "MINIO_PROMETHEUS_JOB_ID": { + Name: "MINIO_PROMETHEUS_JOB_ID", + Value: tenant.PrometheusConfigJobName(), + }, + } + // Specific case of bug in runtimeClass crun where $HOME is not set + for _, pool := range tenant.Spec.Pools { + if pool.RuntimeClassName != nil && *pool.RuntimeClassName == "crun" { + // Set HOME to / + envVarsMap["HOME"] = corev1.EnvVar{ + Name: "HOME", + Value: "/", + } + } + } + var domains []string + // Enable Bucket DNS only if asked for by default turned off + if tenant.BucketDNS() { + domains = append(domains, tenant.MinIOBucketBaseDomain()) + sidecarBucketURL := fmt.Sprintf("http://127.0.0.1:%s%s/%s/%s", + common.WebhookDefaultPort, + common.WebhookAPIBucketService, + tenant.Namespace, + tenant.Name) + envVarsMap[bucketDNSEnv] = corev1.EnvVar{ + Name: bucketDNSEnv, + Value: sidecarBucketURL, + } + } + // Check if any domains are configured + if tenant.HasMinIODomains() { + domains = append(domains, tenant.GetDomainHosts()...) + } + // tell MinIO about all the domains meant to hit it if they are not passed manually via .spec.env + if len(domains) > 0 { + envVarsMap[miniov2.MinIODomain] = corev1.EnvVar{ + Name: miniov2.MinIODomain, + Value: strings.Join(domains, ","), + } + } + + // If no specific server URL is specified we will specify the internal k8s url, but if a list of domains was + // provided we will use the first domain. + serverURL := tenant.MinIOServerEndpoint() + if tenant.HasMinIODomains() { + // Infer schema from tenant TLS, if not explicit + if !strings.HasPrefix(tenant.Spec.Features.Domains.Minio[0], "http") { + useSchema := "http" + if tenant.TLS() { + useSchema = "https" + } + serverURL = fmt.Sprintf("%s://%s", useSchema, tenant.Spec.Features.Domains.Minio[0]) + } else { + serverURL = tenant.Spec.Features.Domains.Minio[0] + } + } + envVarsMap[miniov2.MinIOServerURL] = corev1.EnvVar{ + Name: miniov2.MinIOServerURL, + Value: serverURL, + } + + // Set the redirect url for console + if tenant.HasConsoleDomains() { + consoleDomain := tenant.Spec.Features.Domains.Console + // Infer schema from tenant TLS, if not explicit + if !strings.HasPrefix(consoleDomain, "http") { + useSchema := "http" + if tenant.TLS() { + useSchema = "https" + } + consoleDomain = fmt.Sprintf("%s://%s", useSchema, consoleDomain) + } + envVarsMap[miniov2.MinIOBrowserRedirectURL] = corev1.EnvVar{ + Name: miniov2.MinIOBrowserRedirectURL, + Value: consoleDomain, + } + } + if tenant.HasKESEnabled() { + envVarsMap["MINIO_KMS_KES_ENDPOINT"] = corev1.EnvVar{ + Name: "MINIO_KMS_KES_ENDPOINT", + Value: tenant.KESServiceEndpoint(), + } + envVarsMap["MINIO_KMS_KES_CERT_FILE"] = corev1.EnvVar{ + Name: "MINIO_KMS_KES_CERT_FILE", + Value: miniov2.MinIOCertPath + "/client.crt", + } + envVarsMap["MINIO_KMS_KES_KEY_FILE"] = corev1.EnvVar{ + Name: "MINIO_KMS_KES_KEY_FILE", + Value: miniov2.MinIOCertPath + "/client.key", + } + envVarsMap["MINIO_KMS_KES_CA_PATH"] = corev1.EnvVar{ + Name: "MINIO_KMS_KES_CA_PATH", + Value: miniov2.MinIOCertPath + "/CAs/kes.crt", + } + envVarsMap["MINIO_KMS_KES_CAPATH"] = corev1.EnvVar{ + Name: "MINIO_KMS_KES_CAPATH", + Value: miniov2.MinIOCertPath + "/CAs/kes.crt", + } + envVarsMap["MINIO_KMS_KES_KEY_NAME"] = corev1.EnvVar{ + Name: "MINIO_KMS_KES_KEY_NAME", + Value: tenant.Spec.KES.KeyName, + } + } + + // attach tenant args + args := strings.Join(statefulsets.GetContainerArgs(tenant, ""), " ") + envVarsMap["MINIO_ARGS"] = corev1.EnvVar{ + Name: "MINIO_ARGS", + Value: args, + } + + // Add all the tenant.spec.env environment variables + // User defined environment variables will take precedence over default environment variables + for _, env := range tenant.GetEnvVars() { + envVarsMap[env.Name] = env + } + var envVars []corev1.EnvVar + // transform map to array and skip configurations from config.env + for _, env := range envVarsMap { + if cfgEnvExisting != nil { + if _, ok := cfgEnvExisting[env.Name]; !ok { + envVars = append(envVars, env) + } + } else { + envVars = append(envVars, env) + } + } + // now add everything in the existing config.env + for _, envVar := range cfgEnvExisting { + envVars = append(envVars, envVar) + } + // sort the array to produce the same result everytime + sort.Slice(envVars, func(i, j int) bool { + return envVars[i].Name < envVars[j].Name + }) + + return envVars +} + +func envVarsToFileContent(envVars []corev1.EnvVar) string { + content := "" + for _, env := range envVars { + content += fmt.Sprintf("export %s=\"%s\"\n", env.Name, env.Value) + } + return content +} diff --git a/pkg/configuration/tenant_configuration_test.go b/pkg/configuration/tenant_configuration_test.go new file mode 100644 index 00000000000..9393d888744 --- /dev/null +++ b/pkg/configuration/tenant_configuration_test.go @@ -0,0 +1,385 @@ +// This file is part of MinIO Operator +// Copyright (c) 2023 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package configuration + +import ( + "reflect" + "testing" + + miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestEnvVarsToFileContent(t *testing.T) { + type args struct { + envVars []corev1.EnvVar + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Basic test case", + args: args{ + envVars: []corev1.EnvVar{ + { + Name: "MINIO_UPDATE", + Value: "on", + }, + }, + }, + want: "export MINIO_UPDATE=\"on\"\n", + }, + { + name: "Two Vars test case", + args: args{ + envVars: []corev1.EnvVar{ + { + Name: "MINIO_UPDATE", + Value: "on", + }, + { + Name: "MINIO_UPDATE_MINISIGN_PUBKEY", + Value: "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav", + }, + }, + }, + want: `export MINIO_UPDATE="on" +export MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := envVarsToFileContent(tt.args.envVars); got != tt.want { + t.Errorf("envVarsToFileContent() = `%v`, want `%v`", got, tt.want) + } + }) + } +} + +func TestGetTenantConfiguration(t *testing.T) { + type args struct { + tenant *miniov2.Tenant + cfgEnvExisting map[string]corev1.EnvVar + } + tests := []struct { + name string + args args + want []corev1.EnvVar + }{ + { + name: "Defaulted Values", + args: args{ + tenant: &miniov2.Tenant{}, + cfgEnvExisting: nil, + }, + want: []corev1.EnvVar{ + { + Name: "MINIO_ARGS", + Value: "", + }, + { + Name: "MINIO_PROMETHEUS_JOB_ID", + Value: "minio-job", + }, + { + Name: "MINIO_SERVER_URL", + Value: "https://minio..svc.cluster.local:443", + }, + { + Name: "MINIO_UPDATE", + Value: "on", + }, + { + Name: "MINIO_UPDATE_MINISIGN_PUBKEY", + Value: "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav", + }, + }, + }, + { + name: "Tenant has one env var", + args: args{ + tenant: &miniov2.Tenant{ + Spec: miniov2.TenantSpec{ + Env: []corev1.EnvVar{ + { + Name: "TEST", + Value: "value", + }, + }, + }, + }, + cfgEnvExisting: nil, + }, + want: []corev1.EnvVar{ + { + Name: "MINIO_ARGS", + Value: "", + }, + { + Name: "MINIO_PROMETHEUS_JOB_ID", + Value: "minio-job", + }, + { + Name: "MINIO_SERVER_URL", + Value: "https://minio..svc.cluster.local:443", + }, + { + Name: "MINIO_UPDATE", + Value: "on", + }, + { + Name: "MINIO_UPDATE_MINISIGN_PUBKEY", + Value: "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav", + }, + { + Name: "TEST", + Value: "value", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.args.tenant.EnsureDefaults() + if got := buildTenantEnvs(tt.args.tenant, tt.args.cfgEnvExisting); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildTenantEnvs() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParseConfEnvSecret(t *testing.T) { + type args struct { + secret *corev1.Secret + } + tests := []struct { + name string + args args + want map[string]corev1.EnvVar + }{ + { + name: "Basic case", + args: args{ + secret: &corev1.Secret{ + Data: map[string][]byte{"config.env": []byte(`export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_STORAGE_CLASS_STANDARD="EC:2" +export MINIO_BROWSER="on"`)}, + }, + }, + want: map[string]corev1.EnvVar{ + "MINIO_ROOT_USER": { + Name: "MINIO_ROOT_USER", + Value: "minio", + }, + "MINIO_ROOT_PASSWORD": { + Name: "MINIO_ROOT_PASSWORD", + Value: "minio123", + }, + "MINIO_STORAGE_CLASS_STANDARD": { + Name: "MINIO_STORAGE_CLASS_STANDARD", + Value: "EC:2", + }, + "MINIO_BROWSER": { + Name: "MINIO_BROWSER", + Value: "on", + }, + }, + }, + { + name: "Basic case has tabs", + args: args{ + secret: &corev1.Secret{ + Data: map[string][]byte{"config.env": []byte(` export MINIO_ROOT_USER="minio" + export MINIO_ROOT_PASSWORD="minio123" + export MINIO_STORAGE_CLASS_STANDARD="EC:2" + export MINIO_BROWSER="on"`)}, + }, + }, + want: map[string]corev1.EnvVar{ + "MINIO_ROOT_USER": { + Name: "MINIO_ROOT_USER", + Value: "minio", + }, + "MINIO_ROOT_PASSWORD": { + Name: "MINIO_ROOT_PASSWORD", + Value: "minio123", + }, + "MINIO_STORAGE_CLASS_STANDARD": { + Name: "MINIO_STORAGE_CLASS_STANDARD", + Value: "EC:2", + }, + "MINIO_BROWSER": { + Name: "MINIO_BROWSER", + Value: "on", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := parseConfEnvSecret(tt.args.secret); !reflect.DeepEqual(got, tt.want) { + t.Errorf("parseConfEnvSecret() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetFullTenantConfig(t *testing.T) { + type args struct { + tenant *miniov2.Tenant + configSecret *corev1.Secret + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Empty tenant with one env var", + args: args{ + tenant: &miniov2.Tenant{ + Spec: miniov2.TenantSpec{ + Env: []corev1.EnvVar{ + { + Name: "TEST", + Value: "value", + }, + }, + }, + }, + configSecret: &corev1.Secret{ + Data: map[string][]byte{"config.env": []byte(`export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_STORAGE_CLASS_STANDARD="EC:2" +export MINIO_BROWSER="on"`)}, + }, + }, + want: `export MINIO_ARGS="" +export MINIO_BROWSER="on" +export MINIO_PROMETHEUS_JOB_ID="minio-job" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_ROOT_USER="minio" +export MINIO_SERVER_URL="https://minio..svc.cluster.local:443" +export MINIO_STORAGE_CLASS_STANDARD="EC:2" +export MINIO_UPDATE="on" +export MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" +export TEST="value" +`, + }, + { + name: "Empty tenant; with domains; one env var", + args: args{ + tenant: &miniov2.Tenant{ + Spec: miniov2.TenantSpec{ + Env: []corev1.EnvVar{ + { + Name: "TEST", + Value: "value", + }, + }, + Features: &miniov2.Features{ + Domains: &miniov2.TenantDomains{ + Console: "http://console.minio", + }, + }, + }, + }, + configSecret: &corev1.Secret{ + Data: map[string][]byte{"config.env": []byte(`export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_STORAGE_CLASS_STANDARD="EC:2" +export MINIO_BROWSER="on"`)}, + }, + }, + want: `export MINIO_ARGS="" +export MINIO_BROWSER="on" +export MINIO_BROWSER_REDIRECT_URL="http://console.minio" +export MINIO_PROMETHEUS_JOB_ID="minio-job" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_ROOT_USER="minio" +export MINIO_SERVER_URL="https://minio..svc.cluster.local:443" +export MINIO_STORAGE_CLASS_STANDARD="EC:2" +export MINIO_UPDATE="on" +export MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" +export TEST="value" +`, + }, + { + name: "One Pool Tenant; with domains; one env var", + args: args{ + tenant: &miniov2.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant", + Namespace: "ns-x", + }, + Spec: miniov2.TenantSpec{ + Env: []corev1.EnvVar{ + { + Name: "TEST", + Value: "value", + }, + }, + Features: &miniov2.Features{ + Domains: &miniov2.TenantDomains{ + Console: "http://console.minio", + }, + }, + Pools: []miniov2.Pool{ + { + Name: "pool-0", + Servers: 4, + VolumesPerServer: 4, + VolumeClaimTemplate: nil, + }, + }, + }, + }, + configSecret: &corev1.Secret{ + Data: map[string][]byte{"config.env": []byte(`export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_STORAGE_CLASS_STANDARD="EC:2" +export MINIO_BROWSER="on"`)}, + }, + }, + want: `export MINIO_ARGS="https://tenant-pool-0-{0...3}.tenant-hl.ns-x.svc.cluster.local/export{0...3}" +export MINIO_BROWSER="on" +export MINIO_BROWSER_REDIRECT_URL="http://console.minio" +export MINIO_PROMETHEUS_JOB_ID="minio-job" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_ROOT_USER="minio" +export MINIO_SERVER_URL="https://minio.ns-x.svc.cluster.local:443" +export MINIO_STORAGE_CLASS_STANDARD="EC:2" +export MINIO_UPDATE="on" +export MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" +export TEST="value" +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.args.tenant.EnsureDefaults() + if got, _, _ := GetFullTenantConfig(tt.args.tenant, tt.args.configSecret); got != tt.want { + t.Errorf("GetFullTenantConfig() = `%v`, want `%v`", got, tt.want) + } + }) + } +} diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 71de5f959d4..3719524419b 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -19,157 +19,28 @@ import ( "path/filepath" "sort" "strconv" - "strings" "k8s.io/apimachinery/pkg/util/intstr" - "github.com/minio/operator/pkg/certs" - "github.com/minio/operator/pkg/common" - miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" + "github.com/minio/operator/pkg/certs" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) -const ( - bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT" -) - // Returns the MinIO environment variables set in configuration. // If a user specifies a secret in the spec (for MinIO credentials) we use // that to set MINIO_ROOT_USER & MINIO_ROOT_PASSWORD. func minioEnvironmentVars(t *miniov2.Tenant, skipEnvVars map[string][]byte) []corev1.EnvVar { var envVars []corev1.EnvVar - // Enable `mc admin update` style updates to MinIO binaries - // within the container, only operator is supposed to perform - // these operations. - envVarsMap := map[string]corev1.EnvVar{ - "MINIO_UPDATE": { - Name: "MINIO_UPDATE", - Value: "on", - }, - "MINIO_UPDATE_MINISIGN_PUBKEY": { - Name: "MINIO_UPDATE_MINISIGN_PUBKEY", - Value: "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav", - }, - "MINIO_PROMETHEUS_JOB_ID": { - Name: "MINIO_PROMETHEUS_JOB_ID", - Value: t.PrometheusConfigJobName(), - }, - } - // Specific case of bug in runtimeClass crun where $HOME is not set - for _, pool := range t.Spec.Pools { - if pool.RuntimeClassName != nil && *pool.RuntimeClassName == "crun" { - // Set HOME to / - envVarsMap["HOME"] = corev1.EnvVar{ - Name: "HOME", - Value: "/", - } - } - } - - var domains []string - // Enable Bucket DNS only if asked for by default turned off - if t.BucketDNS() { - domains = append(domains, t.MinIOBucketBaseDomain()) - sidecarBucketURL := fmt.Sprintf("http://127.0.0.1:%s%s/%s/%s", - common.WebhookDefaultPort, - common.WebhookAPIBucketService, - t.Namespace, - t.Name) - envVarsMap[bucketDNSEnv] = corev1.EnvVar{ - Name: bucketDNSEnv, - Value: sidecarBucketURL, - } - } - // Check if any domains are configured - if t.HasMinIODomains() { - domains = append(domains, t.GetDomainHosts()...) - } - // tell MinIO about all the domains meant to hit it if they are not passed manually via .spec.env - if len(domains) > 0 { - envVarsMap[miniov2.MinIODomain] = corev1.EnvVar{ - Name: miniov2.MinIODomain, - Value: strings.Join(domains, ","), - } - } - // If no specific server URL is specified we will specify the internal k8s url, but if a list of domains was - // provided we will use the first domain. - serverURL := t.MinIOServerEndpoint() - if t.HasMinIODomains() { - // Infer schema from tenant TLS, if not explicit - if !strings.HasPrefix(t.Spec.Features.Domains.Minio[0], "http") { - useSchema := "http" - if t.TLS() { - useSchema = "https" - } - serverURL = fmt.Sprintf("%s://%s", useSchema, t.Spec.Features.Domains.Minio[0]) - } else { - serverURL = t.Spec.Features.Domains.Minio[0] - } - } - envVarsMap[miniov2.MinIOServerURL] = corev1.EnvVar{ - Name: miniov2.MinIOServerURL, - Value: serverURL, - } - - // Set the redirect url for console - if t.HasConsoleDomains() { - consoleDomain := t.Spec.Features.Domains.Console - // Infer schema from tenant TLS, if not explicit - if !strings.HasPrefix(consoleDomain, "http") { - useSchema := "http" - if t.TLS() { - useSchema = "https" - } - consoleDomain = fmt.Sprintf("%s://%s", useSchema, consoleDomain) - } - envVarsMap[miniov2.MinIOBrowserRedirectURL] = corev1.EnvVar{ - Name: miniov2.MinIOBrowserRedirectURL, - Value: consoleDomain, - } - } - - if t.HasKESEnabled() { - envVarsMap["MINIO_KMS_KES_ENDPOINT"] = corev1.EnvVar{ - Name: "MINIO_KMS_KES_ENDPOINT", - Value: t.KESServiceEndpoint(), - } - envVarsMap["MINIO_KMS_KES_CERT_FILE"] = corev1.EnvVar{ - Name: "MINIO_KMS_KES_CERT_FILE", - Value: miniov2.MinIOCertPath + "/client.crt", - } - envVarsMap["MINIO_KMS_KES_KEY_FILE"] = corev1.EnvVar{ - Name: "MINIO_KMS_KES_KEY_FILE", - Value: miniov2.MinIOCertPath + "/client.key", - } - envVarsMap["MINIO_KMS_KES_CA_PATH"] = corev1.EnvVar{ - Name: "MINIO_KMS_KES_CA_PATH", - Value: miniov2.MinIOCertPath + "/CAs/kes.crt", - } - envVarsMap["MINIO_KMS_KES_CAPATH"] = corev1.EnvVar{ - Name: "MINIO_KMS_KES_CAPATH", - Value: miniov2.MinIOCertPath + "/CAs/kes.crt", - } - envVarsMap["MINIO_KMS_KES_KEY_NAME"] = corev1.EnvVar{ - Name: "MINIO_KMS_KES_KEY_NAME", - Value: t.Spec.KES.KeyName, - } - } - if t.HasConfigurationSecret() { - envVarsMap["MINIO_CONFIG_ENV_FILE"] = corev1.EnvVar{ - Name: "MINIO_CONFIG_ENV_FILE", - Value: miniov2.CfgFile, - } - } + envVarsMap := map[string]corev1.EnvVar{} - // Add all the tenant.spec.env environment variables - // User defined environment variables will take precedence over default environment variables - for _, env := range t.GetEnvVars() { - envVarsMap[env.Name] = env + envVarsMap["MINIO_CONFIG_ENV_FILE"] = corev1.EnvVar{ + Name: "MINIO_CONFIG_ENV_FILE", + Value: miniov2.CfgFile, } // transform map to array and skip configurations from config.env diff --git a/sidecar/pkg/sidecar/http_handlers.go b/sidecar/pkg/sidecar/bucket_dns_handlers.go similarity index 85% rename from sidecar/pkg/sidecar/http_handlers.go rename to sidecar/pkg/sidecar/bucket_dns_handlers.go index 9d2e498faa1..10f3509741a 100644 --- a/sidecar/pkg/sidecar/http_handlers.go +++ b/sidecar/pkg/sidecar/bucket_dns_handlers.go @@ -23,6 +23,9 @@ import ( "net/http" "strconv" "strings" + "time" + + "github.com/minio/operator/pkg/common" "github.com/gorilla/mux" "github.com/minio/operator/pkg/resources/services" @@ -32,6 +35,27 @@ import ( "k8s.io/klog/v2" ) +func configureWebhookServer(c *Controller) *http.Server { + router := mux.NewRouter().SkipClean(true).UseEncodedPath() + + router.Methods(http.MethodPost). + Path(common.WebhookAPIBucketService + "/{namespace}/{name:.+}"). + HandlerFunc(c.BucketSrvHandler). + Queries(restQueries("bucket")...) + + router.NotFoundHandler = http.NotFoundHandler() + + s := &http.Server{ + Addr: "127.0.0.1:" + common.WebhookDefaultPort, + Handler: router, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + MaxHeaderBytes: 1 << 20, + } + + return s +} + // BucketSrvHandler - POST /webhook/v1/bucketsrv/{namespace}/{name}?bucket={bucket} func (c *Controller) BucketSrvHandler(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) diff --git a/sidecar/pkg/sidecar/handlers_common.go b/sidecar/pkg/sidecar/handlers_common.go new file mode 100644 index 00000000000..4c2fa47d503 --- /dev/null +++ b/sidecar/pkg/sidecar/handlers_common.go @@ -0,0 +1,27 @@ +// This file is part of MinIO Operator +// Copyright (c) 2024 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sidecar + +// Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example) +// If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"] +func restQueries(keys ...string) []string { + var accumulator []string + for _, key := range keys { + accumulator = append(accumulator, key, "{"+key+":.*}") + } + return accumulator +} diff --git a/sidecar/pkg/sidecar/webhook_server.go b/sidecar/pkg/sidecar/probes_handlers.go similarity index 72% rename from sidecar/pkg/sidecar/webhook_server.go rename to sidecar/pkg/sidecar/probes_handlers.go index b8a656b28a2..b67466bbe9b 100644 --- a/sidecar/pkg/sidecar/webhook_server.go +++ b/sidecar/pkg/sidecar/probes_handlers.go @@ -23,42 +23,9 @@ import ( "net/http" "time" - "github.com/minio/operator/pkg/common" - "github.com/gorilla/mux" ) -// Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example) -// If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"] -func restQueries(keys ...string) []string { - var accumulator []string - for _, key := range keys { - accumulator = append(accumulator, key, "{"+key+":.*}") - } - return accumulator -} - -func configureWebhookServer(c *Controller) *http.Server { - router := mux.NewRouter().SkipClean(true).UseEncodedPath() - - router.Methods(http.MethodPost). - Path(common.WebhookAPIBucketService + "/{namespace}/{name:.+}"). - HandlerFunc(c.BucketSrvHandler). - Queries(restQueries("bucket")...) - - router.NotFoundHandler = http.NotFoundHandler() - - s := &http.Server{ - Addr: "127.0.0.1:" + common.WebhookDefaultPort, - Handler: router, - ReadTimeout: time.Minute, - WriteTimeout: time.Minute, - MaxHeaderBytes: 1 << 20, - } - - return s -} - func configureProbesServer(c *Controller, tenantTLS bool) *http.Server { router := mux.NewRouter().SkipClean(true).UseEncodedPath() diff --git a/sidecar/pkg/sidecar/sidecar_handlers.go b/sidecar/pkg/sidecar/sidecar_handlers.go new file mode 100644 index 00000000000..7d702d6e18c --- /dev/null +++ b/sidecar/pkg/sidecar/sidecar_handlers.go @@ -0,0 +1,56 @@ +// This file is part of MinIO Operator +// Copyright (c) 2024 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sidecar + +import ( + "log" + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/minio/operator/pkg/common" +) + +func configureSidecarServer(c *Controller) *http.Server { + router := mux.NewRouter().SkipClean(true).UseEncodedPath() + + router.Methods(http.MethodPost). + Path(common.SidecarAPIConfigEndpoint). + HandlerFunc(c.CheckConfigHandler). + Queries(restQueries("c")...) + + router.NotFoundHandler = http.NotFoundHandler() + + s := &http.Server{ + Addr: "0.0.0.0:" + common.SidecarHTTPPort, + Handler: router, + ReadTimeout: time.Minute, + WriteTimeout: time.Minute, + MaxHeaderBytes: 1 << 20, + } + + return s +} + +// CheckConfigHandler - POST /sidecar/v1/config?c={hash} +func (c *Controller) CheckConfigHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + hash := vars["c"] + + log.Println("Checking config hash: ", hash) +} diff --git a/sidecar/pkg/sidecar/sidecar_utils.go b/sidecar/pkg/sidecar/sidecar_utils.go index e40e4859705..63cb8aef0e7 100644 --- a/sidecar/pkg/sidecar/sidecar_utils.go +++ b/sidecar/pkg/sidecar/sidecar_utils.go @@ -22,17 +22,16 @@ import ( "log" "net/http" "os" - "strings" "time" - common2 "github.com/minio/operator/sidecar/pkg/common" + "github.com/minio/operator/pkg/configuration" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" clientset "github.com/minio/operator/pkg/client/clientset/versioned" minioInformers "github.com/minio/operator/pkg/client/informers/externalversions" v22 "github.com/minio/operator/pkg/client/informers/externalversions/minio.min.io/v2" - "github.com/minio/operator/sidecar/pkg/validator" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" @@ -80,6 +79,7 @@ func StartSideCar(tenantName string, secretName string) { controller := NewSideCarController(kubeClient, controllerClient, namespace, tenantName, secretName) controller.ws = configureWebhookServer(controller) controller.probeServer = configureProbesServer(controller, tenant.TLS()) + controller.sidecar = configureSidecarServer(controller) stopControllerCh := make(chan struct{}) @@ -105,6 +105,14 @@ func StartSideCar(tenantName string, secretName string) { } }() + go func() { + if err = controller.sidecar.ListenAndServe(); err != nil { + // if the web server exits, + klog.Error(err) + close(stopControllerCh) + } + }() + <-stopControllerCh } @@ -121,6 +129,7 @@ type Controller struct { informerFactory informers.SharedInformerFactory ws *http.Server probeServer *http.Server + sidecar *http.Server } // NewSideCarController returns an instance of Controller with the provided clients @@ -143,7 +152,7 @@ func NewSideCarController(kubeClient *kubernetes.Clientset, controllerClient *cl secretInformer: secretInformer, } - tenantInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err := tenantInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ UpdateFunc: func(old, new interface{}) { oldTenant := old.(*v2.Tenant) newTenant := new.(*v2.Tenant) @@ -152,11 +161,15 @@ func NewSideCarController(kubeClient *kubernetes.Clientset, controllerClient *cl // Two different versions of the same Tenant will always have different RVs. return } - c.regenCfg(tenantName, namespace) + c.regenCfgWithTenant(newTenant) }, }) + if err != nil { + log.Println("could not add event handler for tenant informer", err) + return nil + } - secretInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err = secretInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ UpdateFunc: func(old, new interface{}) { oldSecret := old.(*corev1.Secret) // ignore anything that is not what we want @@ -170,65 +183,58 @@ func NewSideCarController(kubeClient *kubernetes.Clientset, controllerClient *cl // Two different versions of the same Tenant will always have different RVs. return } - data := newSecret.Data["config.env"] - // validate root creds in string - rootUserFound := false - rootPwdFound := false - - dataStr := string(data) - if strings.Contains(dataStr, "MINIO_ROOT_USER") { - rootUserFound = true - } - if strings.Contains(dataStr, "MINIO_ACCESS_KEY") { - rootUserFound = true - } - if strings.Contains(dataStr, "MINIO_ROOT_PASSWORD") { - rootPwdFound = true - } - if strings.Contains(dataStr, "MINIO_SECRET_KEY") { - rootPwdFound = true - } - if !rootUserFound || !rootPwdFound { - log.Println("Missing root credentials in the configuration.") - log.Println("MinIO won't start") - os.Exit(1) - } - if !strings.HasSuffix(dataStr, "\n") { - dataStr = dataStr + "\n" - } - c.regenCfgWithCfg(tenantName, namespace, dataStr) + c.regenCfgWithSecret(newSecret) }, }) + if err != nil { + log.Println("could not add event handler for secret informer", err) + return nil + } return c } -func (c *Controller) regenCfg(tenantName string, namespace string) { - rootUserFound, rootPwdFound, fileContents, err := validator.ReadTmpConfig() +func (c *Controller) regenCfgWithTenant(tenant *v2.Tenant) { + // get the tenant secret + tenant.EnsureDefaults() + + configSecret, err := c.secretInformer.Lister().Secrets(c.namespace).Get(tenant.Spec.Configuration.Name) if err != nil { - log.Println(err) + log.Println("could not get secret", err) return } + + fileContents, rootUserFound, rootPwdFound := configuration.GetFullTenantConfig(tenant, configSecret) + if !rootUserFound || !rootPwdFound { log.Println("Missing root credentials in the configuration.") log.Println("MinIO won't start") os.Exit(1) } - c.regenCfgWithCfg(tenantName, namespace, fileContents) -} -func (c *Controller) regenCfgWithCfg(tenantName string, namespace string, fileContents string) { - ctx := context.Background() + err = os.WriteFile(v2.CfgFile, []byte(fileContents), 0o644) + if err != nil { + log.Println(err) + } +} - tenant, err := c.controllerClient.MinioV2().Tenants(namespace).Get(ctx, tenantName, metav1.GetOptions{}) +func (c *Controller) regenCfgWithSecret(configSecret *corev1.Secret) { + // get the tenant + tenant, err := c.tenantInformer.Lister().Tenants(c.namespace).Get(c.tenantName) if err != nil { - log.Println("could not get tenant", err) + log.Println("could not get secret", err) return } tenant.EnsureDefaults() - fileContents = common2.AttachGeneratedConfig(tenant, fileContents) + fileContents, rootUserFound, rootPwdFound := configuration.GetFullTenantConfig(tenant, configSecret) + + if !rootUserFound || !rootPwdFound { + log.Println("Missing root credentials in the configuration.") + log.Println("MinIO won't start") + os.Exit(1) + } err = os.WriteFile(v2.CfgFile, []byte(fileContents), 0o644) if err != nil { diff --git a/sidecar/pkg/validator/validator.go b/sidecar/pkg/validator/validator.go index 79c9496804b..94671a29f33 100644 --- a/sidecar/pkg/validator/validator.go +++ b/sidecar/pkg/validator/validator.go @@ -23,10 +23,11 @@ import ( "os" "strings" - common2 "github.com/minio/operator/sidecar/pkg/common" + "github.com/minio/operator/pkg/configuration" + "k8s.io/client-go/kubernetes" miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" - clientset "github.com/minio/operator/pkg/client/clientset/versioned" + operatorClientset "github.com/minio/operator/pkg/client/clientset/versioned" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/klog/v2" @@ -35,11 +36,6 @@ import ( // Validate checks the configuration on the seeded configuration and issues a valid one for MinIO to // start, however if root credentials are missing, it will exit with error func Validate(tenantName string) { - rootUserFound, rootPwdFound, fileContents, err := ReadTmpConfig() - if err != nil { - panic(err) - } - namespace := miniov2.GetNSFromFile() cfg, err := rest.InClusterConfig() @@ -51,9 +47,14 @@ func Validate(tenantName string) { panic(err) } - controllerClient, err := clientset.NewForConfig(cfg) + kubeClient, err := kubernetes.NewForConfig(cfg) if err != nil { - klog.Fatalf("Error building MinIO clientset: %s", err.Error()) + klog.Fatalf("Error building MinIO operatorClientset: %s", err.Error()) + } + + controllerClient, err := operatorClientset.NewForConfig(cfg) + if err != nil { + klog.Fatalf("Error building MinIO operatorClientset: %s", err.Error()) } ctx := context.Background() @@ -65,8 +66,14 @@ func Validate(tenantName string) { panic(err) } tenant.EnsureDefaults() + // get tenant config secret + configSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(ctx, tenant.Spec.Configuration.Name, metav1.GetOptions{}) + if err != nil { + log.Println(err) + panic(err) + } - fileContents = common2.AttachGeneratedConfig(tenant, fileContents) + fileContents, rootUserFound, rootPwdFound := configuration.GetFullTenantConfig(tenant, configSecret) if !rootUserFound || !rootPwdFound { log.Println("Missing root credentials in the configuration.")