diff --git a/docs/examples.md b/docs/examples.md index 099a16dcc2c..efb4557e6b5 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -78,7 +78,7 @@ This example will deploy a MinIO tenant with TLS using certificates provided by certificate keypairs: ```sh - mkcert "*.minio-tenant.svc.cluster.local" + mkcert "minio.minio-tenant.svc.cluster.local" mkcert "*.myminio.minio-tenant.svc.cluster.local" mkcert "*.myminio-hl.minio-tenant.svc.cluster.local" ``` @@ -90,9 +90,12 @@ inter-node communication. Create `kubernetes secrets` based on the previous certificates ```$xslt -kubectl create secret tls minio-tls-cert --key="_wildcard.minio-tenant.svc.cluster.local-key.pem" --cert="_wildcard.minio-tenant.svc.cluster.local.pem" -n minio-tenant +kubectl create secret tls minio-tls-cert --key="minio.minio-tenant.svc.cluster.local-key.pem" --cert="minio.minio-tenant.svc.cluster.local.pem" -n minio-tenant kubectl create secret tls minio-buckets-cert --key="_wildcard.myminio.minio-tenant.svc.cluster.local-key.pem" --cert="_wildcard.myminio.minio-tenant.svc.cluster.local.pem" -n minio-tenant kubectl create secret tls minio-hl-cert --key="_wildcard.myminio-hl.minio-tenant.svc.cluster.local-key.pem" --cert="_wildcard.myminio-hl.minio-tenant.svc.cluster.local.pem" -n minio-tenant + +# create a new secret for the operator certs +kubectl create secret tls operator-ca-tls-minio-tls-cert --key="minio.minio-tenant.svc.cluster.local-key.pem" --cert="minio.minio-tenant.svc.cluster.local.pem" -n minio-tenant ``` You need to provide those `kubernetes secrets` in your Tenant `YAML` overlay using the `externalCertSecret` fields, ie: diff --git a/pkg/controller/job-controller.go b/pkg/controller/job-controller.go index 642ba1c612d..1db9d42a816 100644 --- a/pkg/controller/job-controller.go +++ b/pkg/controller/job-controller.go @@ -268,7 +268,7 @@ func (c *JobController) SyncHandler(key string) (_ Result, err error) { if err != nil { return WrapResult(Result{}, err) } - err = intervalJob.CreateCommandJob(ctx, c.k8sClient, STSDefaultPort, tenant.TLS()) + err = intervalJob.CreateCommandJob(ctx, c.k8sClient, STSDefaultPort, tenant) if err != nil { return WrapResult(Result{}, fmt.Errorf("create job error: %w", err)) } diff --git a/pkg/utils/miniojob/types.go b/pkg/utils/miniojob/types.go index 70dd7d1a110..bf2b162f73e 100644 --- a/pkg/utils/miniojob/types.go +++ b/pkg/utils/miniojob/types.go @@ -24,6 +24,7 @@ import ( "github.com/minio/operator/pkg/apis/job.min.io/v1alpha1" miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" + "github.com/minio/operator/pkg/certs" "github.com/minio/operator/pkg/runtime" batchjobv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -118,7 +119,7 @@ func (jobCommand *MinIOIntervalJobCommand) Success() bool { } // createJob - create job -func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client.Client, jobCR *v1alpha1.MinIOJob, stsPort int, isTLS bool) (objs []client.Object) { +func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client.Client, jobCR *v1alpha1.MinIOJob, stsPort int, t *miniov2.Tenant) (objs []client.Object) { if jobCommand == nil { return nil } @@ -129,6 +130,7 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client } jobCommand.mutex.RUnlock() jobCommands := []string{} + insecure := false if len(jobCommand.CommandSpec.Command) == 0 { commands := []string{"mc"} commands = append(commands, strings.SplitN(jobCommand.MCOperation, "/", -1)...) @@ -142,6 +144,13 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client } else { jobCommands = append(jobCommands, jobCommand.CommandSpec.Command...) } + + for _, command := range jobCommands { + if command == "--insecure" { + insecure = true + } + } + mcImage := jobCR.Spec.MCImage if mcImage == "" { mcImage = DefaultMCImage @@ -152,7 +161,6 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client MountPath: "/.mc", }, } - baseVolumeMounts = append(baseVolumeMounts, jobCommand.CommandSpec.VolumeMounts...) baseVolumes := []corev1.Volume{ { Name: "config-dir", @@ -161,7 +169,16 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client }, }, } + // if auto cert is not enabled and insecure is not enabled and tenant tls is enabled, add cert volumes + if !t.AutoCert() && !insecure && t.TLS() { + certsVolumes, certsVolumeMounts := getCertVolumes(t) + baseVolumeMounts = append(baseVolumeMounts, certsVolumeMounts...) + baseVolumes = append(baseVolumes, certsVolumes...) + } + + baseVolumeMounts = append(baseVolumeMounts, jobCommand.CommandSpec.VolumeMounts...) baseVolumes = append(baseVolumes, jobCommand.CommandSpec.Volumes...) + baseEnvFrom := []corev1.EnvFromSource{ { SecretRef: &corev1.SecretEnvSource{ @@ -173,7 +190,7 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client } baseEnvFrom = append(baseEnvFrom, jobCommand.CommandSpec.EnvFrom...) scheme := "http" - if isTLS { + if t.TLS() { scheme = "https" } secret := &corev1.Secret{ @@ -239,8 +256,8 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(_ context.Context, _ client } // CreateJob - create job -func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort int, isTLS bool) error { - for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR, stsPort, isTLS) { +func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort int, t *miniov2.Tenant) error { + for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR, stsPort, t) { if obj == nil { continue } @@ -314,10 +331,10 @@ func (intervalJob *MinIOIntervalJob) GetMinioJobStatus(_ context.Context) v1alph } // CreateCommandJob - create command job -func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client, stsPort int, isTLS bool) error { +func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client, stsPort int, t *miniov2.Tenant) error { for _, command := range intervalJob.Command { if len(command.CommandSpec.DependsOn) == 0 { - err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort, isTLS) + err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort, t) if err != nil { return err } @@ -334,7 +351,7 @@ func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sCl } } if allDepsSuccess { - err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort, isTLS) + err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort, t) if err != nil { return err } @@ -379,3 +396,112 @@ func GenerateMinIOIntervalJobCommand(commandSpec v1alpha1.CommandSpec, commandIn } return jobCommand, nil } + +// getCertVolumes - get cert volumes +// from statefulsets.NewPool implementation +func getCertVolumes(t *miniov2.Tenant) (certsVolumes []corev1.Volume, certsVolumeMounts []corev1.VolumeMount) { + var certVolumeSources []corev1.VolumeProjection + for index, secret := range t.Spec.ExternalCertSecret { + crtMountPath := fmt.Sprintf("hostname-%d/%s", index, certs.PublicCertFile) + caMountPath := fmt.Sprintf("CAs/hostname-%d.crt", index) + + if index == 0 { + crtMountPath = certs.PublicCertFile + caMountPath = fmt.Sprintf("%s/%s", certs.CertsCADir, certs.PublicCertFile) + } + + var serverCertPaths []corev1.KeyToPath + if secret.Type == "kubernetes.io/tls" { + serverCertPaths = []corev1.KeyToPath{ + {Key: certs.TLSCertFile, Path: crtMountPath}, + {Key: certs.TLSCertFile, Path: caMountPath}, + } + } else if secret.Type == "cert-manager.io/v1alpha2" || secret.Type == "cert-manager.io/v1" { + serverCertPaths = []corev1.KeyToPath{ + {Key: certs.TLSCertFile, Path: crtMountPath}, + {Key: certs.CAPublicCertFile, Path: caMountPath}, + } + } else { + serverCertPaths = []corev1.KeyToPath{ + {Key: certs.PublicCertFile, Path: crtMountPath}, + {Key: certs.PublicCertFile, Path: caMountPath}, + } + } + certVolumeSources = append(certVolumeSources, corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret.Name, + }, + Items: serverCertPaths, + }, + }) + } + for index, secret := range t.Spec.ExternalClientCertSecrets { + crtMountPath := fmt.Sprintf("client-%d/client.crt", index) + var clientKeyPairPaths []corev1.KeyToPath + if secret.Type == "kubernetes.io/tls" { + clientKeyPairPaths = []corev1.KeyToPath{ + {Key: certs.TLSCertFile, Path: crtMountPath}, + } + } else if secret.Type == "cert-manager.io/v1alpha2" || secret.Type == "cert-manager.io/v1" { + clientKeyPairPaths = []corev1.KeyToPath{ + {Key: certs.TLSCertFile, Path: crtMountPath}, + {Key: certs.CAPublicCertFile, Path: fmt.Sprintf("%s/client-ca-%d.crt", certs.CertsCADir, index)}, + } + } else { + clientKeyPairPaths = []corev1.KeyToPath{ + {Key: certs.PublicCertFile, Path: crtMountPath}, + } + } + certVolumeSources = append(certVolumeSources, corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret.Name, + }, + Items: clientKeyPairPaths, + }, + }) + } + for index, secret := range t.Spec.ExternalCaCertSecret { + var caCertPaths []corev1.KeyToPath + // This covers both secrets of type "kubernetes.io/tls" and + // "cert-manager.io/v1alpha2" because of same keys in both. + if secret.Type == "kubernetes.io/tls" { + caCertPaths = []corev1.KeyToPath{ + {Key: certs.TLSCertFile, Path: fmt.Sprintf("%s/ca-%d.crt", certs.CertsCADir, index)}, + } + } else if secret.Type == "cert-manager.io/v1alpha2" || secret.Type == "cert-manager.io/v1" { + caCertPaths = []corev1.KeyToPath{ + {Key: certs.CAPublicCertFile, Path: fmt.Sprintf("%s/ca-%d.crt", certs.CertsCADir, index)}, + } + } else { + caCertPaths = []corev1.KeyToPath{ + {Key: certs.PublicCertFile, Path: fmt.Sprintf("%s/ca-%d.crt", certs.CertsCADir, index)}, + } + } + certVolumeSources = append(certVolumeSources, corev1.VolumeProjection{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret.Name, + }, + Items: caCertPaths, + }, + }) + } + + if len(certVolumeSources) > 0 { + certsVolumes = append(certsVolumes, corev1.Volume{ + Name: "certs", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: certVolumeSources, + }, + }, + }) + certsVolumeMounts = append(certsVolumeMounts, corev1.VolumeMount{ + Name: "certs", + MountPath: "/root/.mc/certs", + }) + } + return certsVolumes, certsVolumeMounts +}