From fde107fe4b3c4e43cc863e2b031dc3f7297cd718 Mon Sep 17 00:00:00 2001 From: Berkay Tekin Oz Date: Tue, 2 Jul 2024 08:03:04 +0000 Subject: [PATCH 1/3] Initial implementation of passing external datastore parameters --- bootstrap/api/v1beta2/ck8sconfig_types.go | 27 +++++++++++++-- .../api/v1beta2/zz_generated.deepcopy.go | 16 +++++++++ ...ootstrap.cluster.x-k8s.io_ck8sconfigs.yaml | 19 +++++++++++ ....cluster.x-k8s.io_ck8sconfigtemplates.yaml | 19 +++++++++++ .../controllers/ck8sconfig_controller.go | 33 +++++++++++++++++-- ...ne.cluster.x-k8s.io_ck8scontrolplanes.yaml | 19 +++++++++++ ...er.x-k8s.io_ck8scontrolplanetemplates.yaml | 19 +++++++++++ pkg/ck8s/config_init.go | 31 ++++++++++++++--- pkg/secret/certificates.go | 24 +++++++------- 9 files changed, 186 insertions(+), 21 deletions(-) diff --git a/bootstrap/api/v1beta2/ck8sconfig_types.go b/bootstrap/api/v1beta2/ck8sconfig_types.go index f04797e5..14b3866e 100644 --- a/bootstrap/api/v1beta2/ck8sconfig_types.go +++ b/bootstrap/api/v1beta2/ck8sconfig_types.go @@ -60,10 +60,13 @@ type CK8sConfigSpec struct { InitConfig CK8sInitConfiguration `json:"initConfig,omitempty"` } -// TODO -// Will need extend this func when implementing other database options. +// IsEtcdManaged returns true if the control plane is using k8s-dqlite func (c *CK8sConfigSpec) IsEtcdManaged() bool { - return true + switch c.ControlPlaneConfig.DatastoreType { + case "", "k8s-dqlite": + return true + } + return false } // CK8sControlPlaneConfig is configuration for control plane nodes. @@ -80,6 +83,14 @@ type CK8sControlPlaneConfig struct { // +optional NodeTaints []string `json:"nodeTaints,omitempty"` + // DatastoreType is the type of datastore to use for the control plane. + // +optional + DatastoreType string `json:"datastoreType,omitempty"` + + // DatastoreServersSecretRef is a reference to a secret containing the datastore servers. + // +optional + DatastoreServersSecretRef SecretRef `json:"datastoreServersSecretRef,omitempty"` + // K8sDqlitePort is the port to use for k8s-dqlite. If unset, 2379 (etcd) will be used. // +optional K8sDqlitePort int `json:"k8sDqlitePort,omitempty"` @@ -281,6 +292,16 @@ type SecretFileSource struct { Key string `json:"key"` } +// SecretRef is a reference to a secret in the CK8sBootstrapConfig's namespace. +type SecretRef struct { + // Name of the secret in the CK8sBootstrapConfig's namespace to use. + Name string `json:"name"` + + // Key is the key in the secret's data map for this value. + // +optional + Key string `json:"key,omitempty"` +} + func init() { SchemeBuilder.Register(&CK8sConfig{}, &CK8sConfigList{}) } diff --git a/bootstrap/api/v1beta2/zz_generated.deepcopy.go b/bootstrap/api/v1beta2/zz_generated.deepcopy.go index bf61cf62..6fef1f42 100644 --- a/bootstrap/api/v1beta2/zz_generated.deepcopy.go +++ b/bootstrap/api/v1beta2/zz_generated.deepcopy.go @@ -258,6 +258,7 @@ func (in *CK8sControlPlaneConfig) DeepCopyInto(out *CK8sControlPlaneConfig) { *out = make([]string, len(*in)) copy(*out, *in) } + out.DatastoreServersSecretRef = in.DatastoreServersSecretRef if in.MicroclusterPort != nil { in, out := &in.MicroclusterPort, &out.MicroclusterPort *out = new(int) @@ -383,3 +384,18 @@ func (in *SecretFileSource) DeepCopy() *SecretFileSource { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretRef) DeepCopyInto(out *SecretRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretRef. +func (in *SecretRef) DeepCopy() *SecretRef { + if in == nil { + return nil + } + out := new(SecretRef) + in.DeepCopyInto(out) + return out +} diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigs.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigs.yaml index c6d217cd..af0ede5d 100644 --- a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigs.yaml +++ b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigs.yaml @@ -59,6 +59,25 @@ spec: description: CloudProvider is the cloud-provider configuration option to set. type: string + datastoreServersSecretRef: + description: DatastoreServersSecretRef is a reference to a secret + containing the datastore servers. + properties: + key: + description: Key is the key in the secret's data map for this + value. + type: string + name: + description: Name of the secret in the CK8sBootstrapConfig's + namespace to use. + type: string + required: + - name + type: object + datastoreType: + description: DatastoreType is the type of datastore to use for + the control plane. + type: string extraKubeAPIServerArgs: additionalProperties: type: string diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigtemplates.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigtemplates.yaml index 29409ea3..ce558697 100644 --- a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigtemplates.yaml +++ b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigtemplates.yaml @@ -66,6 +66,25 @@ spec: description: CloudProvider is the cloud-provider configuration option to set. type: string + datastoreServersSecretRef: + description: DatastoreServersSecretRef is a reference + to a secret containing the datastore servers. + properties: + key: + description: Key is the key in the secret's data map + for this value. + type: string + name: + description: Name of the secret in the CK8sBootstrapConfig's + namespace to use. + type: string + required: + - name + type: object + datastoreType: + description: DatastoreType is the type of datastore to + use for the control plane. + type: string extraKubeAPIServerArgs: additionalProperties: type: string diff --git a/bootstrap/controllers/ck8sconfig_controller.go b/bootstrap/controllers/ck8sconfig_controller.go index f65a07f8..ae0d35af 100644 --- a/bootstrap/controllers/ck8sconfig_controller.go +++ b/bootstrap/controllers/ck8sconfig_controller.go @@ -387,6 +387,23 @@ func (r *CK8sConfigReconciler) resolveSecretFileContent(ctx context.Context, ns return data, nil } +// resolveSecretFileContent returns file content fetched from a referenced secret object. +func (r *CK8sConfigReconciler) resolveSecretReference(ctx context.Context, ns string, secretRef bootstrapv1.SecretRef) ([]byte, error) { + secret := &corev1.Secret{} + key := types.NamespacedName{Namespace: ns, Name: secretRef.Name} + if err := r.Client.Get(ctx, key, secret); err != nil { + if apierrors.IsNotFound(err) { + return nil, fmt.Errorf("secret not found %s: %w", key, err) + } + return nil, fmt.Errorf("failed to retrieve Secret %q: %w", key, err) + } + data, ok := secret.Data[secretRef.Key] + if !ok { + return nil, fmt.Errorf("secret references non-existent secret key %q: %w", secretRef.Key, ErrInvalidRef) + } + return data, nil +} + func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, reterr error) { // initialize the DataSecretAvailableCondition if missing. // this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing @@ -445,14 +462,26 @@ func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context, return ctrl.Result{}, err } - configStruct, err := ck8s.GenerateInitControlPlaneConfig(ck8s.InitControlPlaneConfig{ + clusterInitConfig := ck8s.InitControlPlaneConfig{ ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host, ControlPlaneConfig: scope.Config.Spec.ControlPlaneConfig, PopulatedCertificates: certificates, InitConfig: scope.Config.Spec.InitConfig, ClusterNetwork: scope.Cluster.Spec.ClusterNetwork, - }) + } + + if !scope.Config.Spec.IsEtcdManaged() { + clusterInitConfig.DatastoreType = scope.Config.Spec.ControlPlaneConfig.DatastoreType + + datastoreServers, err := r.resolveSecretReference(ctx, scope.Config.Namespace, scope.Config.Spec.ControlPlaneConfig.DatastoreServersSecretRef) + if err != nil { + return ctrl.Result{}, err + } + clusterInitConfig.DatastoreServers = string(datastoreServers) + } + + configStruct, err := ck8s.GenerateInitControlPlaneConfig(clusterInitConfig) if err != nil { return ctrl.Result{}, err } diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanes.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanes.yaml index 6dac044d..41c5a889 100644 --- a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanes.yaml +++ b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanes.yaml @@ -254,6 +254,25 @@ spec: description: CloudProvider is the cloud-provider configuration option to set. type: string + datastoreServersSecretRef: + description: DatastoreServersSecretRef is a reference to a + secret containing the datastore servers. + properties: + key: + description: Key is the key in the secret's data map for + this value. + type: string + name: + description: Name of the secret in the CK8sBootstrapConfig's + namespace to use. + type: string + required: + - name + type: object + datastoreType: + description: DatastoreType is the type of datastore to use + for the control plane. + type: string extraKubeAPIServerArgs: additionalProperties: type: string diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanetemplates.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanetemplates.yaml index d868a732..e7585287 100644 --- a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanetemplates.yaml +++ b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanetemplates.yaml @@ -229,6 +229,25 @@ spec: description: CloudProvider is the cloud-provider configuration option to set. type: string + datastoreServersSecretRef: + description: DatastoreServersSecretRef is a reference + to a secret containing the datastore servers. + properties: + key: + description: Key is the key in the secret's data + map for this value. + type: string + name: + description: Name of the secret in the CK8sBootstrapConfig's + namespace to use. + type: string + required: + - name + type: object + datastoreType: + description: DatastoreType is the type of datastore + to use for the control plane. + type: string extraKubeAPIServerArgs: additionalProperties: type: string diff --git a/pkg/ck8s/config_init.go b/pkg/ck8s/config_init.go index 78fc46de..fa3d9097 100644 --- a/pkg/ck8s/config_init.go +++ b/pkg/ck8s/config_init.go @@ -17,6 +17,8 @@ type InitControlPlaneConfig struct { ControlPlaneConfig bootstrapv1.CK8sControlPlaneConfig InitConfig bootstrapv1.CK8sInitConfiguration PopulatedCertificates secret.Certificates + DatastoreType string + DatastoreServers string ClusterNetwork *clusterv1.ClusterNetwork } @@ -41,6 +43,11 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap case secret.FrontProxyCA: out.FrontProxyCACert = ptr.To(string(cert.KeyPair.Cert)) out.FrontProxyCAKey = ptr.To(string(cert.KeyPair.Key)) + case secret.EtcdCA: + out.DatastoreCACert = ptr.To(string(cert.KeyPair.Cert)) + case secret.APIServerEtcdClient: + out.DatastoreClientCert = ptr.To(string(cert.KeyPair.Cert)) + out.DatastoreClientKey = ptr.To(string(cert.KeyPair.Key)) } } // ensure required certificates @@ -56,6 +63,19 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap out.ClusterConfig.CloudProvider = ptr.To(v) } + // Set default datastore type to k8s-dqlite + datastoreType := cfg.DatastoreType + if datastoreType == "" { + datastoreType = "k8s-dqlite" + } + out.DatastoreType = ptr.To(datastoreType) + + if datastoreType != "k8s-dqlite" { + if v := cfg.DatastoreServers; v != "" { + out.DatastoreServers = strings.Split(v, ",") + } + } + // annotations out.ClusterConfig.Annotations = cfg.InitConfig.Annotations @@ -88,12 +108,13 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap // extra SANs out.ExtraSANs = append(out.ExtraSANs, cfg.ControlPlaneEndpoint) - // TODO(neoaggelos): datastore configuration with external etcd (?) - k8sDqlitePort := cfg.ControlPlaneConfig.K8sDqlitePort - if k8sDqlitePort == 0 { - k8sDqlitePort = 2379 + if datastoreType == "k8s-dqlite" { + k8sDqlitePort := cfg.ControlPlaneConfig.K8sDqlitePort + if k8sDqlitePort == 0 { + k8sDqlitePort = 2379 + } + out.K8sDqlitePort = ptr.To(k8sDqlitePort) } - out.K8sDqlitePort = ptr.To(k8sDqlitePort) if v := cfg.ControlPlaneConfig.NodeTaints; len(v) > 0 { out.ControlPlaneTaints = v diff --git a/pkg/secret/certificates.go b/pkg/secret/certificates.go index 095ff0e6..37219729 100644 --- a/pkg/secret/certificates.go +++ b/pkg/secret/certificates.go @@ -68,15 +68,17 @@ func NewCertificatesForInitialControlPlane(config *bootstrapv1.CK8sConfigSpec) C }, } - // TODO(neoaggelos): handle the case of required certificates for external datastore here - // if config.IsEtcdEmbedded() { - // etcdCert := &Certificate{ - // Purpose: EtcdCA, - // CertFile: filepath.Join(certificatesDir, "etcd", "server-ca.crt"), - // KeyFile: filepath.Join(certificatesDir, "etcd", "server-ca.key"), - // } - // certificates = append(certificates, etcdCert) - // } + if !config.IsEtcdManaged() { + etcdCA := &Certificate{ + Purpose: EtcdCA, + External: true, + } + etcdClient := &Certificate{ + Purpose: APIServerEtcdClient, + External: true, + } + certificates = append(certificates, etcdCA, etcdClient) + } return certificates } @@ -237,8 +239,8 @@ func (c *Certificate) AsSecret(clusterName client.ObjectKey, owner metav1.OwnerR } func (c *Certificate) Generate() error { - // Do not generate the APIServerEtcdClient key pair. It is user supplied - if c.Purpose == APIServerEtcdClient { + // Do not generate external certificates key pair. It is user supplied + if c.External { return nil } From ca99a21eec0e14939aa8b103b2df9b29c9f76ea2 Mon Sep 17 00:00:00 2001 From: Berkay Tekin Oz Date: Thu, 11 Jul 2024 12:20:09 +0000 Subject: [PATCH 2/3] Change to use switch in config init, update comment styling --- bootstrap/api/v1beta2/ck8sconfig_types.go | 2 +- pkg/ck8s/config_init.go | 26 ++++++++++------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/bootstrap/api/v1beta2/ck8sconfig_types.go b/bootstrap/api/v1beta2/ck8sconfig_types.go index 14b3866e..a135f715 100644 --- a/bootstrap/api/v1beta2/ck8sconfig_types.go +++ b/bootstrap/api/v1beta2/ck8sconfig_types.go @@ -60,7 +60,7 @@ type CK8sConfigSpec struct { InitConfig CK8sInitConfiguration `json:"initConfig,omitempty"` } -// IsEtcdManaged returns true if the control plane is using k8s-dqlite +// IsEtcdManaged returns true if the control plane is using k8s-dqlite. func (c *CK8sConfigSpec) IsEtcdManaged() bool { switch c.ControlPlaneConfig.DatastoreType { case "", "k8s-dqlite": diff --git a/pkg/ck8s/config_init.go b/pkg/ck8s/config_init.go index fa3d9097..a4995c28 100644 --- a/pkg/ck8s/config_init.go +++ b/pkg/ck8s/config_init.go @@ -63,14 +63,18 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap out.ClusterConfig.CloudProvider = ptr.To(v) } - // Set default datastore type to k8s-dqlite - datastoreType := cfg.DatastoreType - if datastoreType == "" { - datastoreType = "k8s-dqlite" - } - out.DatastoreType = ptr.To(datastoreType) + switch cfg.DatastoreType { + case "", "k8s-dqlite": + // Set default datastore type to k8s-dqlite + out.DatastoreType = ptr.To("k8s-dqlite") - if datastoreType != "k8s-dqlite" { + k8sDqlitePort := cfg.ControlPlaneConfig.K8sDqlitePort + if k8sDqlitePort == 0 { + k8sDqlitePort = 2379 + } + out.K8sDqlitePort = ptr.To(k8sDqlitePort) + default: + out.DatastoreType = ptr.To("external") if v := cfg.DatastoreServers; v != "" { out.DatastoreServers = strings.Split(v, ",") } @@ -108,14 +112,6 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap // extra SANs out.ExtraSANs = append(out.ExtraSANs, cfg.ControlPlaneEndpoint) - if datastoreType == "k8s-dqlite" { - k8sDqlitePort := cfg.ControlPlaneConfig.K8sDqlitePort - if k8sDqlitePort == 0 { - k8sDqlitePort = 2379 - } - out.K8sDqlitePort = ptr.To(k8sDqlitePort) - } - if v := cfg.ControlPlaneConfig.NodeTaints; len(v) > 0 { out.ControlPlaneTaints = v } From 6831ead2d844b3d24c95472fb9b78fdeaa9927b0 Mon Sep 17 00:00:00 2001 From: Berkay Tekin Oz Date: Wed, 17 Jul 2024 11:15:03 +0000 Subject: [PATCH 3/3] Change datastoreServers type --- bootstrap/controllers/ck8sconfig_controller.go | 3 ++- pkg/ck8s/config_init.go | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bootstrap/controllers/ck8sconfig_controller.go b/bootstrap/controllers/ck8sconfig_controller.go index ae0d35af..99bb9b4e 100644 --- a/bootstrap/controllers/ck8sconfig_controller.go +++ b/bootstrap/controllers/ck8sconfig_controller.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/go-logr/logr" @@ -478,7 +479,7 @@ func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context, if err != nil { return ctrl.Result{}, err } - clusterInitConfig.DatastoreServers = string(datastoreServers) + clusterInitConfig.DatastoreServers = strings.Split(string(datastoreServers), ",") } configStruct, err := ck8s.GenerateInitControlPlaneConfig(clusterInitConfig) diff --git a/pkg/ck8s/config_init.go b/pkg/ck8s/config_init.go index a4995c28..5061b248 100644 --- a/pkg/ck8s/config_init.go +++ b/pkg/ck8s/config_init.go @@ -18,7 +18,7 @@ type InitControlPlaneConfig struct { InitConfig bootstrapv1.CK8sInitConfiguration PopulatedCertificates secret.Certificates DatastoreType string - DatastoreServers string + DatastoreServers []string ClusterNetwork *clusterv1.ClusterNetwork } @@ -75,9 +75,7 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap out.K8sDqlitePort = ptr.To(k8sDqlitePort) default: out.DatastoreType = ptr.To("external") - if v := cfg.DatastoreServers; v != "" { - out.DatastoreServers = strings.Split(v, ",") - } + out.DatastoreServers = cfg.DatastoreServers } // annotations