Initial implementation of passing external datastore parameters

bootstrap/api/v1beta2/ck8sconfig_types.go

@@ -60,10 +60,13 @@ type CK8sConfigSpec struct {
 	InitConfig CK8sInitConfiguration `json:"initConfig,omitempty"`
 }
 
-// TODO
-// Will need extend this func when implementing other database options.
+// IsEtcdManaged returns true if the control plane is using k8s-dqlite.
 func (c *CK8sConfigSpec) IsEtcdManaged() bool {
-	return true
+	switch c.ControlPlaneConfig.DatastoreType {
+	case "", "k8s-dqlite":
+		return true
+	}
+	return false
 }
 
 // CK8sControlPlaneConfig is configuration for control plane nodes.
@@ -80,6 +83,14 @@ type CK8sControlPlaneConfig struct {
 	// +optional
 	NodeTaints []string `json:"nodeTaints,omitempty"`
 
+	// DatastoreType is the type of datastore to use for the control plane.
+	// +optional
+	DatastoreType string `json:"datastoreType,omitempty"`
+
+	// DatastoreServersSecretRef is a reference to a secret containing the datastore servers.
+	// +optional
+	DatastoreServersSecretRef SecretRef `json:"datastoreServersSecretRef,omitempty"`
+
 	// K8sDqlitePort is the port to use for k8s-dqlite. If unset, 2379 (etcd) will be used.
 	// +optional
 	K8sDqlitePort int `json:"k8sDqlitePort,omitempty"`
@@ -281,6 +292,16 @@ type SecretFileSource struct {
 	Key string `json:"key"`
 }
 
+// SecretRef is a reference to a secret in the CK8sBootstrapConfig's namespace.
+type SecretRef struct {
+	// Name of the secret in the CK8sBootstrapConfig's namespace to use.
+	Name string `json:"name"`
+
+	// Key is the key in the secret's data map for this value.
+	// +optional
+	Key string `json:"key,omitempty"`
+}
+
 func init() {
 	SchemeBuilder.Register(&CK8sConfig{}, &CK8sConfigList{})
 }

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigs.yaml

@@ -59,6 +59,25 @@ spec:
                     description: CloudProvider is the cloud-provider configuration
                       option to set.
                     type: string
+                  datastoreServersSecretRef:
+                    description: DatastoreServersSecretRef is a reference to a secret
+                      containing the datastore servers.
+                    properties:
+                      key:
+                        description: Key is the key in the secret's data map for this
+                          value.
+                        type: string
+                      name:
+                        description: Name of the secret in the CK8sBootstrapConfig's
+                          namespace to use.
+                        type: string
+                    required:
+                    - name
+                    type: object
+                  datastoreType:
+                    description: DatastoreType is the type of datastore to use for
+                      the control plane.
+                    type: string
                   extraKubeAPIServerArgs:
                     additionalProperties:
                       type: string

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigtemplates.yaml

@@ -66,6 +66,25 @@ spec:
                             description: CloudProvider is the cloud-provider configuration
                               option to set.
                             type: string
+                          datastoreServersSecretRef:
+                            description: DatastoreServersSecretRef is a reference
+                              to a secret containing the datastore servers.
+                            properties:
+                              key:
+                                description: Key is the key in the secret's data map
+                                  for this value.
+                                type: string
+                              name:
+                                description: Name of the secret in the CK8sBootstrapConfig's
+                                  namespace to use.
+                                type: string
+                            required:
+                            - name
+                            type: object
+                          datastoreType:
+                            description: DatastoreType is the type of datastore to
+                              use for the control plane.
+                            type: string
                           extraKubeAPIServerArgs:
                             additionalProperties:
                               type: string

bootstrap/controllers/ck8sconfig_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -387,6 +388,23 @@ func (r *CK8sConfigReconciler) resolveSecretFileContent(ctx context.Context, ns
 	return data, nil
 }
 
+// resolveSecretFileContent returns file content fetched from a referenced secret object.
+func (r *CK8sConfigReconciler) resolveSecretReference(ctx context.Context, ns string, secretRef bootstrapv1.SecretRef) ([]byte, error) {
+	secret := &corev1.Secret{}
+	key := types.NamespacedName{Namespace: ns, Name: secretRef.Name}
+	if err := r.Client.Get(ctx, key, secret); err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil, fmt.Errorf("secret not found %s: %w", key, err)
+		}
+		return nil, fmt.Errorf("failed to retrieve Secret %q: %w", key, err)
+	}
+	data, ok := secret.Data[secretRef.Key]
+	if !ok {
+		return nil, fmt.Errorf("secret references non-existent secret key %q: %w", secretRef.Key, ErrInvalidRef)
+	}
+	return data, nil
+}
+
 func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, reterr error) {
 	// initialize the DataSecretAvailableCondition if missing.
 	// this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing
@@ -445,14 +463,26 @@ func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context,
 		return ctrl.Result{}, err
 	}
 
-	configStruct, err := ck8s.GenerateInitControlPlaneConfig(ck8s.InitControlPlaneConfig{
+	clusterInitConfig := ck8s.InitControlPlaneConfig{
 		ControlPlaneEndpoint:  scope.Cluster.Spec.ControlPlaneEndpoint.Host,
 		ControlPlaneConfig:    scope.Config.Spec.ControlPlaneConfig,
 		PopulatedCertificates: certificates,
 		InitConfig:            scope.Config.Spec.InitConfig,
 
 		ClusterNetwork: scope.Cluster.Spec.ClusterNetwork,
-	})
+	}
+
+	if !scope.Config.Spec.IsEtcdManaged() {
+		clusterInitConfig.DatastoreType = scope.Config.Spec.ControlPlaneConfig.DatastoreType
+
+		datastoreServers, err := r.resolveSecretReference(ctx, scope.Config.Namespace, scope.Config.Spec.ControlPlaneConfig.DatastoreServersSecretRef)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		clusterInitConfig.DatastoreServers = strings.Split(string(datastoreServers), ",")
+	}
+
+	configStruct, err := ck8s.GenerateInitControlPlaneConfig(clusterInitConfig)
 	if err != nil {
 		return ctrl.Result{}, err
 	}

controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanes.yaml

@@ -254,6 +254,25 @@ spec:
                         description: CloudProvider is the cloud-provider configuration
                           option to set.
                         type: string
+                      datastoreServersSecretRef:
+                        description: DatastoreServersSecretRef is a reference to a
+                          secret containing the datastore servers.
+                        properties:
+                          key:
+                            description: Key is the key in the secret's data map for
+                              this value.
+                            type: string
+                          name:
+                            description: Name of the secret in the CK8sBootstrapConfig's
+                              namespace to use.
+                            type: string
+                        required:
+                        - name
+                        type: object
+                      datastoreType:
+                        description: DatastoreType is the type of datastore to use
+                          for the control plane.
+                        type: string
                       extraKubeAPIServerArgs:
                         additionalProperties:
                           type: string

controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanetemplates.yaml

@@ -229,6 +229,25 @@ spec:
                                 description: CloudProvider is the cloud-provider configuration
                                   option to set.
                                 type: string
+                              datastoreServersSecretRef:
+                                description: DatastoreServersSecretRef is a reference
+                                  to a secret containing the datastore servers.
+                                properties:
+                                  key:
+                                    description: Key is the key in the secret's data
+                                      map for this value.
+                                    type: string
+                                  name:
+                                    description: Name of the secret in the CK8sBootstrapConfig's
+                                      namespace to use.
+                                    type: string
+                                required:
+                                - name
+                                type: object
+                              datastoreType:
+                                description: DatastoreType is the type of datastore
+                                  to use for the control plane.
+                                type: string
                               extraKubeAPIServerArgs:
                                 additionalProperties:
                                   type: string

pkg/ck8s/config_init.go

@@ -17,6 +17,8 @@ type InitControlPlaneConfig struct {
 	ControlPlaneConfig    bootstrapv1.CK8sControlPlaneConfig
 	InitConfig            bootstrapv1.CK8sInitConfiguration
 	PopulatedCertificates secret.Certificates
+	DatastoreType         string
+	DatastoreServers      []string
 
 	ClusterNetwork *clusterv1.ClusterNetwork
 }
@@ -41,6 +43,11 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap
 		case secret.FrontProxyCA:
 			out.FrontProxyCACert = ptr.To(string(cert.KeyPair.Cert))
 			out.FrontProxyCAKey = ptr.To(string(cert.KeyPair.Key))
+		case secret.EtcdCA:
+			out.DatastoreCACert = ptr.To(string(cert.KeyPair.Cert))
+		case secret.APIServerEtcdClient:
+			out.DatastoreClientCert = ptr.To(string(cert.KeyPair.Cert))
+			out.DatastoreClientKey = ptr.To(string(cert.KeyPair.Key))
 		}
 	}
 	// ensure required certificates
@@ -56,6 +63,21 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap
 		out.ClusterConfig.CloudProvider = ptr.To(v)
 	}
 
+	switch cfg.DatastoreType {
+	case "", "k8s-dqlite":
+		// Set default datastore type to k8s-dqlite
+		out.DatastoreType = ptr.To("k8s-dqlite")
+
+		k8sDqlitePort := cfg.ControlPlaneConfig.K8sDqlitePort
+		if k8sDqlitePort == 0 {
+			k8sDqlitePort = 2379
+		}
+		out.K8sDqlitePort = ptr.To(k8sDqlitePort)
+	default:
+		out.DatastoreType = ptr.To("external")
+		out.DatastoreServers = cfg.DatastoreServers
+	}
+
 	// annotations
 	out.ClusterConfig.Annotations = cfg.InitConfig.Annotations
 
@@ -88,13 +110,6 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap
 	// extra SANs
 	out.ExtraSANs = append(out.ExtraSANs, cfg.ControlPlaneEndpoint)
 
-	// TODO(neoaggelos): datastore configuration with external etcd (?)
-	k8sDqlitePort := cfg.ControlPlaneConfig.K8sDqlitePort
-	if k8sDqlitePort == 0 {
-		k8sDqlitePort = 2379
-	}
-	out.K8sDqlitePort = ptr.To(k8sDqlitePort)
-
 	if v := cfg.ControlPlaneConfig.NodeTaints; len(v) > 0 {
 		out.ControlPlaneTaints = v
 	}

pkg/secret/certificates.go

@@ -68,15 +68,17 @@ func NewCertificatesForInitialControlPlane(config *bootstrapv1.CK8sConfigSpec) C
 		},
 	}
 
-	// TODO(neoaggelos): handle the case of required certificates for external datastore here
-	// if config.IsEtcdEmbedded() {
-	// 	etcdCert := &Certificate{
-	// 		Purpose:  EtcdCA,
-	// 		CertFile: filepath.Join(certificatesDir, "etcd", "server-ca.crt"),
-	// 		KeyFile:  filepath.Join(certificatesDir, "etcd", "server-ca.key"),
-	// 	}
-	// 	certificates = append(certificates, etcdCert)
-	// }
+	if !config.IsEtcdManaged() {
+		etcdCA := &Certificate{
+			Purpose:  EtcdCA,
+			External: true,
+		}
+		etcdClient := &Certificate{
+			Purpose:  APIServerEtcdClient,
+			External: true,
+		}
+		certificates = append(certificates, etcdCA, etcdClient)
+	}
 
 	return certificates
 }
@@ -237,8 +239,8 @@ func (c *Certificate) AsSecret(clusterName client.ObjectKey, owner metav1.OwnerR
 }
 
 func (c *Certificate) Generate() error {
-	// Do not generate the APIServerEtcdClient key pair. It is user supplied
-	if c.Purpose == APIServerEtcdClient {
+	// Do not generate external certificates key pair. It is user supplied
+	if c.External {
 		return nil
 	}