From b3812bdfc7e1bef64e62756b694b027814d1a3ba Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Thu, 30 May 2024 20:48:30 +0300 Subject: [PATCH] Switch to x509 auth (#450) * separate client ca * create useful CompleteWorkerNodePKI tests * require a client CA to be present for worker nodes * allow preseeding client certificates through bootstrap and join configs --- src/k8s/api/v1/bootstrap_config.go | 61 ++++-- src/k8s/api/v1/join_config.go | 72 +++++-- src/k8s/api/v1/worker.go | 18 +- src/k8s/pkg/k8sd/api/kubeconfig.go | 6 +- src/k8s/pkg/k8sd/api/worker.go | 54 ++---- src/k8s/pkg/k8sd/app/cluster_util.go | 26 ++- src/k8s/pkg/k8sd/app/hooks_bootstrap.go | 58 ++++-- src/k8s/pkg/k8sd/app/hooks_join.go | 4 +- src/k8s/pkg/k8sd/pki/control_plane.go | 93 +++++++-- src/k8s/pkg/k8sd/pki/worker.go | 83 ++++++-- src/k8s/pkg/k8sd/pki/worker_test.go | 178 ++++++++++-------- src/k8s/pkg/k8sd/setup/certificates.go | 8 +- src/k8s/pkg/k8sd/setup/certificates_test.go | 11 +- src/k8s/pkg/k8sd/setup/kube_apiserver.go | 2 +- src/k8s/pkg/k8sd/setup/kube_apiserver_test.go | 4 +- src/k8s/pkg/k8sd/setup/kubelet.go | 2 +- src/k8s/pkg/k8sd/setup/kubelet_test.go | 8 +- src/k8s/pkg/k8sd/setup/util_kubeconfig.go | 13 +- .../pkg/k8sd/setup/util_kubeconfig_test.go | 14 +- .../k8sd/types/cluster_config_certificates.go | 28 ++- .../pkg/k8sd/types/cluster_config_merge.go | 4 + .../k8sd/types/cluster_config_merge_test.go | 4 + 22 files changed, 501 insertions(+), 250 deletions(-) diff --git a/src/k8s/api/v1/bootstrap_config.go b/src/k8s/api/v1/bootstrap_config.go index 819ef2008..c1c661957 100644 --- a/src/k8s/api/v1/bootstrap_config.go +++ b/src/k8s/api/v1/bootstrap_config.go @@ -26,21 +26,34 @@ type BootstrapConfig struct { // Seed configuration for certificates ExtraSANs []string `json:"extra-sans,omitempty" yaml:"extra-sans,omitempty"` - // Seed configuration for external certificates - CACert *string `json:"ca-crt,omitempty" yaml:"ca-crt,omitempty"` - CAKey *string `json:"ca-key,omitempty" yaml:"ca-key,omitempty"` - FrontProxyCACert *string `json:"front-proxy-ca-crt,omitempty" yaml:"front-proxy-ca-crt,omitempty"` - FrontProxyCAKey *string `json:"front-proxy-ca-key,omitempty" yaml:"front-proxy-ca-key,omitempty"` - FrontProxyClientCert *string `json:"front-proxy-client-crt,omitempty" yaml:"front-proxy-client-crt,omitempty"` - FrontProxyClientKey *string `json:"front-proxy-client-key,omitempty" yaml:"front-proxy-client-key,omitempty"` - APIServerKubeletClientCert *string `json:"apiserver-kubelet-client-crt,omitempty" yaml:"apiserver-kubelet-client-crt,omitempty"` - APIServerKubeletClientKey *string `json:"apiserver-kubelet-client-key,omitempty" yaml:"apiserver-kubelet-client-key,omitempty"` - ServiceAccountKey *string `json:"service-account-key,omitempty" yaml:"service-account-key,omitempty"` + // Seed configuration for external certificates (cluster-wide) + CACert *string `json:"ca-crt,omitempty" yaml:"ca-crt,omitempty"` + CAKey *string `json:"ca-key,omitempty" yaml:"ca-key,omitempty"` + ClientCACert *string `json:"client-ca-crt,omitempty" yaml:"client-ca-crt,omitempty"` + ClientCAKey *string `json:"client-ca-key,omitempty" yaml:"client-ca-key,omitempty"` + FrontProxyCACert *string `json:"front-proxy-ca-crt,omitempty" yaml:"front-proxy-ca-crt,omitempty"` + FrontProxyCAKey *string `json:"front-proxy-ca-key,omitempty" yaml:"front-proxy-ca-key,omitempty"` + FrontProxyClientCert *string `json:"front-proxy-client-crt,omitempty" yaml:"front-proxy-client-crt,omitempty"` + FrontProxyClientKey *string `json:"front-proxy-client-key,omitempty" yaml:"front-proxy-client-key,omitempty"` + APIServerKubeletClientCert *string `json:"apiserver-kubelet-client-crt,omitempty" yaml:"apiserver-kubelet-client-crt,omitempty"` + APIServerKubeletClientKey *string `json:"apiserver-kubelet-client-key,omitempty" yaml:"apiserver-kubelet-client-key,omitempty"` + AdminClientCert *string `json:"admin-client-crt,omitempty" yaml:"admin-client-crt,omitempty"` + AdminClientKey *string `json:"admin-client-key,omitempty" yaml:"admin-client-key,omitempty"` + KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"` + KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"` + KubeSchedulerClientCert *string `json:"kube-scheduler-client-crt,omitempty" yaml:"kube-scheduler-client-crt,omitempty"` + KubeSchedulerClientKey *string `json:"kube-scheduler-client-key,omitempty" yaml:"kube-scheduler-client-key,omitempty"` + KubeControllerManagerClientCert *string `json:"kube-controller-manager-client-crt,omitempty" yaml:"kube-controller-manager-client-crt,omitempty"` + KubeControllerManagerClientKey *string `json:"kube-controller-manager-client-key,omitempty" yaml:"kube-ControllerManager-client-key,omitempty"` + ServiceAccountKey *string `json:"service-account-key,omitempty" yaml:"service-account-key,omitempty"` - APIServerCert *string `json:"apiserver-crt,omitempty" yaml:"apiserver-crt,omitempty"` - APIServerKey *string `json:"apiserver-key,omitempty" yaml:"apiserver-key,omitempty"` - KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"` - KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` + // Seed configuration for external certificates (node-specific) + APIServerCert *string `json:"apiserver-crt,omitempty" yaml:"apiserver-crt,omitempty"` + APIServerKey *string `json:"apiserver-key,omitempty" yaml:"apiserver-key,omitempty"` + KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"` + KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` + KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` + KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` } func (b *BootstrapConfig) GetDatastoreType() string { return getField(b.DatastoreType) } @@ -50,6 +63,8 @@ func (b *BootstrapConfig) GetDatastoreClientKey() string { return getField(b.D func (b *BootstrapConfig) GetK8sDqlitePort() int { return getField(b.K8sDqlitePort) } func (b *BootstrapConfig) GetCACert() string { return getField(b.CACert) } func (b *BootstrapConfig) GetCAKey() string { return getField(b.CAKey) } +func (b *BootstrapConfig) GetClientCACert() string { return getField(b.ClientCACert) } +func (b *BootstrapConfig) GetClientCAKey() string { return getField(b.ClientCAKey) } func (b *BootstrapConfig) GetFrontProxyCACert() string { return getField(b.FrontProxyCACert) } func (b *BootstrapConfig) GetFrontProxyCAKey() string { return getField(b.FrontProxyCAKey) } func (b *BootstrapConfig) GetFrontProxyClientCert() string { return getField(b.FrontProxyClientCert) } @@ -60,11 +75,29 @@ func (b *BootstrapConfig) GetAPIServerKubeletClientCert() string { func (b *BootstrapConfig) GetAPIServerKubeletClientKey() string { return getField(b.APIServerKubeletClientKey) } +func (b *BootstrapConfig) GetAdminClientCert() string { return getField(b.AdminClientCert) } +func (b *BootstrapConfig) GetAdminClientKey() string { return getField(b.AdminClientKey) } +func (b *BootstrapConfig) GetKubeProxyClientCert() string { return getField(b.KubeProxyClientCert) } +func (b *BootstrapConfig) GetKubeProxyClientKey() string { return getField(b.KubeProxyClientKey) } +func (b *BootstrapConfig) GetKubeSchedulerClientCert() string { + return getField(b.KubeSchedulerClientCert) +} +func (b *BootstrapConfig) GetKubeSchedulerClientKey() string { + return getField(b.KubeSchedulerClientKey) +} +func (b *BootstrapConfig) GetKubeControllerManagerClientCert() string { + return getField(b.KubeControllerManagerClientCert) +} +func (b *BootstrapConfig) GetKubeControllerManagerClientKey() string { + return getField(b.KubeControllerManagerClientKey) +} func (b *BootstrapConfig) GetServiceAccountKey() string { return getField(b.ServiceAccountKey) } func (b *BootstrapConfig) GetAPIServerCert() string { return getField(b.APIServerCert) } func (b *BootstrapConfig) GetAPIServerKey() string { return getField(b.APIServerKey) } func (b *BootstrapConfig) GetKubeletCert() string { return getField(b.KubeletCert) } func (b *BootstrapConfig) GetKubeletKey() string { return getField(b.KubeletKey) } +func (b *BootstrapConfig) GetKubeletClientCert() string { return getField(b.KubeletClientCert) } +func (b *BootstrapConfig) GetKubeletClientKey() string { return getField(b.KubeletClientKey) } // ToMicrocluster converts a BootstrapConfig to a map[string]string for use in microcluster. func (b *BootstrapConfig) ToMicrocluster() (map[string]string, error) { diff --git a/src/k8s/api/v1/join_config.go b/src/k8s/api/v1/join_config.go index d88dd853d..8324dfdf9 100644 --- a/src/k8s/api/v1/join_config.go +++ b/src/k8s/api/v1/join_config.go @@ -9,32 +9,76 @@ import ( type ControlPlaneNodeJoinConfig struct { ExtraSANS []string `json:"extra-sans,omitempty" yaml:"extra-sans,omitempty"` - APIServerCert *string `json:"apiserver-crt,omitempty" yaml:"apiserver-crt,omitempty"` - APIServerKey *string `json:"apiserver-key,omitempty" yaml:"apiserver-key,omitempty"` - FrontProxyClientCert *string `json:"front-proxy-client-crt,omitempty" yaml:"front-proxy-client-crt,omitempty"` - FrontProxyClientKey *string `json:"front-proxy-client-key,omitempty" yaml:"front-proxy-client-key,omitempty"` - KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"` - KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` + // Seed certificates for external CA + FrontProxyClientCert *string `json:"front-proxy-client-crt,omitempty" yaml:"front-proxy-client-crt,omitempty"` + FrontProxyClientKey *string `json:"front-proxy-client-key,omitempty" yaml:"front-proxy-client-key,omitempty"` + KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"` + KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"` + KubeSchedulerClientCert *string `json:"kube-scheduler-client-crt,omitempty" yaml:"kube-scheduler-client-crt,omitempty"` + KubeSchedulerClientKey *string `json:"kube-scheduler-client-key,omitempty" yaml:"kube-scheduler-client-key,omitempty"` + KubeControllerManagerClientCert *string `json:"kube-controller-manager-client-crt,omitempty" yaml:"kube-controller-manager-client-crt,omitempty"` + KubeControllerManagerClientKey *string `json:"kube-controller-manager-client-key,omitempty" yaml:"kube-ControllerManager-client-key,omitempty"` + + APIServerCert *string `json:"apiserver-crt,omitempty" yaml:"apiserver-crt,omitempty"` + APIServerKey *string `json:"apiserver-key,omitempty" yaml:"apiserver-key,omitempty"` + KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"` + KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` + KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` + KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` } type WorkerNodeJoinConfig struct { - KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"` - KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` + KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"` + KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"` + KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"` + KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"` + KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"` + KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"` } -func (c *ControlPlaneNodeJoinConfig) GetAPIServerCert() string { return getField(c.APIServerCert) } -func (c *ControlPlaneNodeJoinConfig) GetAPIServerKey() string { return getField(c.APIServerKey) } func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientCert() string { return getField(c.FrontProxyClientCert) } func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientKey() string { return getField(c.FrontProxyClientKey) } -func (c *ControlPlaneNodeJoinConfig) GetKubeletCert() string { return getField(c.KubeletCert) } -func (c *ControlPlaneNodeJoinConfig) GetKubeletKey() string { return getField(c.KubeletKey) } +func (b *ControlPlaneNodeJoinConfig) GetKubeProxyClientCert() string { + return getField(b.KubeProxyClientCert) +} +func (b *ControlPlaneNodeJoinConfig) GetKubeProxyClientKey() string { + return getField(b.KubeProxyClientKey) +} +func (b *ControlPlaneNodeJoinConfig) GetKubeSchedulerClientCert() string { + return getField(b.KubeSchedulerClientCert) +} +func (b *ControlPlaneNodeJoinConfig) GetKubeSchedulerClientKey() string { + return getField(b.KubeSchedulerClientKey) +} +func (b *ControlPlaneNodeJoinConfig) GetKubeControllerManagerClientCert() string { + return getField(b.KubeControllerManagerClientCert) +} +func (b *ControlPlaneNodeJoinConfig) GetKubeControllerManagerClientKey() string { + return getField(b.KubeControllerManagerClientKey) +} +func (c *ControlPlaneNodeJoinConfig) GetAPIServerCert() string { return getField(c.APIServerCert) } +func (c *ControlPlaneNodeJoinConfig) GetAPIServerKey() string { return getField(c.APIServerKey) } +func (c *ControlPlaneNodeJoinConfig) GetKubeletCert() string { return getField(c.KubeletCert) } +func (c *ControlPlaneNodeJoinConfig) GetKubeletKey() string { return getField(c.KubeletKey) } +func (c *ControlPlaneNodeJoinConfig) GetKubeletClientCert() string { + return getField(c.KubeletClientCert) +} +func (c *ControlPlaneNodeJoinConfig) GetKubeletClientKey() string { + return getField(c.KubeletClientKey) +} -func (w *WorkerNodeJoinConfig) GetKubeletCert() string { return getField(w.KubeletCert) } -func (w *WorkerNodeJoinConfig) GetKubeletKey() string { return getField(w.KubeletKey) } +func (w *WorkerNodeJoinConfig) GetKubeletCert() string { return getField(w.KubeletCert) } +func (w *WorkerNodeJoinConfig) GetKubeletKey() string { return getField(w.KubeletKey) } +func (w *WorkerNodeJoinConfig) GetKubeletClientCert() string { return getField(w.KubeletClientCert) } +func (w *WorkerNodeJoinConfig) GetKubeletClientKey() string { return getField(w.KubeletClientKey) } +func (w *WorkerNodeJoinConfig) GetKubeProxyClientCert() string { + return getField(w.KubeProxyClientCert) +} +func (w *WorkerNodeJoinConfig) GetKubeProxyClientKey() string { return getField(w.KubeProxyClientKey) } // WorkerJoinConfigFromMicrocluster parses a microcluster map[string]string and retrieves the WorkerNodeJoinConfig. func ControlPlaneJoinConfigFromMicrocluster(m map[string]string) (ControlPlaneNodeJoinConfig, error) { diff --git a/src/k8s/api/v1/worker.go b/src/k8s/api/v1/worker.go index 928b214f3..4b2dd935a 100644 --- a/src/k8s/api/v1/worker.go +++ b/src/k8s/api/v1/worker.go @@ -9,14 +9,20 @@ type WorkerNodeInfoRequest struct { // WorkerNodeInfoResponse is used to return a worker node token. type WorkerNodeInfoResponse struct { - // CA is the PEM encoded certificate authority of the cluster. - CA string `json:"ca,omitempty"` + // CACert is the PEM encoded certificate authority of the cluster. + CACert string `json:"ca,omitempty"` + // ClientCACert is the PEM encoded certificate authority of the cluster clients. + ClientCACert string `json:"client-ca,omitempty"` // APIServers is a list of kube-apiserver endpoints of the cluster. APIServers []string `json:"apiServers"` - // KubeletToken is the token to use for kubelet. - KubeletToken string `json:"kubeletToken"` - // KubeProxyToken is the token to use for kube-proxy. - KubeProxyToken string `json:"kubeProxyToken"` + // KubeletClientCert is the certificate to use in kubelet to authenticate with kube-apiserver. + KubeletClientCert string `json:"kubeletClientCert"` + // KubeletClientKey is the private key to use in kubelet to authenticate with kube-apiserver. + KubeletClientKey string `json:"kubeletClientKey"` + // KubeProxyClientCert is the certificate to use in kube-proxy to authenticate with kube-apiserver. + KubeProxyClientCert string `json:"kubeProxyClientCert"` + // KubeProxyClientKey is the private key to use in kube-proxy to authenticate with kube-apiserver. + KubeProxyClientKey string `json:"kubeProxyClientKey"` // PodCIDR is the configured CIDR for pods in the cluster. PodCIDR string `json:"podCIDR"` // ServiceCIDR is the configured CIDR for services in the cluster. diff --git a/src/k8s/pkg/k8sd/api/kubeconfig.go b/src/k8s/pkg/k8sd/api/kubeconfig.go index fb249866f..6f12e1eb6 100644 --- a/src/k8s/pkg/k8sd/api/kubeconfig.go +++ b/src/k8s/pkg/k8sd/api/kubeconfig.go @@ -27,12 +27,8 @@ func (e *Endpoints) getKubeconfig(s *state.State, r *http.Request) response.Resp if req.Server == "" { server = fmt.Sprintf("%s:%d", s.Address().Hostname(), config.APIServer.GetSecurePort()) } - token, err := databaseutil.GetOrCreateAuthToken(s.Context, s, "kubernetes-admin", []string{"system:masters"}) - if err != nil { - return response.InternalError(fmt.Errorf("failed to get admin token: %w", err)) - } - kubeconfig, err := setup.KubeconfigString(token, server, config.Certificates.GetCACert()) + kubeconfig, err := setup.KubeconfigString(server, config.Certificates.GetCACert(), config.Certificates.GetAdminClientCert(), config.Certificates.GetAdminClientKey()) if err != nil { return response.InternalError(fmt.Errorf("failed to get kubeconfig: %w", err)) } diff --git a/src/k8s/pkg/k8sd/api/worker.go b/src/k8s/pkg/k8sd/api/worker.go index aa900d7af..871a17854 100644 --- a/src/k8s/pkg/k8sd/api/worker.go +++ b/src/k8s/pkg/k8sd/api/worker.go @@ -39,6 +39,8 @@ func (e *Endpoints) postWorkerInfo(s *state.State, r *http.Request) response.Res certificates := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{Years: 10}) certificates.CACert = cfg.Certificates.GetCACert() certificates.CAKey = cfg.Certificates.GetCAKey() + certificates.ClientCACert = cfg.Certificates.GetClientCACert() + certificates.ClientCAKey = cfg.Certificates.GetClientCAKey() workerCertificates, err := certificates.CompleteWorkerNodePKI(workerName, nodeIP, 2048) if err != nil { return response.InternalError(fmt.Errorf("failed to generate worker PKI: %w", err)) @@ -56,31 +58,6 @@ func (e *Endpoints) postWorkerInfo(s *state.State, r *http.Request) response.Res return response.InternalError(fmt.Errorf("failed to retrieve list of known kube-apiserver endpoints: %w", err)) } - var ( - kubeletToken string - proxyToken string - ) - for _, i := range []struct { - token *string - name string - username string - groups []string - }{ - {token: &kubeletToken, name: "kubelet", username: fmt.Sprintf("system:node:%s", workerName), groups: []string{"system:nodes"}}, - {token: &proxyToken, name: "kube-proxy", username: "system:kube-proxy"}, - } { - if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { - t, err := database.GetOrCreateToken(ctx, tx, i.username, i.groups) - if err != nil { - return fmt.Errorf("failed to generate %s token for node %q: %w", i.name, workerName, err) - } - *i.token = t - return nil - }); err != nil { - return response.InternalError(fmt.Errorf("create token transaction failed: %w", err)) - } - } - if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error { return database.AddWorkerNode(ctx, tx, workerName) }); err != nil { @@ -94,17 +71,20 @@ func (e *Endpoints) postWorkerInfo(s *state.State, r *http.Request) response.Res } return response.SyncResponse(true, &apiv1.WorkerNodeInfoResponse{ - CA: cfg.Certificates.GetCACert(), - APIServers: servers, - PodCIDR: cfg.Network.GetPodCIDR(), - ServiceCIDR: cfg.Network.GetServiceCIDR(), - KubeletToken: kubeletToken, - KubeProxyToken: proxyToken, - ClusterDomain: cfg.Kubelet.GetClusterDomain(), - ClusterDNS: cfg.Kubelet.GetClusterDNS(), - CloudProvider: cfg.Kubelet.GetCloudProvider(), - KubeletCert: workerCertificates.KubeletCert, - KubeletKey: workerCertificates.KubeletKey, - K8sdPublicKey: cfg.Certificates.GetK8sdPublicKey(), + CACert: cfg.Certificates.GetCACert(), + ClientCACert: cfg.Certificates.GetClientCACert(), + APIServers: servers, + PodCIDR: cfg.Network.GetPodCIDR(), + ServiceCIDR: cfg.Network.GetServiceCIDR(), + ClusterDomain: cfg.Kubelet.GetClusterDomain(), + ClusterDNS: cfg.Kubelet.GetClusterDNS(), + CloudProvider: cfg.Kubelet.GetCloudProvider(), + KubeletCert: workerCertificates.KubeletCert, + KubeletKey: workerCertificates.KubeletKey, + KubeletClientCert: workerCertificates.KubeletClientCert, + KubeletClientKey: workerCertificates.KubeletClientKey, + KubeProxyClientCert: workerCertificates.KubeProxyClientCert, + KubeProxyClientKey: workerCertificates.KubeProxyClientKey, + K8sdPublicKey: cfg.Certificates.GetK8sdPublicKey(), }) } diff --git a/src/k8s/pkg/k8sd/app/cluster_util.go b/src/k8s/pkg/k8sd/app/cluster_util.go index 8ca695466..2d5f55b5d 100644 --- a/src/k8s/pkg/k8sd/app/cluster_util.go +++ b/src/k8s/pkg/k8sd/app/cluster_util.go @@ -6,7 +6,7 @@ import ( "net" "path" - databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util" + "github.com/canonical/k8s/pkg/k8sd/pki" "github.com/canonical/k8s/pkg/k8sd/setup" "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" @@ -14,24 +14,20 @@ import ( "github.com/canonical/microcluster/state" ) -func setupKubeconfigs(s *state.State, kubeConfigDir string, securePort int, caCert string) error { +func setupKubeconfigs(s *state.State, kubeConfigDir string, securePort int, pki pki.ControlPlanePKI) error { // Generate kubeconfigs for _, kubeconfig := range []struct { - file string - username string - groups []string + file string + crt string + key string }{ - {file: "admin.conf", username: "kubernetes-admin", groups: []string{"system:masters"}}, - {file: "controller.conf", username: "system:kube-controller-manager"}, - {file: "proxy.conf", username: "system:kube-proxy"}, - {file: "scheduler.conf", username: "system:kube-scheduler"}, - {file: "kubelet.conf", username: fmt.Sprintf("system:node:%s", s.Name()), groups: []string{"system:nodes"}}, + {file: "admin.conf", crt: pki.AdminClientCert, key: pki.AdminClientKey}, + {file: "controller.conf", crt: pki.KubeControllerManagerClientCert, key: pki.KubeControllerManagerClientKey}, + {file: "proxy.conf", crt: pki.KubeProxyClientCert, key: pki.KubeProxyClientKey}, + {file: "scheduler.conf", crt: pki.KubeSchedulerClientCert, key: pki.KubeSchedulerClientKey}, + {file: "kubelet.conf", crt: pki.KubeletClientCert, key: pki.KubeletClientKey}, } { - token, err := databaseutil.GetOrCreateAuthToken(s.Context, s, kubeconfig.username, kubeconfig.groups) - if err != nil { - return fmt.Errorf("failed to generate token for username=%s groups=%v: %w", kubeconfig.username, kubeconfig.groups, err) - } - if err := setup.Kubeconfig(path.Join(kubeConfigDir, kubeconfig.file), token, fmt.Sprintf("127.0.0.1:%d", securePort), caCert); err != nil { + if err := setup.Kubeconfig(path.Join(kubeConfigDir, kubeconfig.file), fmt.Sprintf("127.0.0.1:%d", securePort), pki.CACert, kubeconfig.crt, kubeconfig.key); err != nil { return fmt.Errorf("failed to write kubeconfig %s: %w", kubeconfig.file, err) } } diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go index 41be0ccc7..34033075c 100644 --- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go +++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go @@ -120,16 +120,31 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon // Certificates certificates := &pki.WorkerNodePKI{ - CACert: response.CA, - KubeletCert: response.KubeletCert, - KubeletKey: response.KubeletKey, - } - - if v := joinConfig.GetKubeletCert(); v != "" { - certificates.KubeletCert = v - } - if v := joinConfig.GetKubeletKey(); v != "" { - certificates.KubeletKey = v + CACert: response.CACert, + ClientCACert: response.ClientCACert, + KubeletCert: response.KubeletCert, + KubeletKey: response.KubeletKey, + KubeletClientCert: response.KubeletClientCert, + KubeletClientKey: response.KubeletClientKey, + KubeProxyClientCert: response.KubeProxyClientCert, + KubeProxyClientKey: response.KubeProxyClientKey, + } + + // override certificates from JoinConfig + for _, i := range []struct { + target *string + override string + }{ + {target: &certificates.KubeletCert, override: joinConfig.GetKubeletCert()}, + {target: &certificates.KubeletKey, override: joinConfig.GetKubeletKey()}, + {target: &certificates.KubeletClientCert, override: joinConfig.GetKubeletClientCert()}, + {target: &certificates.KubeletClientKey, override: joinConfig.GetKubeletClientKey()}, + {target: &certificates.KubeProxyClientCert, override: joinConfig.GetKubeProxyClientCert()}, + {target: &certificates.KubeProxyClientKey, override: joinConfig.GetKubeProxyClientKey()}, + } { + if i.override != "" { + *i.target = i.override + } } if err := certificates.CompleteCertificates(); err != nil { @@ -140,10 +155,10 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon } // Kubeconfigs - if err := setup.Kubeconfig(path.Join(snap.KubernetesConfigDir(), "kubelet.conf"), response.KubeletToken, "127.0.0.1:6443", certificates.CACert); err != nil { + if err := setup.Kubeconfig(path.Join(snap.KubernetesConfigDir(), "kubelet.conf"), "127.0.0.1:6443", certificates.CACert, certificates.KubeletClientCert, certificates.KubeletClientKey); err != nil { return fmt.Errorf("failed to generate kubelet kubeconfig: %w", err) } - if err := setup.Kubeconfig(path.Join(snap.KubernetesConfigDir(), "proxy.conf"), response.KubeProxyToken, "127.0.0.1:6443", certificates.CACert); err != nil { + if err := setup.Kubeconfig(path.Join(snap.KubernetesConfigDir(), "proxy.conf"), "127.0.0.1:6443", certificates.CACert, certificates.KubeProxyClientCert, certificates.KubeProxyClientKey); err != nil { return fmt.Errorf("failed to generate kube-proxy kubeconfig: %w", err) } @@ -274,6 +289,8 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot certificates.CACert = bootstrapConfig.GetCACert() certificates.CAKey = bootstrapConfig.GetCAKey() + certificates.ClientCACert = bootstrapConfig.GetClientCACert() + certificates.ClientCAKey = bootstrapConfig.GetClientCAKey() certificates.FrontProxyCACert = bootstrapConfig.GetFrontProxyCACert() certificates.FrontProxyCAKey = bootstrapConfig.GetFrontProxyCAKey() certificates.FrontProxyClientCert = bootstrapConfig.GetFrontProxyClientCert() @@ -281,10 +298,21 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot certificates.ServiceAccountKey = bootstrapConfig.GetServiceAccountKey() certificates.APIServerKubeletClientCert = bootstrapConfig.GetAPIServerKubeletClientCert() certificates.APIServerKubeletClientKey = bootstrapConfig.GetAPIServerKubeletClientKey() + certificates.AdminClientCert = bootstrapConfig.GetAdminClientCert() + certificates.AdminClientKey = bootstrapConfig.GetAdminClientKey() + certificates.KubeControllerManagerClientCert = bootstrapConfig.GetKubeControllerManagerClientCert() + certificates.KubeControllerManagerClientKey = bootstrapConfig.GetKubeControllerManagerClientKey() + certificates.KubeSchedulerClientCert = bootstrapConfig.GetKubeSchedulerClientCert() + certificates.KubeSchedulerClientKey = bootstrapConfig.GetKubeSchedulerClientKey() + certificates.KubeProxyClientCert = bootstrapConfig.GetKubeProxyClientCert() + certificates.KubeProxyClientKey = bootstrapConfig.GetKubeProxyClientKey() + certificates.APIServerCert = bootstrapConfig.GetAPIServerCert() certificates.APIServerKey = bootstrapConfig.GetAPIServerKey() certificates.KubeletCert = bootstrapConfig.GetKubeletCert() certificates.KubeletKey = bootstrapConfig.GetKubeletKey() + certificates.KubeletClientCert = bootstrapConfig.GetKubeletClientCert() + certificates.KubeletClientKey = bootstrapConfig.GetKubeletClientKey() if err := certificates.CompleteCertificates(); err != nil { return fmt.Errorf("failed to initialize control plane certificates: %w", err) @@ -296,16 +324,20 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot // Add certificates to the cluster config cfg.Certificates.CACert = utils.Pointer(certificates.CACert) cfg.Certificates.CAKey = utils.Pointer(certificates.CAKey) + cfg.Certificates.ClientCACert = utils.Pointer(certificates.ClientCACert) + cfg.Certificates.ClientCAKey = utils.Pointer(certificates.ClientCAKey) cfg.Certificates.FrontProxyCACert = utils.Pointer(certificates.FrontProxyCACert) cfg.Certificates.FrontProxyCAKey = utils.Pointer(certificates.FrontProxyCAKey) cfg.Certificates.APIServerKubeletClientCert = utils.Pointer(certificates.APIServerKubeletClientCert) cfg.Certificates.APIServerKubeletClientKey = utils.Pointer(certificates.APIServerKubeletClientKey) cfg.Certificates.ServiceAccountKey = utils.Pointer(certificates.ServiceAccountKey) + cfg.Certificates.AdminClientCert = utils.Pointer(certificates.AdminClientCert) + cfg.Certificates.AdminClientKey = utils.Pointer(certificates.AdminClientKey) cfg.Certificates.K8sdPublicKey = utils.Pointer(certificates.K8sdPublicKey) cfg.Certificates.K8sdPrivateKey = utils.Pointer(certificates.K8sdPrivateKey) // Generate kubeconfigs - if err := setupKubeconfigs(s, snap.KubernetesConfigDir(), cfg.APIServer.GetSecurePort(), cfg.Certificates.GetCACert()); err != nil { + if err := setupKubeconfigs(s, snap.KubernetesConfigDir(), cfg.APIServer.GetSecurePort(), *certificates); err != nil { return fmt.Errorf("failed to generate kubeconfigs: %w", err) } diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go index 90e896c3c..2382525f0 100644 --- a/src/k8s/pkg/k8sd/app/hooks_join.go +++ b/src/k8s/pkg/k8sd/app/hooks_join.go @@ -87,6 +87,8 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error { // load shared cluster certificates certificates.CACert = cfg.Certificates.GetCACert() certificates.CAKey = cfg.Certificates.GetCAKey() + certificates.ClientCACert = cfg.Certificates.GetClientCACert() + certificates.ClientCAKey = cfg.Certificates.GetClientCAKey() certificates.FrontProxyCACert = cfg.Certificates.GetFrontProxyCACert() certificates.FrontProxyCAKey = cfg.Certificates.GetFrontProxyCAKey() certificates.APIServerKubeletClientCert = cfg.Certificates.GetAPIServerKubeletClientCert() @@ -111,7 +113,7 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error { return fmt.Errorf("failed to write control plane certificates: %w", err) } - if err := setupKubeconfigs(s, snap.KubernetesConfigDir(), cfg.APIServer.GetSecurePort(), cfg.Certificates.GetCACert()); err != nil { + if err := setupKubeconfigs(s, snap.KubernetesConfigDir(), cfg.APIServer.GetSecurePort(), *certificates); err != nil { return fmt.Errorf("failed to generate kubeconfigs: %w", err) } diff --git a/src/k8s/pkg/k8sd/pki/control_plane.go b/src/k8s/pkg/k8sd/pki/control_plane.go index 4a62e4f91..c842ffa7c 100644 --- a/src/k8s/pkg/k8sd/pki/control_plane.go +++ b/src/k8s/pkg/k8sd/pki/control_plane.go @@ -16,19 +16,35 @@ type ControlPlanePKI struct { years int // how many years the generated certificates will be valid for CACert, CAKey string // CN=kubernetes-ca (self-signed) + ClientCACert, ClientCAKey string // CN=kubernetes-ca-client (self-signed) FrontProxyCACert, FrontProxyCAKey string // CN=kubernetes-front-proxy-ca (self-signed) FrontProxyClientCert, FrontProxyClientKey string // CN=front-proxy-client (signed by kubernetes-front-proxy-ca) ServiceAccountKey string // private key used to sign service account tokens - // CN=kube-apiserver, DNS=hostname,kubernetes.* IP=127.0.0.1,10.152.183.1,address (signed by kubernetes-ca) + // [server] CN=kube-apiserver, DNS=hostname,kubernetes.* IP=127.0.0.1,10.152.183.1,address (signed by kubernetes-ca) APIServerCert, APIServerKey string - // CN=kube-apiserver-kubelet-client, O=system:masters (signed by kubernetes-ca) - APIServerKubeletClientCert, APIServerKubeletClientKey string - - // CN=system:node:hostname, O=system:nodes, DNS=hostname, IP=127.0.0.1,address (signed by kubernetes-ca) + // [server] CN=system:node:$hostname, O=system:nodes, DNS=hostname, IP=127.0.0.1,address (signed by kubernetes-ca) KubeletCert, KubeletKey string + // [client] CN=kubernetes:admin, O=system:masters (signed by kubernetes-ca-client) + AdminClientCert, AdminClientKey string + + // [client] CN=system:kube-controller-manager (signed by kubernetes-ca-client) + KubeControllerManagerClientCert, KubeControllerManagerClientKey string + + // [client] CN=system:kube-scheduler (signed by kubernetes-ca-client) + KubeSchedulerClientCert, KubeSchedulerClientKey string + + // [client] CN=system:kube-proxy (signed by kubernetes-ca-client) + KubeProxyClientCert, KubeProxyClientKey string + + // [client] CN=system:node:$hostname, O=system:nodes (signed by kubernetes-ca-client) + KubeletClientCert, KubeletClientKey string + + // [client] CN=kube-apiserver-kubelet-client, O=system:masters (signed by kubernetes-ca-client) + APIServerKubeletClientCert, APIServerKubeletClientKey string + // Keypair used to verify authenticity of cluster messages (e.g. for configmap/k8sd-config) K8sdPublicKey, K8sdPrivateKey string } @@ -63,6 +79,8 @@ func (c *ControlPlanePKI) CompleteCertificates() error { switch { case c.CACert == "" && c.CAKey != "": return fmt.Errorf("kubernetes CA key is set without a certificate, fail to prevent causing issues") + case c.ClientCACert == "" && c.ClientCAKey != "": + return fmt.Errorf("kubernetes CA client key is set without a certificate, fail to prevent causing issues") case c.FrontProxyCACert == "" && c.FrontProxyCAKey != "": return fmt.Errorf("front-proxy CA key is set without a certificate, fail to prevent causing issues") } @@ -95,7 +113,25 @@ func (c *ControlPlanePKI) CompleteCertificates() error { c.CAKey = key } - caCertificate, caPrivateKey, err := loadCertificate(c.CACert, c.CAKey) + // Generate self-signed client CA (if not set already) + if c.ClientCACert == "" && c.ClientCAKey == "" { + if !c.allowSelfSignedCA { + return fmt.Errorf("kubernetes client CA not specified and generating self-signed CA not allowed") + } + cert, key, err := generateSelfSignedCA(pkix.Name{CommonName: "kubernetes-ca-client"}, c.years, 2048) + if err != nil { + return fmt.Errorf("failed to generate kubernetes client CA: %w", err) + } + c.ClientCACert = cert + c.ClientCAKey = key + } + + serverCACert, serverCAKey, err := loadCertificate(c.CACert, c.CAKey) + if err != nil { + return fmt.Errorf("failed to parse kubernetes CA: %w", err) + } + + clientCACert, clientCAKey, err := loadCertificate(c.ClientCACert, c.ClientCAKey) if err != nil { return fmt.Errorf("failed to parse kubernetes CA: %w", err) } @@ -152,7 +188,7 @@ func (c *ControlPlanePKI) CompleteCertificates() error { // Generate kubelet certificate (if missing) if c.KubeletCert == "" || c.KubeletKey == "" { - if caPrivateKey == nil { + if serverCAKey == nil { return fmt.Errorf("using an external kubernetes CA without providing the kubelet certificate is not possible") } @@ -163,7 +199,7 @@ func (c *ControlPlanePKI) CompleteCertificates() error { if err != nil { return fmt.Errorf("failed to generate kubelet certificate: %w", err) } - cert, key, err := signCertificate(template, 2048, caCertificate, &caPrivateKey.PublicKey, caPrivateKey) + cert, key, err := signCertificate(template, 2048, serverCACert, &serverCAKey.PublicKey, serverCAKey) if err != nil { return fmt.Errorf("failed to sign kubelet certificate: %w", err) } @@ -174,7 +210,7 @@ func (c *ControlPlanePKI) CompleteCertificates() error { // Generate apiserver-kubelet-client certificate (if missing) if c.APIServerKubeletClientCert == "" || c.APIServerKubeletClientKey == "" { - if caPrivateKey == nil { + if clientCAKey == nil { return fmt.Errorf("using an external kubernetes CA without providing the apiserver-kubelet-client certificate is not possible") } @@ -182,7 +218,7 @@ func (c *ControlPlanePKI) CompleteCertificates() error { if err != nil { return fmt.Errorf("failed to generate apiserver-kubelet-client certificate: %w", err) } - cert, key, err := signCertificate(template, 2048, caCertificate, &caPrivateKey.PublicKey, caPrivateKey) + cert, key, err := signCertificate(template, 2048, clientCACert, &clientCAKey.PublicKey, clientCAKey) if err != nil { return fmt.Errorf("failed to sign apiserver-kubelet-client certificate: %w", err) } @@ -193,7 +229,7 @@ func (c *ControlPlanePKI) CompleteCertificates() error { // Generate kube-apiserver certificate (if missing) if c.APIServerCert == "" || c.APIServerKey == "" { - if caPrivateKey == nil { + if serverCAKey == nil { return fmt.Errorf("using an external kubernetes CA without providing the apiserver certificate is not possible") } @@ -205,7 +241,7 @@ func (c *ControlPlanePKI) CompleteCertificates() error { if err != nil { return fmt.Errorf("failed to generate apiserver certificate: %w", err) } - cert, key, err := signCertificate(template, 2048, caCertificate, &caPrivateKey.PublicKey, caPrivateKey) + cert, key, err := signCertificate(template, 2048, serverCACert, &serverCAKey.PublicKey, serverCAKey) if err != nil { return fmt.Errorf("failed to sign apiserver certificate: %w", err) } @@ -214,6 +250,39 @@ func (c *ControlPlanePKI) CompleteCertificates() error { c.APIServerKey = key } + for _, i := range []struct { + name string + cn string + o []string + cert *string + key *string + }{ + {name: "admin", cn: "kubernetes-admin", o: []string{"system:masters"}, cert: &c.AdminClientCert, key: &c.AdminClientKey}, + {name: "controller", cn: "system:kube-controller-manager", cert: &c.KubeControllerManagerClientCert, key: &c.KubeControllerManagerClientKey}, + {name: "proxy", cn: "system:kube-proxy", cert: &c.KubeProxyClientCert, key: &c.KubeProxyClientKey}, + {name: "scheduler", cn: "system:kube-scheduler", cert: &c.KubeSchedulerClientCert, key: &c.KubeSchedulerClientKey}, + {name: "kubelet", cn: fmt.Sprintf("system:node:%s", c.hostname), o: []string{"system:nodes"}, cert: &c.KubeletClientCert, key: &c.KubeletClientKey}, + } { + if *i.cert == "" || *i.key == "" { + if clientCAKey == nil { + return fmt.Errorf("using an external kubernetes CA client without providing the %s certificate is not possible", i.name) + } + + template, err := generateCertificate(pkix.Name{CommonName: i.cn, Organization: i.o}, c.years, false, nil, nil) + if err != nil { + return fmt.Errorf("failed to generate %s client certificate: %w", i.name, err) + } + + cert, key, err := signCertificate(template, 2048, clientCACert, &clientCAKey.PublicKey, clientCAKey) + if err != nil { + return fmt.Errorf("failed to sign %s client certificate: %w", i.name, err) + } + + *i.cert = cert + *i.key = key + } + } + // Generate k8sd cluster key-pair (if missing) if c.K8sdPrivateKey == "" || c.K8sdPublicKey == "" { if !c.allowSelfSignedCA { diff --git a/src/k8s/pkg/k8sd/pki/worker.go b/src/k8s/pkg/k8sd/pki/worker.go index 683c65b39..05db0d4f6 100644 --- a/src/k8s/pkg/k8sd/pki/worker.go +++ b/src/k8s/pkg/k8sd/pki/worker.go @@ -7,37 +7,78 @@ import ( ) type WorkerNodePKI struct { - CACert string - KubeletCert string - KubeletKey string + CACert string // CN=kubernetes-ca + + ClientCACert string // CN=kubernetes-ca-client + + // [server] CN=system:node:hostname, O=system:nodes, DNS=hostname, IP=127.0.0.1,address (signed by kubernetes-ca) + KubeletCert, KubeletKey string + + // [client] CN=system:kube-proxy (signed by kubernetes-ca-client) + KubeProxyClientCert, KubeProxyClientKey string + + // [client] CN=system:node:hostname, O=system:nodes (signed by kubernetes-ca-client) + KubeletClientCert, KubeletClientKey string } // CompleteWorkerNodePKI generates the PKI needed for a worker node. func (c *ControlPlanePKI) CompleteWorkerNodePKI(hostname string, nodeIP net.IP, bits int) (*WorkerNodePKI, error) { - caCert, caKey, err := loadCertificate(c.CACert, c.CAKey) + serverCACert, serverCAKey, err := loadCertificate(c.CACert, c.CAKey) if err != nil { return nil, fmt.Errorf("failed to load kubernetes CA: %w", err) } - // we do not have a CA key to sign the kubelet certificate, only send the cluster CA - if caKey == nil { - return &WorkerNodePKI{CACert: c.CACert}, nil + clientCACert, clientCAKey, err := loadCertificate(c.ClientCACert, c.ClientCAKey) + if err != nil { + return nil, fmt.Errorf("failed to load kubernetes client CA: %w", err) } - template, err := generateCertificate(pkix.Name{CommonName: fmt.Sprintf("system:node:%s", hostname), Organization: []string{"system:nodes"}}, c.years, false, []string{hostname}, []net.IP{{127, 0, 0, 1}, nodeIP}) - if err != nil { - return nil, fmt.Errorf("failed to generate kubelet certificate for hostname=%s address=%s: %w", hostname, nodeIP.String(), err) + pki := &WorkerNodePKI{CACert: c.CACert, ClientCACert: c.ClientCACert} + + // we have a cluster CA key, sign the kubelet server certificate + if serverCAKey != nil { + template, err := generateCertificate(pkix.Name{CommonName: fmt.Sprintf("system:node:%s", hostname), Organization: []string{"system:nodes"}}, c.years, false, []string{hostname}, []net.IP{{127, 0, 0, 1}, nodeIP}) + if err != nil { + return nil, fmt.Errorf("failed to generate kubelet certificate for hostname=%s address=%s: %w", hostname, nodeIP.String(), err) + } + cert, key, err := signCertificate(template, bits, serverCACert, &serverCAKey.PublicKey, serverCAKey) + if err != nil { + return nil, fmt.Errorf("failed to sign kubelet certificate for hostname=%s address=%s: %w", hostname, nodeIP.String(), err) + } + pki.KubeletCert = cert + pki.KubeletKey = key } - cert, key, err := signCertificate(template, bits, caCert, &caKey.PublicKey, caKey) - if err != nil { - return nil, fmt.Errorf("failed to sign kubelet certificate for hostname=%s address=%s: %w", hostname, nodeIP.String(), err) + + // we have a client CA key, sign the kubelet and kube-proxy client certificates + if clientCAKey != nil { + for _, i := range []struct { + name string + cn string + o []string + cert *string + key *string + }{ + {name: "proxy", cn: "system:kube-proxy", cert: &pki.KubeProxyClientCert, key: &pki.KubeProxyClientKey}, + {name: "kubelet", cn: fmt.Sprintf("system:node:%s", hostname), o: []string{"system:nodes"}, cert: &pki.KubeletClientCert, key: &pki.KubeletClientKey}, + } { + if *i.cert == "" || *i.key == "" { + template, err := generateCertificate(pkix.Name{CommonName: i.cn, Organization: i.o}, c.years, false, nil, nil) + if err != nil { + return nil, fmt.Errorf("failed to generate %s client certificate: %w", i.name, err) + } + + cert, key, err := signCertificate(template, 2048, clientCACert, &clientCAKey.PublicKey, clientCAKey) + if err != nil { + return nil, fmt.Errorf("failed to sign %s client certificate: %w", i.name, err) + } + + *i.cert = cert + *i.key = key + } + } } - return &WorkerNodePKI{ - CACert: c.CACert, - KubeletCert: cert, - KubeletKey: key, - }, nil + return pki, nil } func (c *WorkerNodePKI) CompleteCertificates() error { @@ -47,5 +88,11 @@ func (c *WorkerNodePKI) CompleteCertificates() error { if c.KubeletCert == "" || c.KubeletKey == "" { return fmt.Errorf("kubelet certificate not specified") } + if c.KubeletClientCert == "" || c.KubeletClientKey == "" { + return fmt.Errorf("kubelet client certificate not specified") + } + if c.KubeProxyClientCert == "" || c.KubeProxyClientKey == "" { + return fmt.Errorf("kube-proxy client certificate not specified") + } return nil } diff --git a/src/k8s/pkg/k8sd/pki/worker_test.go b/src/k8s/pkg/k8sd/pki/worker_test.go index e3d9b2ed8..bbee1efc9 100644 --- a/src/k8s/pkg/k8sd/pki/worker_test.go +++ b/src/k8s/pkg/k8sd/pki/worker_test.go @@ -1,102 +1,114 @@ -package pki_test +package pki import ( + "crypto/x509/pkix" "net" "testing" - "github.com/canonical/k8s/pkg/k8sd/pki" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" ) -func TestWorkerNodePKI_CompleteWorkerNodePKI(t *testing.T) { - cert := ` ------BEGIN CERTIFICATE----- -MIIDHDCCAgSgAwIBAgIRAMp9M56e6mSaXAkgEkvKbuQwDQYJKoZIhvcNAQELBQAw -GDEWMBQGA1UEAxMNa3ViZXJuZXRlcy1jYTAeFw0yNDAzMTQxNTM3MjdaFw00NDAz -MTQxNTM3MjdaMBgxFjAUBgNVBAMTDWt1YmVybmV0ZXMtY2EwggEiMA0GCSqGSIb3 -DQEBAQUAA4IBDwAwggEKAoIBAQDOph9lBC0hLf2ybOcBfMQQs6AJw6/6MDe06SyY -1uGPOv0CYXsmcku5KzgCruE6Dal2vNK9WQkgTRbxjt84xjHI93/W5IGdB9ZTyGem -SSeEtXD9x71eptKCrHcwbtbbUlLwmRIuAXifVDWZqCp41HwM3HhWgH4cILywFNrp -kHfm6p7CSrFRvzldmU8DAtAUHZ4iGJoVkSKVhSY4Tj18q+5+nkPrUww1QvVJ/QXn -9pc7gwig/qrnF85GjyBCLhO7IghCeImFSRxyMDaOgfa1Fd5mF0i3I5ViHVx3wiQq -IfWzWoO76kTTwugeu6UY88MqV8Y2SPqnEL2lzYNStsJogt+nAgMBAAGjYTBfMA4G -A1UdDwEB/wQEAwIChDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDwYD -VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUKE+9hTdaPuNtXmJzZwmM1MsG0yowDQYJ -KoZIhvcNAQELBQADggEBAIn9XRqXWqQS70fmeIB94UOxmj8TTXql0eFsw8h9NidP -aMFjZbM1ovKVhHId9n09wiTivo/S6kX7n/8IzBPiB9wmlEy6NPpLppfy7VEhUqfK -K1R9leoNoirda0FhQjXoQ1IGdHhA3Gw0woToeIfRlB+J6cMRth88/3bk/aA8ZR63 -bDqf0KtLXVs90UUVehUrWtj14CzSEhsyC7hcd3FKx6yzcviiydPXqBocbdLpzv2w -Zfb6LUptXDMSQxlU+meP6PjZtSxR59HivhrtSqkZd14bW01Pi9zGHvccgOsGI0hi -8+MoDI4x8YcQdkn4uA3wT88spYHOLAsUXLxK9tPlzr8= ------END CERTIFICATE-----` +func TestControlPlanePKI_CompleteWorkerNodePKI(t *testing.T) { - key := ` ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAzqYfZQQtIS39smznAXzEELOgCcOv+jA3tOksmNbhjzr9AmF7 -JnJLuSs4Aq7hOg2pdrzSvVkJIE0W8Y7fOMYxyPd/1uSBnQfWU8hnpkknhLVw/ce9 -XqbSgqx3MG7W21JS8JkSLgF4n1Q1magqeNR8DNx4VoB+HCC8sBTa6ZB35uqewkqx -Ub85XZlPAwLQFB2eIhiaFZEilYUmOE49fKvufp5D61MMNUL1Sf0F5/aXO4MIoP6q -5xfORo8gQi4TuyIIQniJhUkccjA2joH2tRXeZhdItyOVYh1cd8IkKiH1s1qDu+pE -08LoHrulGPPDKlfGNkj6pxC9pc2DUrbCaILfpwIDAQABAoIBAFe1Wo3df+odQxh/ -8GxJME6Gbt62F/LwlDRM44jba1EHkGt6RHLFAC7PkS5SW3XwZoTnD+sd5ym2jo5o -PYYzWN4bbj8fLYQg128oGBYT5poFCLguFsodtCuSV+ROpxLfliRYU8cDCNdXPojB -P4WZai1rRggw8VWu72cs8t0/XCS9nNwzO8er/PEQiMM3RQtAYoANMc/l4GiHW7jk -7+ASZK2WWH+zEPVvtmhNmLrPHtl0AvEoj/PISNoHcUqJ7+ab/YdrRx4Kie8w2rF2 -ieMqSoXb1X5dHoPR5h0maP3XVe4JuhrrukOQgTedkban/cfwZgbvkrNR3t8oMKKa -EH8i8gECgYEA3seCu4FNmavhXxjmw6Zk2CMT5WKcTOmc35jQxOFaEUjsn8+4ONXR -O/JkQfJMG78XX7K3zIaLsV/xQS2JDfXPr1eZGlY1d3GuiZbkMw2rFV50PvssSr17 -OgCLiAj3RbWDCZbCJ5t/x8azEBq8v3cJM81CvACgO0WLvkYovl9tsOECgYEA7XbX -WS77jsmF7YYfzRYhMl7iFYt+zumbz+84xTS8pxgkS6Bk/54rAcpq4vPmeQ0SFuGn -h8CN6TGQ4e8Qt0ea57jMmW74Lpgm4Gk1/lGahELZDJXgfvuoAcjAA67lJi/DheH0 -mfMsxgC5M0BjvRfS+K8Qwh7sn6uDFBkxlfPYuYcCgYBJ52uyIlII8aEhOBSNwSxh -GznlddIeHb2h24MeXRfQ9h0xYupdSGlR9rZVvjiLV9g8MgCRQ+0hmY9iLOXzkKEm -LOwodYLlLfxVvo3TdexUeXIc1pw56yPu+PFQ3pCROobO7olYNFiugHc0l3oYFjgi -TCygS6DcKNUT+RhZFzU/YQKBgQCwagmyh+T7P1vwCiS2CCrBcRwlRWz/6y2GXQKf -/33n5VeRl6dw/+CTg/3Efc5LQBqgRSRhBfxnshsgvqp8fwXmALR/iKF4fDDlp0Ql -nBpfCAqX/wC5VdyK9skv807p/7ISVLuTY8VvlDoCiWOPp5NkjSq2DKNeO901oUHl -VTM9IQKBgQChWOTuFetvVLgMa7kc0y6TYkCRsdb5XxsGkDwQv5EyctG+w5EXB0LM -li9E9xBQfc5nb88jQ4hf+9wjm0Q15LsocSNxtbUN+F5T4cskQAxFB4/djpBSbieu -IAoJNLRY/jIMGkQkzRdS1oXGXZqsAW9ndS5+N6uF7+SaXBO7E5yFKA== ------END RSA PRIVATE KEY-----` - tests := []struct { - name string - pki *pki.ControlPlanePKI - hostname string - nodeIP net.IP - bits int - expectedError bool + g := NewWithT(t) + serverCACert, serverCAKey, err := generateSelfSignedCA(pkix.Name{CommonName: "kubernetes-ca"}, 1, 2048) + g.Expect(err).ToNot(HaveOccurred()) + clientCACert, clientCAKey, err := generateSelfSignedCA(pkix.Name{CommonName: "kubernetes-ca-client"}, 1, 2048) + g.Expect(err).ToNot(HaveOccurred()) + + for _, tc := range []struct { + name string + withCerts func(*ControlPlanePKI) + expectErr bool + expectPKITo types.GomegaMatcher }{ { - name: "CompleteWorkerNodePKI with missing CA certificate", - pki: &pki.ControlPlanePKI{}, - expectedError: true, + name: "WithCACertAndKeys", + withCerts: func(pki *ControlPlanePKI) { + pki.CACert = serverCACert + pki.CAKey = serverCAKey + pki.ClientCACert = clientCACert + pki.ClientCAKey = clientCAKey + }, + expectPKITo: SatisfyAll( + HaveField("CACert", Equal(serverCACert)), + HaveField("ClientCACert", Equal(clientCACert)), + HaveField("KubeletCert", Not(BeEmpty())), + HaveField("KubeletKey", Not(BeEmpty())), + HaveField("KubeletClientCert", Not(BeEmpty())), + HaveField("KubeletClientKey", Not(BeEmpty())), + HaveField("KubeProxyClientCert", Not(BeEmpty())), + HaveField("KubeProxyClientKey", Not(BeEmpty())), + ), }, { - name: "CompleteWorkerNodePKI with CA certificate but without CA key", - pki: &pki.ControlPlanePKI{CACert: cert}, - expectedError: false, + name: "WithoutCerts", + withCerts: func(pki *ControlPlanePKI) {}, + expectErr: true, }, { - name: "CompleteWorkerNodePKI with CA certificate and successful certificate generation", - pki: &pki.ControlPlanePKI{CACert: cert, CAKey: key}, - hostname: "worker-node-1", - nodeIP: net.ParseIP("10.152.183.1"), - bits: 2048, - expectedError: false, + name: "WithoutCACert", + withCerts: func(pki *ControlPlanePKI) { + pki.ClientCACert = clientCACert + }, + expectErr: true, }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pki, err := tt.pki.CompleteWorkerNodePKI(tt.hostname, tt.nodeIP, tt.bits) - - if (err != nil) != tt.expectedError { - t.Errorf("Unexpected error status. Expected error: %v, got error: %v", tt.expectedError, err) - } + { + name: "WithoutClientCACert", + withCerts: func(pki *ControlPlanePKI) { + pki.CACert = serverCACert + }, + expectErr: true, + }, + { + name: "OnlyServerCAKey", + withCerts: func(pki *ControlPlanePKI) { + pki.CACert = serverCACert + pki.CAKey = serverCAKey + pki.ClientCACert = clientCACert + }, + expectPKITo: SatisfyAll( + HaveField("CACert", Equal(serverCACert)), + HaveField("ClientCACert", Equal(clientCACert)), + HaveField("KubeletCert", Not(BeEmpty())), + HaveField("KubeletKey", Not(BeEmpty())), + HaveField("KubeletClientCert", BeEmpty()), + HaveField("KubeletClientKey", BeEmpty()), + HaveField("KubeProxyClientCert", BeEmpty()), + HaveField("KubeProxyClientKey", BeEmpty()), + ), + }, + { + name: "OnlyClientCAKey", + withCerts: func(pki *ControlPlanePKI) { + pki.CACert = serverCACert + pki.ClientCACert = clientCACert + pki.ClientCAKey = clientCAKey + }, + expectPKITo: SatisfyAll( + HaveField("CACert", Equal(serverCACert)), + HaveField("ClientCACert", Equal(clientCACert)), + HaveField("KubeletCert", BeEmpty()), + HaveField("KubeletKey", BeEmpty()), + HaveField("KubeletClientCert", Not(BeEmpty())), + HaveField("KubeletClientKey", Not(BeEmpty())), + HaveField("KubeProxyClientCert", Not(BeEmpty())), + HaveField("KubeProxyClientKey", Not(BeEmpty())), + ), + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + cp := NewControlPlanePKI(ControlPlanePKIOpts{Years: 10}) + tc.withCerts(cp) - if !tt.expectedError { - if pki.CACert == "" { - t.Error("Missing certificate details in completed worker node PKI") - } + pki, err := cp.CompleteWorkerNodePKI("worker", net.IP{10, 0, 0, 1}, 2048) + if tc.expectErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(pki).To(tc.expectPKITo) } }) } diff --git a/src/k8s/pkg/k8sd/setup/certificates.go b/src/k8s/pkg/k8sd/setup/certificates.go index 24bfff46c..f7ed35234 100644 --- a/src/k8s/pkg/k8sd/setup/certificates.go +++ b/src/k8s/pkg/k8sd/setup/certificates.go @@ -100,6 +100,7 @@ func EnsureControlPlanePKI(snap snap.Snap, certificates *pki.ControlPlanePKI) (b path.Join(snap.KubernetesPKIDir(), "apiserver.crt"): certificates.APIServerCert, path.Join(snap.KubernetesPKIDir(), "apiserver.key"): certificates.APIServerKey, path.Join(snap.KubernetesPKIDir(), "ca.crt"): certificates.CACert, + path.Join(snap.KubernetesPKIDir(), "client-ca.crt"): certificates.ClientCACert, path.Join(snap.KubernetesPKIDir(), "ca.key"): certificates.CAKey, path.Join(snap.KubernetesPKIDir(), "front-proxy-ca.crt"): certificates.FrontProxyCACert, path.Join(snap.KubernetesPKIDir(), "front-proxy-ca.key"): certificates.FrontProxyCAKey, @@ -116,8 +117,9 @@ func EnsureControlPlanePKI(snap snap.Snap, certificates *pki.ControlPlanePKI) (b // It returns true if one or more files were updated and any error that occured. func EnsureWorkerPKI(snap snap.Snap, certificates *pki.WorkerNodePKI) (bool, error) { return ensureFiles(snap.UID(), snap.GID(), 0600, map[string]string{ - path.Join(snap.KubernetesPKIDir(), "ca.crt"): certificates.CACert, - path.Join(snap.KubernetesPKIDir(), "kubelet.crt"): certificates.KubeletCert, - path.Join(snap.KubernetesPKIDir(), "kubelet.key"): certificates.KubeletKey, + path.Join(snap.KubernetesPKIDir(), "ca.crt"): certificates.CACert, + path.Join(snap.KubernetesPKIDir(), "client-ca.crt"): certificates.ClientCACert, + path.Join(snap.KubernetesPKIDir(), "kubelet.crt"): certificates.KubeletCert, + path.Join(snap.KubernetesPKIDir(), "kubelet.key"): certificates.KubeletKey, }) } diff --git a/src/k8s/pkg/k8sd/setup/certificates_test.go b/src/k8s/pkg/k8sd/setup/certificates_test.go index b4bd1e421..32526faec 100644 --- a/src/k8s/pkg/k8sd/setup/certificates_test.go +++ b/src/k8s/pkg/k8sd/setup/certificates_test.go @@ -55,6 +55,8 @@ func TestEnsureControlPlanePKI(t *testing.T) { certificates := &pki.ControlPlanePKI{ CACert: "ca_cert", CAKey: "ca_key", + ClientCACert: "client_ca_cert", + ClientCAKey: "client_ca_key", FrontProxyCACert: "front_proxy_ca_cert", FrontProxyCAKey: "front_proxy_ca_key", FrontProxyClientCert: "front_proxy_client_cert", @@ -77,6 +79,7 @@ func TestEnsureControlPlanePKI(t *testing.T) { filepath.Join(tempDir, "apiserver.crt"), filepath.Join(tempDir, "apiserver.key"), filepath.Join(tempDir, "ca.crt"), + filepath.Join(tempDir, "client-ca.crt"), filepath.Join(tempDir, "front-proxy-ca.crt"), filepath.Join(tempDir, "front-proxy-client.crt"), filepath.Join(tempDir, "front-proxy-client.key"), @@ -105,9 +108,10 @@ func TestEnsureWorkerPKI(t *testing.T) { }, } certificates := &pki.WorkerNodePKI{ - CACert: "ca_cert", - KubeletCert: "kubelet_cert", - KubeletKey: "kubelet_key", + CACert: "ca_cert", + ClientCACert: "client_ca_cert", + KubeletCert: "kubelet_cert", + KubeletKey: "kubelet_key", } _, err := setup.EnsureWorkerPKI(mock, certificates) @@ -115,6 +119,7 @@ func TestEnsureWorkerPKI(t *testing.T) { expectedFiles := []string{ filepath.Join(tempDir, "ca.crt"), + filepath.Join(tempDir, "client-ca.crt"), filepath.Join(tempDir, "kubelet.crt"), filepath.Join(tempDir, "kubelet.key"), } diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver.go b/src/k8s/pkg/k8sd/setup/kube_apiserver.go index b48069b59..09d797f27 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver.go @@ -66,7 +66,7 @@ func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, en "--allow-privileged": "true", "--authentication-token-webhook-config-file": authTokenWebhookConfigFile, "--authorization-mode": authorizationMode, - "--client-ca-file": path.Join(snap.KubernetesPKIDir(), "ca.crt"), + "--client-ca-file": path.Join(snap.KubernetesPKIDir(), "client-ca.crt"), "--enable-admission-plugins": "NodeRestriction", "--kubelet-certificate-authority": path.Join(snap.KubernetesPKIDir(), "ca.crt"), "--kubelet-client-certificate": path.Join(snap.KubernetesPKIDir(), "apiserver-kubelet-client.crt"), diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go index c6a236e3f..1749f042c 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go @@ -46,7 +46,7 @@ func TestKubeAPIServer(t *testing.T) { {key: "--allow-privileged", expectedVal: "true"}, {key: "--authentication-token-webhook-config-file", expectedVal: path.Join(s.Mock.ServiceExtraConfigDir, "auth-token-webhook.conf")}, {key: "--authorization-mode", expectedVal: "Node,RBAC"}, - {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, {key: "--enable-admission-plugins", expectedVal: "NodeRestriction"}, {key: "--kubelet-certificate-authority", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, {key: "--kubelet-client-certificate", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver-kubelet-client.crt")}, @@ -101,7 +101,7 @@ func TestKubeAPIServer(t *testing.T) { {key: "--allow-privileged", expectedVal: "true"}, {key: "--authentication-token-webhook-config-file", expectedVal: path.Join(s.Mock.ServiceExtraConfigDir, "auth-token-webhook.conf")}, {key: "--authorization-mode", expectedVal: "Node,RBAC"}, - {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, {key: "--enable-admission-plugins", expectedVal: "NodeRestriction"}, {key: "--kubelet-certificate-authority", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, {key: "--kubelet-client-certificate", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver-kubelet-client.crt")}, diff --git a/src/k8s/pkg/k8sd/setup/kubelet.go b/src/k8s/pkg/k8sd/setup/kubelet.go index b2bb25e1b..7ad85fcdf 100644 --- a/src/k8s/pkg/k8sd/setup/kubelet.go +++ b/src/k8s/pkg/k8sd/setup/kubelet.go @@ -45,7 +45,7 @@ func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, "--anonymous-auth": "false", "--authentication-token-webhook": "true", "--cert-dir": snap.KubernetesPKIDir(), - "--client-ca-file": path.Join(snap.KubernetesPKIDir(), "ca.crt"), + "--client-ca-file": path.Join(snap.KubernetesPKIDir(), "client-ca.crt"), "--container-runtime-endpoint": path.Join(snap.ContainerdSocketDir(), "containerd.sock"), "--containerd": path.Join(snap.ContainerdSocketDir(), "containerd.sock"), "--eviction-hard": "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'", diff --git a/src/k8s/pkg/k8sd/setup/kubelet_test.go b/src/k8s/pkg/k8sd/setup/kubelet_test.go index 71af636b6..59d11e980 100644 --- a/src/k8s/pkg/k8sd/setup/kubelet_test.go +++ b/src/k8s/pkg/k8sd/setup/kubelet_test.go @@ -56,7 +56,7 @@ func TestKubelet(t *testing.T) { {key: "--anonymous-auth", expectedVal: "false"}, {key: "--authentication-token-webhook", expectedVal: "true"}, {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir}, - {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"}, @@ -105,7 +105,7 @@ func TestKubelet(t *testing.T) { {key: "--anonymous-auth", expectedVal: "false"}, {key: "--authentication-token-webhook", expectedVal: "true"}, {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir}, - {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"}, @@ -151,7 +151,7 @@ func TestKubelet(t *testing.T) { {key: "--anonymous-auth", expectedVal: "false"}, {key: "--authentication-token-webhook", expectedVal: "true"}, {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir}, - {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"}, @@ -201,7 +201,7 @@ func TestKubelet(t *testing.T) { {key: "--anonymous-auth", expectedVal: "false"}, {key: "--authentication-token-webhook", expectedVal: "true"}, {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir}, - {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")}, + {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")}, {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")}, {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"}, diff --git a/src/k8s/pkg/k8sd/setup/util_kubeconfig.go b/src/k8s/pkg/k8sd/setup/util_kubeconfig.go index 6e864f484..6bed3a112 100644 --- a/src/k8s/pkg/k8sd/setup/util_kubeconfig.go +++ b/src/k8s/pkg/k8sd/setup/util_kubeconfig.go @@ -9,7 +9,7 @@ import ( ) // createConfig generates a Config suitable for our k8s environment. -func createConfig(token string, server string, caPEM string) *clientcmdapi.Config { +func createConfig(server string, caPEM string, crtPEM string, keyPEM string) *clientcmdapi.Config { config := clientcmdapi.NewConfig() // Default to https:// prefix if no http-like scheme is present. @@ -23,7 +23,8 @@ func createConfig(token string, server string, caPEM string) *clientcmdapi.Confi Server: server, } config.AuthInfos["k8s-user"] = &clientcmdapi.AuthInfo{ - Token: token, + ClientCertificateData: []byte(crtPEM), + ClientKeyData: []byte(keyPEM), } config.Contexts["k8s"] = &clientcmdapi.Context{ Cluster: "k8s", @@ -35,8 +36,8 @@ func createConfig(token string, server string, caPEM string) *clientcmdapi.Confi } // Kubeconfig writes a kubeconfig file to disk. -func Kubeconfig(path string, token string, url string, caPEM string) error { - config := createConfig(token, url, caPEM) +func Kubeconfig(path string, url string, caPEM string, crtPEM string, keyPEM string) error { + config := createConfig(url, caPEM, crtPEM, keyPEM) if err := clientcmd.WriteToFile(*config, path); err != nil { return fmt.Errorf("failed to write kubeconfig: %w", err) } @@ -44,8 +45,8 @@ func Kubeconfig(path string, token string, url string, caPEM string) error { } // KubeconfigString provides a stringified kubeconfig. -func KubeconfigString(token string, url string, caPEM string) (string, error) { - config := createConfig(token, url, caPEM) +func KubeconfigString(url string, caPEM string, crtPEM string, keyPEM string) (string, error) { + config := createConfig(url, caPEM, crtPEM, keyPEM) kubeconfig, err := clientcmd.Write(*config) if err != nil { return "", fmt.Errorf("failed to encode kubeconfig yaml: %w", err) diff --git a/src/k8s/pkg/k8sd/setup/util_kubeconfig_test.go b/src/k8s/pkg/k8sd/setup/util_kubeconfig_test.go index f531830d4..6a51e0f4e 100644 --- a/src/k8s/pkg/k8sd/setup/util_kubeconfig_test.go +++ b/src/k8s/pkg/k8sd/setup/util_kubeconfig_test.go @@ -1,8 +1,6 @@ package setup_test import ( - "encoding/base64" - "fmt" "testing" "github.com/canonical/k8s/pkg/k8sd/setup" @@ -12,11 +10,10 @@ import ( func TestKubeconfigString(t *testing.T) { g := NewWithT(t) - ca := base64.StdEncoding.EncodeToString([]byte("ca")) - expectedConfig := fmt.Sprintf(`apiVersion: v1 + expectedConfig := `apiVersion: v1 clusters: - cluster: - certificate-authority-data: %s + certificate-authority-data: Y2E= server: https://server name: k8s contexts: @@ -30,10 +27,11 @@ preferences: {} users: - name: k8s-user user: - token: token -`, ca) + client-certificate-data: Y3J0 + client-key-data: a2V5 +` - actual, err := setup.KubeconfigString("token", "server", "ca") + actual, err := setup.KubeconfigString("server", "ca", "crt", "key") g.Expect(actual).To(Equal(expectedConfig)) g.Expect(err).To(BeNil()) diff --git a/src/k8s/pkg/k8sd/types/cluster_config_certificates.go b/src/k8s/pkg/k8sd/types/cluster_config_certificates.go index 7c06d0afe..9d2b7d066 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_certificates.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_certificates.go @@ -3,17 +3,35 @@ package types type Certificates struct { CACert *string `json:"ca-crt,omitempty"` CAKey *string `json:"ca-key,omitempty"` + ClientCACert *string `json:"client-ca-crt,omitempty"` + ClientCAKey *string `json:"client-ca-key,omitempty"` FrontProxyCACert *string `json:"front-proxy-ca-crt,omitempty"` FrontProxyCAKey *string `json:"front-proxy-ca-key,omitempty"` ServiceAccountKey *string `json:"service-account-key,omitempty"` APIServerKubeletClientCert *string `json:"apiserver-to-kubelet-client-crt,omitempty"` APIServerKubeletClientKey *string `json:"apiserver-to-kubelet-client-key,omitempty"` + AdminClientCert *string `json:"admin-client-crt,omitempty"` + AdminClientKey *string `json:"admin-client-key,omitempty"` K8sdPublicKey *string `json:"k8sd-public-key,omitempty"` K8sdPrivateKey *string `json:"k8sd-private-key,omitempty"` } -func (c Certificates) GetCACert() string { return getField(c.CACert) } -func (c Certificates) GetCAKey() string { return getField(c.CAKey) } +func (c Certificates) GetCACert() string { return getField(c.CACert) } +func (c Certificates) GetCAKey() string { return getField(c.CAKey) } +func (c Certificates) GetClientCACert() string { + // versions before 1.30.2 were using the same CA for server and client certificates + if v := getField(c.ClientCACert); v != "" { + return v + } + return c.GetCACert() +} +func (c Certificates) GetClientCAKey() string { + // versions before 1.30.2 were using the same CA for server and client certificates + if v := getField(c.ClientCAKey); v != "" { + return v + } + return c.GetCAKey() +} func (c Certificates) GetFrontProxyCACert() string { return getField(c.FrontProxyCACert) } func (c Certificates) GetFrontProxyCAKey() string { return getField(c.FrontProxyCAKey) } func (c Certificates) GetServiceAccountKey() string { return getField(c.ServiceAccountKey) } @@ -23,8 +41,10 @@ func (c Certificates) GetAPIServerKubeletClientCert() string { func (c Certificates) GetAPIServerKubeletClientKey() string { return getField(c.APIServerKubeletClientKey) } -func (c Certificates) GetK8sdPublicKey() string { return getField(c.K8sdPublicKey) } -func (c Certificates) GetK8sdPrivateKey() string { return getField(c.K8sdPrivateKey) } +func (c Certificates) GetAdminClientCert() string { return getField(c.AdminClientCert) } +func (c Certificates) GetAdminClientKey() string { return getField(c.AdminClientKey) } +func (c Certificates) GetK8sdPublicKey() string { return getField(c.K8sdPublicKey) } +func (c Certificates) GetK8sdPrivateKey() string { return getField(c.K8sdPrivateKey) } // Empty returns true if all Certificates fields are unset func (c Certificates) Empty() bool { return c == Certificates{} } diff --git a/src/k8s/pkg/k8sd/types/cluster_config_merge.go b/src/k8s/pkg/k8sd/types/cluster_config_merge.go index 9abd27a91..208c57da7 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_merge.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_merge.go @@ -25,11 +25,15 @@ func MergeClusterConfig(existing ClusterConfig, new ClusterConfig) (ClusterConfi // certificates {name: "CA certificate", val: &config.Certificates.CACert, old: existing.Certificates.CACert, new: new.Certificates.CACert}, {name: "CA key", val: &config.Certificates.CAKey, old: existing.Certificates.CAKey, new: new.Certificates.CAKey}, + {name: "client CA certificate", val: &config.Certificates.ClientCACert, old: existing.Certificates.ClientCACert, new: new.Certificates.ClientCACert}, + {name: "client CA key", val: &config.Certificates.ClientCAKey, old: existing.Certificates.ClientCAKey, new: new.Certificates.ClientCAKey}, {name: "apiserver-kubelet-client certificate", val: &config.Certificates.APIServerKubeletClientCert, old: existing.Certificates.APIServerKubeletClientCert, new: new.Certificates.APIServerKubeletClientCert, allowChange: true}, {name: "apiserver-kubelet-client key", val: &config.Certificates.APIServerKubeletClientKey, old: existing.Certificates.APIServerKubeletClientKey, new: new.Certificates.APIServerKubeletClientKey, allowChange: true}, {name: "front proxy CA certificate", val: &config.Certificates.FrontProxyCACert, old: existing.Certificates.FrontProxyCACert, new: new.Certificates.FrontProxyCACert}, {name: "front proxy CA key", val: &config.Certificates.FrontProxyCAKey, old: existing.Certificates.FrontProxyCAKey, new: new.Certificates.FrontProxyCAKey}, {name: "service account key", val: &config.Certificates.ServiceAccountKey, old: existing.Certificates.ServiceAccountKey, new: new.Certificates.ServiceAccountKey}, + {name: "admin client certificate", val: &config.Certificates.AdminClientCert, old: existing.Certificates.AdminClientCert, new: new.Certificates.AdminClientCert, allowChange: true}, + {name: "admin client key", val: &config.Certificates.AdminClientKey, old: existing.Certificates.AdminClientKey, new: new.Certificates.AdminClientKey, allowChange: true}, {name: "k8sd public key", val: &config.Certificates.K8sdPublicKey, old: existing.Certificates.K8sdPublicKey, new: new.Certificates.K8sdPublicKey}, {name: "k8sd private key", val: &config.Certificates.K8sdPrivateKey, old: existing.Certificates.K8sdPrivateKey, new: new.Certificates.K8sdPrivateKey}, // datastore diff --git a/src/k8s/pkg/k8sd/types/cluster_config_merge_test.go b/src/k8s/pkg/k8sd/types/cluster_config_merge_test.go index 010a222a2..4cdfe77e7 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_merge_test.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_merge_test.go @@ -72,6 +72,8 @@ func TestMergeClusterConfig(t *testing.T) { for _, tcs := range [][]mergeClusterConfigTestCase{ generateMergeClusterConfigTestCases("Certificates/CACert", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.CACert = utils.Pointer(v.(string)) }), generateMergeClusterConfigTestCases("Certificates/CAKey", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.CAKey = utils.Pointer(v.(string)) }), + generateMergeClusterConfigTestCases("Certificates/ClientCACert", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.ClientCACert = utils.Pointer(v.(string)) }), + generateMergeClusterConfigTestCases("Certificates/ClientCAKey", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.ClientCAKey = utils.Pointer(v.(string)) }), generateMergeClusterConfigTestCases("Certificates/FrontProxyCACert", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.FrontProxyCACert = utils.Pointer(v.(string)) }), generateMergeClusterConfigTestCases("Certificates/FrontProxyCAKey", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.FrontProxyCAKey = utils.Pointer(v.(string)) }), generateMergeClusterConfigTestCases("Certificates/ServiceAccountKey", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.ServiceAccountKey = utils.Pointer(v.(string)) }), @@ -81,6 +83,8 @@ func TestMergeClusterConfig(t *testing.T) { generateMergeClusterConfigTestCases("Certificates/APIServerKubeletClientKey", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.APIServerKubeletClientKey = utils.Pointer(v.(string)) }), + generateMergeClusterConfigTestCases("Certificates/AdminClientCert", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.AdminClientCert = utils.Pointer(v.(string)) }), + generateMergeClusterConfigTestCases("Certificates/AdminClientKey", true, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.AdminClientKey = utils.Pointer(v.(string)) }), generateMergeClusterConfigTestCases("Certificates/K8sdPublicKey", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.K8sdPublicKey = utils.Pointer(v.(string)) }), generateMergeClusterConfigTestCases("Certificates/K8sdPrivateKey", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Certificates.K8sdPrivateKey = utils.Pointer(v.(string)) }), generateMergeClusterConfigTestCases("Datastore/Type", false, "v1", "v2", func(c *types.ClusterConfig, v any) { c.Datastore.Type = utils.Pointer(v.(string)) }),