Skip to content

Commit

Permalink
Switch to x509 auth (#450)
Browse files Browse the repository at this point in the history 
* separate client ca
* create useful CompleteWorkerNodePKI tests
* require a client CA to be present for worker nodes
* allow preseeding client certificates through bootstrap and join configs
  • Loading branch information
@neoaggelos
neoaggelos committed May 30, 2024
1 parent c83bc1a commit b3812bd
Show file tree
Hide file tree
Showing 22 changed files with 501 additions and 250 deletions.
61 changes: 47 additions & 14 deletions src/k8s/api/v1/bootstrap_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,34 @@ type BootstrapConfig struct {
// Seed configuration for certificates
ExtraSANs []string `json:"extra-sans,omitempty" yaml:"extra-sans,omitempty"`

// Seed configuration for external certificates
CACert *string `json:"ca-crt,omitempty" yaml:"ca-crt,omitempty"`
CAKey *string `json:"ca-key,omitempty" yaml:"ca-key,omitempty"`
FrontProxyCACert *string `json:"front-proxy-ca-crt,omitempty" yaml:"front-proxy-ca-crt,omitempty"`
FrontProxyCAKey *string `json:"front-proxy-ca-key,omitempty" yaml:"front-proxy-ca-key,omitempty"`
FrontProxyClientCert *string `json:"front-proxy-client-crt,omitempty" yaml:"front-proxy-client-crt,omitempty"`
FrontProxyClientKey *string `json:"front-proxy-client-key,omitempty" yaml:"front-proxy-client-key,omitempty"`
APIServerKubeletClientCert *string `json:"apiserver-kubelet-client-crt,omitempty" yaml:"apiserver-kubelet-client-crt,omitempty"`
APIServerKubeletClientKey *string `json:"apiserver-kubelet-client-key,omitempty" yaml:"apiserver-kubelet-client-key,omitempty"`
ServiceAccountKey *string `json:"service-account-key,omitempty" yaml:"service-account-key,omitempty"`
// Seed configuration for external certificates (cluster-wide)
CACert *string `json:"ca-crt,omitempty" yaml:"ca-crt,omitempty"`
CAKey *string `json:"ca-key,omitempty" yaml:"ca-key,omitempty"`
ClientCACert *string `json:"client-ca-crt,omitempty" yaml:"client-ca-crt,omitempty"`
ClientCAKey *string `json:"client-ca-key,omitempty" yaml:"client-ca-key,omitempty"`
FrontProxyCACert *string `json:"front-proxy-ca-crt,omitempty" yaml:"front-proxy-ca-crt,omitempty"`
FrontProxyCAKey *string `json:"front-proxy-ca-key,omitempty" yaml:"front-proxy-ca-key,omitempty"`
FrontProxyClientCert *string `json:"front-proxy-client-crt,omitempty" yaml:"front-proxy-client-crt,omitempty"`
FrontProxyClientKey *string `json:"front-proxy-client-key,omitempty" yaml:"front-proxy-client-key,omitempty"`
APIServerKubeletClientCert *string `json:"apiserver-kubelet-client-crt,omitempty" yaml:"apiserver-kubelet-client-crt,omitempty"`
APIServerKubeletClientKey *string `json:"apiserver-kubelet-client-key,omitempty" yaml:"apiserver-kubelet-client-key,omitempty"`
AdminClientCert *string `json:"admin-client-crt,omitempty" yaml:"admin-client-crt,omitempty"`
AdminClientKey *string `json:"admin-client-key,omitempty" yaml:"admin-client-key,omitempty"`
KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"`
KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"`
KubeSchedulerClientCert *string `json:"kube-scheduler-client-crt,omitempty" yaml:"kube-scheduler-client-crt,omitempty"`
KubeSchedulerClientKey *string `json:"kube-scheduler-client-key,omitempty" yaml:"kube-scheduler-client-key,omitempty"`
KubeControllerManagerClientCert *string `json:"kube-controller-manager-client-crt,omitempty" yaml:"kube-controller-manager-client-crt,omitempty"`
KubeControllerManagerClientKey *string `json:"kube-controller-manager-client-key,omitempty" yaml:"kube-ControllerManager-client-key,omitempty"`
ServiceAccountKey *string `json:"service-account-key,omitempty" yaml:"service-account-key,omitempty"`

APIServerCert *string `json:"apiserver-crt,omitempty" yaml:"apiserver-crt,omitempty"`
APIServerKey *string `json:"apiserver-key,omitempty" yaml:"apiserver-key,omitempty"`
KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"`
KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
// Seed configuration for external certificates (node-specific)
APIServerCert *string `json:"apiserver-crt,omitempty" yaml:"apiserver-crt,omitempty"`
APIServerKey *string `json:"apiserver-key,omitempty" yaml:"apiserver-key,omitempty"`
KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"`
KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
}

func (b *BootstrapConfig) GetDatastoreType() string { return getField(b.DatastoreType) }
Expand All @@ -50,6 +63,8 @@ func (b *BootstrapConfig) GetDatastoreClientKey() string { return getField(b.D
func (b *BootstrapConfig) GetK8sDqlitePort() int { return getField(b.K8sDqlitePort) }
func (b *BootstrapConfig) GetCACert() string { return getField(b.CACert) }
func (b *BootstrapConfig) GetCAKey() string { return getField(b.CAKey) }
func (b *BootstrapConfig) GetClientCACert() string { return getField(b.ClientCACert) }
func (b *BootstrapConfig) GetClientCAKey() string { return getField(b.ClientCAKey) }
func (b *BootstrapConfig) GetFrontProxyCACert() string { return getField(b.FrontProxyCACert) }
func (b *BootstrapConfig) GetFrontProxyCAKey() string { return getField(b.FrontProxyCAKey) }
func (b *BootstrapConfig) GetFrontProxyClientCert() string { return getField(b.FrontProxyClientCert) }
Expand All @@ -60,11 +75,29 @@ func (b *BootstrapConfig) GetAPIServerKubeletClientCert() string {
func (b *BootstrapConfig) GetAPIServerKubeletClientKey() string {
return getField(b.APIServerKubeletClientKey)
}
func (b *BootstrapConfig) GetAdminClientCert() string { return getField(b.AdminClientCert) }
func (b *BootstrapConfig) GetAdminClientKey() string { return getField(b.AdminClientKey) }
func (b *BootstrapConfig) GetKubeProxyClientCert() string { return getField(b.KubeProxyClientCert) }
func (b *BootstrapConfig) GetKubeProxyClientKey() string { return getField(b.KubeProxyClientKey) }
func (b *BootstrapConfig) GetKubeSchedulerClientCert() string {
return getField(b.KubeSchedulerClientCert)
}
func (b *BootstrapConfig) GetKubeSchedulerClientKey() string {
return getField(b.KubeSchedulerClientKey)
}
func (b *BootstrapConfig) GetKubeControllerManagerClientCert() string {
return getField(b.KubeControllerManagerClientCert)
}
func (b *BootstrapConfig) GetKubeControllerManagerClientKey() string {
return getField(b.KubeControllerManagerClientKey)
}
func (b *BootstrapConfig) GetServiceAccountKey() string { return getField(b.ServiceAccountKey) }
func (b *BootstrapConfig) GetAPIServerCert() string { return getField(b.APIServerCert) }
func (b *BootstrapConfig) GetAPIServerKey() string { return getField(b.APIServerKey) }
func (b *BootstrapConfig) GetKubeletCert() string { return getField(b.KubeletCert) }
func (b *BootstrapConfig) GetKubeletKey() string { return getField(b.KubeletKey) }
func (b *BootstrapConfig) GetKubeletClientCert() string { return getField(b.KubeletClientCert) }
func (b *BootstrapConfig) GetKubeletClientKey() string { return getField(b.KubeletClientKey) }

// ToMicrocluster converts a BootstrapConfig to a map[string]string for use in microcluster.
func (b *BootstrapConfig) ToMicrocluster() (map[string]string, error) {
Expand Down
72 changes: 58 additions & 14 deletions src/k8s/api/v1/join_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,76 @@ import (
type ControlPlaneNodeJoinConfig struct {
ExtraSANS []string `json:"extra-sans,omitempty" yaml:"extra-sans,omitempty"`

APIServerCert *string `json:"apiserver-crt,omitempty" yaml:"apiserver-crt,omitempty"`
APIServerKey *string `json:"apiserver-key,omitempty" yaml:"apiserver-key,omitempty"`
FrontProxyClientCert *string `json:"front-proxy-client-crt,omitempty" yaml:"front-proxy-client-crt,omitempty"`
FrontProxyClientKey *string `json:"front-proxy-client-key,omitempty" yaml:"front-proxy-client-key,omitempty"`
KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"`
KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
// Seed certificates for external CA
FrontProxyClientCert *string `json:"front-proxy-client-crt,omitempty" yaml:"front-proxy-client-crt,omitempty"`
FrontProxyClientKey *string `json:"front-proxy-client-key,omitempty" yaml:"front-proxy-client-key,omitempty"`
KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"`
KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"`
KubeSchedulerClientCert *string `json:"kube-scheduler-client-crt,omitempty" yaml:"kube-scheduler-client-crt,omitempty"`
KubeSchedulerClientKey *string `json:"kube-scheduler-client-key,omitempty" yaml:"kube-scheduler-client-key,omitempty"`
KubeControllerManagerClientCert *string `json:"kube-controller-manager-client-crt,omitempty" yaml:"kube-controller-manager-client-crt,omitempty"`
KubeControllerManagerClientKey *string `json:"kube-controller-manager-client-key,omitempty" yaml:"kube-ControllerManager-client-key,omitempty"`

APIServerCert *string `json:"apiserver-crt,omitempty" yaml:"apiserver-crt,omitempty"`
APIServerKey *string `json:"apiserver-key,omitempty" yaml:"apiserver-key,omitempty"`
KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"`
KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
}

type WorkerNodeJoinConfig struct {
KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"`
KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
KubeletCert *string `json:"kubelet-crt,omitempty" yaml:"kubelet-crt,omitempty"`
KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"`
KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"`
}

func (c *ControlPlaneNodeJoinConfig) GetAPIServerCert() string { return getField(c.APIServerCert) }
func (c *ControlPlaneNodeJoinConfig) GetAPIServerKey() string { return getField(c.APIServerKey) }
func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientCert() string {
return getField(c.FrontProxyClientCert)
}
func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientKey() string {
return getField(c.FrontProxyClientKey)
}
func (c *ControlPlaneNodeJoinConfig) GetKubeletCert() string { return getField(c.KubeletCert) }
func (c *ControlPlaneNodeJoinConfig) GetKubeletKey() string { return getField(c.KubeletKey) }
func (b *ControlPlaneNodeJoinConfig) GetKubeProxyClientCert() string {
return getField(b.KubeProxyClientCert)
}
func (b *ControlPlaneNodeJoinConfig) GetKubeProxyClientKey() string {
return getField(b.KubeProxyClientKey)
}
func (b *ControlPlaneNodeJoinConfig) GetKubeSchedulerClientCert() string {
return getField(b.KubeSchedulerClientCert)
}
func (b *ControlPlaneNodeJoinConfig) GetKubeSchedulerClientKey() string {
return getField(b.KubeSchedulerClientKey)
}
func (b *ControlPlaneNodeJoinConfig) GetKubeControllerManagerClientCert() string {
return getField(b.KubeControllerManagerClientCert)
}
func (b *ControlPlaneNodeJoinConfig) GetKubeControllerManagerClientKey() string {
return getField(b.KubeControllerManagerClientKey)
}
func (c *ControlPlaneNodeJoinConfig) GetAPIServerCert() string { return getField(c.APIServerCert) }
func (c *ControlPlaneNodeJoinConfig) GetAPIServerKey() string { return getField(c.APIServerKey) }
func (c *ControlPlaneNodeJoinConfig) GetKubeletCert() string { return getField(c.KubeletCert) }
func (c *ControlPlaneNodeJoinConfig) GetKubeletKey() string { return getField(c.KubeletKey) }
func (c *ControlPlaneNodeJoinConfig) GetKubeletClientCert() string {
return getField(c.KubeletClientCert)
}
func (c *ControlPlaneNodeJoinConfig) GetKubeletClientKey() string {
return getField(c.KubeletClientKey)
}

func (w *WorkerNodeJoinConfig) GetKubeletCert() string { return getField(w.KubeletCert) }
func (w *WorkerNodeJoinConfig) GetKubeletKey() string { return getField(w.KubeletKey) }
func (w *WorkerNodeJoinConfig) GetKubeletCert() string { return getField(w.KubeletCert) }
func (w *WorkerNodeJoinConfig) GetKubeletKey() string { return getField(w.KubeletKey) }
func (w *WorkerNodeJoinConfig) GetKubeletClientCert() string { return getField(w.KubeletClientCert) }
func (w *WorkerNodeJoinConfig) GetKubeletClientKey() string { return getField(w.KubeletClientKey) }
func (w *WorkerNodeJoinConfig) GetKubeProxyClientCert() string {
return getField(w.KubeProxyClientCert)
}
func (w *WorkerNodeJoinConfig) GetKubeProxyClientKey() string { return getField(w.KubeProxyClientKey) }

// WorkerJoinConfigFromMicrocluster parses a microcluster map[string]string and retrieves the WorkerNodeJoinConfig.
func ControlPlaneJoinConfigFromMicrocluster(m map[string]string) (ControlPlaneNodeJoinConfig, error) {
Expand Down
18 changes: 12 additions & 6 deletions src/k8s/api/v1/worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,20 @@ type WorkerNodeInfoRequest struct {

// WorkerNodeInfoResponse is used to return a worker node token.
type WorkerNodeInfoResponse struct {
// CA is the PEM encoded certificate authority of the cluster.
CA string `json:"ca,omitempty"`
// CACert is the PEM encoded certificate authority of the cluster.
CACert string `json:"ca,omitempty"`
// ClientCACert is the PEM encoded certificate authority of the cluster clients.
ClientCACert string `json:"client-ca,omitempty"`
// APIServers is a list of kube-apiserver endpoints of the cluster.
APIServers []string `json:"apiServers"`
// KubeletToken is the token to use for kubelet.
KubeletToken string `json:"kubeletToken"`
// KubeProxyToken is the token to use for kube-proxy.
KubeProxyToken string `json:"kubeProxyToken"`
// KubeletClientCert is the certificate to use in kubelet to authenticate with kube-apiserver.
KubeletClientCert string `json:"kubeletClientCert"`
// KubeletClientKey is the private key to use in kubelet to authenticate with kube-apiserver.
KubeletClientKey string `json:"kubeletClientKey"`
// KubeProxyClientCert is the certificate to use in kube-proxy to authenticate with kube-apiserver.
KubeProxyClientCert string `json:"kubeProxyClientCert"`
// KubeProxyClientKey is the private key to use in kube-proxy to authenticate with kube-apiserver.
KubeProxyClientKey string `json:"kubeProxyClientKey"`
// PodCIDR is the configured CIDR for pods in the cluster.
PodCIDR string `json:"podCIDR"`
// ServiceCIDR is the configured CIDR for services in the cluster.
Expand Down
6 changes: 1 addition & 5 deletions src/k8s/pkg/k8sd/api/kubeconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,8 @@ func (e *Endpoints) getKubeconfig(s *state.State, r *http.Request) response.Resp
if req.Server == "" {
server = fmt.Sprintf("%s:%d", s.Address().Hostname(), config.APIServer.GetSecurePort())
}
token, err := databaseutil.GetOrCreateAuthToken(s.Context, s, "kubernetes-admin", []string{"system:masters"})
if err != nil {
return response.InternalError(fmt.Errorf("failed to get admin token: %w", err))
}

kubeconfig, err := setup.KubeconfigString(token, server, config.Certificates.GetCACert())
kubeconfig, err := setup.KubeconfigString(server, config.Certificates.GetCACert(), config.Certificates.GetAdminClientCert(), config.Certificates.GetAdminClientKey())
if err != nil {
return response.InternalError(fmt.Errorf("failed to get kubeconfig: %w", err))
}
Expand Down
54 changes: 17 additions & 37 deletions src/k8s/pkg/k8sd/api/worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ func (e *Endpoints) postWorkerInfo(s *state.State, r *http.Request) response.Res
certificates := pki.NewControlPlanePKI(pki.ControlPlanePKIOpts{Years: 10})
certificates.CACert = cfg.Certificates.GetCACert()
certificates.CAKey = cfg.Certificates.GetCAKey()
certificates.ClientCACert = cfg.Certificates.GetClientCACert()
certificates.ClientCAKey = cfg.Certificates.GetClientCAKey()
workerCertificates, err := certificates.CompleteWorkerNodePKI(workerName, nodeIP, 2048)
if err != nil {
return response.InternalError(fmt.Errorf("failed to generate worker PKI: %w", err))
Expand All @@ -56,31 +58,6 @@ func (e *Endpoints) postWorkerInfo(s *state.State, r *http.Request) response.Res
return response.InternalError(fmt.Errorf("failed to retrieve list of known kube-apiserver endpoints: %w", err))
}

var (
kubeletToken string
proxyToken string
)
for _, i := range []struct {
token *string
name string
username string
groups []string
}{
{token: &kubeletToken, name: "kubelet", username: fmt.Sprintf("system:node:%s", workerName), groups: []string{"system:nodes"}},
{token: &proxyToken, name: "kube-proxy", username: "system:kube-proxy"},
} {
if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error {
t, err := database.GetOrCreateToken(ctx, tx, i.username, i.groups)
if err != nil {
return fmt.Errorf("failed to generate %s token for node %q: %w", i.name, workerName, err)
}
*i.token = t
return nil
}); err != nil {
return response.InternalError(fmt.Errorf("create token transaction failed: %w", err))
}
}

if err := s.Database.Transaction(s.Context, func(ctx context.Context, tx *sql.Tx) error {
return database.AddWorkerNode(ctx, tx, workerName)
}); err != nil {
Expand All @@ -94,17 +71,20 @@ func (e *Endpoints) postWorkerInfo(s *state.State, r *http.Request) response.Res
}

return response.SyncResponse(true, &apiv1.WorkerNodeInfoResponse{
CA: cfg.Certificates.GetCACert(),
APIServers: servers,
PodCIDR: cfg.Network.GetPodCIDR(),
ServiceCIDR: cfg.Network.GetServiceCIDR(),
KubeletToken: kubeletToken,
KubeProxyToken: proxyToken,
ClusterDomain: cfg.Kubelet.GetClusterDomain(),
ClusterDNS: cfg.Kubelet.GetClusterDNS(),
CloudProvider: cfg.Kubelet.GetCloudProvider(),
KubeletCert: workerCertificates.KubeletCert,
KubeletKey: workerCertificates.KubeletKey,
K8sdPublicKey: cfg.Certificates.GetK8sdPublicKey(),
CACert: cfg.Certificates.GetCACert(),
ClientCACert: cfg.Certificates.GetClientCACert(),
APIServers: servers,
PodCIDR: cfg.Network.GetPodCIDR(),
ServiceCIDR: cfg.Network.GetServiceCIDR(),
ClusterDomain: cfg.Kubelet.GetClusterDomain(),
ClusterDNS: cfg.Kubelet.GetClusterDNS(),
CloudProvider: cfg.Kubelet.GetCloudProvider(),
KubeletCert: workerCertificates.KubeletCert,
KubeletKey: workerCertificates.KubeletKey,
KubeletClientCert: workerCertificates.KubeletClientCert,
KubeletClientKey: workerCertificates.KubeletClientKey,
KubeProxyClientCert: workerCertificates.KubeProxyClientCert,
KubeProxyClientKey: workerCertificates.KubeProxyClientKey,
K8sdPublicKey: cfg.Certificates.GetK8sdPublicKey(),
})
}
Loading

0 comments on commit b3812bd

Please sign in to comment.