From 07291ac3110efb581ede60fd911bb99450bdaa28 Mon Sep 17 00:00:00 2001 From: austin brown Date: Thu, 22 Aug 2024 12:01:37 -0700 Subject: [PATCH] chore: stage provider-kops progress --- apis/v1alpha1/cluster_types.go | 86 +++++++++----- apis/v1alpha1/zz_generated.deepcopy.go | 32 ++++++ .../{base => base-simple}/cluster.yaml | 0 .../{base => base-simple}/kustomization.yaml | 0 .../overlays/simple/kustomization.yaml | 2 +- internal/controller/cluster/cluster.go | 31 ++++++ internal/controller/cluster/kops_helpers.go | 54 ++++++++- internal/controller/cluster/utils.go | 2 + package/crds/kops.crossplane.io_clusters.yaml | 105 ++++++++++++++++++ 9 files changed, 283 insertions(+), 29 deletions(-) rename examples/cluster/{base => base-simple}/cluster.yaml (100%) rename examples/cluster/{base => base-simple}/kustomization.yaml (100%) diff --git a/apis/v1alpha1/cluster_types.go b/apis/v1alpha1/cluster_types.go index 2fea2b3..ec8b712 100644 --- a/apis/v1alpha1/cluster_types.go +++ b/apis/v1alpha1/cluster_types.go @@ -37,6 +37,8 @@ type ClusterParameters struct { RollingUpdateOpts RollingUpdateOptsSpec `json:"rollingUpdateOpts,omitempty"` + Keypairs []KeypairSpec `json:"keypairs,omitempty"` + Secrets []SecretSpec `json:"secrets,omitempty"` // Cluster is the spec provided for the kops api ClusterSpec; ref: @@ -45,17 +47,28 @@ type ClusterParameters struct { } type RollingUpdateOptsSpec struct { - BastionInterval *string `json:"bastionInterval,omitempty"` - CloudOnly *bool `json:"cloudOnly,omitempty"` + // +kubebuilder:default="15s" + BastionInterval *string `json:"bastionInterval,omitempty"` + // +kubebuilder:default=false + CloudOnly *bool `json:"cloudOnly,omitempty"` + // +kubebuilder:default="15s" ControlPlaneInterval *string `json:"controlPlaneInterval,omitempty"` - DrainTimeout *string `json:"drainTimeout,omitempty"` - FailOnDrainError *bool `json:"failOnDrainError,omitempty"` - FailOnValidateError *bool `json:"failOnValidateError,omitempty"` - Force *bool `json:"force,omitempty"` - NodeInterval *string `json:"nodeInterval,omitempty"` - PostDrainDelay *string `json:"postDrainDelay,omitempty"` - ValidateCount *int32 `json:"validateCount,omitempty"` - ValidationTimeout *string `json:"validationTimeout,omitempty"` + // +kubebuilder:default="15m0s" + DrainTimeout *string `json:"drainTimeout,omitempty"` + // +kubebuilder:default=true + FailOnDrainError *bool `json:"failOnDrainError,omitempty"` + // +kubebuilder:default=true + FailOnValidateError *bool `json:"failOnValidateError,omitempty"` + // +kubebuilder:default=false + Force *bool `json:"force,omitempty"` + // +kubebuilder:default="15s" + NodeInterval *string `json:"nodeInterval,omitempty"` + // +kubebuilder:default="15s" + PostDrainDelay *string `json:"postDrainDelay,omitempty"` + // +kubebuilder:default=2 + ValidateCount *int32 `json:"validateCount,omitempty"` + // +kubebuilder:default="15m0s" + ValidationTimeout *string `json:"validationTimeout,omitempty"` } // ***** @@ -160,10 +173,12 @@ const ( ) type FileAssetSpec struct { - Content string `yaml:"content" json:"content"` - Name string `yaml:"name" json:"name"` - Path string `yaml:"path" json:"path"` - Roles []ClusterFileAssetRole `yaml:"roles" json:"roles"` + Content string `yaml:"content" json:"content"` + // +kubebuilder:default="0440" + Mode string `yaml:"mode" json:"mode"` + Name string `yaml:"name" json:"name"` + Path string `yaml:"path" json:"path"` + Roles []ClusterFileAssetRole `yaml:"roles" json:"roles"` } // +kubebuilder:validation:Enum=Master;Node @@ -210,19 +225,21 @@ type IAMSpec struct { } type KubeAPIServerSpec struct { - APIAudiences []string `yaml:"apiAudiences,omitempty" json:"apiAudiences,omitempty"` - DisableBasicAuth bool `yaml:"disableBasicAuth" json:"disableBasicAuth"` - OidcClientID string `yaml:"oidcClientID" json:"oidcClientID"` - OidcGroupsClaim string `yaml:"oidcGroupsClaim" json:"oidcGroupsClaim"` - OidcIssuerURL string `yaml:"oidcIssuerURL" json:"oidcIssuerURL"` - OidcUsernameClaim string `yaml:"oidcUsernameClaim" json:"oidcUsernameClaim"` - AuditLogMaxAge int `yaml:"auditLogMaxAge" json:"auditLogMaxAge"` - AuditLogMaxBackups int `yaml:"auditLogMaxBackups" json:"auditLogMaxBackups"` - AuditLogMaxSize int `yaml:"auditLogMaxSize" json:"auditLogMaxSize"` - AuditLogPath string `yaml:"auditLogPath" json:"auditLogPath"` - AuditPolicyFile string `yaml:"auditPolicyFile" json:"auditPolicyFile"` - AuditWebhookBatchMaxWait string `yaml:"auditWebhookBatchMaxWait" json:"auditWebhookBatchMaxWait"` - AuditWebhookConfigFile string `yaml:"auditWebhookConfigFile" json:"auditWebhookConfigFile"` + APIAudiences []string `yaml:"apiAudiences,omitempty" json:"apiAudiences,omitempty"` + // +kubebuilder:default=/srv/kubernetes/ca.crt + ClientCAFile string `yaml:"clientCAFile" json:"clientCAFile"` + DisableBasicAuth bool `yaml:"disableBasicAuth" json:"disableBasicAuth"` + OidcClientID string `yaml:"oidcClientID" json:"oidcClientID"` + OidcGroupsClaim string `yaml:"oidcGroupsClaim" json:"oidcGroupsClaim"` + OidcIssuerURL string `yaml:"oidcIssuerURL" json:"oidcIssuerURL"` + OidcUsernameClaim string `yaml:"oidcUsernameClaim" json:"oidcUsernameClaim"` + AuditLogMaxAge int `yaml:"auditLogMaxAge" json:"auditLogMaxAge"` + AuditLogMaxBackups int `yaml:"auditLogMaxBackups" json:"auditLogMaxBackups"` + AuditLogMaxSize int `yaml:"auditLogMaxSize" json:"auditLogMaxSize"` + AuditLogPath string `yaml:"auditLogPath" json:"auditLogPath"` + AuditPolicyFile string `yaml:"auditPolicyFile" json:"auditPolicyFile"` + AuditWebhookBatchMaxWait string `yaml:"auditWebhookBatchMaxWait" json:"auditWebhookBatchMaxWait"` + AuditWebhookConfigFile string `yaml:"auditWebhookConfigFile" json:"auditWebhookConfigFile"` } type KubeletConfigSpec struct { @@ -321,6 +338,21 @@ const ( // ***** END KopsClusterSpec and related ***** // ***** +// ***** +// ***** BEGIN KeypairSpec and related ***** +// ***** + +type KeypairSpec struct { + Keypair string `yaml:"keypair" json:"keypair"` + Cert *string `yaml:"cert,omitempty" json:"cert,omitempty"` + Key *SecretSpec `yaml:"key,omitempty" json:"key,omitempty"` + Primary bool `yaml:"primary" json:"primary"` +} + +// ***** +// ***** END KeypairSpec and related ***** +// ***** + // ***** // ***** BEGIN SecretSpec and related ***** // ***** diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 096974f..7c4dcf2 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -217,6 +217,13 @@ func (in *ClusterParameters) DeepCopyInto(out *ClusterParameters) { } } in.RollingUpdateOpts.DeepCopyInto(&out.RollingUpdateOpts) + if in.Keypairs != nil { + in, out := &in.Keypairs, &out.Keypairs + *out = make([]KeypairSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Secrets != nil { in, out := &in.Secrets, &out.Secrets *out = make([]SecretSpec, len(*in)) @@ -442,6 +449,31 @@ func (in *InstanceGroupSpec) DeepCopy() *InstanceGroupSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KeypairSpec) DeepCopyInto(out *KeypairSpec) { + *out = *in + if in.Cert != nil { + in, out := &in.Cert, &out.Cert + *out = new(string) + **out = **in + } + if in.Key != nil { + in, out := &in.Key, &out.Key + *out = new(SecretSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KeypairSpec. +func (in *KeypairSpec) DeepCopy() *KeypairSpec { + if in == nil { + return nil + } + out := new(KeypairSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KopsClusterSpec) DeepCopyInto(out *KopsClusterSpec) { *out = *in diff --git a/examples/cluster/base/cluster.yaml b/examples/cluster/base-simple/cluster.yaml similarity index 100% rename from examples/cluster/base/cluster.yaml rename to examples/cluster/base-simple/cluster.yaml diff --git a/examples/cluster/base/kustomization.yaml b/examples/cluster/base-simple/kustomization.yaml similarity index 100% rename from examples/cluster/base/kustomization.yaml rename to examples/cluster/base-simple/kustomization.yaml diff --git a/examples/cluster/overlays/simple/kustomization.yaml b/examples/cluster/overlays/simple/kustomization.yaml index 81d0de6..3ff78a4 100644 --- a/examples/cluster/overlays/simple/kustomization.yaml +++ b/examples/cluster/overlays/simple/kustomization.yaml @@ -4,7 +4,7 @@ kind: Kustomization namePrefix: simple- resources: -- ./../../base +- ./../../base-simple configurations: - kustomizeconfig/clusterType.yaml diff --git a/internal/controller/cluster/cluster.go b/internal/controller/cluster/cluster.go index a97b4bf..2c33dc3 100644 --- a/internal/controller/cluster/cluster.go +++ b/internal/controller/cluster/cluster.go @@ -338,6 +338,33 @@ func (c *external) Create(ctx context.Context, mg resource.Managed) (managed.Ext log.Info(fmt.Sprintf("Post create update error: %s; %+v", err.Error(), err)) } + // TODO(ab): we need to provide lifecycle management for keypairs, rather + // than only providing bootstrap creation + for _, kp := range cr.Spec.ForProvider.Keypairs { + privateKeyData := []byte{} + if kp.Key != nil { + privateKeySource, err := resource.CommonCredentialExtractor(bgCtx, kp.Key.Value.Source, c.kube, kp.Key.Value.CommonCredentialSelectors) + privateKeyData = append(privateKeyData, privateKeySource...) + if err != nil { + log.Info(fmt.Sprintf("Post create keypair error: %s; %+v", err.Error(), err)) + } + } + if err := c.service.createKeypair(bgCtx, cr, &kp, privateKeyData); err != nil { + log.Info(fmt.Sprintf("Post create keypair error: %s; %+v", err.Error(), err)) + } + } + if len(cr.Spec.ForProvider.Keypairs) > 0 { + if err := c.service.updateCluster(bgCtx, cr); err != nil { + log.Info(fmt.Sprintf("POST CREATE UPDATE ERROR: %s; %+v", err.Error(), err)) + } + // force cloudonly roll for initial cluster creation when keypairs are introduced + truePtr := bool(true) + cr.Spec.ForProvider.RollingUpdateOpts.CloudOnly = &truePtr + if err := c.service.rollingUpdateCluster(bgCtx, cr); err != nil { + log.Info(fmt.Sprintf("POST CREATE ROLLING UPDATE ERROR: %s; %+v", err.Error(), err)) + } + } + if err := c.annotateCluster(bgCtx, cr, map[string]string{providerKopsCreateComplete: ""}); err != nil { log.Info(fmt.Sprintf("WARNING: %s", err.Error())) } @@ -395,6 +422,10 @@ func (c *external) Update(ctx context.Context, mg resource.Managed) (managed.Ext log.Info(fmt.Sprintf("UPDATE ERROR: %s; %+v", err.Error(), err)) } + if err := c.service.rollingUpdateCluster(ctx, cr); err != nil { + log.Info(fmt.Sprintf("ROLLING UPDATE ERROR: %s; %+v", err.Error(), err)) + } + if err := c.unlockCluster(bgCtx, cr, []string{providerKopsUpdateLocked}); err != nil { log.Info(fmt.Sprintf("WARNING: %s; %+v", err.Error(), err)) } diff --git a/internal/controller/cluster/kops_helpers.go b/internal/controller/cluster/kops_helpers.go index f4f1268..5a82dbb 100644 --- a/internal/controller/cluster/kops_helpers.go +++ b/internal/controller/cluster/kops_helpers.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "os" "os/exec" "strings" @@ -117,6 +118,57 @@ func (k *kopsClient) createCluster(_ context.Context, cr *v1alpha1.Cluster) erro return nil } +func (k *kopsClient) createKeypair(_ context.Context, cr *v1alpha1.Cluster, kp *v1alpha1.KeypairSpec, privateKeyData []byte) error { + args := []string{} + if kp.Primary { + args = append(args, "--primary=true") + } + if kp.Cert != nil { + f, err := os.CreateTemp(tmpDir, "*.crt") + if err != nil { + return err + } + _, err = f.Write([]byte(*kp.Cert)) + if err != nil { + return err + } + args = append(args, fmt.Sprintf("--cert=%s", f.Name())) + defer os.Remove(f.Name()) + } + if len(privateKeyData) > 0 { + f, err := os.CreateTemp(tmpDir, "*.pem") + if err != nil { + return err + } + _, err = f.Write(privateKeyData) + if err != nil { + return err + } + args = append(args, fmt.Sprintf("--key=%s", f.Name())) + defer os.Remove(f.Name()) + } + //nolint:gosec + cmd := exec.Command( + "kops", + "create", + "keypair", + "-v5", + kp.Keypair, + fmt.Sprintf("--name=%s", getClusterExternalName(cr)), + fmt.Sprintf("--state=%s", cr.Spec.ForProvider.State), + ) + cmd.Args = append(cmd.Args, args...) + cmd.Env = append(cmd.Env, getKopsCliEnv(cr, k)...) + + if output, err := cmd.CombinedOutput(); err != nil { + return errors.Wrap(err, string(output)) + } else { + log.Debug(string(output)) + } + + return nil +} + func (k *kopsClient) authenticateToCluster(ctx context.Context, cr *v1alpha1.Cluster, extraArgs []string) error { //nolint:gosec @@ -369,7 +421,7 @@ func (k *kopsClient) updateCluster(ctx context.Context, cr *v1alpha1.Cluster) er log.Debug(fmt.Sprintf("Applied Update:%s", string(output))) } - return k.rollingUpdateCluster(ctx, cr) + return nil } func (k *kopsClient) rollingUpdateCluster(ctx context.Context, cr *v1alpha1.Cluster) error { diff --git a/internal/controller/cluster/utils.go b/internal/controller/cluster/utils.go index 1e3b654..6421bab 100644 --- a/internal/controller/cluster/utils.go +++ b/internal/controller/cluster/utils.go @@ -19,6 +19,8 @@ const ( fileSuffixCreate = "create" fileSuffixObserve = "observe" fileSuffixUpdate = "update" + + tmpDir = "/tmp" ) type kopsClient struct { diff --git a/package/crds/kops.crossplane.io_clusters.yaml b/package/crds/kops.crossplane.io_clusters.yaml index 548b35b..9dc6d91 100644 --- a/package/crds/kops.crossplane.io_clusters.yaml +++ b/package/crds/kops.crossplane.io_clusters.yaml @@ -221,6 +221,9 @@ spec: properties: content: type: string + mode: + default: "0440" + type: string name: type: string path: @@ -234,6 +237,7 @@ spec: type: array required: - content + - mode - name - path - roles @@ -291,6 +295,9 @@ spec: type: string auditWebhookConfigFile: type: string + clientCAFile: + default: /srv/kubernetes/ca.crt + type: string disableBasicAuth: type: boolean oidcClientID: @@ -309,6 +316,7 @@ spec: - auditPolicyFile - auditWebhookBatchMaxWait - auditWebhookConfigFile + - clientCAFile - disableBasicAuth - oidcClientID - oidcGroupsClaim @@ -495,30 +503,127 @@ spec: - spec type: object type: array + keypairs: + items: + properties: + cert: + type: string + key: + properties: + kind: + enum: + - ciliumpassword + - dockerconfig + - encryptionconfig + type: string + name: + type: string + value: + description: ProviderCredentials required to authenticate. + properties: + env: + description: |- + Env is a reference to an environment variable that contains credentials + that must be used to connect to the provider. + properties: + name: + description: Name is the name of an environment + variable. + type: string + required: + - name + type: object + fs: + description: |- + Fs is a reference to a filesystem location that contains credentials that + must be used to connect to the provider. + properties: + path: + description: Path is a filesystem path. + type: string + required: + - path + type: object + secretRef: + description: |- + A SecretRef is a reference to a secret key that contains the credentials + that must be used to connect to the provider. + properties: + key: + description: The key to select. + type: string + name: + description: Name of the secret. + type: string + namespace: + description: Namespace of the secret. + type: string + required: + - key + - name + - namespace + type: object + source: + description: Source of the secret credentials. + enum: + - None + - Secret + - InjectedIdentity + - Environment + - Filesystem + type: string + required: + - source + type: object + required: + - kind + - name + - value + type: object + keypair: + type: string + primary: + type: boolean + required: + - keypair + - primary + type: object + type: array rollingUpdateOpts: properties: bastionInterval: + default: 15s type: string cloudOnly: + default: false type: boolean controlPlaneInterval: + default: 15s type: string drainTimeout: + default: 15m0s type: string failOnDrainError: + default: true type: boolean failOnValidateError: + default: true type: boolean force: + default: false type: boolean nodeInterval: + default: 15s type: string postDrainDelay: + default: 15s type: string validateCount: + default: 2 format: int32 type: integer validationTimeout: + default: 15m0s type: string type: object secrets: