Merge pull request #4700 from damdo/ignition-add-ec2-instance-metadat…

…a-storage-opt

✨ Ignition: add option to store User Data in plain text

api/v1beta1/conversion.go

@@ -98,3 +98,7 @@ func Convert_v1beta2_NetworkSpec_To_v1beta1_NetworkSpec(in *v1beta2.NetworkSpec,
 func Convert_v1beta2_S3Bucket_To_v1beta1_S3Bucket(in *v1beta2.S3Bucket, out *S3Bucket, s conversion.Scope) error {
 	return autoConvert_v1beta2_S3Bucket_To_v1beta1_S3Bucket(in, out, s)
 }
+
+func Convert_v1beta2_Ignition_To_v1beta1_Ignition(in *v1beta2.Ignition, out *Ignition, s conversion.Scope) error {
+	return autoConvert_v1beta2_Ignition_To_v1beta1_Ignition(in, out, s)
+}

api/v1beta2/awsmachine_types.go

@@ -43,6 +43,17 @@ var (
 	SecretBackendSecretsManager = SecretBackend("secrets-manager")
 )
 
+// IgnitionStorageTypeOption defines the different storage types for Ignition.
+type IgnitionStorageTypeOption string
+
+const (
+	// IgnitionStorageTypeOptionClusterObjectStore means the chosen Ignition storage type is ClusterObjectStore.
+	IgnitionStorageTypeOptionClusterObjectStore = IgnitionStorageTypeOption("ClusterObjectStore")
+
+	// IgnitionStorageTypeOptionUnencryptedUserData means the chosen Ignition storage type is UnencryptedUserData.
+	IgnitionStorageTypeOptionUnencryptedUserData = IgnitionStorageTypeOption("UnencryptedUserData")
+)
+
 // AWSMachineSpec defines the desired state of an Amazon EC2 instance.
 type AWSMachineSpec struct {
 	// ProviderID is the unique identifier as specified by the cloud provider.
@@ -206,6 +217,26 @@ type Ignition struct {
 	// +kubebuilder:default="2.3"
 	// +kubebuilder:validation:Enum="2.3";"3.0";"3.1";"3.2";"3.3";"3.4"
 	Version string `json:"version,omitempty"`
+
+	// StorageType defines how to store the boostrap user data for Ignition.
+	// This can be used to instruct Ignition from where to fetch the user data to bootstrap an instance.
+	//
+	// When omitted, the storage option will default to ClusterObjectStore.
+	//
+	// When set to "ClusterObjectStore", if the capability is available and a Cluster ObjectStore configuration
+	// is correctly provided in the Cluster object (under .spec.s3Bucket),
+	// an object store will be used to store bootstrap user data.
+	//
+	// When set to "UnencryptedUserData", EC2 Instance User Data will be used to store the machine bootstrap user data, unencrypted.
+	// This option is considered less secure than others as user data may contain sensitive informations (keys, certificates, etc.)
+	// and users with ec2:DescribeInstances permission or users running pods
+	// that can access the ec2 metadata service have access to this sensitive information.
+	// So this is only to be used at ones own risk, and only when other more secure options are not viable.
+	//
+	// +optional
+	// +kubebuilder:default="ClusterObjectStore"
+	// +kubebuilder:validation:Enum:="ClusterObjectStore";"UnencryptedUserData"
+	StorageType IgnitionStorageTypeOption `json:"storageType,omitempty"`
 }
 
 // AWSMachineStatus defines the observed state of AWSMachine.

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml

@@ -632,6 +632,28 @@ spec:
                 description: Ignition defined options related to the bootstrapping
                   systems where Ignition is used.
                 properties:
+                  storageType:
+                    default: ClusterObjectStore
+                    description: "StorageType defines how to store the boostrap user
+                      data for Ignition. This can be used to instruct Ignition from
+                      where to fetch the user data to bootstrap an instance. \n When
+                      omitted, the storage option will default to ClusterObjectStore.
+                      \n When set to \"ClusterObjectStore\", if the capability is
+                      available and a Cluster ObjectStore configuration is correctly
+                      provided in the Cluster object (under .spec.s3Bucket), an object
+                      store will be used to store bootstrap user data. \n When set
+                      to \"UnencryptedUserData\", EC2 Instance User Data will be used
+                      to store the machine bootstrap user data, unencrypted. This
+                      option is considered less secure than others as user data may
+                      contain sensitive informations (keys, certificates, etc.) and
+                      users with ec2:DescribeInstances permission or users running
+                      pods that can access the ec2 metadata service have access to
+                      this sensitive information. So this is only to be used at ones
+                      own risk, and only when other more secure options are not viable."
+                    enum:
+                    - ClusterObjectStore
+                    - UnencryptedUserData
+                    type: string
                   version:
                     default: "2.3"
                     description: Version defines which version of Ignition will be

config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml

@@ -578,6 +578,30 @@ spec:
                         description: Ignition defined options related to the bootstrapping
                           systems where Ignition is used.
                         properties:
+                          storageType:
+                            default: ClusterObjectStore
+                            description: "StorageType defines how to store the boostrap
+                              user data for Ignition. This can be used to instruct
+                              Ignition from where to fetch the user data to bootstrap
+                              an instance. \n When omitted, the storage option will
+                              default to ClusterObjectStore. \n When set to \"ClusterObjectStore\",
+                              if the capability is available and a Cluster ObjectStore
+                              configuration is correctly provided in the Cluster object
+                              (under .spec.s3Bucket), an object store will be used
+                              to store bootstrap user data. \n When set to \"UnencryptedUserData\",
+                              EC2 Instance User Data will be used to store the machine
+                              bootstrap user data, unencrypted. This option is considered
+                              less secure than others as user data may contain sensitive
+                              informations (keys, certificates, etc.) and users with
+                              ec2:DescribeInstances permission or users running pods
+                              that can access the ec2 metadata service have access
+                              to this sensitive information. So this is only to be
+                              used at ones own risk, and only when other more secure
+                              options are not viable."
+                            enum:
+                            - ClusterObjectStore
+                            - UnencryptedUserData
+                            type: string
                           version:
                             default: "2.3"
                             description: Version defines which version of Ignition

controllers/awsmachine_controller.go

@@ -711,7 +711,21 @@ func (r *AWSMachineReconciler) resolveUserData(machineScope *scope.MachineScope,
 	}
 
 	if machineScope.UseIgnition(userDataFormat) {
-		userData, err = r.ignitionUserData(machineScope, objectStoreSvc, userData)
+		var ignitionStorageType infrav1.IgnitionStorageTypeOption
+		if machineScope.AWSMachine.Spec.Ignition == nil {
+			ignitionStorageType = infrav1.IgnitionStorageTypeOptionClusterObjectStore
+		} else {
+			ignitionStorageType = machineScope.AWSMachine.Spec.Ignition.StorageType
+		}
+
+		switch ignitionStorageType {
+		case infrav1.IgnitionStorageTypeOptionClusterObjectStore:
+			userData, err = r.generateIgnitionWithRemoteStorage(machineScope, objectStoreSvc, userData)
+		case infrav1.IgnitionStorageTypeOptionUnencryptedUserData:
+			// No further modifications to userdata are needed for plain storage in UnencryptedUserData.
+		default:
+			return nil, "", errors.Errorf("unsupported ignition storageType %q", ignitionStorageType)
+		}
 	}
 
 	return userData, userDataFormat, err
@@ -751,9 +765,12 @@ func (r *AWSMachineReconciler) cloudInitUserData(machineScope *scope.MachineScop
 	return encryptedCloudInit, nil
 }
 
-func (r *AWSMachineReconciler) ignitionUserData(scope *scope.MachineScope, objectStoreSvc services.ObjectStoreInterface, userData []byte) ([]byte, error) {
+// generateIgnitionWithRemoteStorage uses a remote object storage (S3 bucket) and stores user data in it,
+// then returns the config to instruct ignition on how to pull the user data from the bucket.
+func (r *AWSMachineReconciler) generateIgnitionWithRemoteStorage(scope *scope.MachineScope, objectStoreSvc services.ObjectStoreInterface, userData []byte) ([]byte, error) {
 	if objectStoreSvc == nil {
-		return nil, errors.New("object store service not available")
+		return nil, errors.New("using Ignition by default requires a cluster wide object storage configured at `AWSCluster.Spec.Ignition.S3Bucket`. " +
+			"You must configure one or instruct Ignition to use EC2 user data instead, by setting `AWSMachine.Spec.Ignition.StorageType` to `UnencryptedUserData`")
 	}
 
 	objectURL, err := objectStoreSvc.Create(scope, userData)
@@ -852,7 +869,10 @@ func (r *AWSMachineReconciler) deleteIgnitionBootstrapDataFromS3(machineScope *s
 		return err
 	}
 
-	if !machineScope.UseIgnition(userDataFormat) {
+	// We only use an S3 bucket to store userdata if we use Ignition with StorageType ClusterObjectStore.
+	if !machineScope.UseIgnition(userDataFormat) ||
+		(machineScope.AWSMachine.Spec.Ignition != nil &&
+			machineScope.AWSMachine.Spec.Ignition.StorageType != infrav1.IgnitionStorageTypeOptionClusterObjectStore) {
 		return nil
 	}