diff --git a/api/v1beta1/conversion.go b/api/v1beta1/conversion.go
index f18fdc678b..2b124f027a 100644
--- a/api/v1beta1/conversion.go
+++ b/api/v1beta1/conversion.go
@@ -98,3 +98,7 @@ func Convert_v1beta2_NetworkSpec_To_v1beta1_NetworkSpec(in *v1beta2.NetworkSpec,
func Convert_v1beta2_S3Bucket_To_v1beta1_S3Bucket(in *v1beta2.S3Bucket, out *S3Bucket, s conversion.Scope) error {
return autoConvert_v1beta2_S3Bucket_To_v1beta1_S3Bucket(in, out, s)
}
+
+func Convert_v1beta2_Ignition_To_v1beta1_Ignition(in *v1beta2.Ignition, out *Ignition, s conversion.Scope) error {
+ return autoConvert_v1beta2_Ignition_To_v1beta1_Ignition(in, out, s)
+}
diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go
index 3a64943cae..6fab23cc8a 100644
--- a/api/v1beta1/zz_generated.conversion.go
+++ b/api/v1beta1/zz_generated.conversion.go
@@ -445,11 +445,6 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
- if err := s.AddGeneratedConversionFunc((*v1beta2.Ignition)(nil), (*Ignition)(nil), func(a, b interface{}, scope conversion.Scope) error {
- return Convert_v1beta2_Ignition_To_v1beta1_Ignition(a.(*v1beta2.Ignition), b.(*Ignition), scope)
- }); err != nil {
- return err
- }
if err := s.AddGeneratedConversionFunc((*IngressRule)(nil), (*v1beta2.IngressRule)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta1_IngressRule_To_v1beta2_IngressRule(a.(*IngressRule), b.(*v1beta2.IngressRule), scope)
}); err != nil {
@@ -560,6 +555,11 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
+ if err := s.AddConversionFunc((*v1beta2.Ignition)(nil), (*Ignition)(nil), func(a, b interface{}, scope conversion.Scope) error {
+ return Convert_v1beta2_Ignition_To_v1beta1_Ignition(a.(*v1beta2.Ignition), b.(*Ignition), scope)
+ }); err != nil {
+ return err
+ }
if err := s.AddConversionFunc((*v1beta2.IngressRule)(nil), (*IngressRule)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta2_IngressRule_To_v1beta1_IngressRule(a.(*v1beta2.IngressRule), b.(*IngressRule), scope)
}); err != nil {
@@ -1360,7 +1360,15 @@ func autoConvert_v1beta1_AWSMachineSpec_To_v1beta2_AWSMachineSpec(in *AWSMachine
if err := Convert_v1beta1_CloudInit_To_v1beta2_CloudInit(&in.CloudInit, &out.CloudInit, s); err != nil {
return err
}
- out.Ignition = (*v1beta2.Ignition)(unsafe.Pointer(in.Ignition))
+ if in.Ignition != nil {
+ in, out := &in.Ignition, &out.Ignition
+ *out = new(v1beta2.Ignition)
+ if err := Convert_v1beta1_Ignition_To_v1beta2_Ignition(*in, *out, s); err != nil {
+ return err
+ }
+ } else {
+ out.Ignition = nil
+ }
out.SpotMarketOptions = (*v1beta2.SpotMarketOptions)(unsafe.Pointer(in.SpotMarketOptions))
out.Tenancy = in.Tenancy
return nil
@@ -1409,7 +1417,15 @@ func autoConvert_v1beta2_AWSMachineSpec_To_v1beta1_AWSMachineSpec(in *v1beta2.AW
if err := Convert_v1beta2_CloudInit_To_v1beta1_CloudInit(&in.CloudInit, &out.CloudInit, s); err != nil {
return err
}
- out.Ignition = (*Ignition)(unsafe.Pointer(in.Ignition))
+ if in.Ignition != nil {
+ in, out := &in.Ignition, &out.Ignition
+ *out = new(Ignition)
+ if err := Convert_v1beta2_Ignition_To_v1beta1_Ignition(*in, *out, s); err != nil {
+ return err
+ }
+ } else {
+ out.Ignition = nil
+ }
out.SpotMarketOptions = (*SpotMarketOptions)(unsafe.Pointer(in.SpotMarketOptions))
// WARNING: in.PlacementGroupName requires manual conversion: does not exist in peer-type
out.Tenancy = in.Tenancy
@@ -1921,14 +1937,10 @@ func Convert_v1beta1_Ignition_To_v1beta2_Ignition(in *Ignition, out *v1beta2.Ign
func autoConvert_v1beta2_Ignition_To_v1beta1_Ignition(in *v1beta2.Ignition, out *Ignition, s conversion.Scope) error {
out.Version = in.Version
+ // WARNING: in.StorageType requires manual conversion: does not exist in peer-type
return nil
}
-// Convert_v1beta2_Ignition_To_v1beta1_Ignition is an autogenerated conversion function.
-func Convert_v1beta2_Ignition_To_v1beta1_Ignition(in *v1beta2.Ignition, out *Ignition, s conversion.Scope) error {
- return autoConvert_v1beta2_Ignition_To_v1beta1_Ignition(in, out, s)
-}
-
func autoConvert_v1beta1_IngressRule_To_v1beta2_IngressRule(in *IngressRule, out *v1beta2.IngressRule, s conversion.Scope) error {
out.Description = in.Description
out.Protocol = v1beta2.SecurityGroupProtocol(in.Protocol)
diff --git a/api/v1beta2/awsmachine_types.go b/api/v1beta2/awsmachine_types.go
index dad3bb0575..10d8ce0dcb 100644
--- a/api/v1beta2/awsmachine_types.go
+++ b/api/v1beta2/awsmachine_types.go
@@ -43,6 +43,17 @@ var (
SecretBackendSecretsManager = SecretBackend("secrets-manager")
)
+// IgnitionStorageTypeOption defines the different storage types for Ignition.
+type IgnitionStorageTypeOption string
+
+const (
+ // IgnitionStorageTypeOptionClusterObjectStore means the chosen Ignition storage type is ClusterObjectStore.
+ IgnitionStorageTypeOptionClusterObjectStore = IgnitionStorageTypeOption("ClusterObjectStore")
+
+ // IgnitionStorageTypeOptionUnencryptedUserData means the chosen Ignition storage type is UnencryptedUserData.
+ IgnitionStorageTypeOptionUnencryptedUserData = IgnitionStorageTypeOption("UnencryptedUserData")
+)
+
// AWSMachineSpec defines the desired state of an Amazon EC2 instance.
type AWSMachineSpec struct {
// ProviderID is the unique identifier as specified by the cloud provider.
@@ -206,6 +217,26 @@ type Ignition struct {
// +kubebuilder:default="2.3"
// +kubebuilder:validation:Enum="2.3";"3.0";"3.1";"3.2";"3.3";"3.4"
Version string `json:"version,omitempty"`
+
+ // StorageType defines how to store the boostrap user data for Ignition.
+ // This can be used to instruct Ignition from where to fetch the user data to bootstrap an instance.
+ //
+ // When omitted, the storage option will default to ClusterObjectStore.
+ //
+ // When set to "ClusterObjectStore", if the capability is available and a Cluster ObjectStore configuration
+ // is correctly provided in the Cluster object (under .spec.s3Bucket),
+ // an object store will be used to store bootstrap user data.
+ //
+ // When set to "UnencryptedUserData", EC2 Instance User Data will be used to store the machine bootstrap user data, unencrypted.
+ // This option is considered less secure than others as user data may contain sensitive informations (keys, certificates, etc.)
+ // and users with ec2:DescribeInstances permission or users running pods
+ // that can access the ec2 metadata service have access to this sensitive information.
+ // So this is only to be used at ones own risk, and only when other more secure options are not viable.
+ //
+ // +optional
+ // +kubebuilder:default="ClusterObjectStore"
+ // +kubebuilder:validation:Enum:="ClusterObjectStore";"UnencryptedUserData"
+ StorageType IgnitionStorageTypeOption `json:"storageType,omitempty"`
}
// AWSMachineStatus defines the observed state of AWSMachine.
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml
index 795f794a56..e356896c1b 100644
--- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml
+++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachines.yaml
@@ -632,6 +632,28 @@ spec:
description: Ignition defined options related to the bootstrapping
systems where Ignition is used.
properties:
+ storageType:
+ default: ClusterObjectStore
+ description: "StorageType defines how to store the boostrap user
+ data for Ignition. This can be used to instruct Ignition from
+ where to fetch the user data to bootstrap an instance. \n When
+ omitted, the storage option will default to ClusterObjectStore.
+ \n When set to \"ClusterObjectStore\", if the capability is
+ available and a Cluster ObjectStore configuration is correctly
+ provided in the Cluster object (under .spec.s3Bucket), an object
+ store will be used to store bootstrap user data. \n When set
+ to \"UnencryptedUserData\", EC2 Instance User Data will be used
+ to store the machine bootstrap user data, unencrypted. This
+ option is considered less secure than others as user data may
+ contain sensitive informations (keys, certificates, etc.) and
+ users with ec2:DescribeInstances permission or users running
+ pods that can access the ec2 metadata service have access to
+ this sensitive information. So this is only to be used at ones
+ own risk, and only when other more secure options are not viable."
+ enum:
+ - ClusterObjectStore
+ - UnencryptedUserData
+ type: string
version:
default: "2.3"
description: Version defines which version of Ignition will be
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml
index 23466b416c..00b85b4969 100644
--- a/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml
+++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_awsmachinetemplates.yaml
@@ -578,6 +578,30 @@ spec:
description: Ignition defined options related to the bootstrapping
systems where Ignition is used.
properties:
+ storageType:
+ default: ClusterObjectStore
+ description: "StorageType defines how to store the boostrap
+ user data for Ignition. This can be used to instruct
+ Ignition from where to fetch the user data to bootstrap
+ an instance. \n When omitted, the storage option will
+ default to ClusterObjectStore. \n When set to \"ClusterObjectStore\",
+ if the capability is available and a Cluster ObjectStore
+ configuration is correctly provided in the Cluster object
+ (under .spec.s3Bucket), an object store will be used
+ to store bootstrap user data. \n When set to \"UnencryptedUserData\",
+ EC2 Instance User Data will be used to store the machine
+ bootstrap user data, unencrypted. This option is considered
+ less secure than others as user data may contain sensitive
+ informations (keys, certificates, etc.) and users with
+ ec2:DescribeInstances permission or users running pods
+ that can access the ec2 metadata service have access
+ to this sensitive information. So this is only to be
+ used at ones own risk, and only when other more secure
+ options are not viable."
+ enum:
+ - ClusterObjectStore
+ - UnencryptedUserData
+ type: string
version:
default: "2.3"
description: Version defines which version of Ignition
diff --git a/controllers/awsmachine_controller.go b/controllers/awsmachine_controller.go
index 9c47adab1a..7ef74fe8c5 100644
--- a/controllers/awsmachine_controller.go
+++ b/controllers/awsmachine_controller.go
@@ -711,7 +711,21 @@ func (r *AWSMachineReconciler) resolveUserData(machineScope *scope.MachineScope,
}
if machineScope.UseIgnition(userDataFormat) {
- userData, err = r.ignitionUserData(machineScope, objectStoreSvc, userData)
+ var ignitionStorageType infrav1.IgnitionStorageTypeOption
+ if machineScope.AWSMachine.Spec.Ignition == nil {
+ ignitionStorageType = infrav1.IgnitionStorageTypeOptionClusterObjectStore
+ } else {
+ ignitionStorageType = machineScope.AWSMachine.Spec.Ignition.StorageType
+ }
+
+ switch ignitionStorageType {
+ case infrav1.IgnitionStorageTypeOptionClusterObjectStore:
+ userData, err = r.generateIgnitionWithRemoteStorage(machineScope, objectStoreSvc, userData)
+ case infrav1.IgnitionStorageTypeOptionUnencryptedUserData:
+ // No further modifications to userdata are needed for plain storage in UnencryptedUserData.
+ default:
+ return nil, "", errors.Errorf("unsupported ignition storageType %q", ignitionStorageType)
+ }
}
return userData, userDataFormat, err
@@ -751,9 +765,12 @@ func (r *AWSMachineReconciler) cloudInitUserData(machineScope *scope.MachineScop
return encryptedCloudInit, nil
}
-func (r *AWSMachineReconciler) ignitionUserData(scope *scope.MachineScope, objectStoreSvc services.ObjectStoreInterface, userData []byte) ([]byte, error) {
+// generateIgnitionWithRemoteStorage uses a remote object storage (S3 bucket) and stores user data in it,
+// then returns the config to instruct ignition on how to pull the user data from the bucket.
+func (r *AWSMachineReconciler) generateIgnitionWithRemoteStorage(scope *scope.MachineScope, objectStoreSvc services.ObjectStoreInterface, userData []byte) ([]byte, error) {
if objectStoreSvc == nil {
- return nil, errors.New("object store service not available")
+ return nil, errors.New("using Ignition by default requires a cluster wide object storage configured at `AWSCluster.Spec.Ignition.S3Bucket`. " +
+ "You must configure one or instruct Ignition to use EC2 user data instead, by setting `AWSMachine.Spec.Ignition.StorageType` to `UnencryptedUserData`")
}
objectURL, err := objectStoreSvc.Create(scope, userData)
@@ -852,7 +869,10 @@ func (r *AWSMachineReconciler) deleteIgnitionBootstrapDataFromS3(machineScope *s
return err
}
- if !machineScope.UseIgnition(userDataFormat) {
+ // We only use an S3 bucket to store userdata if we use Ignition with StorageType ClusterObjectStore.
+ if !machineScope.UseIgnition(userDataFormat) ||
+ (machineScope.AWSMachine.Spec.Ignition != nil &&
+ machineScope.AWSMachine.Spec.Ignition.StorageType != infrav1.IgnitionStorageTypeOptionClusterObjectStore) {
return nil
}
diff --git a/controllers/awsmachine_controller_unit_test.go b/controllers/awsmachine_controller_unit_test.go
index 57af003cfe..dd444c5275 100644
--- a/controllers/awsmachine_controller_unit_test.go
+++ b/controllers/awsmachine_controller_unit_test.go
@@ -706,13 +706,14 @@ func TestAWSMachineReconciler(t *testing.T) {
expectConditions(g, ms.AWSMachine, []conditionAssertion{{infrav1.ELBAttachedCondition, corev1.ConditionTrue, "", ""}})
expectConditions(g, ms.AWSMachine, []conditionAssertion{{infrav1.InstanceReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityWarning, infrav1.InstanceNotReadyReason}})
})
- t.Run("Should store userdata using AWS Secrets Manager", func(t *testing.T) {
+ t.Run("should store userdata for CloudInit using AWS Secrets Manager only when not skipped", func(t *testing.T) {
g := NewWithT(t)
awsMachine := getAWSMachine()
setup(t, g, awsMachine)
defer teardown(t, g)
instanceCreate(t, g)
+ // Explicitly skip AWS Secrets Manager.
ms.AWSMachine.Spec.CloudInit.InsecureSkipSecretsManager = true
ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
@@ -1120,7 +1121,7 @@ func TestAWSMachineReconciler(t *testing.T) {
_, _ = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
})
- t.Run("should delete the secret from the S3 bucket", func(t *testing.T) {
+ t.Run("should delete the secret from the S3 bucket if StorageType ClusterObjectStore is set for Ignition", func(t *testing.T) {
g := NewWithT(t)
awsMachine := getAWSMachine()
setup(t, g, awsMachine)
@@ -1129,7 +1130,8 @@ func TestAWSMachineReconciler(t *testing.T) {
ms.AWSMachine.Spec.CloudInit = infrav1.CloudInit{}
ms.AWSMachine.Spec.Ignition = &infrav1.Ignition{
- Version: "2.3",
+ Version: "2.3",
+ StorageType: infrav1.IgnitionStorageTypeOptionClusterObjectStore,
}
buf := new(bytes.Buffer)
@@ -1138,6 +1140,28 @@ func TestAWSMachineReconciler(t *testing.T) {
objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
+ _, err := reconciler.reconcileDelete(ms, cs, cs, cs, cs)
+ g.Expect(err).To(BeNil())
+ })
+ t.Run("should not delete the secret from the S3 bucket if StorageType UnencryptedUserData is set for Ignition", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ setNodeRef(t, g)
+
+ ms.AWSMachine.Spec.CloudInit = infrav1.CloudInit{}
+ ms.AWSMachine.Spec.Ignition = &infrav1.Ignition{
+ Version: "2.3",
+ StorageType: infrav1.IgnitionStorageTypeOptionUnencryptedUserData,
+ }
+
+ buf := new(bytes.Buffer)
+ klog.SetOutput(buf)
+
+ objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(0)
+ ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
+
_, err := reconciler.reconcileDelete(ms, cs, cs, cs, cs)
g.Expect(err).To(BeNil())
})
@@ -1265,276 +1289,309 @@ func TestAWSMachineReconciler(t *testing.T) {
})
})
- t.Run("Object storage lifecycle", func(t *testing.T) {
- t.Run("creating EC2 instances", func(t *testing.T) {
- var instance *infrav1.Instance
-
- getInstances := func(t *testing.T, g *WithT) {
- t.Helper()
-
- ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil).AnyTimes()
- }
-
- useIgnition := func(t *testing.T, g *WithT) {
+ t.Run("Object storage lifecycle for Ignition's userdata", func(t *testing.T) {
+ t.Run("when Ignition's StorageType is ClusterObjectStore", func(t *testing.T) {
+ useIgnitionWithClusterObjectStore := func(t *testing.T, g *WithT) {
t.Helper()
ms.Machine.Spec.Bootstrap.DataSecretName = ptr.To[string]("bootstrap-data-ignition")
ms.AWSMachine.Spec.CloudInit.SecretCount = 0
ms.AWSMachine.Spec.CloudInit.SecretPrefix = ""
+ ms.AWSMachine.Spec.Ignition = &infrav1.Ignition{
+ Version: "2.3",
+ StorageType: infrav1.IgnitionStorageTypeOptionClusterObjectStore,
+ }
}
- t.Run("should leverage AWS S3", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- getInstances(t, g)
- useIgnition(t, g)
+ t.Run("creating EC2 instances", func(t *testing.T) {
+ var instance *infrav1.Instance
- instance = &infrav1.Instance{
- ID: "myMachine",
- State: infrav1.InstanceStatePending,
+ getInstances := func(t *testing.T, g *WithT) {
+ t.Helper()
+
+ ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil).AnyTimes()
}
- fakeS3URL := "s3://foo"
- objectStoreSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return(fakeS3URL, nil).Times(1)
- ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any(), gomock.Any()).Return(instance, nil).AnyTimes()
- ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
- ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
- ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil)
+ t.Run("should leverage a Cluster Object Store", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ getInstances(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
- ms.AWSMachine.ObjectMeta.Labels = map[string]string{
- clusterv1.MachineControlPlaneLabel: "",
- }
+ instance = &infrav1.Instance{
+ ID: "myMachine",
+ State: infrav1.InstanceStatePending,
+ }
+ fakeS3URL := "s3://foo"
- _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
- g.Expect(err).To(BeNil())
- })
+ objectStoreSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return(fakeS3URL, nil).Times(1)
+ ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any(), gomock.Any()).Return(instance, nil).AnyTimes()
+ ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
+ ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
+ ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil)
- t.Run("should leverage AWS S3 with presigned urls", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- getInstances(t, g)
- useIgnition(t, g)
+ ms.AWSMachine.ObjectMeta.Labels = map[string]string{
+ clusterv1.MachineControlPlaneLabel: "",
+ }
- if cs.AWSCluster.Spec.S3Bucket == nil {
- cs.AWSCluster.Spec.S3Bucket = &infrav1.S3Bucket{}
- }
- cs.AWSCluster.Spec.S3Bucket.PresignedURLDuration = &metav1.Duration{Duration: 1 * time.Hour}
+ _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
+ g.Expect(err).To(BeNil())
+ })
- instance = &infrav1.Instance{
- ID: "myMachine",
- State: infrav1.InstanceStatePending,
- }
+ t.Run("should leverage a Cluster Object Store with presigned urls", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ getInstances(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
- //nolint:gosec
- presigned := "https://cluster-api-aws.s3.us-west-2.amazonaws.com/bootstrap-data.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA3SGQVQG7FGA6KKA6%2F20221104%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20221104T140227Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=b228dbec8c1008c80c162e1210e4503dceead1e4d4751b4d9787314fd6da4d55"
+ if cs.AWSCluster.Spec.S3Bucket == nil {
+ cs.AWSCluster.Spec.S3Bucket = &infrav1.S3Bucket{}
+ }
+ cs.AWSCluster.Spec.S3Bucket.PresignedURLDuration = &metav1.Duration{Duration: 1 * time.Hour}
- objectStoreSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return(presigned, nil).Times(1)
- ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any(), gomock.Any()).Return(instance, nil).AnyTimes()
- ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
- ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
- ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil)
+ instance = &infrav1.Instance{
+ ID: "myMachine",
+ State: infrav1.InstanceStatePending,
+ }
- ms.AWSMachine.ObjectMeta.Labels = map[string]string{
- clusterv1.MachineControlPlaneLabel: "",
- }
+ //nolint:gosec
+ presigned := "https://cluster-api-aws.s3.us-west-2.amazonaws.com/bootstrap-data.yaml?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA3SGQVQG7FGA6KKA6%2F20221104%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20221104T140227Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=b228dbec8c1008c80c162e1210e4503dceead1e4d4751b4d9787314fd6da4d55"
- _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
- g.Expect(err).To(BeNil())
- })
- })
+ objectStoreSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return(presigned, nil).Times(1)
+ ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any(), gomock.Any()).Return(instance, nil).AnyTimes()
+ ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
+ ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
+ ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil)
- t.Run("there's a node ref and a secret ARN", func(t *testing.T) {
- var instance *infrav1.Instance
- setNodeRef := func(t *testing.T, g *WithT) {
- t.Helper()
+ ms.AWSMachine.ObjectMeta.Labels = map[string]string{
+ clusterv1.MachineControlPlaneLabel: "",
+ }
- instance = &infrav1.Instance{
- ID: "myMachine",
- }
+ _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
+ g.Expect(err).To(BeNil())
+ })
+ })
- ms.Machine.Status.NodeRef = &corev1.ObjectReference{
- Kind: "Node",
- Name: "myMachine",
- APIVersion: "v1",
- }
+ t.Run("there's a node ref and a secret ARN", func(t *testing.T) {
+ var instance *infrav1.Instance
+ setNodeRef := func(t *testing.T, g *WithT) {
+ t.Helper()
- ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(instance, nil).AnyTimes()
- }
- useIgnition := func(t *testing.T, g *WithT) {
- t.Helper()
+ instance = &infrav1.Instance{
+ ID: "myMachine",
+ }
- ms.Machine.Spec.Bootstrap.DataSecretName = ptr.To[string]("bootstrap-data-ignition")
- ms.AWSMachine.Spec.CloudInit.SecretCount = 0
- ms.AWSMachine.Spec.CloudInit.SecretPrefix = ""
- }
+ ms.Machine.Status.NodeRef = &corev1.ObjectReference{
+ Kind: "Node",
+ Name: "myMachine",
+ APIVersion: "v1",
+ }
- t.Run("should delete the object if the instance is running", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- setNodeRef(t, g)
- useIgnition(t, g)
+ ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(instance, nil).AnyTimes()
+ }
- instance.State = infrav1.InstanceStateRunning
- ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
- objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
- ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
- ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil)
+ t.Run("should delete the object if the instance is running", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ setNodeRef(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
- _, _ = reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
- })
+ instance.State = infrav1.InstanceStateRunning
+ ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
+ objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
+ ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
+ ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil)
- t.Run("should delete the object if the instance is terminated", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- setNodeRef(t, g)
- useIgnition(t, g)
+ _, _ = reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
+ })
- instance.State = infrav1.InstanceStateTerminated
- objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
+ t.Run("should delete the object if the instance is terminated", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ setNodeRef(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
- _, _ = reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
- })
+ instance.State = infrav1.InstanceStateTerminated
+ objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
- t.Run("should delete the object if the instance is deleted", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- setNodeRef(t, g)
- useIgnition(t, g)
+ _, _ = reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
+ })
- instance.State = infrav1.InstanceStateRunning
- objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
- ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
+ t.Run("should delete the object if the instance is deleted", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ setNodeRef(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
- _, _ = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
- })
+ instance.State = infrav1.InstanceStateRunning
+ objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
+ ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
- t.Run("should delete the object if the AWSMachine is in a failure condition", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- setNodeRef(t, g)
- useIgnition(t, g)
+ _, _ = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
+ })
- // TODO: This seems to have no effect on the test result.
- ms.AWSMachine.Status.FailureReason = capierrors.MachineStatusErrorPtr(capierrors.UpdateMachineError)
+ t.Run("should delete the object if the AWSMachine is in a failure condition", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ setNodeRef(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
- objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
- ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
+ // TODO: This seems to have no effect on the test result.
+ ms.AWSMachine.Status.FailureReason = capierrors.MachineStatusErrorPtr(capierrors.UpdateMachineError)
- _, _ = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
+ objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
+ ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
+
+ _, _ = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
+ })
})
- })
- t.Run("there's only a secret ARN and no node ref", func(t *testing.T) {
- var instance *infrav1.Instance
+ t.Run("there's only a secret ARN and no node ref", func(t *testing.T) {
+ var instance *infrav1.Instance
- getInstances := func(t *testing.T, g *WithT) {
- t.Helper()
+ getInstances := func(t *testing.T, g *WithT) {
+ t.Helper()
- instance = &infrav1.Instance{
- ID: "myMachine",
+ instance = &infrav1.Instance{
+ ID: "myMachine",
+ }
+ ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(instance, nil).AnyTimes()
}
- ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(instance, nil).AnyTimes()
- }
- useIgnition := func(t *testing.T, g *WithT) {
- t.Helper()
-
- ms.Machine.Spec.Bootstrap.DataSecretName = ptr.To[string]("bootstrap-data-ignition")
- ms.AWSMachine.Spec.CloudInit.SecretCount = 0
- ms.AWSMachine.Spec.CloudInit.SecretPrefix = ""
- }
+ t.Run("should not delete the object if the instance is running", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ getInstances(t, g)
- t.Run("should not delete the object if the instance is running", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- getInstances(t, g)
+ instance.State = infrav1.InstanceStateRunning
+ ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
+ ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
+ ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil)
+ objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).MaxTimes(0)
+ _, _ = reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
+ })
- instance.State = infrav1.InstanceStateRunning
- ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
- ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
- ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil)
- objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).MaxTimes(0)
- _, _ = reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
- })
+ t.Run("should delete the object if the instance is terminated", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ getInstances(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
- t.Run("should delete the object if the instance is terminated", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- getInstances(t, g)
- useIgnition(t, g)
+ instance.State = infrav1.InstanceStateTerminated
+ objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
+ _, _ = reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
+ })
- instance.State = infrav1.InstanceStateTerminated
- objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
- _, _ = reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
- })
+ t.Run("should delete the object if the AWSMachine is deleted", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ getInstances(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
- t.Run("should delete the object if the AWSMachine is deleted", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- getInstances(t, g)
- useIgnition(t, g)
+ instance.State = infrav1.InstanceStateRunning
+ objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
+ ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
+ _, _ = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
+ })
- instance.State = infrav1.InstanceStateRunning
- objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
- ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
- _, _ = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
+ t.Run("should delete the object if the AWSMachine is in a failure condition", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ getInstances(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
+
+ // TODO: This seems to have no effect on the test result.
+ ms.AWSMachine.Status.FailureReason = capierrors.MachineStatusErrorPtr(capierrors.UpdateMachineError)
+ objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
+ ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
+ _, _ = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
+ })
})
- t.Run("should delete the object if the AWSMachine is in a failure condition", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- getInstances(t, g)
- useIgnition(t, g)
+ t.Run("there is an intermittent connection issue and no object could be created", func(t *testing.T) {
+ t.Run("should error if object could not be created", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ useIgnitionWithClusterObjectStore(t, g)
- // TODO: This seems to have no effect on the test result.
- ms.AWSMachine.Status.FailureReason = capierrors.MachineStatusErrorPtr(capierrors.UpdateMachineError)
- objectStoreSvc.EXPECT().Delete(gomock.Any()).Return(nil).Times(1)
- ec2Svc.EXPECT().TerminateInstance(gomock.Any()).Return(nil).AnyTimes()
- _, _ = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
+ ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil).AnyTimes()
+ objectStoreSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return("", errors.New("connection error")).Times(1)
+ _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
+ g.Expect(err).ToNot(BeNil())
+ g.Expect(err.Error()).To(ContainSubstring("connection error"))
+ })
})
})
- t.Run("there is an intermittent connection issue and no object could be created", func(t *testing.T) {
- useIgnition := func(t *testing.T, g *WithT) {
+ t.Run("when Ignition's StorageType is UnencryptedUserData", func(t *testing.T) {
+ useIgnitionAndUnencryptedUserData := func(t *testing.T, g *WithT) {
t.Helper()
ms.Machine.Spec.Bootstrap.DataSecretName = ptr.To[string]("bootstrap-data-ignition")
ms.AWSMachine.Spec.CloudInit.SecretCount = 0
ms.AWSMachine.Spec.CloudInit.SecretPrefix = ""
+ ms.AWSMachine.Spec.Ignition = &infrav1.Ignition{
+ Version: "2.3",
+ StorageType: infrav1.IgnitionStorageTypeOptionUnencryptedUserData,
+ }
}
+ t.Run("creating EC2 instances", func(t *testing.T) {
+ var instance *infrav1.Instance
- t.Run("should error if object could not be created", func(t *testing.T) {
- g := NewWithT(t)
- awsMachine := getAWSMachine()
- setup(t, g, awsMachine)
- defer teardown(t, g)
- useIgnition(t, g)
+ getInstances := func(t *testing.T, g *WithT) {
+ t.Helper()
+ ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil).AnyTimes()
+ }
+ t.Run("should NOT leverage a Cluster Object Store", func(t *testing.T) {
+ g := NewWithT(t)
+ awsMachine := getAWSMachine()
+ setup(t, g, awsMachine)
+ defer teardown(t, g)
+ getInstances(t, g)
+ useIgnitionAndUnencryptedUserData(t, g)
- ec2Svc.EXPECT().GetRunningInstanceByTags(gomock.Any()).Return(nil, nil).AnyTimes()
- objectStoreSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return("", errors.New("connection error")).Times(1)
- _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
- g.Expect(err).ToNot(BeNil())
- g.Expect(err.Error()).To(ContainSubstring("connection error"))
+ instance = &infrav1.Instance{
+ ID: "myMachine",
+ State: infrav1.InstanceStatePending,
+ }
+ fakeS3URL := "s3://foo"
+
+ // Expect no Cluster Object Store to be created.
+ objectStoreSvc.EXPECT().Create(gomock.Any(), gomock.Any()).Return(fakeS3URL, nil).Times(0)
+
+ ec2Svc.EXPECT().CreateInstance(gomock.Any(), gomock.Any(), gomock.Any()).Return(instance, nil).AnyTimes()
+ ec2Svc.EXPECT().GetInstanceSecurityGroups(gomock.Any()).Return(map[string][]string{"eid": {}}, nil).Times(1)
+ ec2Svc.EXPECT().GetCoreSecurityGroups(gomock.Any()).Return([]string{}, nil).Times(1)
+ ec2Svc.EXPECT().GetAdditionalSecurityGroupsIDs(gomock.Any()).Return(nil, nil)
+
+ ms.AWSMachine.ObjectMeta.Labels = map[string]string{
+ clusterv1.MachineControlPlaneLabel: "",
+ }
+ _, err := reconciler.reconcileNormal(context.Background(), ms, cs, cs, cs, cs)
+ g.Expect(err).To(BeNil())
+ })
})
})
})
diff --git a/docs/book/src/topics/ignition-support.md b/docs/book/src/topics/ignition-support.md
index fe2c5c90db..0d19e20bdf 100644
--- a/docs/book/src/topics/ignition-support.md
+++ b/docs/book/src/topics/ignition-support.md
@@ -12,8 +12,8 @@ the underlying OS for workload clusters.
Note
-This initial implementation uses Ignition **v2** and was tested with **Flatcar Container Linux** only.
-Future releases are expected to add Ignition **v3** support and cover more Linux distributions.
+This initial implementation used Ignition **v2** and was tested with **Flatcar Container Linux** only.
+Further releases added Ignition **v3** support.
@@ -23,35 +23,21 @@ For more generic information, see [Cluster API documentation on Ignition Bootstr
## Overview
-By default machine controller stores EC2 instance user data using SSM to store it encrypted, which underneath
-use multi part mime types, which are [unlikely to be supported](https://github.com/coreos/ignition/issues/1072)
-by Ignition.
+When using CloudInit for bootstrapping, by default the awsmachine controller stores EC2 instance user data using SSM to store it encrypted, which underneath uses multi part mime types.
+Unfortunately multi part mime types are [not supported](https://github.com/coreos/ignition/issues/1072) by Ignition. Moreover EC2 instance user data storage is also limited to 64 KB, which might not always be enough to provision Kubernetes controlplane because of the size of required certificates and configuration files.
-EC2 user data is also limited to 64 KB, which is often not enough to provision Kubernetes controlplane because
-of the size of required certificates and configuration files.
+To address these limitations, when using Ignition for bootstrapping, by default the awsmachine controller uses a Cluster Object Store (e.g. S3 Bucket), configured in the AWSCluster, to store user data,
+which will be then pulled by the instances during provisioning.
-To address those limitations CAPA can create and use S3 Bucket to store encrypted user data, which will be then
-pulled by the instances during provisioning.
+Optionally, when using Ignition for bootstrapping, users can optionally choose an alternative storageType for user data.
+For now the single available alternative is to store user data unencrypted directly in the EC2 instance user data.
+This storageType option is although discouraged unless strictly necessary, as it is not considered as safe as storing it in the S3 Bucket.
-## IAM Permissions
+## Prerequirements for enabling Ignition bootstrapping
-To manage S3 Buckets and objects inside them, CAPA controllers require additional IAM permissions.
+### Enabling EXP_BOOTSTRAP_FORMAT_IGNITION feature gate
-If you use `clusterawsadm` for managing the IAM roles, you can use the configuration below to create S3-related
-IAM permissions.
-
-``` yaml
-apiVersion: bootstrap.aws.infrastructure.cluster.x-k8s.io/v1beta1
-kind: AWSIAMConfiguration
-spec:
- s3Buckets:
- enable: true
-```
-
-See [Using clusterawsadm to fulfill prerequisites](./using-clusterawsadm-to-fulfill-prerequisites.md) for more
-details.
-
-## Enabling EXP_BOOTSTRAP_FORMAT_IGNITION feature gate
+In order to activate Ignition bootstrap you first need to enable its feature gate.
When deploying CAPA using `clusterctl`, make sure you set `BOOTSTRAP_FORMAT_IGNITION=true` and
`EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true `environment variables to enable experimental Ignition bootstrap
@@ -66,10 +52,31 @@ export EXP_BOOTSTRAP_FORMAT_IGNITION=true # Used by the AWS provider.
clusterctl init --infrastructure aws
```
-## Bucket and object management
+## Choosing a storage type for Ignition user data
+
+S3 is the default storage type when Ignition is enabled for managing machine's bootstrapping.
+But other methods can be choosen for storing Ignition user data.
+
+### Store Ignition config in a Cluster Object Store (e.g. S3 bucket)
+
+To explicitly set ClusterObjectStore as the storage type, provide the following config in the `AWSMachineTemplate`:
+```yaml
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AWSMachineTemplate
+metadata:
+ name: "test"
+spec:
+ template:
+ spec:
+ ignition:
+ storageType: ClusterObjectStore
+```
+
+#### Cluster Object Store and object management
When you want to use Ignition user data format for you machines, you need to configure your cluster to
-specify which S3 bucket to use. Controller will then make sure that the bucket exists and that required policies
+specify which Cluster Object Store to use. Controller will then check that the bucket already exists and that required policies
are in place.
See the configuration snippet below to learn how to configure `AWSCluster` to manage S3 bucket.
@@ -87,13 +94,31 @@ spec:
Buckets are safe to be reused between clusters.
-After successful machine provisioning, bootstrap data is removed from the bucket.
+After successful machine provisioning, the bootstrap data is removed from the object store.
+
+During cluster removal, if the Cluster Object Store is empty, it will be deleted as well.
+
+#### S3 IAM Permissions
-During cluster removal, if S3 bucket is empty, it will be removed as well.
+If you choose to use an S3 bucket as the Cluster Object Store, CAPA controllers require additional IAM permissions.
-## Bucket naming
+If you use `clusterawsadm` for managing the IAM roles, you can use the configuration below to create S3-related
+IAM permissions.
+
+``` yaml
+apiVersion: bootstrap.aws.infrastructure.cluster.x-k8s.io/v1beta1
+kind: AWSIAMConfiguration
+spec:
+ s3Buckets:
+ enable: true
+```
+
+See [Using clusterawsadm to fulfill prerequisites](./using-clusterawsadm-to-fulfill-prerequisites.md) for more
+details.
+
+#### Cluster Object Store naming
-Bucket naming must follow [S3 Bucket naming rules][bucket-naming-rules].
+Cluster Object Store and bucket naming must follow [S3 Bucket naming rules][bucket-naming-rules].
In addition, by default `clusterawsadm` creates IAM roles to only allow interacting with buckets with
`cluster-api-provider-aws-` prefix to reduce the permissions of CAPA controller, so all bucket names should
@@ -109,6 +134,30 @@ spec:
namePrefix: my-custom-secure-bucket-prefix-
```
+### Store Ignition config as UnencryptedUserData
+
+
+
+To instruct the controllers to store the user data directly in the EC2 instance user data unencrypted,
+ provide the following config in the `AWSMachineTemplate`:
+```yaml
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AWSMachineTemplate
+metadata:
+ name: "test"
+spec:
+ template:
+ spec:
+ ignition:
+ storageType: UnencryptedUserData
+```
+
+No further requirements are necessary.
+
## Supported bootstrap providers
At the moment only [CABPK][cabpk] is known to support producing bootstrap data in Ignition format.
diff --git a/test/e2e/suites/unmanaged/helpers_test.go b/test/e2e/suites/unmanaged/helpers_test.go
index ff6ba14202..39457e481c 100644
--- a/test/e2e/suites/unmanaged/helpers_test.go
+++ b/test/e2e/suites/unmanaged/helpers_test.go
@@ -21,6 +21,7 @@ package unmanaged
import (
"context"
+ "encoding/base64"
"fmt"
"io"
"net/http"
@@ -629,6 +630,22 @@ func assertInstanceMetadataOptions(instanceID string, expected infrav1.InstanceM
Expect(metadataOptions.HttpPutResponseHopLimit).To(HaveValue(Equal(expected.HTTPPutResponseHopLimit)))
}
+func assertUnencryptedUserDataIgnition(instanceID string, expected string) {
+ ginkgo.By(fmt.Sprintf("Finding EC2 instance with ID: %s", instanceID))
+ ec2Client := ec2.New(e2eCtx.AWSSession)
+ input := &ec2.DescribeInstanceAttributeInput{
+ Attribute: aws.String(ec2.InstanceAttributeNameUserData),
+ InstanceId: aws.String(instanceID[strings.LastIndex(instanceID, "/")+1:]),
+ }
+
+ result, err := ec2Client.DescribeInstanceAttribute(input)
+ Expect(err).ToNot(HaveOccurred(), "expected DescribeInstanceAttribute call to succeed")
+
+ userData, err := base64.StdEncoding.DecodeString(*result.UserData.Value)
+ Expect(err).ToNot(HaveOccurred(), "expected ec2 instance user data to be base64 decodable")
+ Expect(string(userData)).To(HaveValue(MatchJSON(expected)), "expected userdata to match")
+}
+
func terminateInstance(instanceID string) {
ginkgo.By(fmt.Sprintf("Terminating EC2 instance with ID: %s", instanceID))
ec2Client := ec2.New(e2eCtx.AWSSession)
@@ -868,3 +885,22 @@ func createPodWithEFSMount(clusterClient crclient.Client) {
}
Expect(clusterClient.Create(context.TODO(), pod)).NotTo(HaveOccurred())
}
+
+func getRawBootstrapDataWithFormat(c crclient.Client, m clusterv1.Machine) ([]byte, string, error) {
+ if m.Spec.Bootstrap.DataSecretName == nil {
+ return nil, "", fmt.Errorf("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil")
+ }
+
+ secret := &corev1.Secret{}
+ key := apimachinerytypes.NamespacedName{Namespace: m.Namespace, Name: *m.Spec.Bootstrap.DataSecretName}
+ if err := c.Get(context.TODO(), key, secret); err != nil {
+ return nil, "", fmt.Errorf("failed to retrieve bootstrap data secret for AWSMachine %s/%s: %v", m.Namespace, m.Name, err)
+ }
+
+ value, ok := secret.Data["value"]
+ if !ok {
+ return nil, "", fmt.Errorf("error retrieving bootstrap data: secret value key is missing")
+ }
+
+ return value, string(secret.Data["format"]), nil
+}
diff --git a/test/e2e/suites/unmanaged/unmanaged_functional_test.go b/test/e2e/suites/unmanaged/unmanaged_functional_test.go
index d8d9eaf5a7..aed9e02309 100644
--- a/test/e2e/suites/unmanaged/unmanaged_functional_test.go
+++ b/test/e2e/suites/unmanaged/unmanaged_functional_test.go
@@ -1070,6 +1070,46 @@ var _ = ginkgo.Context("[unmanaged] [functional]", func() {
awsCluster, err := GetAWSClusterByName(ctx, namespace.Name, clusterName)
Expect(err).To(BeNil())
+ ginkgo.By("Creating a MachineDeployment bootstrapped via Ignition with StorageType UnencryptedUserData")
+ unencryptedMDName := clusterName + "-md-unencrypted-userdata"
+ unencryptedUDMachineTemplate := makeAWSMachineTemplate(namespace.Name, unencryptedMDName, e2eCtx.E2EConfig.GetVariable(shared.AwsNodeMachineType), nil)
+ unencryptedUDMachineTemplate.Spec.Template.Spec.ImageLookupBaseOS = "flatcar-stable"
+ unencryptedUDMachineTemplate.Spec.Template.Spec.Ignition = &infrav1.Ignition{
+ StorageType: infrav1.IgnitionStorageTypeOptionUnencryptedUserData,
+ }
+
+ unencryptedUDMachineDeployment := makeMachineDeployment(namespace.Name, unencryptedMDName, clusterName, nil, int32(1))
+ // Use the same bootstrap configuration from one of the existing worker machines,
+ // as that already contains an ignition bootstrap configuration.
+ unencryptedUDMachineDeployment.Spec.Template.Spec.Bootstrap.ConfigRef = md[0].Spec.Template.Spec.Bootstrap.ConfigRef
+
+ framework.CreateMachineDeployment(ctx, framework.CreateMachineDeploymentInput{
+ Creator: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
+ MachineDeployment: unencryptedUDMachineDeployment,
+ BootstrapConfigTemplate: makeJoinBootstrapConfigTemplate(namespace.Name, unencryptedMDName),
+ InfraMachineTemplate: unencryptedUDMachineTemplate,
+ })
+
+ framework.WaitForMachineDeploymentNodesToExist(ctx, framework.WaitForMachineDeploymentNodesToExistInput{
+ Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
+ Cluster: result.Cluster,
+ MachineDeployment: unencryptedUDMachineDeployment,
+ }, e2eCtx.E2EConfig.GetIntervals("", "wait-worker-nodes")...)
+
+ unencryptedUDWorkerMachines := framework.GetMachinesByMachineDeployments(ctx, framework.GetMachinesByMachineDeploymentsInput{
+ Lister: e2eCtx.Environment.BootstrapClusterProxy.GetClient(),
+ ClusterName: clusterName,
+ Namespace: namespace.Name,
+ MachineDeployment: *unencryptedUDMachineDeployment,
+ })
+ Expect(len(unencryptedUDWorkerMachines)).To(Equal(1))
+ // There is only one machine.
+ m := unencryptedUDWorkerMachines[0]
+ machineUserData, userDataFormat, err := getRawBootstrapDataWithFormat(e2eCtx.Environment.BootstrapClusterProxy.GetClient(), m)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(userDataFormat).To(Equal("ignition"))
+ assertUnencryptedUserDataIgnition(*m.Spec.ProviderID, string(machineUserData))
+
ginkgo.By("Validating the s3 endpoint was created")
vpc, err := shared.GetVPCByName(e2eCtx, clusterName+"-vpc")
Expect(err).NotTo(HaveOccurred())