From 571951bbc091fce5a654fd44873aecd2223ac49e Mon Sep 17 00:00:00 2001 From: Mario Valderrama <15158349+avorima@users.noreply.github.com> Date: Tue, 16 Jul 2024 10:08:24 +0200 Subject: [PATCH] :sparkles: Image lookup (#179) **What is the purpose of this pull request/Why do we need it?** Implement image lookup at reconcile time according to label and name selectors. This allows to update the Kubernetes version only by modifying the CAPI resources instead of updating the image ID. **Issue #, if available:** **Description of changes:** **Special notes for your reviewer:** I added the following label to my images: `clusterapi=ionoscloud`. Tested by deploying the new template and updating `KubeadmControlPlane` and `MachineDeployment` versions (1.28.7 -> 1.29.2). ``` kubectl patch kcp test-control-plane --type merge -p '{"spec":{"template":{"spec":{"version":"1.29.2"}}}}' kubectl patch md test-worker --type merge -p '{"spec":{"version":"1.29.2"}}' ``` The PR is big as it is, so I'll look into an e2e test as a follow-up. **Checklist:** - [x] Documentation updated - [x] Unit Tests added - [ ] E2E Tests added - [x] Includes [emojis](https://github.com/kubernetes-sigs/kubebuilder-release-tools?tab=readme-ov-file#kubebuilder-project-versioning) --------- Co-authored-by: Jonas Riedel <138458199+jriedel-ionos@users.noreply.github.com> --- api/v1alpha1/ionoscloudmachine_types.go | 30 +- api/v1alpha1/ionoscloudmachine_types_test.go | 19 +- api/v1alpha1/zz_generated.deepcopy.go | 34 +- ...e.cluster.x-k8s.io_ionoscloudmachines.yaml | 27 +- ...r.x-k8s.io_ionoscloudmachinetemplates.yaml | 29 +- docs/custom-image.md | 19 + docs/quickstart.md | 14 +- internal/ionoscloud/client.go | 4 + internal/ionoscloud/client/client.go | 23 + internal/ionoscloud/client/client_test.go | 26 ++ internal/ionoscloud/clienttest/mock_client.go | 117 +++++ internal/service/cloud/image.go | 155 +++++++ internal/service/cloud/image_test.go | 215 +++++++++ internal/service/cloud/server.go | 12 +- templates/cluster-template-auto-image.yaml | 434 ++++++++++++++++++ 15 files changed, 1139 insertions(+), 19 deletions(-) create mode 100644 internal/service/cloud/image.go create mode 100644 internal/service/cloud/image_test.go create mode 100644 templates/cluster-template-auto-image.yaml diff --git a/api/v1alpha1/ionoscloudmachine_types.go b/api/v1alpha1/ionoscloudmachine_types.go index 5e96e94e..4adf7fba 100644 --- a/api/v1alpha1/ionoscloudmachine_types.go +++ b/api/v1alpha1/ionoscloudmachine_types.go @@ -210,15 +210,37 @@ type Volume struct { AvailabilityZone AvailabilityZone `json:"availabilityZone,omitempty"` // Image is the image to use for the VM. - //+required + //+kubebuilder:validation:XValidation:rule="self.id != '' || has(self.selector)",message="must provide either id or selector" Image *ImageSpec `json:"image"` } // ImageSpec defines the image to use for the VM. type ImageSpec struct { - // ID is the ID of the image to use for the VM. - //+kubebuilder:validation:MinLength=1 - ID string `json:"id"` + // ID is the ID of the image to use for the VM. Has precedence over selector. + // + //+optional + ID string `json:"id,omitempty"` + + // Selector is used to look up images by name and labels. + // Only images in the IonosCloudCluster's location are considered. + // + //+optional + Selector *ImageSelector `json:"selector,omitempty"` +} + +// ImageSelector defines label selectors for looking up images. +type ImageSelector struct { + // MatchLabels is a map of key/value pairs. + // + //+kubebuilder:validation:MinProperties=1 + MatchLabels map[string]string `json:"matchLabels"` + + // UseMachineVersion indicates whether to use the parent Machine's version field to look up image names. + // Enabled by default. + // + //+kubebuilder:default=true + //+optional + UseMachineVersion *bool `json:"useMachineVersion,omitempty"` } // IonosCloudMachineStatus defines the observed state of IonosCloudMachine. diff --git a/api/v1alpha1/ionoscloudmachine_types_test.go b/api/v1alpha1/ionoscloudmachine_types_test.go index a0b23e8d..8ee86382 100644 --- a/api/v1alpha1/ionoscloudmachine_types_test.go +++ b/api/v1alpha1/ionoscloudmachine_types_test.go @@ -311,16 +311,33 @@ var _ = Describe("IonosCloudMachine Tests", func() { m.Spec.Disk.Image = nil Expect(k8sClient.Create(context.Background(), m)).ToNot(Succeed()) }) - It("should fail none is set", func() { + It("should fail if no fields are set", func() { m := defaultMachine() m.Spec.Disk.Image.ID = "" Expect(k8sClient.Create(context.Background(), m)).ToNot(Succeed()) }) + It("should fail if no match labels are set", func() { + m := defaultMachine() + m.Spec.Disk.Image.Selector = &ImageSelector{} + Expect(k8sClient.Create(context.Background(), m)).ToNot(Succeed()) + }) It("should not fail if ID is set", func() { m := defaultMachine() m.Spec.Disk.Image.ID = "1eef-48ec-a246-a51a33aa4f3a" Expect(k8sClient.Create(context.Background(), m)).To(Succeed()) }) + It("should not fail if selector is set", func() { + m := defaultMachine() + m.Spec.Disk.Image.ID = "" + m.Spec.Disk.Image.Selector = &ImageSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + }, + } + Expect(k8sClient.Create(context.Background(), m)).To(Succeed()) + Expect(m.Spec.Disk.Image.Selector.UseMachineVersion).ToNot(BeNil()) + Expect(*m.Spec.Disk.Image.Selector.UseMachineVersion).To(BeTrue()) + }) }) }) Context("Additional Networks", func() { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 75adbd9c..b6978e0d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -27,9 +27,41 @@ import ( "sigs.k8s.io/cluster-api/errors" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageSelector) DeepCopyInto(out *ImageSelector) { + *out = *in + if in.MatchLabels != nil { + in, out := &in.MatchLabels, &out.MatchLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.UseMachineVersion != nil { + in, out := &in.UseMachineVersion, &out.UseMachineVersion + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageSelector. +func (in *ImageSelector) DeepCopy() *ImageSelector { + if in == nil { + return nil + } + out := new(ImageSelector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageSpec) DeepCopyInto(out *ImageSpec) { *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(ImageSelector) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageSpec. @@ -581,7 +613,7 @@ func (in *Volume) DeepCopyInto(out *Volume) { if in.Image != nil { in, out := &in.Image, &out.Image *out = new(ImageSpec) - **out = **in + (*in).DeepCopyInto(*out) } } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachines.yaml index 2419c0c1..cb757a76 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachines.yaml @@ -134,11 +134,32 @@ spec: properties: id: description: ID is the ID of the image to use for the VM. - minLength: 1 + Has precedence over selector. type: string - required: - - id + selector: + description: |- + Selector is used to look up images by name and labels. + Only images in the IonosCloudCluster's location are considered. + properties: + matchLabels: + additionalProperties: + type: string + description: MatchLabels is a map of key/value pairs. + minProperties: 1 + type: object + useMachineVersion: + default: true + description: |- + UseMachineVersion indicates whether to use the parent Machine's version field to look up image names. + Enabled by default. + type: boolean + required: + - matchLabels + type: object type: object + x-kubernetes-validations: + - message: must provide either id or selector + rule: self.id != '' || has(self.selector) name: description: Name is the name of the volume type: string diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachinetemplates.yaml index bcadd90a..98feb6bb 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachinetemplates.yaml @@ -144,12 +144,33 @@ spec: properties: id: description: ID is the ID of the image to use for - the VM. - minLength: 1 + the VM. Has precedence over selector. type: string - required: - - id + selector: + description: |- + Selector is used to look up images by name and labels. + Only images in the IonosCloudCluster's location are considered. + properties: + matchLabels: + additionalProperties: + type: string + description: MatchLabels is a map of key/value + pairs. + minProperties: 1 + type: object + useMachineVersion: + default: true + description: |- + UseMachineVersion indicates whether to use the parent Machine's version field to look up image names. + Enabled by default. + type: boolean + required: + - matchLabels + type: object type: object + x-kubernetes-validations: + - message: must provide either id or selector + rule: self.id != '' || has(self.selector) name: description: Name is the name of the volume type: string diff --git a/docs/custom-image.md b/docs/custom-image.md index 3bfce7c0..d54dc2bb 100644 --- a/docs/custom-image.md +++ b/docs/custom-image.md @@ -89,3 +89,22 @@ Now, you can copy the ID of your image and set it as the `IONOSCLOUD_MACHINE_IMA > [!IMPORTANT] > Please ensure to update the KUBERNETES_VERSION in your environment file (envfile) if it changes. + +### Enabling image lookup + +The provider is able to look up images by label and name for `IonosCloudMachine` resources that make use of an image +selector. +By default, the Kubernetes version of the parent Machine is used, so it's safe to reuse label keys and values for images +that contain the version in their name. + +Currently, it's only possible to label images using the REST API: + +```sh +curl -H "Authorization: Bearer " -H "Content-Type: application/json" -X POST \ + https://api.ionos.com/cloudapi/v6/images//labels -d '{"properties":{"key":"","value":""}}' +``` + +Now, you can set the key and value as `IONOSCLOUD_IMAGE_LABEL_KEY` and `IONOSCLOUD_IMAGE_LABEL_VALUE` environment variables. +Your custom image will then be used when using the [`auto-image`](/templates/cluster-template-auto-image.yaml) template. + +Given the correct labels the Kubernetes version is the only value that needs to be updated for version upgrades. diff --git a/docs/quickstart.md b/docs/quickstart.md index 7a8ba671..4db00a2a 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -164,10 +164,11 @@ export EXP_CLUSTER_RESOURCE_SET="true" We provide the following templates: -| Flavor | Template File | CRS File | -|----------------|----------------------------------------|-------------------------------| -| calico | templates/cluster-template-calico.yaml | templates/crs/cni/calico.yaml | -| default | templates/cluster-template.yaml | - | +| Flavor | Template File | CRS File | +| ---------------- | ---------------------------------------------- | ------------------------------- | +| default | templates/cluster-template.yaml | - | +| calico | templates/cluster-template-calico.yaml | templates/crs/cni/calico.yaml | +| auto-image | templates/cluster-template-auto-image.yaml | - | #### Flavor with Calico CNI @@ -197,6 +198,11 @@ $ clusterctl generate cluster dev-calico \ $ kubectl apply -f cluster.yaml ``` +#### Flavor with auto image lookup + +This template makes use of labels on custom built images. +Please refer to the [custom image docs](/docs/custom-image.md) for more information. + ### Custom Templates If you need anything specific that requires a more complex setup, we recommend to use custom templates: diff --git a/internal/ionoscloud/client.go b/internal/ionoscloud/client.go index 5bf92ff8..72c1a2e6 100644 --- a/internal/ionoscloud/client.go +++ b/internal/ionoscloud/client.go @@ -68,4 +68,8 @@ type Client interface { PatchNIC(ctx context.Context, datacenterID, serverID, nicID string, properties sdk.NicProperties) (string, error) // GetDatacenterLocationByID returns the location of the data center identified by datacenterID. GetDatacenterLocationByID(ctx context.Context, datacenterID string) (string, error) + // GetImage returns the image identified by imageID. + GetImage(ctx context.Context, imageID string) (*sdk.Image, error) + // ListLabels returns a list of all available resource labels. + ListLabels(ctx context.Context) ([]sdk.Label, error) } diff --git a/internal/ionoscloud/client/client.go b/internal/ionoscloud/client/client.go index e8ef3059..f5bf5eab 100644 --- a/internal/ionoscloud/client/client.go +++ b/internal/ionoscloud/client/client.go @@ -477,3 +477,26 @@ func (c *IonosCloudClient) GetDatacenterLocationByID(ctx context.Context, datace return *datacenter.Properties.Location, nil } + +// GetImage returns the image identified by imageID. +func (c *IonosCloudClient) GetImage(ctx context.Context, imageID string) (*sdk.Image, error) { + image, _, err := c.API.ImagesApi.ImagesFindById(ctx, imageID).Execute() + if err != nil { + return nil, fmt.Errorf(apiCallErrWrapper, err) + } + + return &image, nil +} + +// ListLabels returns a list of all available resource labels. +func (c *IonosCloudClient) ListLabels(ctx context.Context) ([]sdk.Label, error) { + labels, _, err := c.API.LabelsApi. + LabelsGet(ctx). + Depth(1). // always use depth 1 because we need the list item properties + Execute() + if err != nil { + return nil, fmt.Errorf(apiCallErrWrapper, err) + } + + return *labels.Items, nil +} diff --git a/internal/ionoscloud/client/client_test.go b/internal/ionoscloud/client/client_test.go index 6cc4f873..c9b31672 100644 --- a/internal/ionoscloud/client/client_test.go +++ b/internal/ionoscloud/client/client_test.go @@ -270,3 +270,29 @@ func TestWithDepth(t *testing.T) { }) } } + +func (s *IonosCloudClientTestSuite) TestGetImage() { + httpmock.RegisterResponder( + http.MethodGet, + catchAllMockURL, + httpmock.NewJsonResponderOrPanic(http.StatusOK, map[string]any{}), + ) + image, err := s.client.GetImage(s.ctx, exampleID) + s.NoError(err) + s.NotNil(image) +} + +func (s *IonosCloudClientTestSuite) TestListLabels() { + httpmock.RegisterResponder( + http.MethodGet, + catchAllMockURL, + httpmock.NewJsonResponderOrPanic(http.StatusOK, map[string]any{ + "items": []map[string]any{ + {}, + }, + }), + ) + labels, err := s.client.ListLabels(s.ctx) + s.NoError(err) + s.NotEmpty(labels) +} diff --git a/internal/ionoscloud/clienttest/mock_client.go b/internal/ionoscloud/clienttest/mock_client.go index 011f0d07..0a9b6d08 100644 --- a/internal/ionoscloud/clienttest/mock_client.go +++ b/internal/ionoscloud/clienttest/mock_client.go @@ -571,6 +571,65 @@ func (_c *MockClient_GetIPBlock_Call) RunAndReturn(run func(context.Context, str return _c } +// GetImage provides a mock function with given fields: ctx, imageID +func (_m *MockClient) GetImage(ctx context.Context, imageID string) (*ionoscloud.Image, error) { + ret := _m.Called(ctx, imageID) + + if len(ret) == 0 { + panic("no return value specified for GetImage") + } + + var r0 *ionoscloud.Image + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (*ionoscloud.Image, error)); ok { + return rf(ctx, imageID) + } + if rf, ok := ret.Get(0).(func(context.Context, string) *ionoscloud.Image); ok { + r0 = rf(ctx, imageID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*ionoscloud.Image) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, imageID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockClient_GetImage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetImage' +type MockClient_GetImage_Call struct { + *mock.Call +} + +// GetImage is a helper method to define mock.On call +// - ctx context.Context +// - imageID string +func (_e *MockClient_Expecter) GetImage(ctx interface{}, imageID interface{}) *MockClient_GetImage_Call { + return &MockClient_GetImage_Call{Call: _e.mock.On("GetImage", ctx, imageID)} +} + +func (_c *MockClient_GetImage_Call) Run(run func(ctx context.Context, imageID string)) *MockClient_GetImage_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *MockClient_GetImage_Call) Return(_a0 *ionoscloud.Image, _a1 error) *MockClient_GetImage_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockClient_GetImage_Call) RunAndReturn(run func(context.Context, string) (*ionoscloud.Image, error)) *MockClient_GetImage_Call { + _c.Call.Return(run) + return _c +} + // GetRequests provides a mock function with given fields: ctx, method, path func (_m *MockClient) GetRequests(ctx context.Context, method string, path string) ([]ionoscloud.Request, error) { ret := _m.Called(ctx, method, path) @@ -808,6 +867,64 @@ func (_c *MockClient_ListLANs_Call) RunAndReturn(run func(context.Context, strin return _c } +// ListLabels provides a mock function with given fields: ctx +func (_m *MockClient) ListLabels(ctx context.Context) ([]ionoscloud.Label, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for ListLabels") + } + + var r0 []ionoscloud.Label + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]ionoscloud.Label, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []ionoscloud.Label); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]ionoscloud.Label) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockClient_ListLabels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListLabels' +type MockClient_ListLabels_Call struct { + *mock.Call +} + +// ListLabels is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockClient_Expecter) ListLabels(ctx interface{}) *MockClient_ListLabels_Call { + return &MockClient_ListLabels_Call{Call: _e.mock.On("ListLabels", ctx)} +} + +func (_c *MockClient_ListLabels_Call) Run(run func(ctx context.Context)) *MockClient_ListLabels_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockClient_ListLabels_Call) Return(_a0 []ionoscloud.Label, _a1 error) *MockClient_ListLabels_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockClient_ListLabels_Call) RunAndReturn(run func(context.Context) ([]ionoscloud.Label, error)) *MockClient_ListLabels_Call { + _c.Call.Return(run) + return _c +} + // ListServers provides a mock function with given fields: ctx, datacenterID func (_m *MockClient) ListServers(ctx context.Context, datacenterID string) (*ionoscloud.Servers, error) { ret := _m.Called(ctx, datacenterID) diff --git a/internal/service/cloud/image.go b/internal/service/cloud/image.go new file mode 100644 index 00000000..a67df5c7 --- /dev/null +++ b/internal/service/cloud/image.go @@ -0,0 +1,155 @@ +/* +Copyright 2024 IONOS Cloud. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud + +import ( + "context" + "errors" + "fmt" + "strings" + + sdk "github.com/ionos-cloud/sdk-go/v6" + "k8s.io/apimachinery/pkg/labels" + + infrav1 "github.com/ionos-cloud/cluster-api-provider-ionoscloud/api/v1alpha1" + "github.com/ionos-cloud/cluster-api-provider-ionoscloud/internal/util/ptr" + "github.com/ionos-cloud/cluster-api-provider-ionoscloud/scope" +) + +var errMissingMachineVersion = errors.New("machine is missing version field") + +// imageMatchError indicates that either 0 or more than 1 images matched a selector. +// This error signals that user action is required. +type imageMatchError struct { + imageIDs []string + selector *infrav1.ImageSelector +} + +func (e imageMatchError) Error() string { + return fmt.Sprintf("found %d images matching selector %q", + len(e.imageIDs), labels.SelectorFromSet(e.selector.MatchLabels)) +} + +func (s *Service) lookupImageID(ctx context.Context, ms *scope.Machine) (string, error) { + imageSpec := ms.IonosMachine.Spec.Disk.Image + + if imageSpec.ID != "" { + return imageSpec.ID, nil + } + + location, err := s.getLocation(ctx, ms) + if err != nil { + return "", err + } + + images, err := s.lookupImagesBySelector(ctx, location, imageSpec.Selector) + if err != nil { + return "", err + } + + if ptr.Deref(imageSpec.Selector.UseMachineVersion, true) { + version := ptr.Deref(ms.Machine.Spec.Version, "") + if version == "" { + return "", errMissingMachineVersion + } + + images = filterImagesByName(images, version) + } + + if n := len(images); n != 1 { + return "", imageMatchError{imageIDs: getImageIDs(images), selector: imageSpec.Selector} + } + + return ptr.Deref(images[0].GetId(), ""), nil +} + +func (s *Service) lookupImagesBySelector( + ctx context.Context, location string, selector *infrav1.ImageSelector, +) ([]*sdk.Image, error) { + resourceLabels, err := s.ionosClient.ListLabels(ctx) + if err != nil { + return nil, err + } + + // mapping from image ID to labels + imageLabelMap := make(map[string]map[string]string) + + for _, label := range resourceLabels { + if ptr.Deref(label.GetProperties().GetResourceType(), "") != "image" { + continue + } + + id := ptr.Deref(label.GetProperties().GetResourceId(), "") + if _, ok := imageLabelMap[id]; !ok { + imageLabelMap[id] = make(map[string]string) + } + key := ptr.Deref(label.GetProperties().GetKey(), "") + value := ptr.Deref(label.GetProperties().GetValue(), "") + imageLabelMap[id][key] = value + } + + var imageIDs []string + for imageID, imageLabels := range imageLabelMap { + if mapContains(imageLabels, selector.MatchLabels) { + imageIDs = append(imageIDs, imageID) + } + } + + var images []*sdk.Image + for _, imageID := range imageIDs { + image, err := s.ionosClient.GetImage(ctx, imageID) + if err != nil { + return nil, err + } + + if ptr.Deref(image.GetProperties().GetLocation(), "") == location { + images = append(images, image) + } + } + + return images, nil +} + +func filterImagesByName(images []*sdk.Image, namePart string) []*sdk.Image { + var result []*sdk.Image + + for _, image := range images { + if strings.Contains(ptr.Deref(image.GetProperties().GetName(), ""), namePart) { + result = append(result, image) + } + } + + return result +} + +func getImageIDs(images []*sdk.Image) []string { + ids := make([]string, len(images)) + for i, image := range images { + ids[i] = *image.Id + } + return ids +} + +// check if b is wholly contained in a. +func mapContains[K, V comparable](a, b map[K]V) bool { + for k, bv := range b { + if av, ok := a[k]; !ok || bv != av { + return false + } + } + return true +} diff --git a/internal/service/cloud/image_test.go b/internal/service/cloud/image_test.go new file mode 100644 index 00000000..4db74623 --- /dev/null +++ b/internal/service/cloud/image_test.go @@ -0,0 +1,215 @@ +/* +Copyright 2024 IONOS Cloud. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud + +import ( + "context" + "fmt" + "slices" + "testing" + + "github.com/go-logr/logr" + sdk "github.com/ionos-cloud/sdk-go/v6" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + infrav1 "github.com/ionos-cloud/cluster-api-provider-ionoscloud/api/v1alpha1" + "github.com/ionos-cloud/cluster-api-provider-ionoscloud/internal/ionoscloud/clienttest" + "github.com/ionos-cloud/cluster-api-provider-ionoscloud/internal/util/ptr" +) + +type imageTestSuite struct { + ServiceTestSuite +} + +func TestImageService(t *testing.T) { + suite.Run(t, new(imageTestSuite)) +} + +func (s *imageTestSuite) SetupTest() { + s.ServiceTestSuite.SetupTest() + s.infraMachine.Spec.Disk.Image.ID = "" + s.infraMachine.Spec.Disk.Image.Selector = &infrav1.ImageSelector{ + MatchLabels: map[string]string{ + "test": "image", + }, + } +} + +func (s *imageTestSuite) TestLookupImageUseImageSpecID() { + s.infraMachine.Spec.Disk.Image.ID = "someid" + imageID, err := s.service.lookupImageID(s.ctx, s.machineScope) + s.NoError(err) + s.Equal("someid", imageID) +} + +func (s *imageTestSuite) TestLookupImageNoMatch() { + s.ionosClient.EXPECT().ListLabels(s.ctx).Return( + []sdk.Label{makeTestLabel("image", "image-1", "no", "match")}, nil, + ).Once() + s.ionosClient.EXPECT().GetDatacenterLocationByID(s.ctx, s.infraMachine.Spec.DatacenterID).Return("loc", nil).Once() + + _, err := s.service.lookupImageID(s.ctx, s.machineScope) + typedErr := new(imageMatchError) + s.ErrorAs(err, typedErr) + s.Empty(typedErr.imageIDs) +} + +func (s *imageTestSuite) TestLookupImageTooManyMatches() { + s.ionosClient.EXPECT().ListLabels(s.ctx).Return( + []sdk.Label{ + makeTestLabel("image", "image-1", "test", "image"), + makeTestLabel("image", "image-2", "test", "image"), + makeTestLabel("image", "image-3", "test", "image"), + }, nil, + ).Once() + s.ionosClient.EXPECT().GetDatacenterLocationByID(s.ctx, s.infraMachine.Spec.DatacenterID).Return("loc", nil).Once() + s.ionosClient.EXPECT().GetImage(s.ctx, "image-1").Return(makeTestImage("image-1", "img-foo-v1.2.3", "test"), nil).Once() + s.ionosClient.EXPECT().GetImage(s.ctx, "image-2").Return(s.makeTestImage("image-2", "img-foo-", "loc"), nil).Once() + s.ionosClient.EXPECT().GetImage(s.ctx, "image-3").Return(s.makeTestImage("image-3", "img-bar-", "loc"), nil).Once() + + _, err := s.service.lookupImageID(s.ctx, s.machineScope) + typedErr := new(imageMatchError) + s.ErrorAs(err, typedErr) + slices.Sort(typedErr.imageIDs) + s.Equal([]string{"image-2", "image-3"}, typedErr.imageIDs) +} + +func (s *imageTestSuite) TestLookupImageMissingMachineVersion() { + s.ionosClient.EXPECT().ListLabels(s.ctx).Return( + []sdk.Label{ + makeTestLabel("image", "image-1", "test", "image"), + }, nil, + ).Once() + s.ionosClient.EXPECT().GetImage(s.ctx, "image-1").Return(s.makeTestImage("image-1", "test", "loc"), nil).Once() + s.ionosClient.EXPECT().GetDatacenterLocationByID(s.ctx, s.infraMachine.Spec.DatacenterID).Return("loc", nil).Once() + + s.capiMachine.Spec.Version = ptr.To("") + + _, err := s.service.lookupImageID(s.ctx, s.machineScope) + s.ErrorIs(err, errMissingMachineVersion) +} + +func (s *imageTestSuite) TestLookupImageIgnoreMissingMachineVersion() { + s.ionosClient.EXPECT().ListLabels(s.ctx).Return( + []sdk.Label{ + makeTestLabel("image", "image-1", "test", "image"), + }, nil, + ).Once() + s.ionosClient.EXPECT().GetImage(s.ctx, "image-1").Return(s.makeTestImage("image-1", "test", "loc"), nil).Once() + s.ionosClient.EXPECT().GetDatacenterLocationByID(s.ctx, s.infraMachine.Spec.DatacenterID).Return("loc", nil).Once() + + s.infraMachine.Spec.Disk.Image.Selector.UseMachineVersion = ptr.To(false) + s.capiMachine.Spec.Version = ptr.To("") + + imageID, err := s.service.lookupImageID(s.ctx, s.machineScope) + s.NoError(err) + s.Equal("image-1", imageID) +} + +func (s *imageTestSuite) TestLookupImageOK() { + s.ionosClient.EXPECT().ListLabels(s.ctx).Return( + []sdk.Label{ + makeTestLabel("image", "image-1", "test", "image"), + }, nil, + ).Once() + s.ionosClient.EXPECT().GetDatacenterLocationByID(s.ctx, s.infraMachine.Spec.DatacenterID).Return("loc", nil).Once() + s.ionosClient.EXPECT().GetImage(s.ctx, "image-1").Return(s.makeTestImage("image-1", "img-", "loc"), nil).Once() + + imageID, err := s.service.lookupImageID(s.ctx, s.machineScope) + s.NoError(err) + s.Equal("image-1", imageID) +} + +func (s *imageTestSuite) makeTestImage(id, namePrefix, location string) *sdk.Image { + return makeTestImage(id, namePrefix+*s.capiMachine.Spec.Version, location) +} + +func TestFilterImagesByName(t *testing.T) { + images := []*sdk.Image{ + makeTestImage("image-1", "img-foo-v1.1.qcow2", "test"), + makeTestImage("image-2", "img-foo-v1.2.qcow2", "test"), + makeTestImage("image-3", "img-bar-1.2.3.qcow2", "test"), + } + expectImages := []*sdk.Image{ + makeTestImage("image-2", "img-foo-v1.2.qcow2", "test"), + makeTestImage("image-3", "img-bar-1.2.3.qcow2", "test"), + } + + require.Equal(t, expectImages, filterImagesByName(images, "1.2")) + require.Equal(t, images, filterImagesByName(images, "")) +} + +func TestLookupImagesBySelector(t *testing.T) { + ctx := context.Background() + ionosClient := clienttest.NewMockClient(t) + ionosClient.EXPECT().ListLabels(ctx).Return([]sdk.Label{ + // wrong resource type + makeTestLabel("server", "server-1", "foo", "bar"), + makeTestLabel("volume", "volume-1", "foo", "bar"), + // no match + makeTestLabel("image", "image-1", "baz", "qux"), + makeTestLabel("image", "image-2", "baz", "qux"), + makeTestLabel("image", "image-2", "lazy", "brown"), + // partial matche + makeTestLabel("image", "image-1", "over", "the"), + makeTestLabel("image", "image-2", "fox", "jumps"), + // full match + makeTestLabel("image", "image-1", "foo", "bar"), + makeTestLabel("image", "image-4", "foo", "bar"), + makeTestLabel("image", "image-1", "fox", "jumps"), + makeTestLabel("image", "image-4", "fox", "jumps"), + }, nil).Once() + ionosClient.EXPECT().GetImage(ctx, "image-1").Return(makeTestImage("image-1", "img-foo-v1.1.qcow2", "loc-1"), nil).Once() + ionosClient.EXPECT().GetImage(ctx, "image-4").Return(makeTestImage("image-4", "img-foo-v1.1.qcow2", "loc-2"), nil).Once() + + selector := &infrav1.ImageSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + "fox": "jumps", + }, + } + + expectImages := []*sdk.Image{makeTestImage("image-1", "img-foo-v1.1.qcow2", "loc-1")} + + svc, _ := NewService(ionosClient, logr.Discard()) + images, err := svc.lookupImagesBySelector(ctx, "loc-1", selector) + require.NoError(t, err) + require.Equal(t, expectImages, images) +} + +func makeTestImage(id, name, location string) *sdk.Image { + return &sdk.Image{ + Id: &id, + Properties: &sdk.ImageProperties{ + Name: &name, + Location: &location, + }, + } +} + +func makeTestLabel(typ, id, key, value string) sdk.Label { + return sdk.Label{ + Id: ptr.To(fmt.Sprintf("urn:label:%s:%s:%s", typ, id, key)), + Properties: &sdk.LabelProperties{ + Key: &key, + Value: &value, + ResourceId: &id, + ResourceType: &typ, + }, + } +} diff --git a/internal/service/cloud/server.go b/internal/service/cloud/server.go index 6b789045..aa471894 100644 --- a/internal/service/cloud/server.go +++ b/internal/service/cloud/server.go @@ -328,12 +328,19 @@ func (s *Service) createServer(ctx context.Context, secret *corev1.Secret, ms *s return fmt.Errorf("unable to parse LAN ID: %w", err) } + imageID, err := s.lookupImageID(ctx, ms) + if err != nil { + // TODO(avorima): Notify users of invalid image selector + return fmt.Errorf("image lookup: %w", err) + } + renderedData := s.renderUserData(ms, string(bootstrapData)) copySpec := ms.IonosMachine.Spec.DeepCopy() entityParams := serverEntityParams{ boostrapData: renderedData, machineSpec: *copySpec, lanID: int32(lanID), + imageID: imageID, } server, requestLocation, err := s.ionosClient.CreateServer( @@ -381,6 +388,7 @@ type serverEntityParams struct { boostrapData string machineSpec infrav1.IonosCloudMachineSpec lanID int32 + imageID string } // buildServerEntities returns the server entities for the expected cloud server resource. @@ -396,8 +404,8 @@ func (s *Service) buildServerEntities(ms *scope.Machine, params serverEntityPara }, } - if machineSpec.Disk.Image.ID != "" { - bootVolume.Properties.Image = &machineSpec.Disk.Image.ID + if params.imageID != "" { + bootVolume.Properties.Image = ¶ms.imageID } serverVolumes := sdk.AttachedVolumes{ diff --git a/templates/cluster-template-auto-image.yaml b/templates/cluster-template-auto-image.yaml new file mode 100644 index 00000000..48f1deb7 --- /dev/null +++ b/templates/cluster-template-auto-image.yaml @@ -0,0 +1,434 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" +type: Opaque +stringData: + token: "${IONOS_TOKEN}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: IonosCloudCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + controlPlaneEndpoint: + host: ${CONTROL_PLANE_ENDPOINT_HOST:-${CONTROL_PLANE_ENDPOINT_IP}} + port: ${CONTROL_PLANE_ENDPOINT_PORT:-6443} + location: ${CONTROL_PLANE_ENDPOINT_LOCATION} + credentialsRef: + name: "${CLUSTER_NAME}-credentials" +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: IonosCloudMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + users: + - name: root + sshAuthorizedKeys: [${IONOSCLOUD_MACHINE_SSH_KEYS}] + ntp: + enabled: true + servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + - 2.de.pool.ntp.org + - 3.de.pool.ntp.org + files: + - path: /etc/ssh/sshd_config.d/ssh-audit_hardening.conf + owner: root:root + permissions: '0644' + content: | + # Restrict key exchange, cipher, and MAC algorithms, as per sshaudit.com + # hardening guide. + KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org,gss-curve25519-sha256-,diffie-hellman-group16-sha512,gss-group16-sha512-,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com + HostKeyAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + CASignatureAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + GSSAPIKexAlgorithms gss-curve25519-sha256-,gss-group16-sha512- + HostbasedAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + PubkeyAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + - path: /etc/sysctl.d/k8s.conf + content: | + fs.inotify.max_user_watches = 65536 + net.netfilter.nf_conntrack_max = 1000000 + - path: /etc/modules-load.d/k8s.conf + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + ip_vs_sed + # Crictl config + - path: /etc/crictl.yaml + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock + timeout: 10 + - path: /etc/kubernetes/manifests/kube-vip.yaml + owner: root:root + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: cp_enable + value: "true" + - name: vip_interface + value: ${VIP_NETWORK_INTERFACE=""} + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: port + value: "${CONTROL_PLANE_ENDPOINT_PORT:-6443}" + - name: vip_arp + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + image: ghcr.io/kube-vip/kube-vip:v0.7.1 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + - localhost + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + type: FileOrCreate + name: kubeconfig + status: {} + - path: /etc/kube-vip-prepare.sh + content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + exit 0 + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + permissions: "0700" + + # CSI Metadata config + - content: | + { + "datacenter-id": "${IONOSCLOUD_DATACENTER_ID}" + } + owner: root:root + path: /etc/ie-csi/cfg.json + permissions: '0644' + + - content: | + #!/bin/bash + set -e + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + exit 0 + fi + + NODE_IPv4_ADDRESS=$(ip -j addr show dev ens6 | jq -r '.[].addr_info[] | select(.family == "inet") | select(.scope=="global") | select(.dynamic) | .local') + if [[ $NODE_IPv4_ADDRESS ]]; then + sed -i '$ s/$/ --node-ip '"$NODE_IPv4_ADDRESS"'/' /etc/default/kubelet + fi + # IPv6 currently not set, the ip is not set then this runs. Needs to be waited for. + NODE_IPv6_ADDRESS=$(ip -j addr show dev ens6 | jq -r '.[].addr_info[] | select(.family == "inet6") | select(.scope=="global") | .local') + if [[ $NODE_IPv6_ADDRESS ]]; then + sed -i '$ s/$/ --node-ip '"$NODE_IPv6_ADDRESS"'/' /etc/default/kubelet + fi + owner: root:root + path: /etc/set-node-ip.sh + permissions: '0700' + + preKubeadmCommands: + - systemctl restart systemd-networkd.service systemd-modules-load.service systemd-journald containerd + # disable swap + - swapoff -a + - sed -i '/ swap / s/^/#/' /etc/fstab + - sysctl --system + - /etc/kube-vip-prepare.sh + # workaround 1.29 IP issue + - /etc/set-node-ip.sh + postKubeadmCommands: + - > + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + - > + systemctl disable --now udisks2 multipathd motd-news.timer fwupd-refresh.timer + packagekit ModemManager snapd snapd.socket snapd.apparmor snapd.seeded + # INFO(schegi-ionos): We decided to not remove this for now, since removing this would require the ccm to be installed for cluster-api + # to continue after the first node. + - export system_uuid=$(kubectl --kubeconfig /etc/kubernetes/kubelet.conf get node $(hostname) -ojsonpath='{..systemUUID }') + - > + kubectl --kubeconfig /etc/kubernetes/kubelet.conf + patch node $(hostname) + --type strategic -p '{"spec": {"providerID": "ionos://'$${system_uuid}'"}}' + - rm /etc/ssh/ssh_host_* + - ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N "" + - ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" + - sed -i 's/^\#HostKey \/etc\/ssh\/ssh_host_\(rsa\|ed25519\)_key$/HostKey \/etc\/ssh\/ssh_host_\1_key/g' /etc/ssh/sshd_config + - awk '$5 >= 3071' /etc/ssh/moduli > /etc/ssh/moduli.safe + - mv /etc/ssh/moduli.safe /etc/ssh/moduli + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - apt-get update + - DEBIAN_FRONTEND=noninteractive apt-get install -q -y netfilter-persistent iptables-persistent + - service netfilter-persistent save + - systemctl restart sshd + initConfiguration: + localAPIEndpoint: + bindPort: ${CONTROL_PLANE_ENDPOINT_PORT:-6443} + nodeRegistration: + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + joinConfiguration: + nodeRegistration: + criSocket: unix:///run/containerd/containerd.sock + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + version: "${KUBERNETES_VERSION}" +--- +kind: IonosCloudMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + datacenterID: ${IONOSCLOUD_DATACENTER_ID} + numCores: ${IONOSCLOUD_MACHINE_NUM_CORES:-4} + memoryMB: ${IONOSCLOUD_MACHINE_MEMORY_MB:-8192} + disk: + image: + selector: + matchLabels: + ${IONOSCLOUD_IMAGE_LABEL_KEY}: ${IONOSCLOUD_IMAGE_LABEL_VALUE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-workers" + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + node-role.kubernetes.io/node: "" + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-worker" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-worker" + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: IonosCloudMachineTemplate +metadata: + name: "${CLUSTER_NAME}-worker" +spec: + template: + spec: + datacenterID: ${IONOSCLOUD_DATACENTER_ID} + numCores: ${IONOSCLOUD_MACHINE_NUM_CORES:-2} + memoryMB: ${IONOSCLOUD_MACHINE_MEMORY_MB:-4096} + disk: + image: + selector: + matchLabels: + ${IONOSCLOUD_IMAGE_LABEL_KEY}: ${IONOSCLOUD_IMAGE_LABEL_VALUE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-worker" +spec: + template: + spec: + users: + - name: root + sshAuthorizedKeys: [${IONOSCLOUD_MACHINE_SSH_KEYS}] + ntp: + enabled: true + servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + - 2.de.pool.ntp.org + - 3.de.pool.ntp.org + files: + - path: /etc/ssh/sshd_config.d/ssh-audit_hardening.conf + owner: root:root + permissions: '0644' + content: | + # Restrict key exchange, cipher, and MAC algorithms, as per sshaudit.com + # hardening guide. + KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org,gss-curve25519-sha256-,diffie-hellman-group16-sha512,gss-group16-sha512-,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com + HostKeyAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + CASignatureAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + GSSAPIKexAlgorithms gss-curve25519-sha256-,gss-group16-sha512- + HostbasedAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + PubkeyAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + - path: /etc/sysctl.d/k8s.conf + content: | + fs.inotify.max_user_watches = 65536 + net.netfilter.nf_conntrack_max = 1000000 + - path: /etc/modules-load.d/k8s.conf + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + ip_vs_sed + # Crictl config + - path: /etc/crictl.yaml + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock + timeout: 10 + # CSI Metadata config + - content: | + { + "datacenter-id": "${IONOSCLOUD_DATACENTER_ID}" + } + owner: root:root + path: /etc/ie-csi/cfg.json + permissions: '0644' + preKubeadmCommands: + - systemctl restart systemd-networkd.service systemd-modules-load.service systemd-journald containerd + # disable swap + - swapoff -a + - sed -i '/ swap / s/^/#/' /etc/fstab + - sysctl --system + postKubeadmCommands: + - > + systemctl disable --now udisks2 multipathd motd-news.timer fwupd-refresh.timer + packagekit ModemManager snapd snapd.socket snapd.apparmor snapd.seeded + # INFO(schegi-ionos): We decided to not remove this for now, since removing this would require the ccm to be + # installed for cluster-api to continue after the first node. + - export system_uuid=$(kubectl --kubeconfig /etc/kubernetes/kubelet.conf get node $(hostname) -ojsonpath='{..systemUUID }') + - > + kubectl --kubeconfig /etc/kubernetes/kubelet.conf + patch node $(hostname) + --type strategic -p '{"spec": {"providerID": "ionos://'$${system_uuid}'"}}' + - rm /etc/ssh/ssh_host_* + - ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N "" + - ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" + - sed -i 's/^\#HostKey \/etc\/ssh\/ssh_host_\(rsa\|ed25519\)_key$/HostKey \/etc\/ssh\/ssh_host_\1_key/g' /etc/ssh/sshd_config + - awk '$5 >= 3071' /etc/ssh/moduli > /etc/ssh/moduli.safe + - mv /etc/ssh/moduli.safe /etc/ssh/moduli + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - apt-get update + - DEBIAN_FRONTEND=noninteractive apt-get install -q -y netfilter-persistent iptables-persistent + - service netfilter-persistent save + - systemctl restart sshd + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + criSocket: unix:///run/containerd/containerd.sock