Rescue trapped resources controlled by ob-operator (#146)

* feat(rescue): add resource rescue machanism to save resources

.dockerignore

@@ -3,3 +3,4 @@
 !**/*.go
 !**/*.mod
 !**/*.sum
+distribution/

PROJECT

@@ -69,15 +69,6 @@ resources:
     defaulting: true
     validation: true
     webhookVersion: v1
-- api:
-    crdVersion: v1
-    namespaced: true
-  controller: true
-  domain: oceanbase.com
-  group: oceanbase
-  kind: OBClusterBackup
-  path: github.com/oceanbase/ob-operator/api/v1alpha1
-  version: v1alpha1
 - api:
     crdVersion: v1
     namespaced: true
@@ -93,7 +84,7 @@ resources:
   controller: true
   domain: oceanbase.com
   group: oceanbase
-  kind: OBClusterRestore
+  kind: OBTenantRestore
   path: github.com/oceanbase/ob-operator/api/v1alpha1
   version: v1alpha1
 - api:
@@ -102,16 +93,20 @@ resources:
   controller: true
   domain: oceanbase.com
   group: oceanbase
-  kind: OBTenantRestore
+  kind: OBTenantBackupPolicy
   path: github.com/oceanbase/ob-operator/api/v1alpha1
   version: v1alpha1
+  webhooks:
+    defaulting: true
+    validation: true
+    webhookVersion: v1
 - api:
     crdVersion: v1
     namespaced: true
   controller: true
   domain: oceanbase.com
   group: oceanbase
-  kind: OBTenantBackupPolicy
+  kind: OBTenantOperation
   path: github.com/oceanbase/ob-operator/api/v1alpha1
   version: v1alpha1
   webhooks:
@@ -124,7 +119,7 @@ resources:
   controller: true
   domain: oceanbase.com
   group: oceanbase
-  kind: OBTenantOperation
+  kind: OBResourceRescue
   path: github.com/oceanbase/ob-operator/api/v1alpha1
   version: v1alpha1
   webhooks:

api/v1alpha1/obresourcerescue_types.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// OBResourceRescueSpec defines the desired state of OBResourceRescue
+type OBResourceRescueSpec struct {
+	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+
+	TargetKind    string `json:"targetKind"`
+	TargetResName string `json:"targetResName"`
+	Type          string `json:"type"`
+	TargetGV      string `json:"targetGV,omitempty"`
+	Namespace     string `json:"namespace,omitempty"`
+	TargetStatus  string `json:"targetStatus,omitempty"`
+}
+
+// OBResourceRescueStatus defines the observed state of OBResourceRescue
+type OBResourceRescueStatus struct {
+	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+	Status string `json:"status"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.status"
+//+kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+
+// OBResourceRescue is the Schema for the obresourcerescues API
+type OBResourceRescue struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   OBResourceRescueSpec   `json:"spec,omitempty"`
+	Status OBResourceRescueStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// OBResourceRescueList contains a list of OBResourceRescue
+type OBResourceRescueList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []OBResourceRescue `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&OBResourceRescue{}, &OBResourceRescueList{})
+}

api/v1alpha1/obresourcerescue_webhook.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+// log is for logging in this package.
+var obresourcerescuelog = logf.Log.WithName("obresourcerescue-resource")
+
+var rescueTypeMapping = map[string]struct{}{
+	"delete": {},
+	"reset":  {},
+	"retry":  {},
+	"skip":   {},
+}
+
+func (r *OBResourceRescue) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(r).
+		Complete()
+}
+
+// TODO(user): EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+
+//+kubebuilder:webhook:path=/mutate-oceanbase-oceanbase-com-v1alpha1-obresourcerescue,mutating=true,failurePolicy=fail,sideEffects=None,groups=oceanbase.oceanbase.com,resources=obresourcerescues,verbs=create;update,versions=v1alpha1,name=mobresourcerescue.kb.io,admissionReviewVersions=v1
+
+var _ webhook.Defaulter = &OBResourceRescue{}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type
+func (r *OBResourceRescue) Default() {
+	r.Spec.Type = strings.ToLower(r.Spec.Type)
+}
+
+// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation.
+//+kubebuilder:webhook:path=/validate-oceanbase-oceanbase-com-v1alpha1-obresourcerescue,mutating=false,failurePolicy=fail,sideEffects=None,groups=oceanbase.oceanbase.com,resources=obresourcerescues,verbs=create;update,versions=v1alpha1,name=vobresourcerescue.kb.io,admissionReviewVersions=v1
+
+var _ webhook.Validator = &OBResourceRescue{}
+
+// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
+func (r *OBResourceRescue) ValidateCreate() (admission.Warnings, error) {
+	return r.validateMutation()
+}
+
+// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
+func (r *OBResourceRescue) ValidateUpdate(old runtime.Object) (admission.Warnings, error) {
+	oldRes := old.(*OBResourceRescue)
+	if r.Status.Status == "Successful" || r.Status.Status == "" {
+		if r.Spec.Type != oldRes.Spec.Type {
+			return nil, field.Invalid(field.NewPath("spec", "type"), r.Spec.Type, "type cannot be changed")
+		}
+		if r.Spec.TargetKind != oldRes.Spec.TargetKind {
+			return nil, field.Invalid(field.NewPath("spec", "targetKind"), r.Spec.TargetKind, "targetKind cannot be changed")
+		}
+		if r.Spec.TargetResName != oldRes.Spec.TargetResName {
+			return nil, field.Invalid(field.NewPath("spec", "targetResName"), r.Spec.TargetResName, "targetResName cannot be changed")
+		}
+		if r.Spec.TargetGV != oldRes.Spec.TargetGV {
+			return nil, field.Invalid(field.NewPath("spec", "targetGV"), r.Spec.TargetGV, "targetGV cannot be changed")
+		}
+		if r.Spec.Namespace != oldRes.Spec.Namespace {
+			return nil, field.Invalid(field.NewPath("spec", "namespace"), r.Spec.Namespace, "namespace cannot be changed")
+		}
+		if r.Spec.TargetStatus != oldRes.Spec.TargetStatus {
+			return nil, field.Invalid(field.NewPath("spec", "targetStatus"), r.Spec.TargetStatus, "targetStatus cannot be changed")
+		}
+	}
+	return r.validateMutation()
+}
+
+// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
+func (r *OBResourceRescue) ValidateDelete() (admission.Warnings, error) {
+	return nil, nil
+}
+
+func (r *OBResourceRescue) validateMutation() (admission.Warnings, error) {
+	var errList field.ErrorList
+	var warnings []string
+
+	if r.Spec.TargetKind == "" {
+		errList = append(errList, field.Required(field.NewPath("spec", "targetKind"), "targetKind is required"))
+	}
+	if r.Spec.TargetResName == "" {
+		errList = append(errList, field.Required(field.NewPath("spec", "targetResName"), "targetResName is required"))
+	}
+	if r.Spec.Type == "" {
+		errList = append(errList, field.Required(field.NewPath("spec", "type"), "type is required"))
+	} else if _, exist := rescueTypeMapping[r.Spec.Type]; !exist {
+		errList = append(errList, field.Invalid(field.NewPath("spec", "type"), r.Spec.Type, "unsupported rescue type"))
+	} else if r.Spec.Type == "reset" && r.Spec.TargetStatus == "" {
+		errList = append(errList, field.Required(field.NewPath("spec", "targetStatus"), "targetStatus is required when type is reset"))
+	}
+
+	return warnings, errList.ToAggregate()
+}

api/v1alpha1/obresourcerescue_webhook_test.go

@@ -0,0 +1,123 @@
+/*
+Copyright (c) 2023 OceanBase
+ob-operator is licensed under Mulan PSL v2.
+You can use this software according to the terms and conditions of the Mulan PSL v2.
+You may obtain a copy of Mulan PSL v2 at:
+         http://license.coscl.org.cn/MulanPSL2
+THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
+EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
+MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
+See the Mulan PSL v2 for more details.
+*/
+
+package v1alpha1
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
+)
+
+var _ = Describe("OBResourceRescueWebhook", func() {
+	It("Validate create", func() {
+		rescue := newOBResourceRescue()
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+	})
+
+	It("Validate wrong types", func() {
+		rescue := newOBResourceRescue()
+		rescue.Spec.Type = "wrong"
+		Expect(k8sClient.Create(ctx, rescue)).ShouldNot(Succeed())
+	})
+
+	It("Validate update", func() {
+		rescue := newOBResourceRescue()
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		rescue.Spec.Type = "reset"
+		rescue.Spec.TargetStatus = "running"
+		Expect(k8sClient.Update(ctx, rescue)).ShouldNot(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+	})
+
+	It("Validate target status field when type is reset", func() {
+		rescue := newOBResourceRescue()
+		rescue.Spec.Type = "reset"
+		Expect(k8sClient.Create(ctx, rescue)).ShouldNot(Succeed())
+		rescue.Spec.TargetStatus = "Running"
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+	})
+
+	It("Validate empty kind, resName, and type", func() {
+		rescue := newOBResourceRescue()
+		rescue.Spec.TargetKind = ""
+		Expect(k8sClient.Create(ctx, rescue)).ShouldNot(Succeed())
+		rescue.Spec.TargetKind = "OBCluster"
+		rescue.Spec.TargetResName = ""
+		Expect(k8sClient.Create(ctx, rescue)).ShouldNot(Succeed())
+		rescue.Spec.TargetResName = "test"
+		rescue.Spec.Type = ""
+		Expect(k8sClient.Create(ctx, rescue)).ShouldNot(Succeed())
+		rescue.Spec.Type = "delete"
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+	})
+
+	It("Validate forbidding to update a resource", func() {
+		rescue := newOBResourceRescue()
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		rescue.Spec.Type = "reset"
+		rescue.Spec.TargetStatus = "working"
+		Expect(k8sClient.Update(ctx, rescue)).ShouldNot(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+
+		rescue = newOBResourceRescue()
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		rescue.Spec.TargetKind = "OBTenant"
+		Expect(k8sClient.Update(ctx, rescue)).ShouldNot(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+
+		rescue = newOBResourceRescue()
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		rescue.Spec.TargetResName = "test2"
+		Expect(k8sClient.Update(ctx, rescue)).ShouldNot(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+
+		rescue = newOBResourceRescue()
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		rescue.Spec.Namespace = "test232"
+		Expect(k8sClient.Update(ctx, rescue)).ShouldNot(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+
+		rescue = newOBResourceRescue()
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		rescue.Spec.TargetGV = "oceanbase.oceanbase.com/v2"
+		Expect(k8sClient.Update(ctx, rescue)).ShouldNot(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+
+		rescue = newOBResourceRescue()
+		Expect(k8sClient.Create(ctx, rescue)).Should(Succeed())
+		rescue.Spec.TargetStatus = "failed"
+		Expect(k8sClient.Update(ctx, rescue)).ShouldNot(Succeed())
+		Expect(k8sClient.Delete(ctx, rescue)).Should(Succeed())
+	})
+})
+
+func newOBResourceRescue() *OBResourceRescue {
+	return &OBResourceRescue{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      rand.String(10),
+			Namespace: defaultNamespace,
+		},
+		Spec: OBResourceRescueSpec{
+			TargetKind:    "OBCluster",
+			TargetResName: "test",
+			Type:          "delete",
+		},
+		Status: OBResourceRescueStatus{
+			Status: "Successful",
+		},
+	}
+}

api/v1alpha1/webhook_suite_test.go

@@ -124,6 +124,9 @@ var _ = BeforeSuite(func() {
 	err = (&OBCluster{}).SetupWebhookWithManager(mgr)
 	Expect(err).NotTo(HaveOccurred())
 
+	err = (&OBResourceRescue{}).SetupWebhookWithManager(mgr)
+	Expect(err).NotTo(HaveOccurred())
+
 	//+kubebuilder:scaffold:webhook
 
 	go func() {