From 8127c9820f17ca59842be3d56f0cab3b5f615799 Mon Sep 17 00:00:00 2001 From: acekingke Date: Tue, 29 Nov 2022 09:48:19 +0800 Subject: [PATCH] *: support juicefs #745 --- Dockerfile.sidecar | 14 ++- api/v1alpha1/backup_types.go | 11 ++ api/v1alpha1/zz_generated.deepcopy.go | 20 ++++ backup/syncer/job.go | 94 ++++++++++++++++ .../crd/bases/mysql.radondb.com_backups.yaml | 16 +++ controllers/mysqlcluster_controller.go | 9 +- mysqlcluster/container/backup.go | 10 +- mysqlcluster/container/container.go | 10 ++ mysqlcluster/mysqlcluster.go | 21 +++- mysqlcluster/mysqlcluster_test.go | 20 ++++ mysqlcluster/syncer/headless_service.go | 8 +- mysqlcluster/syncer/pdb.go | 2 +- mysqlcluster/syncer/sshSecret.go | 59 ++++++++++ mysqlcluster/syncer/sshkey.go | 101 ++++++++++++++++++ mysqlcluster/syncer/statefulset_test.go | 7 ++ utils/constants.go | 14 +++ 16 files changed, 407 insertions(+), 9 deletions(-) create mode 100644 mysqlcluster/syncer/sshSecret.go create mode 100644 mysqlcluster/syncer/sshkey.go diff --git a/Dockerfile.sidecar b/Dockerfile.sidecar index 4eac67be..364e6695 100644 --- a/Dockerfile.sidecar +++ b/Dockerfile.sidecar @@ -43,13 +43,21 @@ RUN set -ex; \ ARG XTRABACKUP_PKG=percona-xtrabackup-24 RUN set -ex; \ apt-get update; \ - apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \ + apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc fuse jq openssh-server; \ wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \ dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \ apt-get update; \ apt-get install -y --no-install-recommends ${XTRABACKUP_PKG}; \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - +#ADD http://mirrors.woqutech.com/download/qfusion/files/bin/juicefs-1.0.0-rc1-linux-amd64 /usr/local/bin/juicefs +# COPY juicefs/juicefs /usr/local/bin/juicefs +RUN wget --no-check-certificate "https://d.juicefs.com/juicefs/releases/download/v1.0.2/juicefs-1.0.2-linux-amd64.tar.gz" && tar -zxf "juicefs-1.0.2-linux-amd64.tar.gz" ;\ + mv juicefs /usr/local/bin/juicefs; \ + chmod +x /usr/local/bin/juicefs ; mkdir -p /run/sshd; \ + mkdir -p /root/.ssh; \ + chmod 700 /root/.ssh WORKDIR / COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar -ENTRYPOINT ["sidecar"] +COPY script/sshd.sh /sshd.sh +COPY script/backup.sh /backup.sh +CMD [ "sidecar" ] diff --git a/api/v1alpha1/backup_types.go b/api/v1alpha1/backup_types.go index 3fc018a2..d177fbde 100644 --- a/api/v1alpha1/backup_types.go +++ b/api/v1alpha1/backup_types.go @@ -21,6 +21,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type JuiceOpt struct { + // sqlite or redis + JuiceMeta string `json:"juiceMeta"` + // backupSecrete name for S3 + BackupSecretName string `json:"backupSecretName"` + JuiceName string `json:"juiceName"` +} + // This is the backup Job CRD. // BackupSpec defines the desired state of Backup type BackupSpec struct { @@ -40,6 +48,9 @@ type BackupSpec struct { // +optional NFSServerAddress string `json:"nfsServerAddress,omitempty"` + // Represents the juicefs parameters which need. + // +optional + JuiceOpt *JuiceOpt `json:"juiceOpt,omitempty"` // ClusterName represents the cluster name to backup ClusterName string `json:"clusterName"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index db850e7c..444caadf 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -104,6 +104,11 @@ func (in *BackupList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { *out = *in + if in.JuiceOpt != nil { + in, out := &in.JuiceOpt, &out.JuiceOpt + *out = new(JuiceOpt) + **out = **in + } if in.HistoryLimit != nil { in, out := &in.HistoryLimit, &out.HistoryLimit *out = new(int32) @@ -174,6 +179,21 @@ func (in *ClusterCondition) DeepCopy() *ClusterCondition { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JuiceOpt) DeepCopyInto(out *JuiceOpt) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JuiceOpt. +func (in *JuiceOpt) DeepCopy() *JuiceOpt { + if in == nil { + return nil + } + out := new(JuiceOpt) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsOpts) DeepCopyInto(out *MetricsOpts) { *out = *in diff --git a/backup/syncer/job.go b/backup/syncer/job.go index b1e7e8a2..96b97972 100644 --- a/backup/syncer/job.go +++ b/backup/syncer/job.go @@ -17,13 +17,16 @@ limitations under the License. package syncer import ( + "context" "fmt" + "strings" "github.com/presslabs/controller-util/pkg/syncer" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" v1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1" @@ -33,6 +36,7 @@ import ( ) type jobSyncer struct { + client client.Client job *batchv1.Job backup *backup.Backup } @@ -50,6 +54,7 @@ func NewJobSyncer(c client.Client, backup *backup.Backup) syncer.Interface { } sync := &jobSyncer{ + client: c, job: obj, backup: backup, } @@ -174,6 +179,10 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec { MountPath: utils.XtrabckupLocal, }, } + } else if s.backup.Spec.JuiceOpt != nil { + // Deal it for juiceOpt + s.buildJuicefsBackPod(&in) + } else { // in.Containers[0].ImagePullPolicy = s.opt.ImagePullPolicy in.Containers[0].Args = []string{ @@ -238,3 +247,88 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec { } return in } + +func (s *jobSyncer) buildJuicefsBackPod(in *corev1.PodSpec) error { + // add volumn about pvc + var defMode int32 = 0600 + var err error + var cmdstr string + in.Volumes = []corev1.Volume{ + { + Name: utils.SShVolumnName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf("%s-ssh-key", s.backup.Spec.ClusterName), + DefaultMode: &defMode, + }, + }, + }, + } + + in.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + Name: utils.SShVolumnName, + MountPath: utils.SshVolumnPath, + }, + } + + // PodName.clusterName-mysql.Namespace + // sample-mysql-0.sample-mysql.default + hostname := fmt.Sprintf("%s.%s-mysql.%s", s.backup.Spec.HostName, s.backup.Spec.ClusterName, s.backup.Namespace) + if cmdstr, err = s.buildJuicefsCmd(s.backup.Spec.JuiceOpt.BackupSecretName); err != nil { + return err + } + + in.Containers[0].Command = []string{"bash", "-c", "--", `cp /etc/secret-ssh/* /root/.ssh +chmod 600 /root/.ssh/authorized_keys ;` + + strings.Join([]string{ + "ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", hostname, cmdstr, + }, " ")} + + return nil +} + +func (s *jobSyncer) buildJuicefsCmd(secName string) (string, error) { + juiceopt := s.backup.Spec.JuiceOpt + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secName, + Namespace: s.backup.Namespace, + }, + } + err := s.client.Get(context.TODO(), + types.NamespacedName{Namespace: s.backup.Namespace, + Name: secName}, secret) + + if err != nil { + return "", err + } + url, bucket := secret.Data["s3-endpoint"], secret.Data["s3-bucket"] + accesskey, secretkey := secret.Data["s3-access-key"], secret.Data["s3-secret-key"] + juicebucket := installBucket(string(url), string(bucket)) + cmdstr := fmt.Sprintf(`<