Skip to content

Commit

Permalink
Merge branch 'cfsturbo-client' into 'master' (merge request !36)
Browse files Browse the repository at this point in the history
feat[cfsturbo-v1.2.4]: auto install turbo client kernel module
  • Loading branch information
binqiniu committed May 25, 2023
2 parents ba76247 + 54f0a3c commit 317195f
Show file tree
Hide file tree
Showing 8 changed files with 128 additions and 34 deletions.
7 changes: 4 additions & 3 deletions cmd/cfsturbo/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import (
)

var (
endpoint = flag.String("endpoint", "unix://plugin/csi.sock", "CSI endpoint")
nodeID = flag.String("nodeID", "", "node ID")
endpoint = flag.String("endpoint", "unix://plugin/csi.sock", "CSI endpoint")
nodeID = flag.String("nodeID", "", "node ID")
componentType = flag.String("component_type", "", "component type")
)

func main() {
Expand All @@ -20,6 +21,6 @@ func main() {
glog.Fatal("nodeID is empty")
}

drv := cfsturbo.NewDriver(*nodeID, *endpoint)
drv := cfsturbo.NewDriver(*nodeID, *endpoint, *componentType)
drv.Run()
}
2 changes: 1 addition & 1 deletion deploy/cfsturbo/kubernetes/csi-controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ spec:
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: ccr.ccs.tencentyun.com/tkeimages/csi-tencentcloud-cfsturbo:v1.2.3
image: ccr.ccs.tencentyun.com/tkeimages/csi-tencentcloud-cfsturbo:v1.2.4
args:
- "--nodeID=$(NODE_ID)"
- "--endpoint=$(ADDRESS)"
Expand Down
8 changes: 7 additions & 1 deletion deploy/cfsturbo/kubernetes/csi-node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,11 @@ spec:
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: ccr.ccs.tencentyun.com/tkeimages/csi-tencentcloud-cfsturbo:v1.2.3
image: ccr.ccs.tencentyun.com/tkeimages/csi-tencentcloud-cfsturbo:v1.2.4
args :
- "--nodeID=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--component_type=node"
env:
- name: NODE_ID
valueFrom:
Expand All @@ -65,6 +66,8 @@ spec:
- name: global-mount-dir
mountPath: /etc/cfsturbo/global
mountPropagation: "Bidirectional"
- name: host-root
mountPath: /host
volumes:
- name: plugin-dir
hostPath:
Expand All @@ -82,3 +85,6 @@ spec:
hostPath:
path: /etc/cfsturbo/global
type: DirectoryOrCreate
- name: host-root
hostPath:
path: /
3 changes: 0 additions & 3 deletions docs/README_CFSTURBO_zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@ cfsturbo-csi-node-ztc7k 2/2 Running 0 10m

首先需要在[腾讯云文件存储控制台]( https://console.cloud.tencent.com/cfs )完成 turbo 类型文件系统的创建。

**注意**
使用 cfsturbo 插件挂载 turbo 类型文件系统,需预先使用 [CFS 客户端工具](https://console.cloud.tencent.com/cfs/fs/cvmInitialize) 在集群节点内安装对应客户端。

### Dynamic Provisioning

完成 sc、pvc 创建后,插件会在文件系统中创建如 `/cfs/$CLUSTER_ID/$pvname` 格式的子目录;完成 pod 创建后,插件会将上一步创建的子目录挂载到容器对应目录下;完成 pod、pvc 删除后,插件会根据 sc 中配置的回收策略来选择是否进行子目录清理。
Expand Down
14 changes: 10 additions & 4 deletions driver/cfsturbo/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@ limitations under the License.
package cfsturbo

import (
"os"

"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/golang/glog"
csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common"
"github.com/tencentcloud/kubernetes-csi-tencentcloud/driver/util"
"k8s.io/utils/mount"
"os"
)

type driver struct {
Expand All @@ -33,11 +34,11 @@ type driver struct {
const (
ClusterId = "CLUSTER_ID"
DriverName = "com.tencent.cloud.csi.cfsturbo"
DriverVersion = "v1.0.0"
DriverVersion = "cfsturbo"
)

func NewDriver(nodeID, endpoint string) *driver {
glog.Infof("Driver: %v version: %v", DriverName, DriverVersion)
func NewDriver(nodeID, endpoint, componentType string) *driver {
glog.Infof("Driver: %v version: %v region: %v", DriverName, DriverVersion, componentType)

csiDriver := csicommon.NewCSIDriver(DriverName, DriverVersion, nodeID)
csiDriver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
Expand All @@ -46,6 +47,11 @@ func NewDriver(nodeID, endpoint string) *driver {
csiDriver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
})
if componentType == "node" { //只在ds中启动
install := newInstaller()
// start a Goroutine loop
go install.loop()
}

return &driver{
endpoint: endpoint,
Expand Down
104 changes: 104 additions & 0 deletions driver/cfsturbo/install_cfs_client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package cfsturbo

import (
"fmt"
"os"
"syscall"
"time"

"os/exec"

"github.com/golang/glog"
)

const (
cfsTurboUrl = "https://cfsturbo-client-1251013638.cos.ap-guangzhou.myqcloud.com/tools/cfs_turbo_client_setup"
)

type installer struct {
}

func newInstaller() *installer {
glog.Infof("Installer is initialization...")
return &installer{}
}

// Install cfs turbo client
func (in *installer) Install() error {
//check cfs lustre core kmod install
err := exec.Command("/bin/bash", "-c", fmt.Sprintf("lsmod | grep %s", CFSTurboLustreKernelModule)).Run()
if err == nil {
//已经安装了内核模块
glog.Infof("node has alreay install kernel mod in node before mount cfs turbo lustre, skip install")
return nil
}
//没有的话直接安装
glog.Infof("node has not install kernel mod, start install")

// hold真实的文件系统 root
oldRootF, err := os.Open("/")
defer oldRootF.Close()
if err != nil {
return fmt.Errorf("hold root system err, message: %s", err.Error())
}

// exec chroot
err = syscall.Chroot("/host")
defer holdBack(oldRootF)
if err != nil {
return fmt.Errorf("chroot /host exec err, message: %s", err.Error())
}

// rm -fr
err = exec.Command("/bin/bash", "-c", "rm -fr /tmp/cfs-turbo*; rm -fr /tmp/cfs_turbo*; rm -fr turbo*").Run()
if err != nil {
// rm -fr fail
return fmt.Errorf("rm -fr /tmp/cfs-turbo err, message: %s", err.Error())
}
//wget
err = exec.Command("/bin/bash", "-c", fmt.Sprintf("wget -O %s %s", "/tmp/cfs_turbo_client_setup", cfsTurboUrl)).Run()
if err != nil {
// wget fail
return fmt.Errorf("wget cfs_turbo_client_setup err, message: %s", err.Error())
}
//chmod
err = exec.Command("/bin/bash", "-c", "chmod +x /tmp/cfs_turbo_client_setup").Run()
if err != nil {
// chmod fail
return fmt.Errorf("chmod +x for cfs_turbo_client_setup err, message: %s", err.Error())
}
//exec
err = exec.Command("/bin/bash", "-c", "/tmp/cfs_turbo_client_setup").Run()
if err != nil {
// exec fail
return fmt.Errorf("exec cfs_turbo_client_setup err, message: %s", err.Error())
}

glog.Infof(" install cfs client success !")
return nil
}

func holdBack(oldRootF *os.File) {
// switch back
err := oldRootF.Chdir()
if err != nil {
glog.Warningf("chdir() err: %v", err)
}
err = syscall.Chroot(".")
if err != nil {
glog.Warningf("chroot back err: %v", err)
}
}

func (in *installer) loop() error {
for {
err := in.Install()
if err == nil {
glog.Infof("install cfs client success, return")
return nil
}
// need loop again
glog.Errorf("loop install cfs client failed, message: %s", err.Error())
time.Sleep(5 * time.Minute)
}
}
23 changes: 1 addition & 22 deletions driver/util/secret_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,32 +2,11 @@ package util

import (
"os"

"github.com/golang/glog"
"github.com/tencentcloud/kubernetes-csi-tencentcloud/driver/cloud"
)

func GetSercet() (secretID, secretKey, token string, isTokenUpdate bool) {
secretID = os.Getenv("TENCENTCLOUD_API_SECRET_ID")
secretKey = os.Getenv("TENCENTCLOUD_API_SECRET_KEY")

if secretID != "" && secretKey != "" {
return
}

var err error
if secretID == "" || secretKey == "" {
glog.Info("Get secretID or secretKey from env failed, will use cloud norm!")
secretID, secretKey, token, isTokenUpdate, err = GetNormTmpSecret()
if err != nil {
glog.Errorf("GetNormTmpSecret error %v", err)
return "", "", "", false
}
}

return
return secretID, secretKey, "", false
}

func GetNormTmpSecret() (string, string, string, bool, error) {
return cloud.GetNormCredentialInstance().GetCredential()
}
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocm
github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8=
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14=
Expand Down

0 comments on commit 317195f

Please sign in to comment.