Merge pull request #19769 from zexi/container-lifecycle-devices

Container lifecycle devices

pkg/apis/compute/api.go

@@ -287,6 +287,7 @@ type IsolatedDeviceConfig struct {
 	NetworkIndex *int8  `json:"network_index"`
 	WireId       string `json:"wire_id"`
 	DiskIndex    *int8  `json:"disk_index"`
+	DevicePath   string `json:"device_path"`
 }
 
 type BaremetalDiskConfig struct {

pkg/apis/container.go

@@ -21,6 +21,25 @@ type ContainerKeyValue struct {
 	Value string `json:"value"`
 }
 
+type ContainerLifecyleHandlerType string
+
+const (
+	ContainerLifecyleHandlerTypeExec ContainerLifecyleHandlerType = "exec"
+)
+
+type ContainerLifecyleHandlerExecAction struct {
+	Command []string `json:"command"`
+}
+
+type ContainerLifecyleHandler struct {
+	Type ContainerLifecyleHandlerType        `json:"type"`
+	Exec *ContainerLifecyleHandlerExecAction `json:"exec"`
+}
+
+type ContainerLifecyle struct {
+	PostStart *ContainerLifecyleHandler `json:"post_start"`
+}
+
 type ContainerSpec struct {
 	// Image to use.
 	Image string `json:"image"`
@@ -37,9 +56,11 @@ type ContainerSpec struct {
 	// Enable lxcfs
 	EnableLxcfs bool `json:"enable_lxcfs"`
 	// Volume mounts
-	VolumeMounts []*ContainerVolumeMount `json:"volume_mounts"`
-	Capabilities *ContainerCapability    `json:"capabilities"`
-	Privileged   bool                    `json:"privileged"`
+	VolumeMounts       []*ContainerVolumeMount `json:"volume_mounts"`
+	Capabilities       *ContainerCapability    `json:"capabilities"`
+	Privileged         bool                    `json:"privileged"`
+	Lifecyle           *ContainerLifecyle      `json:"lifecyle"`
+	CgroupDevicesAllow []string                `json:"cgroup_devices_allow"`
 }
 
 type ContainerCapability struct {

pkg/cloudcommon/cmdline/parser.go

@@ -362,6 +362,8 @@ func ParseIsolatedDevice(desc string, idx int) (*compute.IsolatedDeviceConfig, e
 			dev.DevType = p
 		} else if strings.HasPrefix(p, "vendor=") {
 			dev.Vendor = p[len("vendor="):]
+		} else if strings.HasPrefix(p, "device_path=") {
+			dev.DevicePath = p[len("device_path="):]
 		} else {
 			dev.Model = p
 		}

pkg/cloudcommon/cmdline/parser_test.go

@@ -195,3 +195,49 @@ func TestFetchDiskConfigsByJSON(t *testing.T) {
 		})
 	}
 }
+
+func TestParseIsolatedDevice(t *testing.T) {
+	tests := []struct {
+		desc    string
+		idx     int
+		want    *compute.IsolatedDeviceConfig
+		wantErr bool
+	}{
+		{
+			desc: "device_path=/dev/nvme0",
+			idx:  0,
+			want: &compute.IsolatedDeviceConfig{
+				DevicePath: "/dev/nvme0",
+			},
+		},
+		{
+			desc: "GPU_HPC:device_path=/dev/nvme0",
+			idx:  0,
+			want: &compute.IsolatedDeviceConfig{
+				DevicePath: "/dev/nvme0",
+				Model:      "GPU_HPC",
+			},
+		},
+		{
+			desc: "GPU_HPC:device_path=/dev/nvme0:1d3ce781-2b64-4ee7-8d23-6bbcf478c54b",
+			idx:  0,
+			want: &compute.IsolatedDeviceConfig{
+				Id:         "1d3ce781-2b64-4ee7-8d23-6bbcf478c54b",
+				DevicePath: "/dev/nvme0",
+				Model:      "GPU_HPC",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			got, err := ParseIsolatedDevice(tt.desc, tt.idx)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseIsolatedDevice() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ParseIsolatedDevice() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/compute/container_drivers/lifecycle/doc.go

@@ -0,0 +1 @@
+package lifecycle // import "yunion.io/x/onecloud/pkg/compute/container_drivers/lifecycle"

pkg/compute/container_drivers/lifecycle/exec.go

@@ -0,0 +1,48 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lifecycle
+
+import (
+	"context"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func init() {
+	models.RegisterContainerLifecyleDriver(newExec())
+}
+
+type execDriver struct{}
+
+func newExec() models.IContainerLifecyleDriver {
+	return &execDriver{}
+}
+
+func (e execDriver) GetType() apis.ContainerLifecyleHandlerType {
+	return apis.ContainerLifecyleHandlerTypeExec
+}
+
+func (e execDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerLifecyleHandler) error {
+	if input.Exec == nil {
+		return httperrors.NewNotEmptyError("exec field")
+	}
+	if len(input.Exec.Command) == 0 {
+		return httperrors.NewNotEmptyError("command is required")
+	}
+	return nil
+}

pkg/compute/models/container_drivers.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"context"
+	"sync"
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -11,28 +12,67 @@ import (
 )
 
 var (
-	containerVolumeDrivers = make(map[apis.ContainerVolumeMountType]IContainerVolumeMountDriver)
-	containerDeviceDrivers = make(map[apis.ContainerDeviceType]IContainerDeviceDriver)
+	containerVolumeDrivers    = newContainerDrivers()
+	containerDeviceDrivers    = newContainerDrivers()
+	containerLifecycleDrivers = newContainerDrivers()
 )
 
-func RegisterContainerVolumeMountDriver(drv IContainerVolumeMountDriver) {
-	containerVolumeDrivers[drv.GetType()] = drv
+type containerDrivers struct {
+	drivers *sync.Map
 }
 
-func GetContainerVolumeMountDriver(typ apis.ContainerVolumeMountType) IContainerVolumeMountDriver {
-	drv, err := GetContainerVolumeMountDriverWithError(typ)
+func newContainerDrivers() *containerDrivers {
+	return &containerDrivers{
+		drivers: new(sync.Map),
+	}
+}
+
+func (cd *containerDrivers) GetWithError(typ string) (interface{}, error) {
+	drv, ok := cd.drivers.Load(typ)
+	if !ok {
+		return drv, httperrors.NewNotFoundError("not found driver by type %q", typ)
+	}
+	return drv, nil
+}
+
+func (cd *containerDrivers) Get(typ string) interface{} {
+	drv, err := cd.GetWithError(typ)
 	if err != nil {
 		panic(err.Error())
 	}
 	return drv
 }
 
-func GetContainerVolumeMountDriverWithError(typ apis.ContainerVolumeMountType) (IContainerVolumeMountDriver, error) {
-	drv, ok := containerVolumeDrivers[typ]
-	if !ok {
-		return nil, httperrors.NewNotFoundError("not found driver by type %q", typ)
+func (cd *containerDrivers) Register(typ string, drv interface{}) {
+	cd.drivers.Store(typ, drv)
+}
+
+func registerContainerDriver[K ~string, D any](drvs *containerDrivers, typ K, drv D) {
+	drvs.Register(string(typ), drv)
+}
+
+func getContainerDriver[K ~string, D any](drvs *containerDrivers, typ K) D {
+	return drvs.Get(string(typ)).(D)
+}
+
+func getContainerDriverWithError[K ~string, D any](drvs *containerDrivers, typ K) (D, error) {
+	drv, err := drvs.GetWithError(string(typ))
+	if err != nil {
+		return drv.(D), err
 	}
-	return drv, nil
+	return drv.(D), nil
+}
+
+func RegisterContainerVolumeMountDriver(drv IContainerVolumeMountDriver) {
+	registerContainerDriver(containerVolumeDrivers, drv.GetType(), drv)
+}
+
+func GetContainerVolumeMountDriver(typ apis.ContainerVolumeMountType) IContainerVolumeMountDriver {
+	return getContainerDriver[apis.ContainerVolumeMountType, IContainerVolumeMountDriver](containerVolumeDrivers, typ)
+}
+
+func GetContainerVolumeMountDriverWithError(typ apis.ContainerVolumeMountType) (IContainerVolumeMountDriver, error) {
+	return getContainerDriverWithError[apis.ContainerVolumeMountType, IContainerVolumeMountDriver](containerVolumeDrivers, typ)
 }
 
 type IContainerVolumeMountDriver interface {
@@ -42,23 +82,15 @@ type IContainerVolumeMountDriver interface {
 }
 
 func RegisterContainerDeviceDriver(drv IContainerDeviceDriver) {
-	containerDeviceDrivers[drv.GetType()] = drv
+	registerContainerDriver(containerDeviceDrivers, drv.GetType(), drv)
 }
 
 func GetContainerDeviceDriver(typ apis.ContainerDeviceType) IContainerDeviceDriver {
-	drv, err := GetContainerDeviceDriverWithError(typ)
-	if err != nil {
-		panic(err.Error())
-	}
-	return drv
+	return getContainerDriver[apis.ContainerDeviceType, IContainerDeviceDriver](containerDeviceDrivers, typ)
 }
 
 func GetContainerDeviceDriverWithError(typ apis.ContainerDeviceType) (IContainerDeviceDriver, error) {
-	drv, ok := containerDeviceDrivers[typ]
-	if !ok {
-		return nil, httperrors.NewNotFoundError("not found driver by type %q", typ)
-	}
-	return drv, nil
+	return getContainerDriverWithError[apis.ContainerDeviceType, IContainerDeviceDriver](containerDeviceDrivers, typ)
 }
 
 type IContainerDeviceDriver interface {
@@ -67,3 +99,20 @@ type IContainerDeviceDriver interface {
 	ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error)
 	ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error)
 }
+
+type IContainerLifecyleDriver interface {
+	GetType() apis.ContainerLifecyleHandlerType
+	ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerLifecyleHandler) error
+}
+
+func RegisterContainerLifecyleDriver(drv IContainerLifecyleDriver) {
+	registerContainerDriver(containerLifecycleDrivers, drv.GetType(), drv)
+}
+
+func GetContainerLifecyleDriver(typ apis.ContainerLifecyleHandlerType) IContainerLifecyleDriver {
+	return getContainerDriver[apis.ContainerLifecyleHandlerType, IContainerLifecyleDriver](containerLifecycleDrivers, typ)
+}
+
+func GetContainerLifecyleDriverWithError(typ apis.ContainerLifecyleHandlerType) (IContainerLifecyleDriver, error) {
+	return getContainerDriverWithError[apis.ContainerLifecyleHandlerType, IContainerLifecyleDriver](containerLifecycleDrivers, typ)
+}

pkg/compute/models/containers.go

@@ -148,13 +148,35 @@ func (m *SContainerManager) ValidateSpec(ctx context.Context, userCred mcclient.
 		}
 	}
 
+	if err := m.ValidateSpecLifecycle(ctx, userCred, spec); err != nil {
+		return errors.Wrap(err, "validate lifecycle")
+	}
+
 	return nil
 }
 
+func (m *SContainerManager) ValidateSpecLifecycle(ctx context.Context, cred mcclient.TokenCredential, spec *api.ContainerSpec) error {
+	if spec.Lifecyle == nil {
+		return nil
+	}
+	if err := m.ValidateSpecLifecyclePostStart(ctx, cred, spec.Lifecyle.PostStart); err != nil {
+		return errors.Wrap(err, "validate post start")
+	}
+	return nil
+}
+
+func (m *SContainerManager) ValidateSpecLifecyclePostStart(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerLifecyleHandler) error {
+	drv, err := GetContainerLifecyleDriverWithError(input.Type)
+	if err != nil {
+		return httperrors.NewInputParameterError("get lifecycle driver: %v", err)
+	}
+	return drv.ValidateCreateData(ctx, userCred, input)
+}
+
 func (m *SContainerManager) ValidateSpecDevice(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error) {
 	drv, err := GetContainerDeviceDriverWithError(dev.Type)
 	if err != nil {
-		return nil, httperrors.NewInputParameterError("get device drvice: %v", err)
+		return nil, httperrors.NewInputParameterError("get device driver: %v", err)
 	}
 	return drv.ValidateCreateData(ctx, userCred, pod, dev)
 }

pkg/compute/models/isolated_devices.go

@@ -556,6 +556,8 @@ func (manager *SIsolatedDeviceManager) _isValidDeviceInfo(config *api.IsolatedDe
 func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error {
 	if len(devConfig.Id) > 0 {
 		return manager.attachSpecificDeviceToGuest(ctx, guest, devConfig, userCred)
+	} else if len(devConfig.DevicePath) > 0 {
+		return manager.attachHostDeviceToGuestByDevicePath(ctx, guest, host, devConfig, userCred)
 	} else {
 		return manager.attachHostDeviceToGuestByModel(ctx, guest, host, devConfig, userCred)
 	}
@@ -573,6 +575,19 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C
 	return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex)
 }
 
+func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error {
+	if len(devConfig.Model) == 0 || len(devConfig.DevicePath) == 0 {
+		return fmt.Errorf("Model or DevicePath is empty: %#v", devConfig)
+	}
+	// if dev type is not nic, wire is empty string
+	devs, err := manager.findHostUnusedByDevAttr(devConfig.Model, "device_path", devConfig.DevicePath, host.Id, devConfig.WireId)
+	if err != nil || len(devs) == 0 {
+		return fmt.Errorf("Can't found model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id)
+	}
+	selectedDev := devs[0]
+	return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex)
+}
+
 func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error {
 	if len(devConfig.Model) == 0 {
 		return fmt.Errorf("Not found model from info: %#v", devConfig)
@@ -642,11 +657,15 @@ func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SI
 }
 
 func (manager *SIsolatedDeviceManager) findHostUnusedByDevConfig(model, devType, hostId, wireId string) ([]SIsolatedDevice, error) {
+	return manager.findHostUnusedByDevAttr(model, "dev_type", devType, hostId, wireId)
+}
+
+func (manager *SIsolatedDeviceManager) findHostUnusedByDevAttr(model, attrKey, attrVal, hostId, wireId string) ([]SIsolatedDevice, error) {
 	devs := make([]SIsolatedDevice, 0)
 	q := manager.findUnusedQuery()
 	q = q.Equals("model", model).Equals("host_id", hostId)
-	if devType != "" {
-		q.Equals("dev_type", devType)
+	if attrVal != "" {
+		q.Equals(attrKey, attrVal)
 	}
 	if wireId != "" {
 		wire := WireManager.FetchWireById(wireId)

pkg/compute/service/service.go

@@ -40,6 +40,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	_ "yunion.io/x/onecloud/pkg/compute/container_drivers/device"
+	_ "yunion.io/x/onecloud/pkg/compute/container_drivers/lifecycle"
 	_ "yunion.io/x/onecloud/pkg/compute/container_drivers/volume_mount"
 	_ "yunion.io/x/onecloud/pkg/compute/guestdrivers"
 	_ "yunion.io/x/onecloud/pkg/compute/hostdrivers"

pkg/hostman/container/lifecycle/doc.go

@@ -0,0 +1 @@
+package lifecycle // import "yunion.io/x/onecloud/pkg/hostman/container/lifecycle"