From 366340e74c61ab6ff4aabc875374fcb87c32c5b7 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Thu, 21 Mar 2024 16:54:55 +0800 Subject: [PATCH] feat(region,scheduler,host): container lifecycle and cgroup devices permissions --- pkg/apis/compute/api.go | 1 + pkg/apis/container.go | 27 +++++- pkg/cloudcommon/cmdline/parser.go | 2 + pkg/cloudcommon/cmdline/parser_test.go | 46 +++++++++ .../container_drivers/lifecycle/doc.go | 1 + .../container_drivers/lifecycle/exec.go | 48 ++++++++++ pkg/compute/models/container_drivers.go | 93 ++++++++++++++----- pkg/compute/models/containers.go | 24 ++++- pkg/compute/models/isolated_devices.go | 23 ++++- pkg/compute/service/service.go | 1 + pkg/hostman/container/lifecycle/doc.go | 1 + pkg/hostman/container/lifecycle/exec.go | 44 +++++++++ pkg/hostman/container/lifecycle/lifecycle.go | 30 ++++++ pkg/hostman/guestman/pod.go | 53 ++++++++++- pkg/mcclient/options/compute/containers.go | 49 ++++++---- .../predicates/isolated_device_predicate.go | 19 ++++ pkg/scheduler/cache/candidate/base.go | 15 +++ pkg/scheduler/core/types.go | 2 + pkg/scheduler/test/mock/core.go | 8 ++ pkg/util/pod/pod.go | 38 ++++---- 20 files changed, 458 insertions(+), 67 deletions(-) create mode 100644 pkg/compute/container_drivers/lifecycle/doc.go create mode 100644 pkg/compute/container_drivers/lifecycle/exec.go create mode 100644 pkg/hostman/container/lifecycle/doc.go create mode 100644 pkg/hostman/container/lifecycle/exec.go create mode 100644 pkg/hostman/container/lifecycle/lifecycle.go diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index 305c778eb88..9359a03008a 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -287,6 +287,7 @@ type IsolatedDeviceConfig struct { NetworkIndex *int8 `json:"network_index"` WireId string `json:"wire_id"` DiskIndex *int8 `json:"disk_index"` + DevicePath string `json:"device_path"` } type BaremetalDiskConfig struct { diff --git a/pkg/apis/container.go b/pkg/apis/container.go index 239540670f5..a1735fbfc62 100644 --- a/pkg/apis/container.go +++ b/pkg/apis/container.go @@ -21,6 +21,25 @@ type ContainerKeyValue struct { Value string `json:"value"` } +type ContainerLifecyleHandlerType string + +const ( + ContainerLifecyleHandlerTypeExec ContainerLifecyleHandlerType = "exec" +) + +type ContainerLifecyleHandlerExecAction struct { + Command []string `json:"command"` +} + +type ContainerLifecyleHandler struct { + Type ContainerLifecyleHandlerType `json:"type"` + Exec *ContainerLifecyleHandlerExecAction `json:"exec"` +} + +type ContainerLifecyle struct { + PostStart *ContainerLifecyleHandler `json:"post_start"` +} + type ContainerSpec struct { // Image to use. Image string `json:"image"` @@ -37,9 +56,11 @@ type ContainerSpec struct { // Enable lxcfs EnableLxcfs bool `json:"enable_lxcfs"` // Volume mounts - VolumeMounts []*ContainerVolumeMount `json:"volume_mounts"` - Capabilities *ContainerCapability `json:"capabilities"` - Privileged bool `json:"privileged"` + VolumeMounts []*ContainerVolumeMount `json:"volume_mounts"` + Capabilities *ContainerCapability `json:"capabilities"` + Privileged bool `json:"privileged"` + Lifecyle *ContainerLifecyle `json:"lifecyle"` + CgroupDevicesAllow []string `json:"cgroup_devices_allow"` } type ContainerCapability struct { diff --git a/pkg/cloudcommon/cmdline/parser.go b/pkg/cloudcommon/cmdline/parser.go index 07cdb28a023..0e4a3c966a4 100644 --- a/pkg/cloudcommon/cmdline/parser.go +++ b/pkg/cloudcommon/cmdline/parser.go @@ -362,6 +362,8 @@ func ParseIsolatedDevice(desc string, idx int) (*compute.IsolatedDeviceConfig, e dev.DevType = p } else if strings.HasPrefix(p, "vendor=") { dev.Vendor = p[len("vendor="):] + } else if strings.HasPrefix(p, "device_path=") { + dev.DevicePath = p[len("device_path="):] } else { dev.Model = p } diff --git a/pkg/cloudcommon/cmdline/parser_test.go b/pkg/cloudcommon/cmdline/parser_test.go index 8b7d2e87851..b736ede84d3 100644 --- a/pkg/cloudcommon/cmdline/parser_test.go +++ b/pkg/cloudcommon/cmdline/parser_test.go @@ -195,3 +195,49 @@ func TestFetchDiskConfigsByJSON(t *testing.T) { }) } } + +func TestParseIsolatedDevice(t *testing.T) { + tests := []struct { + desc string + idx int + want *compute.IsolatedDeviceConfig + wantErr bool + }{ + { + desc: "device_path=/dev/nvme0", + idx: 0, + want: &compute.IsolatedDeviceConfig{ + DevicePath: "/dev/nvme0", + }, + }, + { + desc: "GPU_HPC:device_path=/dev/nvme0", + idx: 0, + want: &compute.IsolatedDeviceConfig{ + DevicePath: "/dev/nvme0", + Model: "GPU_HPC", + }, + }, + { + desc: "GPU_HPC:device_path=/dev/nvme0:1d3ce781-2b64-4ee7-8d23-6bbcf478c54b", + idx: 0, + want: &compute.IsolatedDeviceConfig{ + Id: "1d3ce781-2b64-4ee7-8d23-6bbcf478c54b", + DevicePath: "/dev/nvme0", + Model: "GPU_HPC", + }, + }, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + got, err := ParseIsolatedDevice(tt.desc, tt.idx) + if (err != nil) != tt.wantErr { + t.Errorf("ParseIsolatedDevice() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseIsolatedDevice() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/compute/container_drivers/lifecycle/doc.go b/pkg/compute/container_drivers/lifecycle/doc.go new file mode 100644 index 00000000000..7703ece1e83 --- /dev/null +++ b/pkg/compute/container_drivers/lifecycle/doc.go @@ -0,0 +1 @@ +package lifecycle // import "yunion.io/x/onecloud/pkg/compute/container_drivers/lifecycle" diff --git a/pkg/compute/container_drivers/lifecycle/exec.go b/pkg/compute/container_drivers/lifecycle/exec.go new file mode 100644 index 00000000000..309d1e185da --- /dev/null +++ b/pkg/compute/container_drivers/lifecycle/exec.go @@ -0,0 +1,48 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lifecycle + +import ( + "context" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + models.RegisterContainerLifecyleDriver(newExec()) +} + +type execDriver struct{} + +func newExec() models.IContainerLifecyleDriver { + return &execDriver{} +} + +func (e execDriver) GetType() apis.ContainerLifecyleHandlerType { + return apis.ContainerLifecyleHandlerTypeExec +} + +func (e execDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerLifecyleHandler) error { + if input.Exec == nil { + return httperrors.NewNotEmptyError("exec field") + } + if len(input.Exec.Command) == 0 { + return httperrors.NewNotEmptyError("command is required") + } + return nil +} diff --git a/pkg/compute/models/container_drivers.go b/pkg/compute/models/container_drivers.go index 3c02b8de9e6..ae4427e2524 100644 --- a/pkg/compute/models/container_drivers.go +++ b/pkg/compute/models/container_drivers.go @@ -2,6 +2,7 @@ package models import ( "context" + "sync" "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -11,28 +12,67 @@ import ( ) var ( - containerVolumeDrivers = make(map[apis.ContainerVolumeMountType]IContainerVolumeMountDriver) - containerDeviceDrivers = make(map[apis.ContainerDeviceType]IContainerDeviceDriver) + containerVolumeDrivers = newContainerDrivers() + containerDeviceDrivers = newContainerDrivers() + containerLifecycleDrivers = newContainerDrivers() ) -func RegisterContainerVolumeMountDriver(drv IContainerVolumeMountDriver) { - containerVolumeDrivers[drv.GetType()] = drv +type containerDrivers struct { + drivers *sync.Map } -func GetContainerVolumeMountDriver(typ apis.ContainerVolumeMountType) IContainerVolumeMountDriver { - drv, err := GetContainerVolumeMountDriverWithError(typ) +func newContainerDrivers() *containerDrivers { + return &containerDrivers{ + drivers: new(sync.Map), + } +} + +func (cd *containerDrivers) GetWithError(typ string) (interface{}, error) { + drv, ok := cd.drivers.Load(typ) + if !ok { + return drv, httperrors.NewNotFoundError("not found driver by type %q", typ) + } + return drv, nil +} + +func (cd *containerDrivers) Get(typ string) interface{} { + drv, err := cd.GetWithError(typ) if err != nil { panic(err.Error()) } return drv } -func GetContainerVolumeMountDriverWithError(typ apis.ContainerVolumeMountType) (IContainerVolumeMountDriver, error) { - drv, ok := containerVolumeDrivers[typ] - if !ok { - return nil, httperrors.NewNotFoundError("not found driver by type %q", typ) +func (cd *containerDrivers) Register(typ string, drv interface{}) { + cd.drivers.Store(typ, drv) +} + +func registerContainerDriver[K ~string, D any](drvs *containerDrivers, typ K, drv D) { + drvs.Register(string(typ), drv) +} + +func getContainerDriver[K ~string, D any](drvs *containerDrivers, typ K) D { + return drvs.Get(string(typ)).(D) +} + +func getContainerDriverWithError[K ~string, D any](drvs *containerDrivers, typ K) (D, error) { + drv, err := drvs.GetWithError(string(typ)) + if err != nil { + return drv.(D), err } - return drv, nil + return drv.(D), nil +} + +func RegisterContainerVolumeMountDriver(drv IContainerVolumeMountDriver) { + registerContainerDriver(containerVolumeDrivers, drv.GetType(), drv) +} + +func GetContainerVolumeMountDriver(typ apis.ContainerVolumeMountType) IContainerVolumeMountDriver { + return getContainerDriver[apis.ContainerVolumeMountType, IContainerVolumeMountDriver](containerVolumeDrivers, typ) +} + +func GetContainerVolumeMountDriverWithError(typ apis.ContainerVolumeMountType) (IContainerVolumeMountDriver, error) { + return getContainerDriverWithError[apis.ContainerVolumeMountType, IContainerVolumeMountDriver](containerVolumeDrivers, typ) } type IContainerVolumeMountDriver interface { @@ -42,23 +82,15 @@ type IContainerVolumeMountDriver interface { } func RegisterContainerDeviceDriver(drv IContainerDeviceDriver) { - containerDeviceDrivers[drv.GetType()] = drv + registerContainerDriver(containerDeviceDrivers, drv.GetType(), drv) } func GetContainerDeviceDriver(typ apis.ContainerDeviceType) IContainerDeviceDriver { - drv, err := GetContainerDeviceDriverWithError(typ) - if err != nil { - panic(err.Error()) - } - return drv + return getContainerDriver[apis.ContainerDeviceType, IContainerDeviceDriver](containerDeviceDrivers, typ) } func GetContainerDeviceDriverWithError(typ apis.ContainerDeviceType) (IContainerDeviceDriver, error) { - drv, ok := containerDeviceDrivers[typ] - if !ok { - return nil, httperrors.NewNotFoundError("not found driver by type %q", typ) - } - return drv, nil + return getContainerDriverWithError[apis.ContainerDeviceType, IContainerDeviceDriver](containerDeviceDrivers, typ) } type IContainerDeviceDriver interface { @@ -67,3 +99,20 @@ type IContainerDeviceDriver interface { ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error) ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error) } + +type IContainerLifecyleDriver interface { + GetType() apis.ContainerLifecyleHandlerType + ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerLifecyleHandler) error +} + +func RegisterContainerLifecyleDriver(drv IContainerLifecyleDriver) { + registerContainerDriver(containerLifecycleDrivers, drv.GetType(), drv) +} + +func GetContainerLifecyleDriver(typ apis.ContainerLifecyleHandlerType) IContainerLifecyleDriver { + return getContainerDriver[apis.ContainerLifecyleHandlerType, IContainerLifecyleDriver](containerLifecycleDrivers, typ) +} + +func GetContainerLifecyleDriverWithError(typ apis.ContainerLifecyleHandlerType) (IContainerLifecyleDriver, error) { + return getContainerDriverWithError[apis.ContainerLifecyleHandlerType, IContainerLifecyleDriver](containerLifecycleDrivers, typ) +} diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go index aed1b2a7f2e..0ec8f76829f 100644 --- a/pkg/compute/models/containers.go +++ b/pkg/compute/models/containers.go @@ -148,13 +148,35 @@ func (m *SContainerManager) ValidateSpec(ctx context.Context, userCred mcclient. } } + if err := m.ValidateSpecLifecycle(ctx, userCred, spec); err != nil { + return errors.Wrap(err, "validate lifecycle") + } + return nil } +func (m *SContainerManager) ValidateSpecLifecycle(ctx context.Context, cred mcclient.TokenCredential, spec *api.ContainerSpec) error { + if spec.Lifecyle == nil { + return nil + } + if err := m.ValidateSpecLifecyclePostStart(ctx, cred, spec.Lifecyle.PostStart); err != nil { + return errors.Wrap(err, "validate post start") + } + return nil +} + +func (m *SContainerManager) ValidateSpecLifecyclePostStart(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerLifecyleHandler) error { + drv, err := GetContainerLifecyleDriverWithError(input.Type) + if err != nil { + return httperrors.NewInputParameterError("get lifecycle driver: %v", err) + } + return drv.ValidateCreateData(ctx, userCred, input) +} + func (m *SContainerManager) ValidateSpecDevice(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error) { drv, err := GetContainerDeviceDriverWithError(dev.Type) if err != nil { - return nil, httperrors.NewInputParameterError("get device drvice: %v", err) + return nil, httperrors.NewInputParameterError("get device driver: %v", err) } return drv.ValidateCreateData(ctx, userCred, pod, dev) } diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 548836b106e..0116c755c87 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -556,6 +556,8 @@ func (manager *SIsolatedDeviceManager) _isValidDeviceInfo(config *api.IsolatedDe func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { if len(devConfig.Id) > 0 { return manager.attachSpecificDeviceToGuest(ctx, guest, devConfig, userCred) + } else if len(devConfig.DevicePath) > 0 { + return manager.attachHostDeviceToGuestByDevicePath(ctx, guest, host, devConfig, userCred) } else { return manager.attachHostDeviceToGuestByModel(ctx, guest, host, devConfig, userCred) } @@ -573,6 +575,19 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex) } +func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { + if len(devConfig.Model) == 0 || len(devConfig.DevicePath) == 0 { + return fmt.Errorf("Model or DevicePath is empty: %#v", devConfig) + } + // if dev type is not nic, wire is empty string + devs, err := manager.findHostUnusedByDevAttr(devConfig.Model, "device_path", devConfig.DevicePath, host.Id, devConfig.WireId) + if err != nil || len(devs) == 0 { + return fmt.Errorf("Can't found model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id) + } + selectedDev := devs[0] + return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex) +} + func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error { if len(devConfig.Model) == 0 { return fmt.Errorf("Not found model from info: %#v", devConfig) @@ -642,11 +657,15 @@ func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SI } func (manager *SIsolatedDeviceManager) findHostUnusedByDevConfig(model, devType, hostId, wireId string) ([]SIsolatedDevice, error) { + return manager.findHostUnusedByDevAttr(model, "dev_type", devType, hostId, wireId) +} + +func (manager *SIsolatedDeviceManager) findHostUnusedByDevAttr(model, attrKey, attrVal, hostId, wireId string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) q := manager.findUnusedQuery() q = q.Equals("model", model).Equals("host_id", hostId) - if devType != "" { - q.Equals("dev_type", devType) + if attrVal != "" { + q.Equals(attrKey, attrVal) } if wireId != "" { wire := WireManager.FetchWireById(wireId) diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index fae44008a0d..9c613aae9f8 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -40,6 +40,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" _ "yunion.io/x/onecloud/pkg/compute/container_drivers/device" + _ "yunion.io/x/onecloud/pkg/compute/container_drivers/lifecycle" _ "yunion.io/x/onecloud/pkg/compute/container_drivers/volume_mount" _ "yunion.io/x/onecloud/pkg/compute/guestdrivers" _ "yunion.io/x/onecloud/pkg/compute/hostdrivers" diff --git a/pkg/hostman/container/lifecycle/doc.go b/pkg/hostman/container/lifecycle/doc.go new file mode 100644 index 00000000000..97042f40572 --- /dev/null +++ b/pkg/hostman/container/lifecycle/doc.go @@ -0,0 +1 @@ +package lifecycle // import "yunion.io/x/onecloud/pkg/hostman/container/lifecycle" diff --git a/pkg/hostman/container/lifecycle/exec.go b/pkg/hostman/container/lifecycle/exec.go new file mode 100644 index 00000000000..63928e74de9 --- /dev/null +++ b/pkg/hostman/container/lifecycle/exec.go @@ -0,0 +1,44 @@ +package lifecycle + +import ( + "context" + + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/util/pod" +) + +func init() { + RegisterDriver(newExec()) +} + +type execDriver struct{} + +func newExec() ILifecycle { + return &execDriver{} +} + +func (e execDriver) GetType() apis.ContainerLifecyleHandlerType { + return apis.ContainerLifecyleHandlerTypeExec +} + +func (e execDriver) Run(ctx context.Context, input *apis.ContainerLifecyleHandler, cri pod.CRI, id string) error { + cfg := input.Exec + cli := cri.GetRuntimeClient() + resp, err := cli.ExecSync(ctx, &runtimeapi.ExecSyncRequest{ + ContainerId: id, + Cmd: cfg.Command, + }) + if err != nil { + return errors.Wrapf(err, "exec sync") + } + if resp.GetExitCode() != 0 { + return errors.Wrapf(err, "stdout: %s, stderr: %s, exited: %d", resp.GetStdout(), resp.GetStderr(), resp.GetExitCode()) + } + log.Infof("run command %v: stdout: %s, stderr: %s", cfg.Command, resp.Stdout, resp.Stderr) + return nil +} diff --git a/pkg/hostman/container/lifecycle/lifecycle.go b/pkg/hostman/container/lifecycle/lifecycle.go new file mode 100644 index 00000000000..bfaf0cc8cd5 --- /dev/null +++ b/pkg/hostman/container/lifecycle/lifecycle.go @@ -0,0 +1,30 @@ +package lifecycle + +import ( + "context" + "fmt" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/util/pod" +) + +var ( + drivers = make(map[apis.ContainerLifecyleHandlerType]ILifecycle) +) + +func RegisterDriver(drv ILifecycle) { + drivers[drv.GetType()] = drv +} + +func GetDriver(typ apis.ContainerLifecyleHandlerType) ILifecycle { + drv, ok := drivers[typ] + if !ok { + panic(fmt.Sprintf("not found driver by type %s", typ)) + } + return drv +} + +type ILifecycle interface { + GetType() apis.ContainerLifecyleHandlerType + Run(ctx context.Context, input *apis.ContainerLifecyleHandler, cri pod.CRI, id string) error +} diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index a6f69dedd63..9b31c7bf921 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -33,6 +33,7 @@ import ( computeapi "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/container/device" + "yunion.io/x/onecloud/pkg/hostman/container/lifecycle" "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" @@ -47,6 +48,7 @@ import ( "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/netutils2/getport" "yunion.io/x/onecloud/pkg/util/pod" + "yunion.io/x/onecloud/pkg/util/procutils" ) type PodInstance interface { @@ -444,9 +446,15 @@ func (s *sPodGuestInstance) stopPod(ctx context.Context, timeout int64) error { ctx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) defer cancel() - return s.getCRI().StopPod(ctx, &runtimeapi.StopPodSandboxRequest{ + /*if err := s.getCRI().StopPod(ctx, &runtimeapi.StopPodSandboxRequest{ PodSandboxId: s.getCRIId(), - }) + }); err != nil { + return errors.Wrapf(err, "stop cri pod: %s", s.getCRIId()) + }*/ + if err := s.getCRI().RemovePod(ctx, s.getCRIId()); err != nil { + return errors.Wrapf(err, "remove cri pod: %s", s.getCRIId()) + } + return nil } func (s *sPodGuestInstance) LoadDesc() error { @@ -528,9 +536,30 @@ func (s *sPodGuestInstance) StartContainer(ctx context.Context, userCred mcclien if err := s.getCRI().StartContainer(ctx, criId); err != nil { return nil, errors.Wrap(err, "CRI.StartContainer") } + if err := s.setContainerCgroupDevicesAllow(criId, input.Spec.CgroupDevicesAllow); err != nil { + return nil, errors.Wrap(err, "set cgroup devices allow") + } + if err := s.doContainerStartPostLifecycle(ctx, criId, input); err != nil { + return nil, errors.Wrap(err, "do container lifecycle") + } return nil, nil } +func (s *sPodGuestInstance) doContainerStartPostLifecycle(ctx context.Context, criId string, input *hostapi.ContainerCreateInput) error { + ls := input.Spec.Lifecyle + if ls == nil { + return nil + } + if ls.PostStart == nil { + return nil + } + drv := lifecycle.GetDriver(ls.PostStart.Type) + if err := drv.Run(ctx, ls.PostStart, s.getCRI(), criId); err != nil { + return errors.Wrapf(err, "run %s", ls.PostStart.Type) + } + return nil +} + func (s *sPodGuestInstance) StopContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) { criId, err := s.getContainerCRIId(ctrId) if err != nil { @@ -736,6 +765,26 @@ func (s *sPodGuestInstance) getContainerMounts(input *hostapi.ContainerCreateInp return mounts, nil } +func (s *sPodGuestInstance) getContainerCgroupDir(dirType string, ctrId string) string { + cgroupDir := "/sys/fs/cgroup" + return filepath.Join(cgroupDir, dirType, s.getCgroupParent(), ctrId) +} + +func (s *sPodGuestInstance) getContainerCgroupDevicesDir(ctrId string) string { + return s.getContainerCgroupDir("devices", ctrId) +} + +func (s *sPodGuestInstance) setContainerCgroupDevicesAllow(ctrId string, allowStrs []string) error { + for _, allowStr := range allowStrs { + deviceAllowFile := filepath.Join(s.getContainerCgroupDevicesDir(ctrId), "devices.allow") + out, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", fmt.Sprintf("echo '%s' > %s", allowStr, deviceAllowFile)).Output() + if err != nil { + return errors.Wrapf(err, "echo %s to %s: %s", deviceAllowFile, allowStr, out) + } + } + return nil +} + func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerCreateInput) (string, error) { log.Infof("=====container input: %s", jsonutils.Marshal(input).PrettyString()) podCfg, err := s.getPodSandboxConfig() diff --git a/pkg/mcclient/options/compute/containers.go b/pkg/mcclient/options/compute/containers.go index e3fd9b0293a..c42ddf4731c 100644 --- a/pkg/mcclient/options/compute/containers.go +++ b/pkg/mcclient/options/compute/containers.go @@ -44,31 +44,44 @@ type ContainerDeleteOptions struct { } type ContainerCreateCommonOptions struct { - IMAGE string `help:"Image of container" json:"image"` - Command []string `help:"Command to execute (i.e., entrypoint for docker)" json:"command"` - Args []string `help:"Args for the Command (i.e. command for docker)" json:"args"` - WorkingDir string `help:"Current working directory of the command" json:"working_dir"` - Env []string `help:"List of environment variable to set in the container and the format is: ="` - VolumeMount []string `help:"Volume mount of the container and the format is: name=,mount=,readonly=,disk_index=,disk_id="` - Device []string `help:"Host device: ::, e.g.: /dev/snd:/dev/snd:rwm"` - Privileged bool `help:"Privileged mode"` - Caps string `help:"Container capabilities, e.g.: SETPCAP,AUDIT_WRITE,SYS_CHROOT,CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID,SYSLOG,SYS_ADMIN,WAKE_ALARM,SYS_PTRACE,BLOCK_SUSPEND,MKNOD,KILL,SYS_RESOURCE,NET_RAW,NET_ADMIN,NET_BIND_SERVICE,SYS_NICE"` - DropCaps string `help:"Container dropped capabilities, split by ','"` - EnableLxcfs bool `help:"Enable lxcfs"` + IMAGE string `help:"Image of container" json:"image"` + Command []string `help:"Command to execute (i.e., entrypoint for docker)" json:"command"` + Args []string `help:"Args for the Command (i.e. command for docker)" json:"args"` + WorkingDir string `help:"Current working directory of the command" json:"working_dir"` + Env []string `help:"List of environment variable to set in the container and the format is: ="` + VolumeMount []string `help:"Volume mount of the container and the format is: name=,mount=,readonly=,disk_index=,disk_id="` + Device []string `help:"Host device: ::, e.g.: /dev/snd:/dev/snd:rwm"` + Privileged bool `help:"Privileged mode"` + Caps string `help:"Container capabilities, e.g.: SETPCAP,AUDIT_WRITE,SYS_CHROOT,CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID,SYSLOG,SYS_ADMIN,WAKE_ALARM,SYS_PTRACE,BLOCK_SUSPEND,MKNOD,KILL,SYS_RESOURCE,NET_RAW,NET_ADMIN,NET_BIND_SERVICE,SYS_NICE"` + DropCaps string `help:"Container dropped capabilities, split by ','"` + EnableLxcfs bool `help:"Enable lxcfs"` + PostStartExec string `help:"Post started execution command"` + CgroupDeviceAllow []string `help:"Cgroup devices.allow, e.g.: 'c 13:* rwm'"` } func (o ContainerCreateCommonOptions) getCreateSpec() (*computeapi.ContainerSpec, error) { req := &computeapi.ContainerSpec{ ContainerSpec: apis.ContainerSpec{ - Image: o.IMAGE, - Command: o.Command, - Args: o.Args, - WorkingDir: o.WorkingDir, - EnableLxcfs: o.EnableLxcfs, - Privileged: o.Privileged, - Capabilities: &apis.ContainerCapability{}, + Image: o.IMAGE, + Command: o.Command, + Args: o.Args, + WorkingDir: o.WorkingDir, + EnableLxcfs: o.EnableLxcfs, + Privileged: o.Privileged, + Capabilities: &apis.ContainerCapability{}, + CgroupDevicesAllow: o.CgroupDeviceAllow, }, } + if len(o.PostStartExec) != 0 { + req.Lifecyle = &apis.ContainerLifecyle{ + PostStart: &apis.ContainerLifecyleHandler{ + Type: apis.ContainerLifecyleHandlerTypeExec, + Exec: &apis.ContainerLifecyleHandlerExecAction{ + Command: strings.Split(o.PostStartExec, " "), + }, + }, + } + } if len(o.Caps) != 0 { req.Capabilities.Add = strings.Split(o.Caps, ",") } diff --git a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go index 8acd4c50627..b5642414a52 100644 --- a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go +++ b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go @@ -146,6 +146,25 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c } } + // check host device by device_path + devicePathReq := make(map[string]int, 0) + for _, dev := range reqIsoDevs { + if len(dev.DevicePath) != 0 { + devicePathReq[dev.DevicePath] += 1 + } + } + for devPath, reqCnt := range devicePathReq { + freeCount := len(getter.UnusedIsolatedDevicesByDevicePath(devPath)) + if freeCount < reqCount { + h.Exclude(fmt.Sprintf("IsolatedDevice device_path %q not enough, request: %d, hostFree: %d", devPath, reqCount, freeCount)) + return h.GetResult() + } + cap := freeCount / reqCnt + if int64(cap) < minCapacity { + minCapacity = int64(cap) + } + } + h.SetCapacity(minCapacity) return h.GetResult() } diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go index 5a0c5eea30f..1246482f060 100644 --- a/pkg/scheduler/cache/candidate/base.go +++ b/pkg/scheduler/cache/candidate/base.go @@ -299,6 +299,10 @@ func (b baseHostGetter) UnusedIsolatedDevicesByVendorModel(vendorModel string) [ return b.h.UnusedIsolatedDevicesByVendorModel(vendorModel) } +func (b baseHostGetter) UnusedIsolatedDevicesByDevicePath(devPath string) []*core.IsolatedDeviceDesc { + return b.h.UnusedIsolatedDevicesByDevicePath(devPath) +} + func (b baseHostGetter) UnusedIsolatedDevicesByModel(model string) []*core.IsolatedDeviceDesc { return b.h.UnusedIsolatedDevicesByModel(model) } @@ -507,6 +511,16 @@ func (h *BaseHostDesc) UnusedIsolatedDevicesByModel(model string) []*core.Isolat return ret } +func (h *BaseHostDesc) UnusedIsolatedDevicesByDevicePath(devPath string) []*core.IsolatedDeviceDesc { + ret := make([]*core.IsolatedDeviceDesc, 0) + for _, dev := range h.UnusedIsolatedDevices() { + if devPath == dev.DevicePath { + ret = append(ret, dev) + } + } + return ret +} + func (h *BaseHostDesc) UnusedIsolatedDevicesByModelAndWire(model, wire string) []*core.IsolatedDeviceDesc { ret := make([]*core.IsolatedDeviceDesc, 0) for _, dev := range h.UnusedIsolatedDevices() { @@ -558,6 +572,7 @@ func (h *BaseHostDesc) fillIsolatedDevices(b *baseBuilder, host *computemodels.S Addr: devModel.Addr, VendorDeviceID: devModel.VendorDeviceId, WireId: devModel.WireId, + DevicePath: devModel.DevicePath, } devs[index] = dev } diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index c7b6833bffd..d1507999a3e 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -122,6 +122,7 @@ type CandidatePropertyGetter interface { UnusedIsolatedDevicesByVendorModel(vendorModel string) []*IsolatedDeviceDesc UnusedIsolatedDevicesByModel(model string) []*IsolatedDeviceDesc UnusedIsolatedDevicesByModelAndWire(model, wire string) []*IsolatedDeviceDesc + UnusedIsolatedDevicesByDevicePath(devPath string) []*IsolatedDeviceDesc GetIsolatedDevice(devID string) *IsolatedDeviceDesc UnusedGpuDevices() []*IsolatedDeviceDesc GetIsolatedDevices() []*IsolatedDeviceDesc @@ -270,6 +271,7 @@ type IsolatedDeviceDesc struct { Addr string VendorDeviceID string WireId string + DevicePath string } func (i *IsolatedDeviceDesc) VendorID() string { diff --git a/pkg/scheduler/test/mock/core.go b/pkg/scheduler/test/mock/core.go index 5acfd828881..99a5b6934af 100644 --- a/pkg/scheduler/test/mock/core.go +++ b/pkg/scheduler/test/mock/core.go @@ -700,6 +700,14 @@ func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByModel(arg0 string) return ret0 } +// UnusedIsolatedDevicesByDevicePath mocks base method +func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByDevicePath(arg0 string) []*core.IsolatedDeviceDesc { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnusedIsolatedDevicesByDevicePath", arg0) + ret0, _ := ret[0].([]*core.IsolatedDeviceDesc) + return ret0 +} + // UnusedIsolatedDevicesByModel indicates an expected call of UnusedIsolatedDevicesByModel func (mr *MockCandidatePropertyGetterMockRecorder) UnusedIsolatedDevicesByModel(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() diff --git a/pkg/util/pod/pod.go b/pkg/util/pod/pod.go index 7f917c48bca..523bc41022d 100644 --- a/pkg/util/pod/pod.go +++ b/pkg/util/pod/pod.go @@ -33,8 +33,8 @@ type CRI interface { ImageStatus(ctx context.Context, req *runtimeapi.ImageStatusRequest) (*runtimeapi.ImageStatusResponse, error) // lower layer client - // getImageClient() runtimeapi.ImageServiceClient - // getRuntimeClient() runtimeapi.RuntimeServiceClient + GetImageClient() runtimeapi.ImageServiceClient + GetRuntimeClient() runtimeapi.RuntimeServiceClient } type ListContainerOptions struct { @@ -92,23 +92,23 @@ func NewCRI(endpoint string, timeout time.Duration) (CRI, error) { }, nil } -func (c crictl) getImageClient() runtimeapi.ImageServiceClient { +func (c crictl) GetImageClient() runtimeapi.ImageServiceClient { return c.imgCli } -func (c crictl) getRuntimeClient() runtimeapi.RuntimeServiceClient { +func (c crictl) GetRuntimeClient() runtimeapi.RuntimeServiceClient { return c.runCli } func (c crictl) Version(ctx context.Context) (*runtimeapi.VersionResponse, error) { - return c.getRuntimeClient().Version(ctx, &runtimeapi.VersionRequest{}) + return c.GetRuntimeClient().Version(ctx, &runtimeapi.VersionRequest{}) } func (c crictl) ListImages(ctx context.Context, filter *runtimeapi.ImageFilter) ([]*runtimeapi.Image, error) { ctx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() - resp, err := c.getImageClient().ListImages(ctx, &runtimeapi.ListImagesRequest{ + resp, err := c.GetImageClient().ListImages(ctx, &runtimeapi.ListImagesRequest{ Filter: filter, }) if err != nil { @@ -123,7 +123,7 @@ func (c crictl) RunPod(ctx context.Context, podConfig *runtimeapi.PodSandboxConf RuntimeHandler: runtimeHandler, } log.Infof("RunPodSandboxRequest: %v", req) - r, err := c.getRuntimeClient().RunPodSandbox(ctx, req) + r, err := c.GetRuntimeClient().RunPodSandbox(ctx, req) if err != nil { return "", errors.Wrapf(err, "RunPod with request: %s", req.String()) } @@ -131,7 +131,7 @@ func (c crictl) RunPod(ctx context.Context, podConfig *runtimeapi.PodSandboxConf } func (c crictl) StopPod(ctx context.Context, req *runtimeapi.StopPodSandboxRequest) error { - _, err := c.getRuntimeClient().StopPodSandbox(ctx, req) + _, err := c.GetRuntimeClient().StopPodSandbox(ctx, req) if err != nil { return errors.Wrap(err, "StopPodSandbox") } @@ -150,7 +150,7 @@ func (c crictl) PullImageWithSandbox(ctx context.Context, image string, auth *ru SandboxConfig: sandbox, } log.Infof("PullImageRequest: %v", req) - r, err := c.getImageClient().PullImage(ctx, req) + r, err := c.GetImageClient().PullImage(ctx, req) if err != nil { return nil, errors.Wrapf(err, "PullImage with %s", req) } @@ -188,7 +188,7 @@ func (c crictl) CreateContainer(ctx context.Context, } log.Infof("CreateContainerRequest: %v", req) - r, err := c.getRuntimeClient().CreateContainer(ctx, req) + r, err := c.GetRuntimeClient().CreateContainer(ctx, req) if err != nil { return "", errors.Wrapf(err, "CreateContainer with: %s", req) } @@ -201,7 +201,7 @@ func (c crictl) StartContainer(ctx context.Context, id string) error { if id == "" { return errors.Error("Id can't be empty") } - if _, err := c.getRuntimeClient().StartContainer(ctx, &runtimeapi.StartContainerRequest{ + if _, err := c.GetRuntimeClient().StartContainer(ctx, &runtimeapi.StartContainerRequest{ ContainerId: id, }); err != nil { return errors.Wrapf(err, "StartContainer %s", id) @@ -270,7 +270,7 @@ func (c crictl) ListContainers(ctx context.Context, opts ListContainerOptions) ( req := &runtimeapi.ListContainersRequest{ Filter: filter, } - r, err := c.getRuntimeClient().ListContainers(ctx, req) + r, err := c.GetRuntimeClient().ListContainers(ctx, req) if err != nil { return nil, errors.Wrap(err, "ListContainers") } @@ -302,7 +302,7 @@ func (c crictl) ListPods(ctx context.Context, opts ListPodOptions) ([]*runtimeap req := &runtimeapi.ListPodSandboxRequest{ Filter: filter, } - ret, err := c.getRuntimeClient().ListPodSandbox(ctx, req) + ret, err := c.GetRuntimeClient().ListPodSandbox(ctx, req) if err != nil { return nil, errors.Wrap(err, "ListPodSandbox") } @@ -310,7 +310,7 @@ func (c crictl) ListPods(ctx context.Context, opts ListPodOptions) ([]*runtimeap } func (c crictl) RemovePod(ctx context.Context, podId string) error { - if _, err := c.getRuntimeClient().RemovePodSandbox(ctx, &runtimeapi.RemovePodSandboxRequest{ + if _, err := c.GetRuntimeClient().RemovePodSandbox(ctx, &runtimeapi.RemovePodSandboxRequest{ PodSandboxId: podId, }); err != nil { return errors.Wrap(err, "RemovePodSandbox") @@ -319,7 +319,7 @@ func (c crictl) RemovePod(ctx context.Context, podId string) error { } func (c crictl) StopContainer(ctx context.Context, ctrId string, timeout int64) error { - if _, err := c.getRuntimeClient().StopContainer(ctx, &runtimeapi.StopContainerRequest{ + if _, err := c.GetRuntimeClient().StopContainer(ctx, &runtimeapi.StopContainerRequest{ ContainerId: ctrId, Timeout: timeout, }); err != nil { @@ -329,7 +329,7 @@ func (c crictl) StopContainer(ctx context.Context, ctrId string, timeout int64) } func (c crictl) RemoveContainer(ctx context.Context, ctrId string) error { - _, err := c.getRuntimeClient().RemoveContainer(ctx, &runtimeapi.RemoveContainerRequest{ + _, err := c.GetRuntimeClient().RemoveContainer(ctx, &runtimeapi.RemoveContainerRequest{ ContainerId: ctrId, }) if err != nil { @@ -343,11 +343,11 @@ func (c crictl) ContainerStatus(ctx context.Context, ctrId string) (*runtimeapi. ContainerId: ctrId, Verbose: false, } - return c.getRuntimeClient().ContainerStatus(ctx, req) + return c.GetRuntimeClient().ContainerStatus(ctx, req) } func (c crictl) PullImage(ctx context.Context, req *runtimeapi.PullImageRequest) (*runtimeapi.PullImageResponse, error) { - resp, err := c.getImageClient().PullImage(ctx, req) + resp, err := c.GetImageClient().PullImage(ctx, req) if err != nil { return nil, errors.Wrapf(err, "PullImage") } @@ -355,5 +355,5 @@ func (c crictl) PullImage(ctx context.Context, req *runtimeapi.PullImageRequest) } func (c crictl) ImageStatus(ctx context.Context, req *runtimeapi.ImageStatusRequest) (*runtimeapi.ImageStatusResponse, error) { - return c.getImageClient().ImageStatus(ctx, req) + return c.GetImageClient().ImageStatus(ctx, req) }