Skip to content

Commit

Permalink
Merge pull request #19950 from zexi/pod-image-overlay
Browse files Browse the repository at this point in the history 
fix(region): container overlay volume_mount over disk image
  • Loading branch information
@zexi
zexi authored Apr 11, 2024
2 parents 5a4d2d9 + 3d95b90 commit 9e89730
Show file tree
Hide file tree
Showing 8 changed files with 135 additions and 63 deletions.
11 changes: 8 additions & 3 deletions pkg/apis/container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,18 +131,23 @@ type ContainerOverlayDiskImage struct {
type ContainerDiskOverlayType string

const (
CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY ContainerDiskOverlayType = "directory"
CONTAINER_DISK_OVERLAY_TYPE_UNKNOWN ContainerDiskOverlayType = "unknown"
CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY ContainerDiskOverlayType = "directory"
CONTAINER_DISK_OVERLAY_TYPE_DISK_IMAGE ContainerDiskOverlayType = "disk_image"
CONTAINER_DISK_OVERLAY_TYPE_UNKNOWN ContainerDiskOverlayType = "unknown"
)

type ContainerVolumeMountDiskOverlay struct {
LowerDir []string `json:"lower_dir"`
LowerDir []string `json:"lower_dir"`
UseDiskImage bool `json:"use_disk_image"`
}

func (o ContainerVolumeMountDiskOverlay) GetType() ContainerDiskOverlayType {
if len(o.LowerDir) != 0 {
return CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY
}
if o.UseDiskImage {
return CONTAINER_DISK_OVERLAY_TYPE_DISK_IMAGE
}
return CONTAINER_DISK_OVERLAY_TYPE_UNKNOWN
}

Expand Down
68 changes: 48 additions & 20 deletions pkg/compute/container_drivers/volume_mount/disk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ func init() {
}

type iDiskOverlay interface {
validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay) error
validatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay, disk *api.DiskConfig) error
validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay, obj *models.SDisk) error
}

type disk struct {
Expand All @@ -27,7 +28,8 @@ type disk struct {
func newDisk() models.IContainerVolumeMountDriver {
return &disk{
overlayDrivers: map[apis.ContainerDiskOverlayType]iDiskOverlay{
apis.CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY: newDiskOverlayDir(),
apis.CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY: newDiskOverlayDir(),
apis.CONTAINER_DISK_OVERLAY_TYPE_DISK_IMAGE: newDiskOverlayImage(),
},
}
}
Expand Down Expand Up @@ -61,20 +63,16 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre
return nil, errors.Wrap(err, "get pod disks")
}
disk := vm.Disk
var diskObj models.SDisk
if disk.Index != nil {
diskIndex := *disk.Index
if diskIndex >= len(disks) {
return nil, httperrors.NewInputParameterError("disk.index %d is large than disk size %d", diskIndex, len(disks))
}
diskObj := disks[diskIndex]
diskObj = disks[diskIndex]
vm.Disk.Id = diskObj.GetId()
// remove index
vm.Disk.Index = nil
if diskObj.TemplateId != "" {
if vm.Disk.SubDirectory == "" {
return nil, httperrors.NewInputParameterError("sub_directory is required when disk has template_id %s", diskObj.TemplateId)
}
}
} else {
if disk.Id == "" {
return nil, httperrors.NewNotEmptyError("disk.id is empty")
Expand All @@ -83,11 +81,7 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre
for _, d := range disks {
if d.GetId() == disk.Id || d.GetName() == disk.Id {
disk.Id = d.GetId()
if d.TemplateId != "" {
if vm.Disk.SubDirectory == "" {
return nil, httperrors.NewInputParameterError("sub_directory is required when disk has template_id %s", d.TemplateId)
}
}
diskObj = d
foundDisk = true
break
}
Expand All @@ -96,7 +90,7 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre
return nil, httperrors.NewNotFoundError("not found pod disk by %s", disk.Id)
}
}
if err := d.validateOverlay(ctx, userCred, vm); err != nil {
if err := d.validateOverlay(ctx, userCred, vm, &diskObj); err != nil {
return nil, errors.Wrapf(err, "validate overlay")
}
return vm, nil
Expand All @@ -122,9 +116,9 @@ func (d disk) ValidatePodCreateData(ctx context.Context, userCred mcclient.Token
return httperrors.NewInputParameterError("disk.index %d is large than disk size %d", diskIndex, len(disks))
}
inputDisk := disks[diskIndex]
if inputDisk.ImageId != "" {
if disk.SubDirectory == "" {
return httperrors.NewInputParameterError("sub_directory is required when disk has image_id %s", inputDisk.ImageId)
if vm.Disk.Overlay != nil {
if err := d.getOverlayDriver(vm.Disk.Overlay).validatePodCreateData(ctx, userCred, vm.Disk.Overlay, inputDisk); err != nil {
return httperrors.NewInputParameterError("valid overlay %v", err)
}
}
return nil
Expand All @@ -134,15 +128,15 @@ func (d disk) getOverlayDriver(ov *apis.ContainerVolumeMountDiskOverlay) iDiskOv
return d.overlayDrivers[ov.GetType()]
}

func (d disk) validateOverlay(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount) error {
func (d disk) validateOverlay(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount, diskObj *models.SDisk) error {
if vm.Disk.Overlay == nil {
return nil
}
ov := vm.Disk.Overlay
if err := ov.IsValid(); err != nil {
return httperrors.NewInputParameterError("invalid overlay input: %v", err)
}
if err := d.getOverlayDriver(ov).validateCreateData(ctx, userCred, ov); err != nil {
if err := d.getOverlayDriver(ov).validateCreateData(ctx, userCred, ov, diskObj); err != nil {
return errors.Wrapf(err, "validate overlay %s", ov.GetType())
}
return nil
Expand All @@ -154,7 +148,7 @@ func newDiskOverlayDir() iDiskOverlay {
return &diskOverlayDir{}
}

func (d diskOverlayDir) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay) error {
func (d diskOverlayDir) validateCommonCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay) error {
if len(input.LowerDir) == 0 {
return httperrors.NewNotEmptyError("lower_dir is required")
}
Expand All @@ -168,3 +162,37 @@ func (d diskOverlayDir) validateCreateData(ctx context.Context, userCred mcclien
}
return nil
}

func (d diskOverlayDir) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay, _ *models.SDisk) error {
return d.validateCommonCreateData(ctx, userCred, input)
}

func (d diskOverlayDir) validatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay, disk *api.DiskConfig) error {
return d.validateCommonCreateData(ctx, userCred, input)
}

type diskOverlayImage struct{}

func newDiskOverlayImage() iDiskOverlay {
return &diskOverlayImage{}
}

func (d diskOverlayImage) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay, diskObj *models.SDisk) error {
if !input.UseDiskImage {
return nil
}
if diskObj.TemplateId == "" {
return httperrors.NewInputParameterError("disk %s must have template_id", diskObj.GetId())
}
return nil
}

func (d diskOverlayImage) validatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerVolumeMountDiskOverlay, disk *api.DiskConfig) error {
if !input.UseDiskImage {
return nil
}
if disk.ImageId == "" {
return httperrors.NewInputParameterError("disk %#v must have image_id", disk)
}
return nil
}
18 changes: 11 additions & 7 deletions pkg/compute/models/containers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,19 +305,23 @@ func (vm *ContainerVolumeMountRelation) toHostDiskMount(disk *apis.ContainerVolu
}

func (vm *ContainerVolumeMountRelation) ToHostMount() (*hostapi.ContainerVolumeMount, error) {
disk, err := vm.toHostDiskMount(vm.VolumeMount.Disk)
if err != nil {
return nil, errors.Wrap(err, "toHostDiskMount")
}
return &hostapi.ContainerVolumeMount{
ret := &hostapi.ContainerVolumeMount{
Type: vm.VolumeMount.Type,
Disk: disk,
Disk: nil,
HostPath: vm.VolumeMount.HostPath,
ReadOnly: vm.VolumeMount.ReadOnly,
MountPath: vm.VolumeMount.MountPath,
SelinuxRelabel: vm.VolumeMount.SelinuxRelabel,
Propagation: vm.VolumeMount.Propagation,
}, nil
}
if vm.VolumeMount.Disk != nil {
disk, err := vm.toHostDiskMount(vm.VolumeMount.Disk)
if err != nil {
return nil, errors.Wrap(err, "toHostDiskMount")
}
ret.Disk = disk
}
return ret, nil
}

func (m *SContainerManager) GetVolumeMountRelations(pod *SGuest, spec *api.ContainerSpec) ([]*ContainerVolumeMountRelation, error) {
Expand Down
19 changes: 16 additions & 3 deletions pkg/compute/models/disks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1377,11 +1377,24 @@ func (self *SDisk) GetFsFormat() string {
return self.FsFormat
}

func (self *SDisk) GetCacheImageFormat() string {
func (self *SDisk) GetCacheImageFormat(ctx context.Context) (string, error) {
if self.TemplateId != "" {
s := auth.GetAdminSession(ctx, options.Options.Region)
img, err := image.Images.Get(s, self.TemplateId, nil)
if err == nil {
diskFmt, err := img.GetString("disk_format")
if err != nil {
return "", errors.Wrapf(err, "not found disk_format of image %s", self.TemplateId)
}
if diskFmt == imageapi.IMAGE_DISK_FORMAT_TGZ {
return imageapi.IMAGE_DISK_FORMAT_TGZ, nil
}
}
}
if self.DiskFormat == "raw" {
return "qcow2"
return "qcow2", nil
}
return self.DiskFormat
return self.DiskFormat, nil
}

func (manager *SDiskManager) getDisksByStorage(storage *SStorage) ([]SDisk, error) {
Expand Down
7 changes: 6 additions & 1 deletion pkg/compute/tasks/disk_create_task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,14 @@ func (self *DiskCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel,
// use image only if disk not created from snapshot or backup
if len(imageId) > 0 && len(disk.SnapshotId) == 0 && len(disk.BackupId) == 0 {
self.SetStage("OnStorageCacheImageComplete", nil)
cacheImageFmt, err := disk.GetCacheImageFormat(ctx)
if err != nil {
self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(err.Error()))
return
}
input := api.CacheImageInput{
ImageId: imageId,
Format: disk.GetCacheImageFormat(),
Format: cacheImageFmt,
ParentTaskId: self.GetTaskId(),
}
guest := disk.GetGuest()
Expand Down
51 changes: 22 additions & 29 deletions pkg/hostman/container/volume_mount/disk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ type disk struct {
func newDisk() IVolumeMount {
return &disk{
overlayDrivers: map[apis.ContainerDiskOverlayType]iDiskOverlay{
apis.CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY: newDiskOverlayDir(),
apis.CONTAINER_DISK_OVERLAY_TYPE_DIRECTORY: newDiskOverlayDir(),
apis.CONTAINER_DISK_OVERLAY_TYPE_DISK_IMAGE: newDiskOverlayImage(),
},
}
}
Expand Down Expand Up @@ -160,11 +161,6 @@ func (d disk) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount
return errors.Wrapf(err, "mount container %s overlay dir: %#v", ctrId, vmDisk.Overlay)
}
}
if vmDisk.TemplateId != "" {
if err := d.mountTemplateOverlay(context.Background(), pod, ctrId, vm); err != nil {
return errors.Wrapf(err, "mount container %s template overlay: %#v", ctrId, vmDisk.TemplateId)
}
}
return nil
}

Expand Down Expand Up @@ -197,11 +193,6 @@ func (d disk) Unmount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMou
return errors.Wrapf(err, "umount overlay")
}
}
if vm.Disk.TemplateId != "" {
if err := d.unmountTemplateOverlay(context.Background(), pod, ctrId, vm); err != nil {
return errors.Wrapf(err, "unmount container %s template overlay: %#v", ctrId, vm.Disk.TemplateId)
}
}
mntPoint := pod.GetDiskMountPoint(iDisk)
if err := container_storage.Unmount(mntPoint); err != nil {
return errors.Wrapf(err, "unmount %s", mntPoint)
Expand Down Expand Up @@ -258,24 +249,6 @@ func (d disk) doTemplateOverlayAction(
return nil
}

func (d disk) mountTemplateOverlay(
ctx context.Context,
pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error {
if err := d.doTemplateOverlayAction(ctx, pod, ctrId, vm, newDiskOverlayDir().mount); err != nil {
return errors.Wrapf(err, "mount template overlay")
}
return nil
}

func (d disk) unmountTemplateOverlay(
ctx context.Context,
pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error {
if err := d.doTemplateOverlayAction(ctx, pod, ctrId, vm, newDiskOverlayDir().unmount); err != nil {
return errors.Wrapf(err, "unmount template overlay")
}
return nil
}

type diskOverlayDir struct{}

func newDiskOverlayDir() iDiskOverlay {
Expand Down Expand Up @@ -314,3 +287,23 @@ func (dod diskOverlayDir) unmount(d disk, pod IPodInfo, ctrId string, vm *hostap
overlayDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir)
return container_storage.Unmount(overlayDir)
}

type diskOverlayImage struct{}

func newDiskOverlayImage() iDiskOverlay {
return &diskOverlayImage{}
}

func (di diskOverlayImage) mount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error {
if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().mount); err != nil {
return errors.Wrapf(err, "mount template overlay")
}
return nil
}

func (di diskOverlayImage) unmount(d disk, pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error {
if err := d.doTemplateOverlayAction(context.Background(), pod, ctrId, vm, newDiskOverlayDir().unmount); err != nil {
return errors.Wrapf(err, "unmount template overlay")
}
return nil
}
9 changes: 9 additions & 0 deletions pkg/mcclient/options/compute/containers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,15 @@ func parseContainerVolumeMount(vmStr string) (*apis.ContainerVolumeMount, error)
vm.Disk.Overlay = &apis.ContainerVolumeMountDiskOverlay{
LowerDir: strings.Split(val, ":"),
}
case "overlay_disk_image":
if strings.ToLower(val) == "true" {
if vm.Disk == nil {
vm.Disk = &apis.ContainerVolumeMountDisk{}
}
vm.Disk.Overlay = &apis.ContainerVolumeMountDiskOverlay{
UseDiskImage: true,
}
}
}
}
return vm, nil
Expand Down
15 changes: 15 additions & 0 deletions pkg/mcclient/options/compute/containers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,21 @@ func Test_parseContainerVolumeMount(t *testing.T) {
Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
},
},
{
args: "readonly=true,mount_path=/data,disk_index=0,disk_subdir=data,overlay_disk_image=true",
want: &apis.ContainerVolumeMount{
ReadOnly: true,
MountPath: "/data",
Disk: &apis.ContainerVolumeMountDisk{
Index: &index0,
SubDirectory: "data",
Overlay: &apis.ContainerVolumeMountDiskOverlay{
UseDiskImage: true,
},
},
Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
},
},
{
args: "readonly=true,mount_path=/storage_size,disk_index=0,disk_ssf=storage_size",
want: &apis.ContainerVolumeMount{
Expand Down

0 comments on commit 9e89730

Please sign in to comment.