feat(region,host): guest sync os info (#19100)

cmd/climc/shell/compute/servers.go

@@ -125,6 +125,7 @@ func init() {
 	cmd.BatchPerform("set-os-info", &options.ServerSetOSInfoOptions{})
 	cmd.BatchPerform("start-rescue", &options.ServerStartOptions{})
 	cmd.BatchPerform("stop-rescue", &options.ServerStartOptions{})
+	cmd.BatchPerform("sync-os-info", &options.ServerIdsOptions{})
 
 	cmd.Get("vnc", new(options.ServerVncOptions))
 	cmd.Get("desc", new(options.ServerIdOptions))

pkg/apis/compute/guest_const.go

@@ -173,6 +173,7 @@ const (
 	VM_QGA_SET_PASSWORD        = "qga_set_password"
 	VM_QGA_COMMAND_EXECUTING   = "qga_command_executing"
 	VM_QGA_EXEC_COMMAND_FAILED = "qga_exec_command_failed"
+	VM_QGA_SYNC_OS_INFO        = "qga_sync_os_info"
 
 	VM_QGA_SET_NETWORK        = "qga_set_network"
 	VM_QGA_SET_NETWORK_FAILED = "qga_set_network_failed"

pkg/apis/compute/guests.go

@@ -1201,4 +1201,5 @@ type ServerSetOSInfoInput struct {
 	Distribution string `json:"distribution" help:"OS distribution, e.g.: CentOS, Ubuntu, Windows Server 2016 Datacenter"`
 	// OS version, e.g: 7.9, 22.04, 6.3
 	Version string `json:"version" help:"OS version, e.g.: 7.9, 22.04, 6.3"`
+	Arch    string `json:"arch" help:"OS arch, e.g.: x86_64, aarch64"`
 }

pkg/cloudcommon/db/opslog_const.go

@@ -163,6 +163,9 @@ const (
 	ACT_SET_USER_PASSWORD      = "set_user_password"
 	ACT_SET_USER_PASSWORD_FAIL = "set_user_password_fail"
 
+	ACT_SYNC_OS_INFO      = "sync_os_info"
+	ACT_SYNC_OS_INFO_FAIL = "sync_os_info_fail"
+
 	ACT_VM_IO_THROTTLE      = "io_throttle"
 	ACT_VM_IO_THROTTLE_FAIL = "io_throttle_fail"
 

pkg/compute/guestdrivers/base.go

@@ -520,6 +520,10 @@ func (self *SBaseGuestDriver) QgaRequestGetNetwork(ctx context.Context, userCred
 	return nil, httperrors.ErrNotImplemented
 }
 
+func (drv *SBaseGuestDriver) QgaRequestGetOsInfo(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) (jsonutils.JSONObject, error) {
+	return nil, httperrors.ErrNotImplemented
+}
+
 func (drv *SBaseGuestDriver) RequestQgaCommand(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) (jsonutils.JSONObject, error) {
 	return nil, httperrors.ErrNotImplemented
 }
@@ -550,6 +554,10 @@ func (self *SBaseGuestDriver) ValidateSetOSInfo(ctx context.Context, userCred mc
 	return nil
 }
 
+func (self *SBaseGuestDriver) ValidateSyncOSInfo(ctx context.Context, userCred mcclient.TokenCredential, _ *models.SGuest) error {
+	return httperrors.ErrNotImplemented
+}
+
 func (self *SBaseGuestDriver) RequestStartRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) error {
 	return httperrors.ErrNotImplemented
 }

pkg/compute/guestdrivers/kvm.go

@@ -1118,6 +1118,17 @@ func (self *SKVMGuestDriver) QgaRequestGetNetwork(ctx context.Context, userCred
 	return res, nil
 }
 
+func (self *SKVMGuestDriver) QgaRequestGetOsInfo(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) (jsonutils.JSONObject, error) {
+	url := fmt.Sprintf("%s/servers/%s/qga-get-os-info", host.ManagerUri, guest.Id)
+	httpClient := httputils.GetDefaultClient()
+	header := mcclient.GetTokenHeaders(userCred)
+	_, res, err := httputils.JSONRequest(httpClient, ctx, "POST", url, header, nil, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "host request")
+	}
+	return res, nil
+}
+
 func (self *SKVMGuestDriver) QgaRequestSetUserPassword(ctx context.Context, task taskman.ITask, host *models.SHost, guest *models.SGuest, input *api.ServerQgaSetPasswordInput) error {
 	url := fmt.Sprintf("%s/servers/%s/qga-set-password", host.ManagerUri, guest.Id)
 	httpClient := httputils.GetDefaultClient()
@@ -1195,3 +1206,10 @@ func (self *SKVMGuestDriver) RequestStopRescue(ctx context.Context, task taskman
 
 	return nil
 }
+
+func (self *SKVMGuestDriver) ValidateSyncOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest) error {
+	if !utils.IsInStringArray(guest.Status, []string{api.VM_RUNNING, api.VM_READY}) {
+		return httperrors.NewBadRequestError("can't sync guest os info in status %s", guest.Status)
+	}
+	return nil
+}

pkg/compute/models/guest_actions.go

@@ -6051,6 +6051,7 @@ func (self *SGuest) PerformSetOsInfo(ctx context.Context, userCred mcclient.Toke
 		api.VM_METADATA_OS_NAME:    input.Type,
 		api.VM_METADATA_OS_VERSION: input.Version,
 		api.VM_METADATA_OS_DISTRO:  input.Distribution,
+		api.VM_METADATA_OS_ARCH:    input.Arch,
 	} {
 		if len(v) == 0 {
 			continue
@@ -6061,3 +6062,21 @@ func (self *SGuest) PerformSetOsInfo(ctx context.Context, userCred mcclient.Toke
 	}
 	return nil, nil
 }
+
+func (self *SGuest) PerformSyncOsInfo(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if err := self.GetDriver().ValidateSyncOSInfo(ctx, userCred, self); err != nil {
+		return nil, err
+	}
+	if self.Status == api.VM_READY {
+		// start guest deploy task to sync os info
+		return nil, self.StartGuestDeployTask(ctx, userCred, nil, "deploy", "")
+	} else {
+		res, err := self.PerformQgaPing(ctx, userCred, nil, nil)
+		if err != nil || res.Contains("ping_error") {
+			return nil, httperrors.NewBadRequestError("qga ping failed is qga running?")
+		}
+
+		// try qga get os info
+		return nil, self.startQgaSyncOsInfoTask(ctx, userCred, "")
+	}
+}

pkg/compute/models/guestdrivers.go

@@ -238,6 +238,7 @@ type IGuestDriver interface {
 	QgaRequestGuestInfoTask(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *SHost, guest *SGuest) (jsonutils.JSONObject, error)
 	QgaRequestSetNetwork(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *SHost, guest *SGuest) (jsonutils.JSONObject, error)
 	QgaRequestGetNetwork(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *SHost, guest *SGuest) (jsonutils.JSONObject, error)
+	QgaRequestGetOsInfo(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *SHost, guest *SGuest) (jsonutils.JSONObject, error)
 
 	FetchMonitorUrl(ctx context.Context, guest *SGuest) string
 	RequestResetNicTrafficLimit(ctx context.Context, task taskman.ITask, host *SHost, guest *SGuest, input []api.ServerNicTrafficLimit) error
@@ -246,6 +247,7 @@ type IGuestDriver interface {
 	SyncOsInfo(ctx context.Context, userCred mcclient.TokenCredential, g *SGuest, extVM cloudprovider.IOSInfo) error
 
 	ValidateSetOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, input *api.ServerSetOSInfoInput) error
+	ValidateSyncOSInfo(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest) error
 	RequestStartRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *SHost, guest *SGuest) error
 	RequestStopRescue(ctx context.Context, task taskman.ITask, body jsonutils.JSONObject, host *SHost, guest *SGuest) error
 }

pkg/compute/models/qemu_guest_agent.go

@@ -129,3 +129,14 @@ func (self *SGuest) PerformQgaGetNetwork(
 	host, _ := self.GetHost()
 	return self.GetDriver().QgaRequestGetNetwork(ctx, userCred, nil, host, self)
 }
+
+func (self *SGuest) startQgaSyncOsInfoTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
+	self.SetStatus(userCred, api.VM_QGA_SYNC_OS_INFO, "")
+	kwargs := jsonutils.NewDict()
+	task, err := taskman.TaskManager.NewTask(ctx, "GuestQgaSyncOsInfoTask", self, userCred, kwargs, parentTaskId, "", nil)
+	if err != nil {
+		return err
+	}
+	task.ScheduleRun(nil)
+	return nil
+}

pkg/compute/tasks/guest_qga_sync_os_info_task.go

@@ -0,0 +1,107 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tasks
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+func init() {
+	taskman.RegisterTask(GuestQgaSyncOsInfoTask{})
+}
+
+type GuestQgaSyncOsInfoTask struct {
+	SGuestBaseTask
+}
+
+func (self *GuestQgaSyncOsInfoTask) guestPing(ctx context.Context, guest *models.SGuest) error {
+	host, err := guest.GetHost()
+	if err != nil {
+		return err
+	}
+	return guest.GetDriver().QgaRequestGuestPing(ctx, self.GetTaskRequestHeader(), host, guest, true, nil)
+}
+
+func (self *GuestQgaSyncOsInfoTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	host, _ := guest.GetHost()
+	res, err := guest.GetDriver().QgaRequestGetOsInfo(ctx, self.UserCred, nil, host, guest)
+	if err != nil {
+		self.taskFailed(ctx, guest, err.Error())
+		return
+	}
+	self.updateOsInfo(ctx, guest, res)
+}
+
+type GuestOsInfo struct {
+	Id            string `json:"id"`
+	KernelRelease string `json:"kernel-release"`
+	KernelVersion string `json:"kernel-version"`
+	Machine       string `json:"machine"`
+	Name          string `json:"name"`
+	PrettyName    string `json:"pretty-name"`
+	Version       string `json:"version"`
+	VersionId     string `json:"version-id"`
+}
+
+func (self *GuestQgaSyncOsInfoTask) updateOsInfo(ctx context.Context, guest *models.SGuest, res jsonutils.JSONObject) {
+	osInfo := new(GuestOsInfo)
+	err := res.Unmarshal(osInfo)
+	if err != nil {
+		self.taskFailed(ctx, guest, fmt.Sprintf("failed unmarshal osinfo %s", err))
+		return
+	}
+	osType := "Linux"
+	if osInfo.Id == "mswindows" {
+		osType = "Windows"
+	}
+	osInput := api.ServerSetOSInfoInput{
+		Type:         osType,
+		Distribution: osInfo.PrettyName,
+		Version:      osInfo.Version,
+		Arch:         osInfo.Machine,
+	}
+	_, err = guest.PerformSetOsInfo(ctx, self.UserCred, nil, osInput)
+	if err != nil {
+		self.taskFailed(ctx, guest, fmt.Sprintf("failed set osinfo %s", err))
+		return
+	}
+	self.OnUpdateOsInfoComplete(ctx, guest, osInput)
+}
+
+func (self *GuestQgaSyncOsInfoTask) taskFailed(ctx context.Context, guest *models.SGuest, reason string) {
+	guest.SetStatus(self.UserCred, api.VM_QGA_EXEC_COMMAND_FAILED, reason)
+	guest.UpdateQgaStatus(api.QGA_STATUS_EXECUTE_FAILED)
+	db.OpsLog.LogEvent(guest, db.ACT_SYNC_OS_INFO_FAIL, reason, self.UserCred)
+	logclient.AddActionLogWithContext(ctx, guest, logclient.ACT_SET_USER_PASSWORD, reason, self.UserCred, false)
+	self.SetStageFailed(ctx, jsonutils.NewString(reason))
+}
+
+func (self *GuestQgaSyncOsInfoTask) OnUpdateOsInfoComplete(ctx context.Context, guest *models.SGuest, osInput api.ServerSetOSInfoInput) {
+	guest.SetStatus(self.UserCred, api.VM_RUNNING, "on qga set user password success")
+	guest.UpdateQgaStatus(api.QGA_STATUS_AVAILABLE)
+	db.OpsLog.LogEvent(guest, db.ACT_SYNC_OS_INFO, jsonutils.Marshal(osInput), self.UserCred)
+	logclient.AddActionLogWithContext(ctx, guest, logclient.ACT_SET_USER_PASSWORD, jsonutils.Marshal(osInput), self.UserCred, false)
+	self.SetStageComplete(ctx, nil)
+}

pkg/hostman/guestman/guest-agent.go

@@ -160,3 +160,19 @@ func (m *SGuestManager) QgaGetNetwork(sid string) (string, error) {
 	}
 	return "", errors.Errorf("qga unfinished last cmd, is qga unavailable?")
 }
+
+func (m *SGuestManager) QgaGetOsInfo(sid string) (jsonutils.JSONObject, error) {
+	guest, err := m.checkAndInitGuestQga(sid)
+	if err != nil {
+		return nil, err
+	}
+	if guest.guestAgent.TryLock() {
+		defer guest.guestAgent.Unlock()
+		res, err := guest.guestAgent.QgaGuestGetOsInfo()
+		if err != nil {
+			return nil, errors.Wrap(err, "qga get os info fail")
+		}
+		return jsonutils.Marshal(res), nil
+	}
+	return nil, errors.Errorf("qga unfinished last cmd, is qga unavailable?")
+}

pkg/hostman/guestman/guest_create_from_remote.go

@@ -23,10 +23,9 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
-	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
-	hostutils "yunion.io/x/onecloud/pkg/hostman/hostutils"
+	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
 )
 
@@ -38,12 +37,7 @@ func (m *SGuestManager) GuestCreateFromEsxi(
 		return nil, hostutils.ParamsError
 	}
 	guest, _ := m.GetServer(createConfig.Sid)
-	if err := guest.SaveSourceDesc(createConfig.GuestDesc); err != nil {
-		return nil, err
-	}
-	guest.Desc = new(desc.SGuestDesc)
-	jsonutils.Marshal(createConfig.GuestDesc).Unmarshal(guest.Desc)
-	if err := guest.SaveLiveDesc(guest.Desc); err != nil {
+	if err := guest.SaveDesc(createConfig.GuestDesc); err != nil {
 		return nil, err
 	}
 
@@ -128,12 +122,7 @@ func (m *SGuestManager) GuestCreateFromCloudpods(
 		return nil, hostutils.ParamsError
 	}
 	guest, _ := m.GetServer(createConfig.Sid)
-	if err := guest.SaveSourceDesc(createConfig.GuestDesc); err != nil {
-		return nil, err
-	}
-	guest.Desc = new(desc.SGuestDesc)
-	jsonutils.Marshal(createConfig.GuestDesc).Unmarshal(guest.Desc)
-	if err := guest.SaveLiveDesc(guest.Desc); err != nil {
+	if err := guest.SaveDesc(createConfig.GuestDesc); err != nil {
 		return nil, err
 	}
 	var err error

pkg/hostman/guestman/guesthandlers/guesthandler.go

@@ -104,6 +104,7 @@ func AddGuestTaskHandler(prefix string, app *appsrv.Application) {
 			"qga-guest-info-task":      qgaGuestInfoTask,
 			"qga-get-network":          qgaGetNetwork,
 			"qga-set-network":          qgaSetNetwork,
+			"qga-get-os-info":          qgaGetOsInfo,
 			"start-rescue":             guestStartRescue,
 			"stop-rescue":              guestStopRescue,
 		} {
@@ -944,6 +945,11 @@ func qgaSetNetwork(ctx context.Context, userCred mcclient.TokenCredential, sid s
 	return gm.QgaSetNetwork(qgaNetMod, sid, input.Timeout)
 }
 
+func qgaGetOsInfo(ctx context.Context, userCred mcclient.TokenCredential, sid string, body jsonutils.JSONObject) (interface{}, error) {
+	gm := guestman.GetGuestManager()
+	return gm.QgaGetOsInfo(sid)
+}
+
 // guestStartRescue prepare rescue files
 func guestStartRescue(ctx context.Context, userCred mcclient.TokenCredential, sid string, body jsonutils.JSONObject) (interface{}, error) {
 	// Start rescue guest

pkg/hostman/guestman/guestman.go

@@ -778,7 +778,9 @@ func (m *SGuestManager) GuestDeploy(ctx context.Context, params interface{}) (js
 			if err != nil {
 				return nil, httperrors.NewBadRequestError("Failed unmarshal guest desc %s", err)
 			}
-			guest.SaveSourceDesc(guestDesc)
+			if err := guest.SaveDesc(guestDesc); err != nil {
+				return nil, errors.Wrap(err, "failed save desc")
+			}
 		}
 		return m.startDeploy(ctx, deployParams, guest)
 	} else {
@@ -867,7 +869,9 @@ func (m *SGuestManager) GuestStart(ctx context.Context, userCred mcclient.TokenC
 	if guest, ok := m.GetServer(sid); ok {
 		guestDesc := new(desc.SGuestDesc)
 		if err := body.Unmarshal(guestDesc, "desc"); err == nil {
-			guest.SaveSourceDesc(guestDesc)
+			if err = guest.SaveDesc(guestDesc); err != nil {
+				return nil, errors.Wrap(err, "save desc")
+			}
 		}
 		if guest.IsStopped() {
 			data, err := body.Get("params")
@@ -1058,7 +1062,7 @@ func (m *SGuestManager) DestPrepareMigrate(ctx context.Context, params interface
 				return nil, fmt.Errorf("dest prepare migrate failed %s", err)
 			}
 		}
-		if err := guest.SaveSourceDesc(migParams.Desc); err != nil {
+		if err := guest.SaveDesc(migParams.Desc); err != nil {
 			log.Errorln(err)
 			return nil, err
 		}
@@ -1349,7 +1353,7 @@ func (m *SGuestManager) StartBlockReplication(ctx context.Context, params interf
 	}
 	guest, _ := m.GetServer(mirrorParams.Sid)
 	// TODO: check desc
-	if err := guest.SaveSourceDesc(mirrorParams.Desc); err != nil {
+	if err := guest.SaveDesc(mirrorParams.Desc); err != nil {
 		return nil, err
 	}
 	onSucc := func() {

pkg/hostman/guestman/libvirt.go

@@ -31,7 +31,6 @@ import (
 	"yunion.io/x/pkg/util/netutils"
 
 	"yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
@@ -68,12 +67,7 @@ func (m *SGuestManager) GuestCreateFromLibvirt(
 		disksPath.Set(disk.DiskId, jsonutils.NewString(iDisk.GetPath()))
 	}
 	guest, _ := m.GetServer(createConfig.Sid)
-	if err := guest.SaveSourceDesc(createConfig.GuestDesc); err != nil {
-		return nil, err
-	}
-	guest.Desc = new(desc.SGuestDesc)
-	jsonutils.Marshal(createConfig.GuestDesc).Unmarshal(guest.Desc)
-	if err := guest.SaveLiveDesc(guest.Desc); err != nil {
+	if err := guest.SaveDesc(createConfig.GuestDesc); err != nil {
 		return nil, err
 	}