Domain info mismatch from vm runtime log and actual dumpxml for the domain

[yb01]

this is found in testing vertical scaling of memory: the below are the logs from the virtlet.log after the memory was detached without errors, and the dumpxml from the virsh command for the same domain. One of them seems has not refresh its cache from the libvirt metadata store.

`I0728 04:59:46.511827 27016 virtualization.go:1088] Update Domain Resources fb62a563-2ea0-53b3-7a4e-e07cd0bdb657, &LinuxContainerResources{CpuPeriod:100000,CpuQuota:200000,CpuShares:2048,MemoryLimitInBytes:4294967296,OomScoreAdj:-998,CpusetCpus:,CpusetMems:,}
I0728 04:59:46.512044 27016 virtualization.go:1107] Update Domain Memory configuration
I0728 04:59:46.512890 27016 virtualization.go:1182] Memory Info in KiB: current memory: 4456448, newMemory: 4194304. Max memory: 8388608
I0728 04:59:46.512906 27016 virtualization.go:1189] Update domain memory in KiB: 4456448 -> 4194304
I0728 04:59:46.512913 27016 libvirt_domain.go:315] MemoryChanges in KiB: -262144
I0728 04:59:46.512919 27016 libvirt_domain.go:322] isAttach: false
I0728 04:59:46.512925 27016 libvirt_domain.go:325] Number of device needed : 2
I0728 04:59:46.512932 27016 libvirt_domain.go:334] Detach memory device to domain
I0728 04:59:46.567247 27016 libvirt_domain.go:334] Detach memory device to domain
I0728 04:59:51.572437 27016 virtualization.go:1131] Update runtime metadata with config: {8109f3b5-169d-4c48-abb3-dc30e03890c4 vmdefault default system vm sha256:c4110030e2edf06db87f5b6e4efc27300977683d53f040996d15dcc0ad49bb5a 0 4294967296 2048 100000 200000 map[VirtletCPUModel:host-model VirtletRootVolumeSize:4Gi VirtletVCPUCount:2 kubernetes.io/config.seen:2020-07-28T04:57:34.390612769Z kubernetes.io/config.source:api kubernetes.io/target-runtime:virtlet.cloud] map[io.kubernetes.container.hash:28beced7 io.kubernetes.container.restartCount:0 io.kubernetes.container.terminationMessagePath: io.kubernetes.container.terminationMessagePolicy: io.kubernetes.pod.terminationGracePeriod:30] map[io.kubernetes.container.name:vm io.kubernetes.pod.name:vmdefault io.kubernetes.pod.namespace:default io.kubernetes.pod.tenant:system io.kubernetes.pod.uid:8109f3b5-169d-4c48-abb3-dc30e03890c4] 0xc4207526e0 fb62a563-2ea0-53b3-7a4e-e07cd0bdb657 [{KUBERNETES_SERVICE_PORT 443} {KUBERNETES_SERVICE_PORT_HTTPS 443} {KUBERNETES_PORT tcp://10.0.0.1:443} {KUBERNETES_PORT_443_TCP tcp://10.0.0.1:443} {KUBERNETES_PORT_443_TCP_PROTO tcp} {KUBERNETES_PORT_443_TCP_PORT 443} {KUBERNETES_PORT_443_TCP_ADDR 10.0.0.1} {KUBERNETES_SERVICE_HOST 10.0.0.1}] [{/etc/hosts /var/lib/kubelet/pods/8109f3b5-169d-4c48-abb3-dc30e03890c4/etc-hosts false}] [] 0xc4200113e0 /var/log/pods/system_default_vmdefault_8109f3b5-169d-4c48-abb3-dc30e03890c4 vm/0.log /kubepods/pod8109f3b5-169d-4c48-abb3-dc30e03890c4}
I0728 04:59:51.574446 27016 grpc.go:119] LEAVE: /runtime.v1alpha2.RuntimeService/UpdateContainerResources()
{}
I0728 04:59:51.575280 27016 grpc.go:110] ENTER: /runtime.v1alpha2.RuntimeService/PodSandboxStatus():
pod_sandbox_id: 8109f3b5-169d-4c48-abb3-dc30e03890c4
I0728 04:59:51.575652 27016 grpc.go:119] LEAVE: /runtime.v1alpha2.RuntimeService/PodSandboxStatus()
status:
annotations:
VirtletCPUModel: host-model
VirtletRootVolumeSize: 4Gi
VirtletVCPUCount: "2"
kubernetes.io/config.seen: 2020-07-28T04:57:34.390612769Z
kubernetes.io/config.source: api
kubernetes.io/target-runtime: virtlet.cloud
created_at: 1595912255016461234
id: 8109f3b5-169d-4c48-abb3-dc30e03890c4
labels:
io.kubernetes.pod.name: vmdefault
io.kubernetes.pod.namespace: default
io.kubernetes.pod.tenant: system
io.kubernetes.pod.uid: 8109f3b5-169d-4c48-abb3-dc30e03890c4
linux:
namespaces:
options: {}
metadata:
name: vmdefault
namespace: default
tenant: system
uid: 8109f3b5-169d-4c48-abb3-dc30e03890c4
network:
ip: 10.88.1.79
I0728 04:59:51.577258 27016 grpc.go:110] ENTER: /runtime.v1alpha2.RuntimeService/ContainerStatus():
container_id: fb62a563-2ea0-53b3-7a4e-e07cd0bdb657
I0728 04:59:51.577278 27016 virtualization.go:1204] Sync vm config with libvirt domain settings for VM: fb62a563-2ea0-53b3-7a4e-e07cd0bdb657
I0728 04:59:51.578158 27016 virtualization.go:1215] domain info memory in KiB: memory: 4194304, memoryUnit: KiB, cpuShares: 2048, cpuQuota: 100000, cpuPeriod: 100000, vcpus: 2
I0728 04:59:51.578310 27016 virtualization.go:1225] Runtime metadata config Memory in KiB: 4194304, CPUShares: 2048, CPUQuota: 200000, CPUPeriod: 100000
I0728 04:59:51.578320 27016 virtualization.go:1259] All resource are in sync, return
I0728 04:59:51.578651 27016 grpc.go:119] LEAVE: /runtime.v1alpha2.RuntimeService/ContainerStatus()
status:
annotations:
io.kubernetes.container.hash: 28beced7
io.kubernetes.container.restartCount: "0"
io.kubernetes.container.terminationMessagePath: ""
io.kubernetes.container.terminationMessagePolicy: ""
io.kubernetes.pod.terminationGracePeriod: "30"
created_at: 1595912255079354384
id: fb62a563-2ea0-53b3-7a4e-e07cd0bdb657
image:
image: sha256:c4110030e2edf06db87f5b6e4efc27300977683d53f040996d15dcc0ad49bb5a
image_ref: sha256:c4110030e2edf06db87f5b6e4efc27300977683d53f040996d15dcc0ad49bb5a
labels:
io.kubernetes.container.name: vm
io.kubernetes.pod.name: vmdefault
io.kubernetes.pod.namespace: default
io.kubernetes.pod.tenant: system
io.kubernetes.pod.uid: 8109f3b5-169d-4c48-abb3-dc30e03890c4
log_path: /var/log/pods/system_default_vmdefault_8109f3b5-169d-4c48-abb3-dc30e03890c4/vm/0.log
metadata:
name: vm
mounts:

• container_path: /etc/hosts
  host_path: /var/lib/kubelet/pods/8109f3b5-169d-4c48-abb3-dc30e03890c4/etc-hosts
  resources:
  cpu_period: 100000
  cpu_quota: 200000
  cpu_shares: 2048
  memory_limit_in_bytes: 4294967296
  started_at: 1595912255533392077
  state: 1
  `

root@ip-172-31-10-115:/# virsh dumpxml 20 <domain type='qemu' id='20' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'> <name>arktosRT--fb62a563-2ea0-vm</name> <uuid>fb62a563-2ea0-53b3-7a4e-e07cd0bdb657</uuid> <maxMemory slots='16' unit='KiB'>8388608</maxMemory> <memory unit='KiB'>4325376</memory> <currentMemory unit='KiB'>4325376</currentMemory> <vcpu placement='static' current='2'>4</vcpu> <cputune> <shares>2048</shares> <period>100000</period> <quota>100000</quota> </cputune>