feat(host): container nvidia gpu metrics

pkg/hostman/guestman/pod.go

@@ -134,6 +134,8 @@ type PodInstance interface {
 
 	IsInternalStopped(ctrCriId string) (*ContainerExpectedStatus, bool)
 	IsInternalRemoved(ctrCriId string) bool
+
+	GetPodContainerCriIds() []string
 }
 
 type sContainer struct {
@@ -588,6 +590,24 @@ func (s *sPodGuestInstance) GetContainers() []*hostapi.ContainerDesc {
 	return s.GetDesc().Containers
 }
 
+func (s *sPodGuestInstance) GetPodContainerCriIds() []string {
+	criids := make([]string, 0)
+	for i := range s.containers {
+		criids = append(criids, s.containers[i].CRIId)
+	}
+	return criids
+}
+
+func (s *sPodGuestInstance) HasContainerNvidiaGpu() bool {
+	for i := range s.Desc.IsolatedDevices {
+		if s.Desc.IsolatedDevices[i].DevType == computeapi.CONTAINER_DEV_NVIDIA_MPS ||
+			s.Desc.IsolatedDevices[i].DevType == computeapi.CONTAINER_DEV_NVIDIA_GPU {
+			return true
+		}
+	}
+	return false
+}
+
 func (s *sPodGuestInstance) GetContainerById(ctrId string) *hostapi.ContainerDesc {
 	ctrs := s.GetContainers()
 	for i := range ctrs {
@@ -1700,7 +1720,12 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie
 		if err := s.getIsolatedDeviceExtraConfig(spec, ctrCfg); err != nil {
 			return "", err
 		}
+	} else {
+		if hostinfo.Instance().HasContainerNvidiaGpu() {
+			ctrCfg.Envs = append(ctrCfg.Envs, &runtimeapi.KeyValue{Key: "NVIDIA_VISIBLE_DEVICES", Value: "void"})
+		}
 	}
+
 	if len(spec.Command) != 0 {
 		ctrCfg.Command = spec.Command
 	}

pkg/hostman/hostinfo/container.go

@@ -22,7 +22,9 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
+	apis "yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/util/pod"
 	"yunion.io/x/onecloud/pkg/util/pod/cadvisor"
@@ -76,3 +78,70 @@ func (h *SHostInfo) GetContainerCPUMap() *pod.HostContainerCPUMap {
 func (h *SHostInfo) GetContainerStatsProvider() stats.ContainerStatsProvider {
 	return h.containerStatsProvider
 }
+
+type INvidiaGpuIndexMemoryInterface interface {
+	GetNvidiaDevMemSize() int
+	GetNvidiaDevIndex() string
+}
+
+func (h *SHostInfo) GetNvidiaGpuIndexMemoryMap() map[string]int {
+	res := map[string]int{}
+	for i := range h.containerNvidiaGpus {
+		iDev, ok := h.containerNvidiaGpus[i].(INvidiaGpuIndexMemoryInterface)
+		if !ok {
+			continue
+		}
+		index := iDev.GetNvidiaDevIndex()
+		memSize := iDev.GetNvidiaDevMemSize()
+		res[index] = memSize
+	}
+	return res
+}
+
+func (h *SHostInfo) HasContainerVastaitechGpu() bool {
+	if h.hasVastaitechGpus != nil {
+		return *h.hasVastaitechGpus
+	}
+	hasVastaitechGpus := false
+	devs := h.IsolatedDeviceMan.GetDevices()
+	for i := range devs {
+		if devs[i].GetDeviceType() == apis.CONTAINER_DEV_VASTAITECH_GPU {
+			hasVastaitechGpus = true
+		}
+	}
+	h.hasVastaitechGpus = &hasVastaitechGpus
+	return *h.hasVastaitechGpus
+}
+
+func (h *SHostInfo) HasContainerCphAmdGpu() bool {
+	if h.hasCphAmdGpus != nil {
+		return *h.hasCphAmdGpus
+	}
+	hasCphAmdGpus := false
+	devs := h.IsolatedDeviceMan.GetDevices()
+	for i := range devs {
+		if devs[i].GetDeviceType() == apis.CONTAINER_DEV_CPH_AMD_GPU {
+			hasCphAmdGpus = true
+		}
+	}
+	h.hasCphAmdGpus = &hasCphAmdGpus
+	return *h.hasCphAmdGpus
+}
+
+func (h *SHostInfo) HasContainerNvidiaGpu() bool {
+	if h.hasNvidiaGpus != nil {
+		return *h.hasNvidiaGpus
+	}
+	hasNvidiaGpus := false
+	nvDevs := make([]isolated_device.IDevice, 0)
+	devs := h.IsolatedDeviceMan.GetDevices()
+	for i := range devs {
+		if devs[i].GetDeviceType() == apis.CONTAINER_DEV_NVIDIA_GPU || devs[i].GetDeviceType() == apis.CONTAINER_DEV_NVIDIA_MPS {
+			hasNvidiaGpus = true
+			nvDevs = append(nvDevs, devs[i])
+		}
+	}
+	h.hasNvidiaGpus = &hasNvidiaGpus
+	h.containerNvidiaGpus = nvDevs
+	return *h.hasNvidiaGpus
+}

pkg/hostman/hostinfo/hostinfo.go

@@ -129,6 +129,10 @@ type SHostInfo struct {
 	containerCPUMap                *pod.HostContainerCPUMap
 	containerStatsProvider         stats.ContainerStatsProvider
 	containerCpufreqSimulateConfig *jsonutils.JSONDict
+	containerNvidiaGpus            []isolated_device.IDevice
+	hasNvidiaGpus                  *bool
+	hasVastaitechGpus              *bool
+	hasCphAmdGpus                  *bool
 }
 
 func (h *SHostInfo) GetContainerDeviceConfigurationFilePath() string {
@@ -2552,6 +2556,7 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 	// group dev
 	hasNetint := false
 	hasVasmi := false
+	hasNvidiasmi := false
 	for _, dev := range devs {
 		devType := dev.GetDeviceType()
 		switch devType {
@@ -2575,18 +2580,23 @@ func (h *SHostInfo) injectTelegrafDeviceConfig(conf map[string]interface{}) {
 		case string(isolated_device.ContainerDeviceTypeVastaitechGpu):
 			hasVasmi = true
 			continue
+		case string(isolated_device.ContainerDeviceTypeNvidiaGpu), string(isolated_device.ContainerDeviceTypeNvidiaMps):
+			hasNvidiasmi = true
 		}
 	}
 	if hasNetint {
-		conf[system_service.TELEGAF_INPUT_NETDEV] = map[string]interface{}{
+		conf[system_service.TELEGRAF_INPUT_NETDEV] = map[string]interface{}{
 			system_service.TELEGRAF_INPUT_CONF_BIN_PATH: "/usr/bin/ni_rsrc_mon",
 		}
 	}
 	if hasVasmi {
-		conf[system_service.TELEGAF_INPUT_VASMI] = map[string]interface{}{
+		conf[system_service.TELEGRAF_INPUT_VASMI] = map[string]interface{}{
 			system_service.TELEGRAF_INPUT_CONF_BIN_PATH: "/usr/bin/vasmi",
 		}
 	}
+	if hasNvidiasmi {
+		conf[system_service.TELEGRAF_INPUT_NVIDIASMI] = struct{}{}
+	}
 }
 
 func (h *SHostInfo) getNicsTelegrafConf() []map[string]interface{} {

pkg/hostman/hostmetrics/container_cph_amd_gpu_metrics.go

@@ -0,0 +1,120 @@
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostmetrics
+
+import (
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+)
+
+type CphAmdGpuProcessMetrics struct {
+	Pid   string // Process ID
+	DevId string
+	Mem   float64 // Memory Utilization
+}
+
+/*
+pid  2088269 command [email protected]:
+        0x00000001:         4096 byte  GTT CPU_ACCESS_REQUIRED
+        0x00000002:      2097152 byte  GTT CPU_ACCESS_REQUIRED
+        0x00000003:      2097152 byte VRAM VRAM_CLEARED
+        0x00000004:      2097152 byte VRAM NO_CPU_ACCESS VRAM_CLEARED
+        0x00000006:      2097152 byte  GTT CPU_ACCESS_REQUIRED VRAM_CLEARED
+        0x00000007:      2097152 byte  GTT CPU_ACCESS_REQUIRED VRAM_CLEARED
+*/
+
+func GetCphAmdGpuProcessMetrics() ([]CphAmdGpuProcessMetrics, error) {
+	debugDriDir := "/sys/kernel/debug/dri"
+	entrys, err := os.ReadDir(debugDriDir)
+	if err != nil {
+		return nil, errors.Wrap(err, "os.ReadDir")
+	}
+
+	res := make([]CphAmdGpuProcessMetrics, 0)
+	for i := range entrys {
+		if entrys[i].IsDir() {
+			fpath := path.Join(debugDriDir, entrys[i].Name(), "amdgpu_gem_info")
+			if fileutils2.Exists(fpath) {
+				content, err := fileutils2.FileGetContents(fpath)
+				if err != nil {
+					log.Errorf("failed FileGetContents %s: %s", fpath, err)
+					continue
+				}
+				metrics := parseCphAmdGpuGemInfo(content, entrys[i].Name())
+				if len(metrics) > 0 {
+					res = append(res, metrics...)
+				}
+			}
+		}
+	}
+	return res, nil
+}
+
+func parseCphAmdGpuGemInfo(content string, devId string) []CphAmdGpuProcessMetrics {
+	res := make([]CphAmdGpuProcessMetrics, 0)
+	lines := strings.Split(content, "\n")
+	var i, length = 0, len(lines)
+	for i < length {
+		line := strings.TrimSpace(lines[i])
+		segs := strings.Fields(line)
+		if len(segs) < 2 {
+			i++
+			continue
+		}
+		if segs[0] != "pid" {
+			i++
+			continue
+		}
+		pid := segs[1]
+		var vramTotal int64 = 0
+		j := i + 1
+		for j < length {
+			line := strings.TrimSpace(lines[j])
+			if len(line) == 0 {
+				break
+			}
+			segs := strings.Fields(line)
+			if len(segs) < 4 {
+				log.Errorf("unknown output line %s", line)
+				break
+			}
+			if segs[0] == "pid" {
+				break
+			}
+			memUsedStr, memType := segs[1], segs[3]
+			if memType == "VRAM" {
+				memUsed, err := strconv.ParseInt(memUsedStr, 10, 64)
+				if err != nil {
+					log.Errorf("failed parse memused %s %s: %s", line, memUsedStr, err)
+					break
+				}
+				vramTotal += memUsed
+			}
+			j++
+		}
+		res = append(res, CphAmdGpuProcessMetrics{
+			Pid:   pid,
+			DevId: devId,
+			Mem:   float64(vramTotal) / 1024.0 / 1024.0,
+		})
+		i = j
+	}
+
+	return res
+}