From e01a76afa3b3f5c1a96449366de8559945a577d3 Mon Sep 17 00:00:00 2001 From: vsoch Date: Fri, 23 Feb 2024 20:35:44 -0700 Subject: [PATCH] feat: add memory extractor Problem: ndf does not appear to parse /proc/meminfo, and I did not think to do it. Solution: add a system[memory] parser that does that! Signed-off-by: vsoch --- README.md | 3 +- docs/usage.md | 43 ++++++++++++++++++++--------- go.mod | 4 +-- go.sum | 8 +++--- plugins/extractors/system/memory.go | 41 +++++++++++++++++++++++++++ plugins/extractors/system/system.go | 13 +++++++-- 6 files changed, 90 insertions(+), 22 deletions(-) create mode 100644 plugins/extractors/system/memory.go diff --git a/README.md b/README.md index 806483d..b06ad2b 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ github.com/compspec/compspec-go/plugins/extractors/nfd imports And moving forward we will be working from this WIP branch: ```bash -go get -u github.com/converged-computing/nfd-source/source@35294a0e91702fc0c9dcac48c3013a705625314b +# This is the develop branch +go get -u github.com/converged-computing/nfd-source/source@20d686e64926b80421637e82fb68e6c5f3f9242a ``` Note that the above is the main branch on February 22, 2024! diff --git a/docs/usage.md b/docs/usage.md index f3def94..aaf134d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -54,17 +54,33 @@ The list command lists each extractor, and sections available for it. $ ./bin/compspec list ``` ```console - Compatibility Plugins - TYPE NAME SECTION - extractor kernel boot - extractor kernel config - extractor kernel modules - extractor system cpu - extractor system processor - extractor system os - extractor system arch - extractor library mpi - TOTAL 8 + Compatibility Plugins + TYPE NAME SECTION + generic kernel extractor + extractor kernel boot + extractor kernel config + extractor kernel modules +---------------------------------------------------------- + generic system extractor + extractor system processor + extractor system os + extractor system arch + extractor system memory +---------------------------------------------------------- + generic library extractor + extractor library mpi +---------------------------------------------------------- + node feature discovery + extractor nfd cpu + extractor nfd kernel + extractor nfd local + extractor nfd memory + extractor nfd network + extractor nfd pci + extractor nfd storage + extractor nfd system + extractor nfd usb + TOTAL 4 17 ``` Note that we will eventually add a description column - it's not really warranted yet! @@ -451,9 +467,9 @@ the full ability to specify: Current Extractors include: - Library: library-specific metadata (e.g., mpi) - - System: system-specific metadata (e.g., processor, cpu, arch, os) + - System: system-specific metadata (e.g., processor, cpu, arch, os, memory) - Kernel: kernel-speific metadata (e.g., boot, config, modules) - + - Node Feature Discovery: uses the [source](https://github.com/converged-computing/nfd-source) of NFD to derive metadata across many domains (cpu, kernel, local, memory, network, pci, storage, system, usb) #### Library @@ -508,6 +524,7 @@ The system extractor supports three sections - processor: detailed information on every processor - os: operating system information - arch: architecture + - memory: parses /proc/meminfo and gives results primarily in KB For example: diff --git a/go.mod b/go.mod index 6c8cca3..b5d091c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/akamensky/argparse v1.4.0 github.com/converged-computing/jsongraph-go v0.0.0-20231221142916-249fef6889b3 - github.com/converged-computing/nfd-source v0.0.0-20240222201246-35294a0e9170 + github.com/converged-computing/nfd-source v0.0.0-20240224025007-20d686e64926 github.com/jedib0t/go-pretty/v6 v6.5.4 github.com/moby/moby v25.0.3+incompatible github.com/opencontainers/image-spec v1.1.0 @@ -33,7 +33,7 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/stretchr/objx v0.5.1 // indirect github.com/stretchr/testify v1.8.4 // indirect - golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/sync v0.6.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect diff --git a/go.sum b/go.sum index 17ca825..44b9560 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/converged-computing/jsongraph-go v0.0.0-20231221142916-249fef6889b3 h1:frJJfyARuHmF2eohDCyltBLE6tRJKvA1shuS2aWQaf8= github.com/converged-computing/jsongraph-go v0.0.0-20231221142916-249fef6889b3/go.mod h1:+DhVyLXGVfBsfta4185jd33jqa94inshCcdvsXK2Irk= -github.com/converged-computing/nfd-source v0.0.0-20240222201246-35294a0e9170 h1:I2SCLorK+p4eP6GwIkuZgckq1BNDQDbpK+Ojn38ypQU= -github.com/converged-computing/nfd-source v0.0.0-20240222201246-35294a0e9170/go.mod h1:I15nBsQqBTUsc3A4a6cuQmZjQ8lYUZSZ2a7UAE5SZ3g= +github.com/converged-computing/nfd-source v0.0.0-20240224025007-20d686e64926 h1:VZmgK3t4564vdHNpE//q6kuPlugOrojkDHP4Gqd4A1g= +github.com/converged-computing/nfd-source v0.0.0-20240224025007-20d686e64926/go.mod h1:I15nBsQqBTUsc3A4a6cuQmZjQ8lYUZSZ2a7UAE5SZ3g= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= @@ -59,8 +59,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/plugins/extractors/system/memory.go b/plugins/extractors/system/memory.go new file mode 100644 index 0000000..db9b283 --- /dev/null +++ b/plugins/extractors/system/memory.go @@ -0,0 +1,41 @@ +package system + +import ( + "os" + "strings" + + "github.com/compspec/compspec-go/pkg/extractor" +) + +const ( + memoryInfoFile = "/proc/meminfo" +) + +// getMemoryInformation parses /proc/meminfo to get node memory metadata +func getMemoryInformation() (extractor.ExtractorSection, error) { + info := extractor.ExtractorSection{} + + raw, err := os.ReadFile(memoryInfoFile) + if err != nil { + return nil, err + } + + lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + + // We need custom parsing, the sections per processor are split by newlines + for _, line := range lines { + + // I don't see any empty lines, etc. + line = strings.Trim(line, " ") + parts := strings.Split(line, ":") + + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + // Replace parens with underscore. Leave camel case for the rest... + key = strings.ReplaceAll(key, "(", "_") + key = strings.ToLower(strings.ReplaceAll(key, ")", "")) + info[key] = value + } + return info, nil +} diff --git a/plugins/extractors/system/system.go b/plugins/extractors/system/system.go index ff5be1a..bcf2a07 100644 --- a/plugins/extractors/system/system.go +++ b/plugins/extractors/system/system.go @@ -16,10 +16,11 @@ const ( ProcessorSection = "processor" ArchSection = "arch" OsSection = "os" + MemorySection = "memory" ) var ( - validSections = []string{ProcessorSection, OsSection, ArchSection} + validSections = []string{ProcessorSection, OsSection, ArchSection, MemorySection} ) type SystemExtractor struct { @@ -78,6 +79,14 @@ func (e SystemExtractor) Extract(interface{}) (extractor.ExtractorData, error) { sections[ArchSection] = section } + if name == MemorySection { + section, err := getMemoryInformation() + if err != nil { + return data, err + } + sections[MemorySection] = section + } + } data.Sections = sections return data, nil @@ -90,7 +99,7 @@ func NewPlugin(sections []string) (extractor.Extractor, error) { } e := SystemExtractor{sections: sections} if !e.Validate() { - return nil, fmt.Errorf("plugin %s is not valid\n", e.Name()) + return nil, fmt.Errorf("plugin %s is not valid", e.Name()) } return e, nil }