Merge pull request #24 from compspec/add-memory-extractor

feat: add memory extractor
compspec · Feb 24, 2024 · 51ffdbe · 51ffdbe
2 parents 314caec + e01a76a
commit 51ffdbe
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -28,7 +28,8 @@ github.com/compspec/compspec-go/plugins/extractors/nfd imports
 And moving forward we will be working from this WIP branch:
 
 ```bash
-go get -u github.com/converged-computing/nfd-source/source@35294a0e91702fc0c9dcac48c3013a705625314b
+# This is the develop branch
+go get -u github.com/converged-computing/nfd-source/source@20d686e64926b80421637e82fb68e6c5f3f9242a
 ```
 
 Note that the above is the main branch on February 22, 2024!

diff --git a/docs/usage.md b/docs/usage.md
@@ -54,17 +54,33 @@ The list command lists each extractor, and sections available for it.
 $ ./bin/compspec list
 ```
 ```console
- Compatibility Plugins               
-        TYPE      NAME     SECTION   
-        extractor  kernel   boot      
-        extractor  kernel   config    
-        extractor  kernel   modules   
-        extractor  system   cpu       
-        extractor  system   processor 
-        extractor  system   os        
-        extractor  system   arch      
-        extractor  library  mpi       
- TOTAL  8                            
+ Compatibility Plugins                                    
+                            TYPE       NAME     SECTION   
+ generic kernel extractor                                 
+                            extractor  kernel   boot      
+                            extractor  kernel   config    
+                            extractor  kernel   modules   
+----------------------------------------------------------
+ generic system extractor                                 
+                            extractor  system   processor 
+                            extractor  system   os        
+                            extractor  system   arch      
+                            extractor  system   memory    
+----------------------------------------------------------
+ generic library extractor                                
+                            extractor  library  mpi       
+----------------------------------------------------------
+ node feature discovery                                   
+                            extractor  nfd      cpu       
+                            extractor  nfd      kernel    
+                            extractor  nfd      local     
+                            extractor  nfd      memory    
+                            extractor  nfd      network   
+                            extractor  nfd      pci       
+                            extractor  nfd      storage   
+                            extractor  nfd      system    
+                            extractor  nfd      usb       
+ TOTAL                                 4        17        
 ```
 
 Note that we will eventually add a description column - it's not really warranted yet!
@@ -451,9 +467,9 @@ the full ability to specify:
 Current Extractors include:
 
  - Library: library-specific metadata (e.g., mpi)
- - System: system-specific metadata (e.g., processor, cpu, arch, os)
+ - System: system-specific metadata (e.g., processor, cpu, arch, os, memory)
  - Kernel: kernel-speific metadata (e.g., boot, config, modules)
-
+ - Node Feature Discovery: uses the [source](https://github.com/converged-computing/nfd-source) of NFD to derive metadata across many domains (cpu, kernel, local, memory, network, pci, storage, system, usb)
 
 #### Library
 
@@ -508,6 +524,7 @@ The system extractor supports three sections
  - processor: detailed information on every processor
  - os: operating system information
  - arch: architecture
+ - memory: parses /proc/meminfo and gives results primarily in KB
 
 For example:
 

diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/akamensky/argparse v1.4.0
 	github.com/converged-computing/jsongraph-go v0.0.0-20231221142916-249fef6889b3
-	github.com/converged-computing/nfd-source v0.0.0-20240222201246-35294a0e9170
+	github.com/converged-computing/nfd-source v0.0.0-20240224025007-20d686e64926
 	github.com/jedib0t/go-pretty/v6 v6.5.4
 	github.com/moby/moby v25.0.3+incompatible
 	github.com/opencontainers/image-spec v1.1.0
@@ -33,7 +33,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/stretchr/objx v0.5.1 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
-	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect
+	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.1 // indirect

diff --git a/go.sum b/go.sum
@@ -4,8 +4,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/converged-computing/jsongraph-go v0.0.0-20231221142916-249fef6889b3 h1:frJJfyARuHmF2eohDCyltBLE6tRJKvA1shuS2aWQaf8=
 github.com/converged-computing/jsongraph-go v0.0.0-20231221142916-249fef6889b3/go.mod h1:+DhVyLXGVfBsfta4185jd33jqa94inshCcdvsXK2Irk=
-github.com/converged-computing/nfd-source v0.0.0-20240222201246-35294a0e9170 h1:I2SCLorK+p4eP6GwIkuZgckq1BNDQDbpK+Ojn38ypQU=
-github.com/converged-computing/nfd-source v0.0.0-20240222201246-35294a0e9170/go.mod h1:I15nBsQqBTUsc3A4a6cuQmZjQ8lYUZSZ2a7UAE5SZ3g=
+github.com/converged-computing/nfd-source v0.0.0-20240224025007-20d686e64926 h1:VZmgK3t4564vdHNpE//q6kuPlugOrojkDHP4Gqd4A1g=
+github.com/converged-computing/nfd-source v0.0.0-20240224025007-20d686e64926/go.mod h1:I15nBsQqBTUsc3A4a6cuQmZjQ8lYUZSZ2a7UAE5SZ3g=
 github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
@@ -59,8 +59,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE=
-golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
+golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ=
+golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

diff --git a/plugins/extractors/system/memory.go b/plugins/extractors/system/memory.go
@@ -0,0 +1,41 @@
+package system
+
+import (
+	"os"
+	"strings"
+
+	"github.com/compspec/compspec-go/pkg/extractor"
+)
+
+const (
+	memoryInfoFile = "/proc/meminfo"
+)
+
+// getMemoryInformation parses /proc/meminfo to get node memory metadata
+func getMemoryInformation() (extractor.ExtractorSection, error) {
+	info := extractor.ExtractorSection{}
+
+	raw, err := os.ReadFile(memoryInfoFile)
+	if err != nil {
+		return nil, err
+	}
+
+	lines := strings.Split(strings.TrimSpace(string(raw)), "\n")
+
+	// We need custom parsing, the sections per processor are split by newlines
+	for _, line := range lines {
+
+		// I don't see any empty lines, etc.
+		line = strings.Trim(line, " ")
+		parts := strings.Split(line, ":")
+
+		key := strings.TrimSpace(parts[0])
+		value := strings.TrimSpace(parts[1])
+
+		// Replace parens with underscore. Leave camel case for the rest...
+		key = strings.ReplaceAll(key, "(", "_")
+		key = strings.ToLower(strings.ReplaceAll(key, ")", ""))
+		info[key] = value
+	}
+	return info, nil
+}
diff --git a/plugins/extractors/system/system.go b/plugins/extractors/system/system.go
@@ -16,10 +16,11 @@ const (
 	ProcessorSection = "processor"
 	ArchSection      = "arch"
 	OsSection        = "os"
+	MemorySection    = "memory"
 )
 
 var (
-	validSections = []string{ProcessorSection, OsSection, ArchSection}
+	validSections = []string{ProcessorSection, OsSection, ArchSection, MemorySection}
 )
 
 type SystemExtractor struct {
@@ -78,6 +79,14 @@ func (e SystemExtractor) Extract(interface{}) (extractor.ExtractorData, error) {
 			sections[ArchSection] = section
 		}
 
+		if name == MemorySection {
+			section, err := getMemoryInformation()
+			if err != nil {
+				return data, err
+			}
+			sections[MemorySection] = section
+		}
+
 	}
 	data.Sections = sections
 	return data, nil
@@ -90,7 +99,7 @@ func NewPlugin(sections []string) (extractor.Extractor, error) {
 	}
 	e := SystemExtractor{sections: sections}
 	if !e.Validate() {
-		return nil, fmt.Errorf("plugin %s is not valid\n", e.Name())
+		return nil, fmt.Errorf("plugin %s is not valid", e.Name())
 	}
 	return e, nil
 }