feat: add creator plugins

Problem: we did not have a way to define a creator as a plugin.
Solution: add a plugin interface to create. I originally was going
to create separate plugin types, but I like the idea that one plugin
family can decide to define both easily. When this is refactored to
have a more "register" design (to make it flexible to changing the set
available) it will be nice to provide Create/Extract from the same
interface and keep the number of interfaces / functions for them
minimal. I was going to add hwloc now, but there seems to be a
bug so we will need to move forward prototyping with the current
proxy for nodes, which is just using the go runtime package. We
obviously need to improve upon this.

Signed-off-by: vsoch <[email protected]>

.gitignore

@@ -17,4 +17,7 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 bin
-vendor
+vendor
+cache
+lib
+*.json

Makefile.hwloc

@@ -0,0 +1,35 @@
+# This makefile will be used when we can add hwloc - there is currently a bug.
+HERE ?= $(shell pwd)
+LOCALBIN ?= $(shell pwd)/bin
+
+# Install hwloc here for use to compile, etc.
+LOCALLIB ?= $(shell pwd)/lib
+HWLOC_INCLUDE ?= $(LOCALLIB)/include/hwloc.h
+BUILDENVVAR=CGO_CFLAGS="-I$(LOCALLIB)/include" CGO_LDFLAGS="-L$(LOCALLIB)/lib -lhwloc"
+
+.PHONY: all
+
+all: build
+
+.PHONY: $(LOCALBIN)
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+.PHONY: $(LOCALLIB)
+$(LOCALLIB):
+	mkdir -p $(LOCALLIB)
+
+$(HWLOC_INCLUDE):
+	git clone --depth 1 https://github.com/open-mpi/hwloc /tmp/hwloc || true && \
+	  cd /tmp/hwloc && ./autogen.sh && \
+	  ./configure --enable-static --disable-shared LDFLAGS="-static" --prefix=$(LOCALLIB)/ && \
+  	  make LDFLAGS=-all-static && make install
+
+build: $(LOCALBIN) $(HWLOC_INCLUDE)
+	GO111MODULE="on" $(BUILDENVVAR) go build -ldflags '-w' -o $(LOCALBIN)/compspec cmd/compspec/compspec.go
+
+build-arm: $(LOCALBIN) $(HWLOC_INCLUDE)
+	GO111MODULE="on" $(BUILDENVVAR) GOARCH=arm64 go build -ldflags '-w' -o $(LOCALBIN)/compspec-arm cmd/compspec/compspec.go
+
+build-ppc: $(LOCALBIN) $(HWLOC_INCLUDE)
+	GO111MODULE="on" $(BUILDENVVAR) GOARCH=ppc64le go build -ldflags '-w' -o $(LOCALBIN)/compspec-ppc cmd/compspec/compspec.go

README.md

@@ -11,6 +11,8 @@ This is a prototype compatibility checking tool. Right now our aim is to use in
 
  - I'm starting with just Linux. I know there are those "other" platforms, but if it doesn't run on HPC or Kubernetes easily I'm not super interested (ahem, Mac and Windows)!
  - not all extractors work in containers (e.g., kernel needs to be on the host)
+ - The node feature discovery source doesn't provide mapping of socket -> cores, nor does it give details about logical vs. physical CPU.
+  - We will likely want to add hwloc go bindings, but there is a bug currently. 
 
 Note that for development we are using nfd-source that does not require kubernetes:
 

cmd/compspec/create/artifact.go

@@ -1,127 +1,31 @@
 package create
 
 import (
-	"fmt"
-	"os"
+	"strings"
 
-	"github.com/compspec/compspec-go/pkg/types"
-	ep "github.com/compspec/compspec-go/plugins/extractors"
-
-	p "github.com/compspec/compspec-go/plugins"
+	"github.com/compspec/compspec-go/plugins/creators/artifact"
 )
 
 // Artifact will create a compatibility artifact based on a request in YAML
 // TODO likely want to refactor this into a proper create plugin
 func Artifact(specname string, fields []string, saveto string, allowFail bool) error {
 
-	// Cut out early if a spec not provided
-	if specname == "" {
-		return fmt.Errorf("a spec input -i/--input is required")
-	}
-	request, err := loadRequest(specname)
-	if err != nil {
-		return err
-	}
-
-	// Right now we only know about extractors, when we define subfields
-	// we can further filter here.
-	extractors := request.GetExtractors()
-	plugins, err := ep.GetPlugins(extractors)
-	if err != nil {
-		return err
-	}
-
-	// Finally, add custom fields and extract metadata
-	result, err := plugins.Extract(allowFail)
-	if err != nil {
-		return err
+	// This is janky, oh well
+	allowFailFlag := "false"
+	if allowFail {
+		allowFailFlag = "true"
 	}
 
-	// Update with custom fields (either new or overwrite)
-	result.AddCustomFields(fields)
-
-	// The compspec returned is the populated Compatibility request!
-	compspec, err := PopulateExtractors(&result, request)
+	// assemble options for node creator
+	creator, err := artifact.NewPlugin()
 	if err != nil {
 		return err
 	}
-
-	output, err := compspec.ToJson()
-	if err != nil {
-		return err
+	options := map[string]string{
+		"specname":  specname,
+		"fields":    strings.Join(fields, "||"),
+		"saveto":    saveto,
+		"allowFail": allowFailFlag,
 	}
-	if saveto == "" {
-		fmt.Println(string(output))
-	} else {
-		err = os.WriteFile(saveto, output, 0644)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// LoadExtractors loads a compatibility result into a compatibility request
-// After this we can save the populated thing into an artifact (json DUMP)
-func PopulateExtractors(result *ep.Result, request *types.CompatibilityRequest) (*types.CompatibilityRequest, error) {
-
-	// Every metadata attribute must be known under a schema
-	schemas := request.Metadata.Schemas
-	if len(schemas) == 0 {
-		return nil, fmt.Errorf("the request must have one or more schemas")
-	}
-	for i, compat := range request.Compatibilities {
-
-		// The compatibility section name is a schema, and must be defined
-		url, ok := schemas[compat.Name]
-		if !ok {
-			return nil, fmt.Errorf("%s is missing a schema", compat.Name)
-		}
-		if url == "" {
-			return nil, fmt.Errorf("%s has an empty schema", compat.Name)
-		}
-
-		for key, extractorKey := range compat.Attributes {
-
-			// Get the extractor, section, and subfield from the extractor lookup key
-			f, err := p.ParseField(extractorKey)
-			if err != nil {
-				fmt.Printf("warning: cannot parse %s: %s, setting to empty\n", key, extractorKey)
-				compat.Attributes[key] = ""
-				continue
-			}
-
-			// If we get here, we can parse it and look it up in our result metadata
-			extractor, ok := result.Results[f.Extractor]
-			if !ok {
-				fmt.Printf("warning: extractor %s is unknown, setting to empty\n", f.Extractor)
-				compat.Attributes[key] = ""
-				continue
-			}
-
-			// Now get the section
-			section, ok := extractor.Sections[f.Section]
-			if !ok {
-				fmt.Printf("warning: section %s.%s is unknown, setting to empty\n", f.Extractor, f.Section)
-				compat.Attributes[key] = ""
-				continue
-			}
-
-			// Now get the value!
-			value, ok := section[f.Field]
-			if !ok {
-				fmt.Printf("warning: field %s.%s.%s is unknown, setting to empty\n", f.Extractor, f.Section, f.Field)
-				compat.Attributes[key] = ""
-				continue
-			}
-
-			// If we get here - we found it! Hooray!
-			compat.Attributes[key] = value
-		}
-
-		// Update the compatibiity
-		request.Compatibilities[i] = compat
-	}
-
-	return request, nil
+	return creator.Create(options)
 }