feat: add experimental jobspec (#4)

This adds the resources section to tasks, and it is intended
to be flexible to allow serializing only.

Signed-off-by: vsoch <[email protected]>
Co-authored-by: vsoch <[email protected]>

.gitignore

@@ -1,2 +1,3 @@
 vendor
 examples/v1/bin
+examples/experimental/bin

Makefile

@@ -3,12 +3,13 @@ COMMONENVVAR=GOOS=$(shell uname -s | tr A-Z a-z)
 RELEASE_VERSION?=v$(shell date +%Y%m%d)-$(shell git describe --tags --match "v*")
 
 .PHONY: all
-all: example1 example2 example3 example4 example5 example6 createnew
+all: example1 example2 example3 example4 example5 example6 createnew experimental1
 
 .PHONY: build
 build: 
 	go mod tidy
 	mkdir -p ./examples/v1/bin
+	mkdir -p ./examples/experimental/bin
 
 # Build examples
 .PHONY: createnew
@@ -39,6 +40,10 @@ example5: build
 example6: build
 	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o ./examples/v1/bin/example6 examples/v1/example6/example.go
 
+.PHONY: experimental1
+experimental1: build
+	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o ./examples/experimental/bin/example1 examples/experimental/example1/example.go
+
 .PHONY: test
 test:
 	./examples/v1/bin/example1
@@ -48,6 +53,7 @@ test:
 	./examples/v1/bin/example5
 	./examples/v1/bin/example6
 	./examples/v1/bin/createnew
+	./examples/experimental/bin/example1
 
 .PHONY: clean
 clean:

examples/experimental/example1/example.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	v1 "github.com/compspec/jobspec-go/pkg/jobspec/experimental"
+)
+
+func main() {
+	fmt.Println("This example reads, parses, and validates a Jobspec")
+
+	// Assumes running from the root
+	fileName := flag.String("json", "examples/v1/example1/jobspec.yaml", "yaml file")
+	flag.Parse()
+
+	yamlFile := *fileName
+	if yamlFile == "" {
+		flag.Usage()
+		os.Exit(0)
+	}
+	js, err := v1.LoadJobspecYaml(yamlFile)
+	if err != nil {
+		fmt.Printf("error reading %s:%s\n", yamlFile, err)
+		os.Exit(1)
+	}
+
+	// Validate the jobspec
+	valid, err := js.Validate()
+	if !valid || err != nil {
+		fmt.Printf("schema is not valid:%s\n", err)
+		os.Exit(1)
+	} else {
+		fmt.Println("schema is valid")
+	}
+	fmt.Println(js)
+
+	out, err := js.JobspecToYaml()
+	if err != nil {
+		fmt.Printf("error marshalling %s:%s\n", yamlFile, err)
+		os.Exit(1)
+	}
+	fmt.Println(string(out))
+
+	// One example of json
+	out, err = js.JobspecToJson()
+	if err != nil {
+		fmt.Printf("error marshalling %s:%s\n", yamlFile, err)
+		os.Exit(1)
+	}
+	fmt.Println(string(out))
+}

examples/experimental/example1/jobspec.yaml

@@ -0,0 +1,23 @@
+version: 1
+resources:
+  - type: node
+    count: 4
+    with:
+      - type: slot
+        count: 1
+        label: default
+        with:
+          - type: core
+            count: 2
+tasks:
+- command:
+  - ior
+  count:
+    per_slot: 1
+  resources:
+    io:
+      storage:
+      - priority: 1
+        storage: mtl2unit
+      - priority: 2
+        storage: shm

pkg/jobspec/experimental/convert.go

@@ -0,0 +1,73 @@
+package experimental
+
+import (
+	"fmt"
+	"strings"
+)
+
+// NewSimpleJobSpec generates a simple jobspec for nodes, command, tasks, and (optionally) a name
+func NewSimpleJobspec(name, command string, nodes, tasks int32) (*Jobspec, error) {
+
+	// If no name provided for the slot, use the first
+	// work of the command
+	if name == "" {
+		parts := strings.Split(command, " ")
+		name = strings.ToLower(parts[0])
+	}
+	if nodes < 1 {
+		return nil, fmt.Errorf("nodes for the job must be >= 1")
+	}
+	if command == "" {
+		return nil, fmt.Errorf("a command must be provided")
+	}
+
+	// The node resource is what we are asking for
+	nodeResource := Resource{
+		Type:  "node",
+		Count: nodes,
+	}
+
+	// The slot is where we are doing an assessment for scheduling
+	slot := Resource{
+		Type:  "slot",
+		Count: int32(1),
+		Label: name,
+	}
+
+	// If tasks are defined, this is total tasks across the nodes
+	// We add to the slot
+	if tasks != 0 {
+		taskResource := Resource{
+			Type:  "core",
+			Count: tasks,
+		}
+		slot.With = []Resource{taskResource}
+	}
+
+	// And then the entire resource spec is added to the top level node resource
+	nodeResource.With = []Resource{slot}
+
+	// Tasks reference the slot and command
+	// Note: if we need better split can use "github.com/google/shlex"
+	cmd := strings.Split(command, " ")
+	taskResource := []Tasks{
+		{
+			Command: cmd,
+			Slot:    name,
+			Count:   Count{PerSlot: int32(1)},
+		}}
+
+	// Attributes are for the system, we aren't going to add them yet
+	// attributes:
+	// system:
+	//   duration: 3600.
+	//   cwd: "/home/flux"
+	//   environment:
+	// 	HOME: "/home/flux"
+	// This is verison 1 as defined by v1 above
+	return &Jobspec{
+		Version:   jobspecVersion,
+		Resources: []Resource{nodeResource},
+		Tasks:     taskResource,
+	}, nil
+}

pkg/jobspec/experimental/jobspec.go

@@ -0,0 +1,76 @@
+package experimental
+
+import (
+	"encoding/json"
+	"os"
+
+	"sigs.k8s.io/yaml"
+
+	"github.com/compspec/jobspec-go/pkg/schema"
+)
+
+// LoadJobspecYaml loads a jobspec from a yaml file path
+func LoadJobspecYaml(yamlFile string) (*Jobspec, error) {
+	js := Jobspec{}
+	file, err := os.ReadFile(yamlFile)
+	if err != nil {
+		return &js, err
+	}
+
+	err = yaml.Unmarshal([]byte(file), &js)
+	if err != nil {
+		return &js, err
+	}
+	return &js, nil
+}
+
+// JobspectoYaml convets back to yaml (as string)
+func (js *Jobspec) JobspecToYaml() (string, error) {
+	out, err := yaml.Marshal(js)
+	if err != nil {
+		return "", err
+	}
+	return string(out), nil
+}
+
+// JobspectoJson convets back to json string
+func (js *Jobspec) JobspecToJson() (string, error) {
+	out, err := json.MarshalIndent(js, "", " ")
+	if err != nil {
+		return "", err
+	}
+	return string(out), nil
+}
+
+// Validate converts to bytes and validate with jsonschema
+func (js *Jobspec) Validate() (bool, error) {
+
+	// Get back into bytes form
+	out, err := yaml.Marshal(js)
+	if err != nil {
+		return false, err
+	}
+	// Validate the jobspec
+	return schema.Validate(out, schema.SchemaUrl, Schema)
+
+}
+
+// Helper function to get a job name, derived from the command
+func (js *Jobspec) GetJobName() string {
+
+	// Generic name to fall back tp
+	name := "app"
+
+	// If we have tasks, we can get from the command
+	// This entire set of checks is meant to be conservative
+	// and avoid any errors with nil / empty arrays, etc.
+	if js.Tasks != nil {
+		if len(js.Tasks) > 0 {
+			command := js.Tasks[0].Command
+			if len(command) > 0 {
+				name = command[0]
+			}
+		}
+	}
+	return name
+}

pkg/jobspec/experimental/schema.go

@@ -0,0 +1,8 @@
+package experimental
+
+import (
+	_ "embed"
+)
+
+//go:embed schema.json
+var Schema string