osbuild: start stage separation support

This commit prepares for stage separation support. The old import
mechanism is still supported but if it fails the new way of
just running the binary and outputing the data via stdout is
used.

Makefile

@@ -85,6 +85,10 @@ help:
 	@echo "    help:               Print this usage information."
 	@echo "    man:                Generate all man-pages"
 	@echo
+	@echo "    go:                 Generate all go stages"
+	@echo "    go-test:            Test all go stages"
+	@echo "    go-clean:           Clean all go stages"
+	@echo
 	@echo "    coverity-download:  Force a new download of the coverity tool"
 	@echo "    coverity-check:     Run the coverity test suite"
 	@echo "    coverity-submit:    Run coverity and submit the results"
@@ -121,6 +125,23 @@ $(MANPAGES_TROFF): $(BUILDDIR)/docs/%: $(SRCDIR)/docs/%.rst | $(BUILDDIR)/docs/
 .PHONY: man
 man: $(MANPAGES_TROFF)
 
+#
+## Go stages
+#
+GO_STAGES_IN = $(wildcard $(SRCDIR)/stages/go/*/main.go)
+GO_STAGES_OUT = $(patsubst %/,%, $(subst /go/,/,$(dir $(GO_STAGES_IN))))
+$(GO_STAGES_OUT): $(GO_STAGES_IN)
+	go build -o "$@" "$<"
+.PHONY: go go-clean go-test
+go: $(GO_STAGES_OUT)
+go-clean:
+	rm -f $(GO_STAGES_OUT)
+go-test: go
+	# XXX: ugly
+	for d in $(wildcard $(SRCDIR)/stages/go/*); do \
+		(cd $$d && go test ); \
+	done
+
 #
 # Coverity
 #

osbuild/meta.py

@@ -27,6 +27,7 @@
 import json
 import os
 import pkgutil
+import subprocess
 import sys
 from collections import deque
 from typing import (Any, Deque, Dict, List, Optional, Sequence, Set, Tuple,
@@ -410,6 +411,35 @@ def _parse_caps(cls, _klass, _name, node):
 
     @classmethod
     def load(cls, root, klass, name) -> Optional["ModuleInfo"]:
+        base = cls.MODULES.get(klass)
+        if not base:
+            raise ValueError(f"Unsupported type: {klass}")
+        path = os.path.join(root, base, name)
+
+        try:
+            return cls._load_py(path, klass, name)
+        except (SyntaxError, UnicodeDecodeError):
+            pass
+        return cls._load_generic(path, klass, name)
+
+    @classmethod
+    def _load_generic(cls, path, klass, name) -> Optional["ModuleInfo"]:
+        output = subprocess.check_output([path, "--all"])
+        p = json.loads(output)
+        doclist = p.get("doc", "").split("\n")
+        info = {
+            "schema": {
+                "1": json.loads(p.get("schema", "{}")),
+                "2": json.loads(p.get("schema_2", "{}")),
+            },
+            'desc': doclist[0],
+            'info': "\n".join(doclist[1:]),
+            'caps': p.get("capabilities", set()),
+        }
+        return cls(klass, name, path, info)
+
+    @classmethod
+    def _load_py(cls, path, klass, name) -> Optional["ModuleInfo"]:
         names = ["SCHEMA", "SCHEMA_2", "CAPABILITIES"]
 
         def filter_type(lst, target):
@@ -418,11 +448,6 @@ def filter_type(lst, target):
         def targets(a):
             return [t.id for t in filter_type(a.targets, ast.Name)]
 
-        base = cls.MODULES.get(klass)
-        if not base:
-            raise ValueError(f"Unsupported type: {klass}")
-
-        path = os.path.join(root, base, name)
         try:
             with open(path, encoding="utf8") as f:
                 data = f.read()

stages/go/org.osbuild.staticgzip/export_test.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+)
+
+func MockApiArguments(t *testing.T, apiArgs []byte) (restore func()) {
+	saved := apiArgumentsPath
+
+	tmpdir := t.TempDir()
+	apiArgumentsPath = filepath.Join(tmpdir, "apiArguments")
+	if err := ioutil.WriteFile(apiArgumentsPath, apiArgs, 0644); err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	return func() {
+		apiArgumentsPath = saved
+	}
+}
+
+var (
+	ApiArguments = apiArguments
+)

stages/go/org.osbuild.staticgzip/go.mod

@@ -0,0 +1,6 @@
+module org.osbuild.stages/staticgzip
+
+go 1.19
+
+require "org.osbuild.stages/staticgzip" v0.0.0
+replace "org.osbuild.stages/staticgzip" v0.0.0 => "./"

stages/go/org.osbuild.staticgzip/main.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+var info = map[string]string{
+	"schema_2": `
+{
+  "inputs": {
+    "type": "object",
+    "additionalProperties": false,
+    "required": ["file"],
+    "properties": {
+      "file": {
+	"type": "object",
+	"additionalProperties": true
+      }
+    }
+  },
+  "options": {
+    "additionalProperties": false,
+    "required": ["filename"],
+    "properties": {
+      "filename": {
+	"description": "Filename to use for the compressed file",
+	"type": "string"
+      }
+    }
+  }
+}`,
+	"doc": `
+Compress a file using gzip
+
+No external dependencies.`,
+}
+
+type apiArgs struct {
+	Tree   string `json:"tree"`
+	Inputs struct {
+		File struct {
+			Path string `json:"path"`
+			Data struct {
+				Files map[string]interface{} `json:"files"`
+			} `json:"data"`
+		} `json:"file"`
+	} `json:"inputs"`
+	Options struct {
+		Filename string `json:"filename"`
+	} `json:"options"`
+}
+
+var apiArgumentsPath = "/run/osbuild/api/arguments"
+
+func parseInputs(args *apiArgs) (string, error) {
+	files := args.Inputs.File.Data.Files
+	if len(files) != 1 {
+		return "", fmt.Errorf("unexpected amount of destination files %q", files)
+	}
+	// XXX: fugly
+	var file string
+	for k, _ := range files {
+		file = k
+	}
+
+	path := filepath.Join(args.Inputs.File.Path, file)
+	return path, nil
+}
+
+func apiArguments() (*apiArgs, error) {
+	f, err := os.Open(apiArgumentsPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	// alternatively we could unmarshal to map[string]interface{} and
+	// poke around via type assertions but that is also not fun
+	var data apiArgs
+	dec := json.NewDecoder(f)
+	if err := dec.Decode(&data); err != nil {
+		return nil, err
+	}
+
+	return &data, nil
+}
+
+func run() error {
+	args, err := apiArguments()
+	if err != nil {
+		return err
+	}
+	output := args.Tree
+	filename := args.Options.Filename
+	source, err := parseInputs(args)
+	if err != nil {
+		return err
+	}
+	target := filepath.Join(output, filename)
+
+	inf, err := os.Open(source)
+	if err != nil {
+		return err
+	}
+	defer inf.Close()
+	outf, err := os.Create(target)
+	if err != nil {
+		return err
+	}
+	w := gzip.NewWriter(outf)
+	if _, err := io.Copy(w, inf); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	if len(os.Args) == 2 && os.Args[1] == "--all" {
+		enc := json.NewEncoder(os.Stdout)
+		if err := enc.Encode(&info); err != nil {
+			fmt.Fprintf(os.Stderr, "error: %v\n", err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	}
+
+	if err := run(); err != nil {
+		panic(err)
+	}
+}

stages/go/org.osbuild.staticgzip/main_test.go

@@ -0,0 +1,60 @@
+package main_test
+
+import (
+	"reflect"
+	"testing"
+
+	main "org.osbuild.stages/staticgzip"
+)
+
+var fakeInput = []byte(`
+{
+  "tree": "/run/osbuild/tree",
+  "paths": {
+    "devices": "/dev",
+    "inputs": "/run/osbuild/inputs",
+    "mounts": "/run/osbuild/mounts"
+  },
+  "devices": {},
+  "inputs": {
+    "file": {
+      "path": "/run/osbuild/inputs/file",
+      "data": {
+        "files": {
+          "sha256:f950375066d74787f31cbd8f9f91c71819357cad243fb9d4a0d9ef4fa76709e0": {}
+        }
+      }
+    }
+  },
+  "mounts": {},
+  "options": {
+    "filename": "compressed.gz"
+  },
+  "meta": {
+    "id": "6dc907e7cf7b6436938d55eabad9209d4d4b7c4f338f3eef20f1d212aca48c79"
+  }
+}
+`)
+
+func TestApiArguments(t *testing.T) {
+	restore := main.MockApiArguments(t, fakeInput)
+	defer restore()
+
+	args, err := main.ApiArguments()
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	if args.Inputs.File.Path != "/run/osbuild/inputs/file" {
+		t.Fatalf("unexpected args %v", args)
+	}
+	expected := map[string]interface{}{
+		"sha256:f950375066d74787f31cbd8f9f91c71819357cad243fb9d4a0d9ef4fa76709e0": map[string]interface{}{},
+	}
+	input := args.Inputs.File.Data.Files
+	if !reflect.DeepEqual(input, expected) {
+		t.Fatalf("unexpected args %v", args)
+	}
+	if args.Options.Filename != "compressed.gz" {
+		t.Fatalf("unexpected args %v", args)
+	}
+}