osbuild: start stage separation support

This commit prepares for stage separation support. The old import
mechanism is still supported but if it fails the new way of
just running the binary and outputing the data via stdout is
used.

Makefile

@@ -85,6 +85,8 @@ help:
 	@echo "    help:               Print this usage information."
 	@echo "    man:                Generate all man-pages"
 	@echo
+	@echo "    go:                 Generate all go stages"
+	@echo
 	@echo "    coverity-download:  Force a new download of the coverity tool"
 	@echo "    coverity-check:     Run the coverity test suite"
 	@echo "    coverity-submit:    Run coverity and submit the results"
@@ -121,6 +123,19 @@ $(MANPAGES_TROFF): $(BUILDDIR)/docs/%: $(SRCDIR)/docs/%.rst | $(BUILDDIR)/docs/
 .PHONY: man
 man: $(MANPAGES_TROFF)
 
+#
+## Go stages
+#
+GO_STAGES_IN = $(wildcard $(SRCDIR)/stages/go/*/*.go)
+GO_STAGES_OUT = $(patsubst %/,%, $(subst /go/,/,$(dir $(GO_STAGES_IN))))
+$(GO_STAGES_OUT): $(GO_STAGES_IN)
+	go build -o "$@" "$<"
+.PHONY: go go-clean
+go: $(GO_STAGES_OUT)
+
+go-clean:
+	rm -f $(GO_STAGES_OUT)
+
 #
 # Coverity
 #

go.mod

@@ -0,0 +1,3 @@
+module github.com/osbuild/osbuild
+
+go 1.18

osbuild/meta.py

@@ -27,6 +27,7 @@
 import json
 import os
 import pkgutil
+import subprocess
 import sys
 from collections import deque
 from typing import (Any, Deque, Dict, List, Optional, Sequence, Set, Tuple,
@@ -410,6 +411,35 @@ def _parse_caps(cls, _klass, _name, node):
 
     @classmethod
     def load(cls, root, klass, name) -> Optional["ModuleInfo"]:
+        base = cls.MODULES.get(klass)
+        if not base:
+            raise ValueError(f"Unsupported type: {klass}")
+        path = os.path.join(root, base, name)
+
+        try:
+            return cls._load_py(path, klass, name)
+        except (SyntaxError, UnicodeDecodeError):
+            pass
+        return cls._load_generic(path, klass, name)
+
+    @classmethod
+    def _load_generic(cls, path, klass, name) -> Optional["ModuleInfo"]:
+        output = subprocess.check_output([path, "--all"])
+        p = json.loads(output)
+        doclist = p.get("doc", "").split("\n")
+        info = {
+            "schema": {
+                "1": json.loads(p.get("schema", "{}")),
+                "2": json.loads(p.get("schema_2", "{}")),
+            },
+            'desc': doclist[0],
+            'info': "\n".join(doclist[1:]),
+            'caps': p.get("capabilities", set()),
+        }
+        return cls(klass, name, path, info)
+
+    @classmethod
+    def _load_py(cls, path, klass, name) -> Optional["ModuleInfo"]:
         names = ["SCHEMA", "SCHEMA_2", "CAPABILITIES"]
 
         def filter_type(lst, target):
@@ -418,11 +448,6 @@ def filter_type(lst, target):
         def targets(a):
             return [t.id for t in filter_type(a.targets, ast.Name)]
 
-        base = cls.MODULES.get(klass)
-        if not base:
-            raise ValueError(f"Unsupported type: {klass}")
-
-        path = os.path.join(root, base, name)
         try:
             with open(path, encoding="utf8") as f:
                 data = f.read()

stages/go/org.osbuild.staticgzip/main.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+var info = map[string]string{
+	"schema_2": `
+{
+  "inputs": {
+    "type": "object",
+    "additionalProperties": false,
+    "required": ["file"],
+    "properties": {
+      "file": {
+	"type": "object",
+	"additionalProperties": true
+      }
+    }
+  },
+  "options": {
+    "additionalProperties": false,
+    "required": ["filename"],
+    "properties": {
+      "filename": {
+	"description": "Filename to use for the compressed file",
+	"type": "string"
+      }
+    }
+  }
+}`,
+	"doc": `
+Compress a file using gzip
+
+No external dependencies.`,
+}
+
+var apiArgumentsPath = "/run/osbuild/api/arguments"
+
+func parseInputs(inputs map[string]interface{}) (string, error) {
+	image := inputs["file"].(map[string]interface{})
+	files := image["data"].(map[string]interface{})["files"].(map[string]interface{})
+	if len(files) != 1 {
+		return "", fmt.Errorf("unexpected amount of destination files %q", files)
+	}
+	// XXX: fugly
+	var file string
+	for k, _ := range files {
+		file = k
+	}
+
+	path := filepath.Join(image["path"].(string), file)
+	return path, nil
+}
+
+func apiArguments() (map[string]interface{}, error) {
+	f, err := os.Open(apiArgumentsPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var data = make(map[string]interface{})
+	dec := json.NewDecoder(f)
+	if err := dec.Decode(&data); err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+func run() error {
+	args, err := apiArguments()
+	if err != nil {
+		return err
+	}
+	// XXX: use proper structs
+	inputs := args["inputs"].(map[string]interface{})
+	output := args["tree"].(string)
+	options := args["options"].(map[string]interface{})
+
+	filename := options["filename"].(string)
+	source, err := parseInputs(inputs)
+	if err != nil {
+		return err
+	}
+	target := filepath.Join(output, filename)
+
+	inf, err := os.Open(source)
+	if err != nil {
+		return err
+	}
+	defer inf.Close()
+	outf, err := os.Create(target)
+	if err != nil {
+		return err
+	}
+	w := gzip.NewWriter(outf)
+	if _, err := io.Copy(w, inf); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func main() {
+	if len(os.Args) == 2 && os.Args[1] == "--all" {
+		enc := json.NewEncoder(os.Stdout)
+		if err := enc.Encode(&info); err != nil {
+			fmt.Fprintf(os.Stderr, "error: %v\n", err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	}
+
+	if err := run(); err != nil {
+		panic(err)
+	}
+}

test/data/stages/gzip/b.json

@@ -864,7 +864,7 @@
       "build": "name:build",
       "stages": [
         {
-          "type": "org.osbuild.gzip",
+          "type": "org.osbuild.staticgzip",
           "inputs": {
             "file": {
               "type": "org.osbuild.files",