config,storage: support populating directories from archives

Tarballs are ubiquitous as a binary release format, and without any
builtin ability to fetch, validate, and extract them, we are left
with half-baked hacks to do all of this from the confines of a oneshot
systemd service, or worse, extracting and providing the entirety of the
archive contents as files and directory entries in the ignition config,
resulting in very large json documents.

Let's not do this. Instead, this commit formalizes the use of archives
via adding a new (optional) "contents" key under a directory entry.

This new contents key is identical in function as its eponymous version
in the "files" entries, except that it incorporates a new "archive"
subkey to specify the archive format rather than guessing with
heuristics. Today, only "archive": "tar" is supported, though this
commit is structured to allow the addition of other archive types if
needed.

config/shared/errors/errors.go

@@ -84,6 +84,9 @@ var (
 	ErrInvalidProxy              = errors.New("proxies must be http(s)")
 	ErrInsecureProxy             = errors.New("insecure plaintext HTTP proxy specified for HTTPS resources")
 	ErrPathConflictsSystemd      = errors.New("path conflicts with systemd unit or dropin")
+	ErrUnsupportedArchiveType    = errors.New("unsupported archive type")
+	ErrArchiveTypeRequired       = errors.New("archive type is required")
+	ErrOverwriteMustBeTrue       = errors.New("overwrite must be true when specifying directory contents")
 
 	// Systemd section errors
 	ErrInvalidSystemdExt       = errors.New("invalid systemd unit extension")

config/v3_4_experimental/schema/ignition.json

@@ -40,6 +40,21 @@
         }
       }
     },
+    "archiveResource": {
+      "allOf": [
+        {
+          "$ref": "#/definitions/resource"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "archive": {
+              "type": ["string", "null"]
+            }
+          }
+        }
+      ]
+    },
     "verification": {
       "type": "object",
       "properties": {
@@ -394,6 +409,9 @@
               "properties": {
                 "mode": {
                   "type": ["integer", "null"]
+                },
+                "contents": {
+                  "$ref": "#/definitions/archiveResource"
                 }
               }
             }

config/v3_4_experimental/types/directory.go

@@ -15,12 +15,18 @@
 package types
 
 import (
+	"github.com/coreos/ignition/v2/config/shared/errors"
+	"github.com/coreos/ignition/v2/config/util"
+
 	"github.com/coreos/vcontext/path"
 	"github.com/coreos/vcontext/report"
 )
 
 func (d Directory) Validate(c path.ContextPath) (r report.Report) {
 	r.Merge(d.Node.Validate(c))
 	r.AddOnError(c.Append("mode"), validateMode(d.Mode))
+	if !util.NilOrEmpty(d.Contents.Archive) && (d.Overwrite == nil || !*d.Overwrite) {
+		r.AddOnError(c.Append("overwrite"), errors.ErrOverwriteMustBeTrue)
+	}
 	return
 }

config/v3_4_experimental/types/resource.go

@@ -89,3 +89,24 @@ func (res Resource) validateRequiredSource() error {
 	}
 	return validateURL(*res.Source)
 }
+
+func (res ArchiveResource) Validate(c path.ContextPath) (r report.Report) {
+	r.Merge(res.Resource.Validate(c))
+	r.AddOnError(c.Append("archive"), res.validateArchive())
+	return
+}
+
+func (res ArchiveResource) validateArchive() error {
+	if util.NilOrEmpty(res.Source) {
+		// archive can be omitted iff the contents are omitted
+		return nil
+	}
+	if util.NilOrEmpty(res.Archive) {
+		return errors.ErrArchiveTypeRequired
+	}
+	switch *res.Archive {
+	case "tar":
+		return nil
+	}
+	return errors.ErrUnsupportedArchiveType
+}

config/v3_4_experimental/types/schema.go

@@ -2,6 +2,15 @@ package types
 
 // generated by "schematyper --package=types config/v3_4_experimental/schema/ignition.json -o config/v3_4_experimental/types/schema.go --root-type=Config" -- DO NOT EDIT
 
+type ArchiveResource struct {
+	Resource
+	ArchiveResourceEmbedded1
+}
+
+type ArchiveResourceEmbedded1 struct {
+	Archive *string `json:"archive,omitempty"`
+}
+
 type Clevis struct {
 	Custom    ClevisCustom `json:"custom,omitempty"`
 	Tang      []Tang       `json:"tang,omitempty"`
@@ -31,7 +40,8 @@ type Directory struct {
 }
 
 type DirectoryEmbedded1 struct {
-	Mode *int `json:"mode,omitempty"`
+	Contents ArchiveResource `json:"contents,omitempty"`
+	Mode     *int            `json:"mode,omitempty"`
 }
 
 type Disk struct {

docs/configuration-v3_4_experimental.md

@@ -111,6 +111,15 @@ The Ignition configuration is a JSON document conforming to the following specif
     * **_group_** (object): specifies the directory's group.
       * **_id_** (integer): the group ID of the group.
       * **_name_** (string): the group name of the group.
+    * **_contents_** (object): options related to the contents of the directory. If specified, `overwrite` must be `true`.
+      * **archive** (string): format of the archive to extract into the directory. must be `tar`.
+      * **_compression_** (string): the type of compression used on the archive (null or gzip). Compression cannot be used with S3.
+      * **_source_** (string): the URL of the archive to extract. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a directory already exists at the path, Ignition will do nothing. If source is omitted and no directory exists, an empty directory will be created.
+      * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
+        * **name** (string): the header name.
+        * **_value_** (string): the header contents.
+      * **_verification_** (object): options related to the verification of the archive file.
+        * **_hash_** (string): the hash of the archive file, in the form `<type>-<value>` where type is either `sha512` or `sha256`.
   * **_links_** (list of objects): the list of links to be created. Every file, directory, and link must have a unique `path`.
     * **path** (string): the absolute path to the link
     * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If overwrite is false and a matching link exists at the path, Ignition will only set the owner and group. Defaults to false.

internal/exec/stages/files/filesystemEntries.go

@@ -17,6 +17,7 @@ package files
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -285,7 +286,7 @@ func (tmp fileEntry) create(l *log.Logger, u util.Util) error {
 
 	for _, op := range fetchOps {
 		msg := "writing file %q"
-		if op.Append {
+		if op.Mode == util.FetchAppend {
 			msg = "appending to file %q"
 		}
 		if err := l.LogOp(
@@ -323,6 +324,33 @@ func (tmp dirEntry) create(l *log.Logger, u util.Util) error {
 		return fmt.Errorf("error creating directory %s: A non-directory already exists and overwrite is false", d.Path)
 	}
 
+	if d.Contents.Archive != nil {
+		dirf, err := os.Open(d.Path)
+		if err != nil {
+			return fmt.Errorf("open() failed on %s: %v", d.Path, err)
+		}
+		switch _, err := dirf.Readdirnames(1); {
+		case err == nil:
+			return fmt.Errorf("refusing to populate directory %s: directory is not empty and overwrite is false", d.Path)
+		case err != io.EOF:
+			return fmt.Errorf("readdirnames() failed on %s: %v", d.Path, err)
+		}
+
+		fetch, err := util.MakeFetchOp(l, d.Node, d.Contents.Resource)
+		if err != nil {
+			return fmt.Errorf("failed to resolve directory %q: %v", d.Path, err)
+		}
+		fetch.Mode = util.FetchExtract
+		fetch.ArchiveType = util.ArchiveType(*d.Contents.Archive)
+
+		op := func() error {
+			return u.PerformFetch(fetch)
+		}
+		if err := l.LogOp(op, "populating directory %q", d.Path); err != nil {
+			return fmt.Errorf("failed to populate directory %q: %v", d.Path, err)
+		}
+	}
+
 	if err := u.SetPermissions(d.Mode, d.Node); err != nil {
 		return fmt.Errorf("error setting directory permissions for %s: %v", d.Path, err)
 	}