config,storage: support populating directories from archives

config/shared/errors/errors.go

@@ -84,6 +84,9 @@ var (
 	ErrInvalidProxy              = errors.New("proxies must be http(s)")
 	ErrInsecureProxy             = errors.New("insecure plaintext HTTP proxy specified for HTTPS resources")
 	ErrPathConflictsSystemd      = errors.New("path conflicts with systemd unit or dropin")
+	ErrUnsupportedArchiveType    = errors.New("unsupported archive type")
+	ErrArchiveTypeRequired       = errors.New("archive type is required")
+	ErrOverwriteMustBeTrue       = errors.New("overwrite must be true when specifying directory contents")
 
 	// Systemd section errors
 	ErrInvalidSystemdExt       = errors.New("invalid systemd unit extension")

config/util/interfaces.go

@@ -30,10 +30,22 @@ type Keyed interface {
 	Key() string
 }
 
+type KeyPrefixed interface {
+	KeyPrefix() string
+}
+
 // CallKey is a helper to call the Key() function since this needs to happen a lot
 func CallKey(v reflect.Value) string {
 	if v.Kind() == reflect.String {
 		return v.Convert(reflect.TypeOf("")).Interface().(string)
 	}
 	return v.Interface().(Keyed).Key()
 }
+
+// CallKeyPrefix is a helper to call the KeyPrefix() method.
+func CallKeyPrefix(v reflect.Value) string {
+	if prefixable, ok := v.Interface().(KeyPrefixed); ok {
+		return prefixable.KeyPrefix()
+	}
+	return ""
+}

config/v3_4_experimental/schema/ignition.json

@@ -40,6 +40,21 @@
         }
       }
     },
+    "archiveResource": {
+      "allOf": [
+        {
+          "$ref": "#/definitions/resource"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "archive": {
+              "type": ["string", "null"]
+            }
+          }
+        }
+      ]
+    },
     "verification": {
       "type": "object",
       "properties": {
@@ -394,6 +409,9 @@
               "properties": {
                 "mode": {
                   "type": ["integer", "null"]
+                },
+                "contents": {
+                  "$ref": "#/definitions/archiveResource"
                 }
               }
             }

config/v3_4_experimental/types/directory.go

@@ -15,12 +15,27 @@
 package types
 
 import (
+	"github.com/coreos/ignition/v2/config/shared/errors"
+	"github.com/coreos/ignition/v2/config/util"
+
 	"github.com/coreos/vcontext/path"
 	"github.com/coreos/vcontext/report"
 )
 
 func (d Directory) Validate(c path.ContextPath) (r report.Report) {
 	r.Merge(d.Node.Validate(c))
 	r.AddOnError(c.Append("mode"), validateMode(d.Mode))
+	if !util.NilOrEmpty(d.Contents.Archive) && (d.Overwrite == nil || !*d.Overwrite) {
+		r.AddOnError(c.Append("overwrite"), errors.ErrOverwriteMustBeTrue)
+	}
 	return
 }
+
+func (d Directory) KeyPrefix() string {
+	if util.NilOrEmpty(d.Contents.Archive) {
+		return ""
+	}
+	// If a directory is populated by an archive, all other file/directory entries
+	// in the config must conflict with any files under said directory.
+	return d.Path
+}

config/v3_4_experimental/types/resource.go

@@ -89,3 +89,24 @@ func (res Resource) validateRequiredSource() error {
 	}
 	return validateURL(*res.Source)
 }
+
+func (res ArchiveResource) Validate(c path.ContextPath) (r report.Report) {
+	r.Merge(res.Resource.Validate(c))
+	r.AddOnError(c.Append("archive"), res.validateArchive())
+	return
+}
+
+func (res ArchiveResource) validateArchive() error {
+	if util.NilOrEmpty(res.Source) {
+		// archive can be omitted iff the contents are omitted
+		return nil
+	}
+	if util.NilOrEmpty(res.Archive) {
+		return errors.ErrArchiveTypeRequired
+	}
+	switch *res.Archive {
+	case "tar":
+		return nil
+	}
+	return errors.ErrUnsupportedArchiveType
+}

config/v3_4_experimental/types/schema.go

@@ -2,6 +2,15 @@ package types
 
 // generated by "schematyper --package=types config/v3_4_experimental/schema/ignition.json -o config/v3_4_experimental/types/schema.go --root-type=Config" -- DO NOT EDIT
 
+type ArchiveResource struct {
+	Resource
+	ArchiveResourceEmbedded1
+}
+
+type ArchiveResourceEmbedded1 struct {
+	Archive *string `json:"archive,omitempty"`
+}
+
 type Clevis struct {
 	Custom    ClevisCustom `json:"custom,omitempty"`
 	Tang      []Tang       `json:"tang,omitempty"`
@@ -31,7 +40,8 @@ type Directory struct {
 }
 
 type DirectoryEmbedded1 struct {
-	Mode *int `json:"mode,omitempty"`
+	Contents ArchiveResource `json:"contents,omitempty"`
+	Mode     *int            `json:"mode,omitempty"`
 }
 
 type Disk struct {

config/validate/validate.go

@@ -17,6 +17,7 @@ package validate
 import (
 	"fmt"
 	"reflect"
+	"strings"
 
 	"github.com/coreos/ignition/v2/config/shared/errors"
 	"github.com/coreos/ignition/v2/config/util"
@@ -33,6 +34,12 @@ func ValidateDups(v reflect.Value, c path.ContextPath) (r report.Report) {
 		return
 	}
 	dupsLists := map[string]map[string]struct{}{}
+
+	// This should probably be a collection of prefix trees, but this would
+	// either require implementing one or adding a new dependency for what
+	// amounts to a minute amount of gain in performance, given that we do
+	// not have thousands of key prefixes to manage.
+	prefixLists := map[string][]string{}
 	ignoreDups := map[string]struct{}{}
 	if i, ok := v.Interface().(util.IgnoresDups); ok {
 		ignoreDups = i.IgnoreDuplicates()
@@ -59,13 +66,23 @@ func ValidateDups(v reflect.Value, c path.ContextPath) (r report.Report) {
 			dupsLists[dupListName] = make(map[string]struct{}, field.Value.Len())
 			dupList = dupsLists[dupListName]
 		}
+		prefixList := prefixLists[dupListName]
 		for i := 0; i < field.Value.Len(); i++ {
 			key := util.CallKey(field.Value.Index(i))
 			if _, isDup := dupList[key]; isDup {
 				r.AddOnError(c.Append(validate.FieldName(field, c.Tag), i), errors.ErrDuplicate)
 			}
+			for _, prefix := range prefixList {
+				if strings.HasPrefix(key, prefix) {
+					r.AddOnError(c.Append(validate.FieldName(field, c.Tag), i), errors.ErrDuplicate)
+				}
+			}
+			if prefix := util.CallKeyPrefix(field.Value.Index(i)); prefix != "" {
+				prefixList = append(prefixList, prefix)
+			}
 			dupList[key] = struct{}{}
 		}
+		prefixLists[dupListName] = prefixList
 	}
 	return
 }

docs/configuration-v3_4_experimental.md

@@ -111,6 +111,15 @@ The Ignition configuration is a JSON document conforming to the following specif
     * **_group_** (object): specifies the directory's group.
       * **_id_** (integer): the group ID of the group.
       * **_name_** (string): the group name of the group.
+    * **_contents_** (object): options related to the contents of the directory. If specified, `overwrite` must be `true`. Directories populated from an archive own all files under it. This means that specifying files, directories, and links under the path of this directory always result in a conflict error during config validation.
+      * **archive** (string): format of the archive to extract into the directory. Must be `tar`. If `tar` is specified, the source must be a USTAR, PAX, or GNU tarball. Only regular files, directories, and links (both hard links and symlinks) are extracted, other file types are ignored and emit a warning. Note that for `tar` archives, sparse files are not supported and processing an archive with one will result in an error.
+      * **_compression_** (string): the type of compression used on the archive (null or gzip). Compression cannot be used with S3.
+      * **_source_** (string): the URL of the archive to extract. Supported schemes are `http`, `https`, `tftp`, `s3`, `arn`, `gs`, and [`data`][rfc2397]. When using `http`, it is advisable to use the verification option to ensure the contents haven't been modified. If source is omitted and a directory already exists at the path, Ignition will do nothing. If source is omitted and no directory exists, an empty directory will be created.
+      * **_httpHeaders_** (list of objects): a list of HTTP headers to be added to the request. Available for `http` and `https` source schemes only.
+        * **name** (string): the header name.
+        * **_value_** (string): the header contents.
+      * **_verification_** (object): options related to the verification of the archive file.
+        * **_hash_** (string): the hash of the archive file, in the form `<type>-<value>` where type is either `sha512` or `sha256`.
   * **_links_** (list of objects): the list of links to be created. Every file, directory, and link must have a unique `path`.
     * **path** (string): the absolute path to the link
     * **_overwrite_** (boolean): whether to delete preexisting nodes at the path. If overwrite is false and a matching link exists at the path, Ignition will only set the owner and group. Defaults to false.

docs/release-notes.md

@@ -12,6 +12,7 @@ nav_order: 9
 ### Features
 
 - Ship aarch64 macOS ignition-validate binary in GitHub release artifacts
+- Add the ability to populate directory from tar archives.
 
 ### Changes
 

internal/exec/stages/files/filesystemEntries.go

@@ -17,6 +17,7 @@ package files
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -285,7 +286,7 @@ func (tmp fileEntry) create(l *log.Logger, u util.Util) error {
 
 	for _, op := range fetchOps {
 		msg := "writing file %q"
-		if op.Append {
+		if op.Mode == util.FetchAppend {
 			msg = "appending to file %q"
 		}
 		if err := l.LogOp(
@@ -323,6 +324,33 @@ func (tmp dirEntry) create(l *log.Logger, u util.Util) error {
 		return fmt.Errorf("error creating directory %s: A non-directory already exists and overwrite is false", d.Path)
 	}
 
+	if d.Contents.Archive != nil {
+		dirf, err := os.Open(d.Path)
+		if err != nil {
+			return fmt.Errorf("open() failed on %s: %v", d.Path, err)
+		}
+		switch _, err := dirf.Readdirnames(1); {
+		case err == nil:
+			return fmt.Errorf("refusing to populate directory %s: directory is not empty and overwrite is false", d.Path)
+		case err != io.EOF:
+			return fmt.Errorf("readdirnames() failed on %s: %v", d.Path, err)
+		}
+
+		fetch, err := util.MakeFetchOp(l, d.Node, d.Contents.Resource)
+		if err != nil {
+			return fmt.Errorf("failed to resolve directory %q: %v", d.Path, err)
+		}
+		fetch.Mode = util.FetchExtract
+		fetch.ArchiveType = util.ArchiveType(*d.Contents.Archive)
+
+		op := func() error {
+			return u.PerformFetch(fetch)
+		}
+		if err := l.LogOp(op, "populating directory %q", d.Path); err != nil {
+			return fmt.Errorf("failed to populate directory %q: %v", d.Path, err)
+		}
+	}
+
 	if err := u.SetPermissions(d.Mode, d.Node); err != nil {
 		return fmt.Errorf("error setting directory permissions for %s: %v", d.Path, err)
 	}