chunked: support writing files in a flat dir format

so that they can be stored by their digest

Signed-off-by: Giuseppe Scrivano <[email protected]>

drivers/driver.go

@@ -191,8 +191,21 @@ type DriverWithDifferOutput struct {
 	TOCDigest          digest.Digest
 }
 
+type DifferOutputFormat int
+
+const (
+	// DifferOutputFormatDir means the output is a directory and it will
+	// keep the original layout.
+	DifferOutputFormatDir = iota
+	// DifferOutputFormatFlat will store the files by their checksum, in the form
+	// checksum[0:2]/checksum[2:]
+	DifferOutputFormatFlat
+)
+
 // DifferOptions overrides how the differ work
 type DifferOptions struct {
+	// Format defines the destination directory layout format
+	Format DifferOutputFormat
 }
 
 // Differ defines the interface for using a custom differ.

pkg/chunked/storage_linux.go

@@ -1317,6 +1317,42 @@ func (c *chunkedDiffer) findAndCopyFile(dirfd int, r *internal.FileMetadata, cop
 	return false, nil
 }
 
+func makeEntriesFlat(dirfd int, mergedEntries []internal.FileMetadata) ([]internal.FileMetadata, error) {
+	var new []internal.FileMetadata
+
+	hashes := make(map[string]string)
+	for i := range mergedEntries {
+		if mergedEntries[i].Type != TypeReg {
+			continue
+		}
+		if mergedEntries[i].Digest == "" {
+			continue
+		}
+		digest, err := digest.Parse(mergedEntries[i].Digest)
+		if err != nil {
+			return nil, err
+		}
+		d := digest.Encoded()
+
+		if hashes[d] != "" {
+			continue
+		}
+		hashes[d] = d
+
+		mergedEntries[i].Name = fmt.Sprintf("%s/%s", d[0:2], d[2:])
+
+		// check if it already exists
+		var stat unix.Stat_t
+		if unix.Fstatat(dirfd, mergedEntries[i].Name, &stat, 0) == nil {
+			continue
+		}
+
+		unix.Mkdirat(dirfd, d[0:2], 0o755)
+		new = append(new, mergedEntries[i])
+	}
+	return new, nil
+}
+
 func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, differOpts *graphdriver.DifferOptions) (graphdriver.DriverWithDifferOutput, error) {
 	defer c.layersCache.release()
 	defer func() {
@@ -1389,6 +1425,13 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 	}
 	defer unix.Close(dirfd)
 
+	if differOpts != nil && differOpts.Format == graphdriver.DifferOutputFormatFlat {
+		mergedEntries, err = makeEntriesFlat(dirfd, mergedEntries)
+		if err != nil {
+			return output, err
+		}
+	}
+
 	// hardlinks can point to missing files.  So create them after all files
 	// are retrieved
 	var hardLinks []hardLinkToCreate