feat(publish): feat/top level associations (#75)

* feat: catalog mirror bug fix

pkg/archive/archive.go

@@ -1,10 +1,13 @@
 package archive
 
 import (
+	"archive/tar"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/RedHatGov/bundle/pkg/config"
@@ -24,6 +27,7 @@ type Archiver interface {
 	Walk(string, archiver.WalkFunc) error
 	Open(io.Reader, int64) error
 	Read() (archiver.File, error)
+	CheckPath(string, string) error
 }
 
 type packager struct {
@@ -212,3 +216,182 @@ func includeFile(fpath string) bool {
 	split := strings.Split(filepath.Clean(fpath), string(filepath.Separator))
 	return split[0] == config.InternalDir || split[0] == config.PublishDir || split[0] == "catalogs"
 }
+
+// Copied from mholt archiver repo. Temporary and can
+// add back to repo
+// Changes include the additional of the excludePath variable and is
+// passed to the untarNext function
+func Unarchive(a Archiver, source, destination string, excludePaths []string) error {
+
+	if !fileExists(destination) {
+		err := mkdir(destination, 0755)
+		if err != nil {
+			return fmt.Errorf("preparing destination: %v", err)
+		}
+	}
+
+	file, err := os.Open(source)
+	if err != nil {
+		return fmt.Errorf("opening source archive: %v", err)
+	}
+	defer file.Close()
+
+	err = a.Open(file, 0)
+	if err != nil {
+		return fmt.Errorf("opening tar archive for reading: %v", err)
+	}
+	defer a.Close()
+
+	for {
+		err := untarNext(a, destination, excludePaths)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			if archiver.IsIllegalPathError(err) {
+				log.Printf("[ERROR] Reading file in tar archive: %v", err)
+				continue
+			}
+			return fmt.Errorf("reading file in tar archive: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func fileExists(name string) bool {
+	_, err := os.Stat(name)
+	return !os.IsNotExist(err)
+}
+
+func mkdir(dirPath string, dirMode os.FileMode) error {
+	err := os.MkdirAll(dirPath, dirMode)
+	if err != nil {
+		return fmt.Errorf("%s: making directory: %v", dirPath, err)
+	}
+	return nil
+}
+
+func untarNext(a Archiver, destination string, exclude []string) error {
+	f, err := a.Read()
+	if err != nil {
+		return err // don't wrap error; calling loop must break on io.EOF
+	}
+	defer f.Close()
+
+	header, ok := f.Header.(*tar.Header)
+	if !ok {
+		return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header)
+	}
+
+	errPath := a.CheckPath(destination, header.Name)
+	if errPath != nil {
+		return fmt.Errorf("checking path traversal attempt: %v", errPath)
+	}
+
+	// Added change here to check if
+	// current path is in the exclusion
+	// list
+	for _, path := range exclude {
+		if within(path, header.Name) {
+			return nil
+		}
+
+	}
+
+	return untarFile(f, destination, header)
+}
+
+func untarFile(f archiver.File, destination string, hdr *tar.Header) error {
+	to := filepath.Join(destination, hdr.Name)
+
+	switch hdr.Typeflag {
+	case tar.TypeDir:
+		return mkdir(to, f.Mode())
+	case tar.TypeReg, tar.TypeRegA, tar.TypeChar, tar.TypeBlock, tar.TypeFifo, tar.TypeGNUSparse:
+		return writeNewFile(to, f, f.Mode())
+	case tar.TypeSymlink:
+		return writeNewSymbolicLink(to, hdr.Linkname)
+	case tar.TypeLink:
+		return writeNewHardLink(to, filepath.Join(destination, hdr.Linkname))
+	case tar.TypeXGlobalHeader:
+		return nil // ignore the pax global header from git-generated tarballs
+	default:
+		return fmt.Errorf("%s: unknown type flag: %c", hdr.Name, hdr.Typeflag)
+	}
+}
+
+func writeNewFile(fpath string, in io.Reader, fm os.FileMode) error {
+	err := os.MkdirAll(filepath.Dir(fpath), 0755)
+	if err != nil {
+		return fmt.Errorf("%s: making directory for file: %v", fpath, err)
+	}
+
+	out, err := os.Create(fpath)
+	if err != nil {
+		return fmt.Errorf("%s: creating new file: %v", fpath, err)
+	}
+	defer out.Close()
+
+	err = out.Chmod(fm)
+	if err != nil && runtime.GOOS != "windows" {
+		return fmt.Errorf("%s: changing file mode: %v", fpath, err)
+	}
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return fmt.Errorf("%s: writing file: %v", fpath, err)
+	}
+	return nil
+}
+
+func writeNewSymbolicLink(fpath string, target string) error {
+	err := os.MkdirAll(filepath.Dir(fpath), 0755)
+	if err != nil {
+		return fmt.Errorf("%s: making directory for file: %v", fpath, err)
+	}
+
+	_, err = os.Lstat(fpath)
+	if err == nil {
+		err = os.Remove(fpath)
+		if err != nil {
+			return fmt.Errorf("%s: failed to unlink: %+v", fpath, err)
+		}
+	}
+
+	err = os.Symlink(target, fpath)
+	if err != nil {
+		return fmt.Errorf("%s: making symbolic link for: %v", fpath, err)
+	}
+	return nil
+}
+
+func writeNewHardLink(fpath string, target string) error {
+	err := os.MkdirAll(filepath.Dir(fpath), 0755)
+	if err != nil {
+		return fmt.Errorf("%s: making directory for file: %v", fpath, err)
+	}
+
+	_, err = os.Lstat(fpath)
+	if err == nil {
+		err = os.Remove(fpath)
+		if err != nil {
+			return fmt.Errorf("%s: failed to unlink: %+v", fpath, err)
+		}
+	}
+
+	err = os.Link(target, fpath)
+	if err != nil {
+		return fmt.Errorf("%s: making hard link for: %v", fpath, err)
+	}
+	return nil
+}
+
+// within returns true if sub is within or equal to parent.
+func within(parent, sub string) bool {
+	rel, err := filepath.Rel(parent, sub)
+	if err != nil {
+		return false
+	}
+	return !strings.Contains(rel, "..")
+}

pkg/bundle/publish/publish.go

@@ -253,13 +253,15 @@ func (o *Options) Run(ctx context.Context, cmd *cobra.Command, f kcmdutil.Factor
 		case image.TypeOCPRelease:
 			m.Destination.Ref.Tag = ""
 			m.Destination.Ref.ID = ""
-			// Only add top level release images to
-			// release mapping
-			if strings.Contains(assoc.Name, "ocp-release") {
+			if assoc.TopLevel {
 				releaseMappings = append(releaseMappings, m)
 			}
 		case image.TypeOperatorCatalog:
-			catalogMappings = append(catalogMappings, m)
+			if assoc.TopLevel {
+				catalogMappings = append(catalogMappings, m)
+			} else {
+				logrus.Infof("skipping %v", assoc)
+			}
 		case image.TypeOperatorBundle, image.TypeOperatorRelatedImage:
 			// Let the `catalog mirror` API call mirror all bundle and related images in the catalog.
 			// TODO(estroz): this may be incorrect if bundle and related images not in a catalog can be archived,
@@ -374,13 +376,14 @@ func (o *Options) Run(ctx context.Context, cmd *cobra.Command, f kcmdutil.Factor
 		catOpts.DryRun = o.DryRun
 		catOpts.MaxPathComponents = 2
 		catOpts.SecurityOptions.Insecure = o.SkipTLS
-		//catOpts.FilterOptions = imagemanifest.FilterOptions{FilterByOS: ".*"}
+		catOpts.FilterOptions = imagemanifest.FilterOptions{FilterByOS: o.FilterByOS}
+		catOpts.MaxICSPSize = 250000
 
 		args := []string{
 			m.Source.String(),
 			o.ToMirror,
 		}
-		if err := catOpts.Complete(&cobra.Command{}, args); err != nil {
+		if err := catOpts.Complete(cmd, args); err != nil {
 			return fmt.Errorf("error constructing catalog options: %v", err)
 		}
 		if err := catOpts.Validate(); err != nil {
@@ -443,7 +446,7 @@ func (o *Options) unpackImageSet(a archive.Archiver, dest string) error {
 
 			if extension == a.String() {
 				logrus.Debugf("Extracting archive %s", path)
-				if err := a.Unarchive(path, dest); err != nil {
+				if err := archive.Unarchive(a, path, dest, []string{"blobs"}); err != nil {
 					return err
 				}
 			}
@@ -454,7 +457,7 @@ func (o *Options) unpackImageSet(a archive.Archiver, dest string) error {
 	} else {
 
 		logrus.Infof("Extracting archive %s", o.ArchivePath)
-		if err := a.Unarchive(o.ArchivePath, dest); err != nil {
+		if err := archive.Unarchive(a, o.ArchivePath, dest, []string{"blobs"}); err != nil {
 			return err
 		}
 	}

pkg/bundle/release.go

@@ -245,6 +245,10 @@ func (o *ReleaseOptions) downloadMirror(secret []byte, toDir, from string) (imag
 		return nil, err
 	}
 	for k, assoc := range assocs {
+		if strings.Contains(assoc.Name, "ocp-release") {
+			assoc.TopLevel = true
+		}
+
 		assoc.Type = image.TypeOCPRelease
 		assocs[k] = assoc
 	}

pkg/cli/options.go

@@ -20,6 +20,7 @@ type RootOptions struct {
 	SkipTLS          bool
 	SkipVerification bool
 	SkipCleanup      bool
+	FilterByOS       string
 
 	logfileCleanup func()
 }
@@ -32,6 +33,7 @@ func (o *RootOptions) BindFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&o.SkipTLS, "skip-tls", false, "skip client-side TLS validation")
 	fs.BoolVar(&o.SkipVerification, "skip-verification", false, "skip digest verification")
 	fs.BoolVar(&o.SkipCleanup, "skip-cleanup", false, "skip removal of artifact directories")
+	fs.StringVar(&o.FilterByOS, "filter-by-os", "linux/amd64", "A regular expression to control which index image is picked when multiple variants are available")
 }
 
 func (o *RootOptions) LogfilePreRun(cmd *cobra.Command, _ []string) {

pkg/image/association.go

@@ -53,6 +53,9 @@ type Association struct {
 	// Type of the image in the context of this tool.
 	// See the ImageType enum for options.
 	Type ImageType `json:"type"`
+	// TopLeveLevel will determine if the association is a parent
+	// or child
+	TopLevel bool `json:"toplevel"`
 	// ManifestDigests of images if the image is a docker manifest list or OCI index.
 	// These manifests refer to image manifests by content SHA256 digest.
 	// LayerDigests and Manifests are mutually exclusive.
@@ -210,7 +213,7 @@ func AssociateImageLayers(rootDir string, imgMappings map[string]string, images
 		}
 
 		// TODO(estroz): parallelize
-		associations, err := associateImageLayers(image, localRoot, dirRef, tagOrID, skipParse)
+		associations, err := associateImageLayers(image, localRoot, dirRef, tagOrID, true, skipParse)
 		if err != nil {
 			errs = append(errs, err)
 		}
@@ -222,7 +225,7 @@ func AssociateImageLayers(rootDir string, imgMappings map[string]string, images
 	return bundleAssociations, utilerrors.NewAggregate(errs)
 }
 
-func associateImageLayers(image, localRoot, dirRef, tagOrID string, skipParse func(string) bool) (associations []Association, err error) {
+func associateImageLayers(image, localRoot, dirRef, tagOrID string, toplevel bool, skipParse func(string) bool) (associations []Association, err error) {
 	if skipParse(image) {
 		return nil, nil
 	}
@@ -283,7 +286,7 @@ func associateImageLayers(image, localRoot, dirRef, tagOrID string, skipParse fu
 			association.ManifestDigests = append(association.ManifestDigests, digestStr)
 			// Recurse on child manifests, which should be in the same directory
 			// with the same file name as it's digest.
-			childAssocs, err := associateImageLayers(digestStr, localRoot, dirRef, digestStr, skipParse)
+			childAssocs, err := associateImageLayers(digestStr, localRoot, dirRef, digestStr, false, skipParse)
 			if err != nil {
 				return nil, err
 			}

pkg/operator/mirror.go

@@ -425,6 +425,9 @@ func (o *MirrorOptions) associateDeclarativeConfigImageLayers(ctlgRef imagesourc
 		}
 
 		for k, assoc := range assocs {
+			if assoc.Name == ctlgRef.Ref.Exact() {
+				assoc.TopLevel = true
+			}
 			assoc.Type = typ
 			assocs[k] = assoc
 		}