diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 2e8095b51..8a84a2166 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -1,10 +1,13 @@ package archive import ( + "archive/tar" "fmt" "io" + "log" "os" "path/filepath" + "runtime" "strings" "github.com/RedHatGov/bundle/pkg/config" @@ -24,6 +27,7 @@ type Archiver interface { Walk(string, archiver.WalkFunc) error Open(io.Reader, int64) error Read() (archiver.File, error) + CheckPath(string, string) error } type packager struct { @@ -212,3 +216,182 @@ func includeFile(fpath string) bool { split := strings.Split(filepath.Clean(fpath), string(filepath.Separator)) return split[0] == config.InternalDir || split[0] == config.PublishDir || split[0] == "catalogs" } + +// Copied from mholt archiver repo. Temporary and can +// add back to repo +// Changes include the additional of the excludePath variable and is +// passed to the untarNext function +func Unarchive(a Archiver, source, destination string, excludePaths []string) error { + + if !fileExists(destination) { + err := mkdir(destination, 0755) + if err != nil { + return fmt.Errorf("preparing destination: %v", err) + } + } + + file, err := os.Open(source) + if err != nil { + return fmt.Errorf("opening source archive: %v", err) + } + defer file.Close() + + err = a.Open(file, 0) + if err != nil { + return fmt.Errorf("opening tar archive for reading: %v", err) + } + defer a.Close() + + for { + err := untarNext(a, destination, excludePaths) + if err == io.EOF { + break + } + if err != nil { + if archiver.IsIllegalPathError(err) { + log.Printf("[ERROR] Reading file in tar archive: %v", err) + continue + } + return fmt.Errorf("reading file in tar archive: %v", err) + } + } + + return nil +} + +func fileExists(name string) bool { + _, err := os.Stat(name) + return !os.IsNotExist(err) +} + +func mkdir(dirPath string, dirMode os.FileMode) error { + err := os.MkdirAll(dirPath, dirMode) + if err != nil { + return fmt.Errorf("%s: making directory: %v", dirPath, err) + } + return nil +} + +func untarNext(a Archiver, destination string, exclude []string) error { + f, err := a.Read() + if err != nil { + return err // don't wrap error; calling loop must break on io.EOF + } + defer f.Close() + + header, ok := f.Header.(*tar.Header) + if !ok { + return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header) + } + + errPath := a.CheckPath(destination, header.Name) + if errPath != nil { + return fmt.Errorf("checking path traversal attempt: %v", errPath) + } + + // Added change here to check if + // current path is in the exclusion + // list + for _, path := range exclude { + if within(path, header.Name) { + return nil + } + + } + + return untarFile(f, destination, header) +} + +func untarFile(f archiver.File, destination string, hdr *tar.Header) error { + to := filepath.Join(destination, hdr.Name) + + switch hdr.Typeflag { + case tar.TypeDir: + return mkdir(to, f.Mode()) + case tar.TypeReg, tar.TypeRegA, tar.TypeChar, tar.TypeBlock, tar.TypeFifo, tar.TypeGNUSparse: + return writeNewFile(to, f, f.Mode()) + case tar.TypeSymlink: + return writeNewSymbolicLink(to, hdr.Linkname) + case tar.TypeLink: + return writeNewHardLink(to, filepath.Join(destination, hdr.Linkname)) + case tar.TypeXGlobalHeader: + return nil // ignore the pax global header from git-generated tarballs + default: + return fmt.Errorf("%s: unknown type flag: %c", hdr.Name, hdr.Typeflag) + } +} + +func writeNewFile(fpath string, in io.Reader, fm os.FileMode) error { + err := os.MkdirAll(filepath.Dir(fpath), 0755) + if err != nil { + return fmt.Errorf("%s: making directory for file: %v", fpath, err) + } + + out, err := os.Create(fpath) + if err != nil { + return fmt.Errorf("%s: creating new file: %v", fpath, err) + } + defer out.Close() + + err = out.Chmod(fm) + if err != nil && runtime.GOOS != "windows" { + return fmt.Errorf("%s: changing file mode: %v", fpath, err) + } + + _, err = io.Copy(out, in) + if err != nil { + return fmt.Errorf("%s: writing file: %v", fpath, err) + } + return nil +} + +func writeNewSymbolicLink(fpath string, target string) error { + err := os.MkdirAll(filepath.Dir(fpath), 0755) + if err != nil { + return fmt.Errorf("%s: making directory for file: %v", fpath, err) + } + + _, err = os.Lstat(fpath) + if err == nil { + err = os.Remove(fpath) + if err != nil { + return fmt.Errorf("%s: failed to unlink: %+v", fpath, err) + } + } + + err = os.Symlink(target, fpath) + if err != nil { + return fmt.Errorf("%s: making symbolic link for: %v", fpath, err) + } + return nil +} + +func writeNewHardLink(fpath string, target string) error { + err := os.MkdirAll(filepath.Dir(fpath), 0755) + if err != nil { + return fmt.Errorf("%s: making directory for file: %v", fpath, err) + } + + _, err = os.Lstat(fpath) + if err == nil { + err = os.Remove(fpath) + if err != nil { + return fmt.Errorf("%s: failed to unlink: %+v", fpath, err) + } + } + + err = os.Link(target, fpath) + if err != nil { + return fmt.Errorf("%s: making hard link for: %v", fpath, err) + } + return nil +} + +// within returns true if sub is within or equal to parent. +func within(parent, sub string) bool { + rel, err := filepath.Rel(parent, sub) + if err != nil { + return false + } + return !strings.Contains(rel, "..") +} diff --git a/pkg/bundle/publish/publish.go b/pkg/bundle/publish/publish.go index d950fe62a..673441861 100644 --- a/pkg/bundle/publish/publish.go +++ b/pkg/bundle/publish/publish.go @@ -253,13 +253,15 @@ func (o *Options) Run(ctx context.Context, cmd *cobra.Command, f kcmdutil.Factor case image.TypeOCPRelease: m.Destination.Ref.Tag = "" m.Destination.Ref.ID = "" - // Only add top level release images to - // release mapping - if strings.Contains(assoc.Name, "ocp-release") { + if assoc.TopLevel { releaseMappings = append(releaseMappings, m) } case image.TypeOperatorCatalog: - catalogMappings = append(catalogMappings, m) + if assoc.TopLevel { + catalogMappings = append(catalogMappings, m) + } else { + logrus.Infof("skipping %v", assoc) + } case image.TypeOperatorBundle, image.TypeOperatorRelatedImage: // Let the `catalog mirror` API call mirror all bundle and related images in the catalog. // TODO(estroz): this may be incorrect if bundle and related images not in a catalog can be archived, @@ -374,13 +376,14 @@ func (o *Options) Run(ctx context.Context, cmd *cobra.Command, f kcmdutil.Factor catOpts.DryRun = o.DryRun catOpts.MaxPathComponents = 2 catOpts.SecurityOptions.Insecure = o.SkipTLS - //catOpts.FilterOptions = imagemanifest.FilterOptions{FilterByOS: ".*"} + catOpts.FilterOptions = imagemanifest.FilterOptions{FilterByOS: o.FilterByOS} + catOpts.MaxICSPSize = 250000 args := []string{ m.Source.String(), o.ToMirror, } - if err := catOpts.Complete(&cobra.Command{}, args); err != nil { + if err := catOpts.Complete(cmd, args); err != nil { return fmt.Errorf("error constructing catalog options: %v", err) } if err := catOpts.Validate(); err != nil { @@ -443,7 +446,7 @@ func (o *Options) unpackImageSet(a archive.Archiver, dest string) error { if extension == a.String() { logrus.Debugf("Extracting archive %s", path) - if err := a.Unarchive(path, dest); err != nil { + if err := archive.Unarchive(a, path, dest, []string{"blobs"}); err != nil { return err } } @@ -454,7 +457,7 @@ func (o *Options) unpackImageSet(a archive.Archiver, dest string) error { } else { logrus.Infof("Extracting archive %s", o.ArchivePath) - if err := a.Unarchive(o.ArchivePath, dest); err != nil { + if err := archive.Unarchive(a, o.ArchivePath, dest, []string{"blobs"}); err != nil { return err } } diff --git a/pkg/bundle/release.go b/pkg/bundle/release.go index 9a3a1b797..a9100ae1d 100644 --- a/pkg/bundle/release.go +++ b/pkg/bundle/release.go @@ -245,6 +245,10 @@ func (o *ReleaseOptions) downloadMirror(secret []byte, toDir, from string) (imag return nil, err } for k, assoc := range assocs { + if strings.Contains(assoc.Name, "ocp-release") { + assoc.TopLevel = true + } + assoc.Type = image.TypeOCPRelease assocs[k] = assoc } diff --git a/pkg/cli/options.go b/pkg/cli/options.go index 3bcd67f4c..94327d417 100644 --- a/pkg/cli/options.go +++ b/pkg/cli/options.go @@ -20,6 +20,7 @@ type RootOptions struct { SkipTLS bool SkipVerification bool SkipCleanup bool + FilterByOS string logfileCleanup func() } @@ -32,6 +33,7 @@ func (o *RootOptions) BindFlags(fs *pflag.FlagSet) { fs.BoolVar(&o.SkipTLS, "skip-tls", false, "skip client-side TLS validation") fs.BoolVar(&o.SkipVerification, "skip-verification", false, "skip digest verification") fs.BoolVar(&o.SkipCleanup, "skip-cleanup", false, "skip removal of artifact directories") + fs.StringVar(&o.FilterByOS, "filter-by-os", "linux/amd64", "A regular expression to control which index image is picked when multiple variants are available") } func (o *RootOptions) LogfilePreRun(cmd *cobra.Command, _ []string) { diff --git a/pkg/image/association.go b/pkg/image/association.go index 084f7aeba..afe30ef73 100644 --- a/pkg/image/association.go +++ b/pkg/image/association.go @@ -53,6 +53,9 @@ type Association struct { // Type of the image in the context of this tool. // See the ImageType enum for options. Type ImageType `json:"type"` + // TopLeveLevel will determine if the association is a parent + // or child + TopLevel bool `json:"toplevel"` // ManifestDigests of images if the image is a docker manifest list or OCI index. // These manifests refer to image manifests by content SHA256 digest. // LayerDigests and Manifests are mutually exclusive. @@ -210,7 +213,7 @@ func AssociateImageLayers(rootDir string, imgMappings map[string]string, images } // TODO(estroz): parallelize - associations, err := associateImageLayers(image, localRoot, dirRef, tagOrID, skipParse) + associations, err := associateImageLayers(image, localRoot, dirRef, tagOrID, true, skipParse) if err != nil { errs = append(errs, err) } @@ -222,7 +225,7 @@ func AssociateImageLayers(rootDir string, imgMappings map[string]string, images return bundleAssociations, utilerrors.NewAggregate(errs) } -func associateImageLayers(image, localRoot, dirRef, tagOrID string, skipParse func(string) bool) (associations []Association, err error) { +func associateImageLayers(image, localRoot, dirRef, tagOrID string, toplevel bool, skipParse func(string) bool) (associations []Association, err error) { if skipParse(image) { return nil, nil } @@ -283,7 +286,7 @@ func associateImageLayers(image, localRoot, dirRef, tagOrID string, skipParse fu association.ManifestDigests = append(association.ManifestDigests, digestStr) // Recurse on child manifests, which should be in the same directory // with the same file name as it's digest. - childAssocs, err := associateImageLayers(digestStr, localRoot, dirRef, digestStr, skipParse) + childAssocs, err := associateImageLayers(digestStr, localRoot, dirRef, digestStr, false, skipParse) if err != nil { return nil, err } diff --git a/pkg/operator/mirror.go b/pkg/operator/mirror.go index 45f051896..9826225df 100644 --- a/pkg/operator/mirror.go +++ b/pkg/operator/mirror.go @@ -425,6 +425,9 @@ func (o *MirrorOptions) associateDeclarativeConfigImageLayers(ctlgRef imagesourc } for k, assoc := range assocs { + if assoc.Name == ctlgRef.Ref.Exact() { + assoc.TopLevel = true + } assoc.Type = typ assocs[k] = assoc }