feat(create, pusblish): add support for customer Helm charts (#105)

* feat: helm support and release image pinning

* fix: updated go.sum

README.md

@@ -31,7 +31,7 @@ Replace this value with a real registry host, or create a `docker.io/library/reg
 1. Create then publish to your mirror registry:
     ```sh
     ./bin/oc-bundle create full --config imageset-config.yaml --dir test-create --output archives --log-level debug
-    ./bin/oc-bundle publish --archive archives --dir test-publish --to-mirror reg.mirror.com
+    ./bin/oc-bundle publish --archive archives --dir test-publish --to-mirror reg.mirror.com --output results
     ```
 
 For configuration and options, see the [expanded overview](./docs/overview.md) and [usage](./docs/usage.md) docs.

data/imageset-config.yaml

@@ -28,6 +28,16 @@ mirror:
     - name: nginx
   additionalimages:
     - name: registry.redhat.io/ubi8/ubi:latest
+  helm:
+    local:
+      - name: podinfo
+        path: /test/podinfo-5.0.0.tar.gz
+    repos: 
+      - name: podinfo
+        url: https://stefanprodan.github.io/podinfo
+        charts: 
+          - name: podinfo
+            version: 5.0.0
   blockedimages:
     - name: alpine
     - name: redis

go.mod

@@ -5,7 +5,7 @@ go 1.16
 require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/bshuster-repo/logrus-logstash-hook v1.0.2 // indirect
-	github.com/containerd/containerd v1.5.2
+	github.com/containerd/containerd v1.5.4
 	github.com/containers/image/v5 v5.15.2
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/go-git/go-git/v5 v5.4.2
@@ -28,13 +28,14 @@ require (
 	github.com/stretchr/testify v1.7.0
 	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
 	gopkg.in/yaml.v2 v2.4.0
+	helm.sh/helm/v3 v3.7.0
 	k8s.io/apimachinery v0.22.1
-	k8s.io/cli-runtime v0.22.0
+	k8s.io/cli-runtime v0.22.1
 	k8s.io/client-go v0.22.1
 	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.9.0
-	k8s.io/kubectl v0.22.0
-	rsc.io/letsencrypt v0.0.3 // indirect
+	k8s.io/kubectl v0.22.1
+	sigs.k8s.io/kustomize/kyaml v0.11.0
 	sigs.k8s.io/yaml v1.2.0
 )
 

pkg/archive/archive.go

@@ -11,10 +11,11 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/RedHatGov/bundle/pkg/config"
-	"github.com/RedHatGov/bundle/pkg/config/v1alpha1"
 	"github.com/mholt/archiver/v3"
 	"github.com/sirupsen/logrus"
+
+	"github.com/RedHatGov/bundle/pkg/config"
+	"github.com/RedHatGov/bundle/pkg/config/v1alpha1"
 )
 
 type Archiver interface {
@@ -227,7 +228,7 @@ func blobInArchive(file string) string {
 
 func includeFile(fpath string) bool {
 	split := strings.Split(filepath.Clean(fpath), string(filepath.Separator))
-	return split[0] == config.InternalDir || split[0] == config.PublishDir || split[0] == "catalogs"
+	return split[0] == config.InternalDir || split[0] == config.PublishDir || split[0] == "catalogs" || split[0] == config.HelmDir
 }
 
 func shouldRemove(fpath string, info fs.FileInfo) bool {

pkg/archive/archive_test.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/RedHatGov/bundle/pkg/bundle"
 	"github.com/RedHatGov/bundle/pkg/config"
 	"github.com/RedHatGov/bundle/pkg/config/v1alpha1"
 )
@@ -57,8 +56,8 @@ func Test_SplitArchive(t *testing.T) {
 
 		packager := NewPackager(tt.manifests, tt.blobs)
 
-		if err := bundle.MakeCreateDirs(testdir); err != nil {
-			t.Fatal(err)
+		if err := os.MkdirAll(filepath.Join(testdir, config.SourceDir), os.ModePerm); err != nil {
+			t.Fail()
 		}
 
 		cwd, err := os.Getwd()

pkg/bundle/additional.go

@@ -1,6 +1,7 @@
 package bundle
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 
@@ -23,7 +24,7 @@ func NewAdditionalOptions(ro cli.RootOptions) *AdditionalOptions {
 }
 
 // GetAdditional downloads specified images in the imageset-config.yaml under mirror.additonalImages
-func (o *AdditionalOptions) GetAdditional(_ v1alpha1.PastMirror, cfg v1alpha1.ImageSetConfiguration) (image.AssociationSet, error) {
+func (o *AdditionalOptions) GetAdditional(cfg v1alpha1.ImageSetConfiguration, imageList []v1alpha1.AdditionalImages) (image.AssociationSet, error) {
 
 	opts := mirror.NewMirrorImageOptions(o.IOStreams)
 	opts.DryRun = o.DryRun
@@ -32,12 +33,12 @@ func (o *AdditionalOptions) GetAdditional(_ v1alpha1.PastMirror, cfg v1alpha1.Im
 	opts.FileDir = filepath.Join(o.Dir, config.SourceDir)
 	opts.FilterOptions = o.FilterOptions
 
-	logrus.Infof("Downloading %d image(s) to %s", len(cfg.Mirror.AdditionalImages), opts.FileDir)
+	logrus.Infof("Downloading %d image(s) to %s", len(imageList), opts.FileDir)
 
 	var mappings []mirror.Mapping
-	images := make([]string, len(cfg.Mirror.AdditionalImages))
-	assocMappings := make(map[string]string, len(cfg.Mirror.AdditionalImages))
-	for i, img := range cfg.Mirror.AdditionalImages {
+	images := make([]string, len(imageList))
+	assocMappings := make(map[string]string, len(imageList))
+	for i, img := range imageList {
 
 		// If the pullSecret is not empty create a cached context
 		// else let `oc mirror` use the default docker config location
@@ -74,7 +75,11 @@ func (o *AdditionalOptions) GetAdditional(_ v1alpha1.PastMirror, cfg v1alpha1.Im
 
 		// Add mapping and image for image association.
 		// The registry component is not included in the final path.
-		srcImage := srcRef.Ref.String()
+		srcImage, err := pinImages(context.TODO(), srcRef.Ref.Exact(), "", o.SkipTLS)
+		if err != nil {
+			return nil, err
+		}
+
 		dstRef.Ref.Registry = ""
 		assocMappings[srcImage] = dstRef.String()
 		images[i] = srcImage

pkg/bundle/additional_test.go

@@ -1,6 +1,7 @@
 package bundle
 
 import (
+	"context"
 	"os"
 	"testing"
 
@@ -16,7 +17,6 @@ import (
 
 func Test_GetAdditional(t *testing.T) {
 
-	mirror := v1alpha1.PastMirror{}
 	cfg := v1alpha1.ImageSetConfiguration{}
 	cfg.Mirror = v1alpha1.Mirror{
 		BlockedImages: []v1alpha1.BlockedImages{
@@ -39,9 +39,12 @@ func Test_GetAdditional(t *testing.T) {
 	}
 	opts := NewAdditionalOptions(ro)
 
-	assocs, err := opts.GetAdditional(mirror, cfg)
+	assocs, err := opts.GetAdditional(cfg, cfg.Mirror.AdditionalImages)
+	require.NoError(t, err)
+
+	testerImg, err := pinImages(context.TODO(), "quay.io/estroz/pull-tester-additional:latest", "", false)
 	require.NoError(t, err)
 	if assert.Len(t, assocs, 1) {
-		require.Contains(t, assocs, "quay.io/estroz/pull-tester-additional:latest")
+		require.Contains(t, assocs, testerImg)
 	}
 }

pkg/bundle/create/create.go

@@ -79,7 +79,16 @@ func (o *Options) RunFull(ctx context.Context, flags *pflag.FlagSet) error {
 
 		if len(cfg.Mirror.AdditionalImages) != 0 {
 			opts := bundle.NewAdditionalOptions(*o.RootOptions)
-			assocs, err := opts.GetAdditional(run, cfg)
+			assocs, err := opts.GetAdditional(cfg, cfg.Mirror.AdditionalImages)
+			if err != nil {
+				return meta, run, err
+			}
+			allAssocs.Merge(assocs)
+		}
+
+		if len(cfg.Mirror.Helm.Local) != 0 || len(cfg.Mirror.Helm.Repos) != 0 {
+			opts := bundle.NewHelmOptions(*o.RootOptions)
+			assocs, err := opts.PullCharts(cfg)
 			if err != nil {
 				return meta, run, err
 			}
@@ -142,7 +151,16 @@ func (o *Options) RunDiff(ctx context.Context, flags *pflag.FlagSet) error {
 
 		if len(cfg.Mirror.AdditionalImages) != 0 {
 			opts := bundle.NewAdditionalOptions(*o.RootOptions)
-			assocs, err := opts.GetAdditional(lastRun, cfg)
+			assocs, err := opts.GetAdditional(cfg, cfg.Mirror.AdditionalImages)
+			if err != nil {
+				return meta, run, err
+			}
+			allAssocs.Merge(assocs)
+		}
+
+		if len(cfg.Mirror.Helm.Local) != 0 || len(cfg.Mirror.Helm.Repos) != 0 {
+			opts := bundle.NewHelmOptions(*o.RootOptions)
+			assocs, err := opts.PullCharts(cfg)
 			if err != nil {
 				return meta, run, err
 			}