Merge pull request #502 from zong-zhe/oci-downloader-cache

feat: add OciCache to support the new local file structure for OCI

go.mod

@@ -150,6 +150,7 @@ require (
 
 require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/dchest/siphash v1.2.3
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/gofrs/flock v0.12.1
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3

go.sum

@@ -286,6 +286,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
+github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/distribution/distribution/v3 v3.0.0-beta.1 h1:X+ELTxPuZ1Xe5MsD3kp2wfGUhc8I+MPfRis8dZ818Ic=

pkg/client/deperated.go

@@ -217,20 +217,10 @@ func (c *KpmClient) DownloadFromOci(dep *downloader.Oci, localPath string) (stri
 		return "", err
 	}
 
-	matches, _ := filepath.Glob(filepath.Join(localPath, "*.tar"))
-	if matches == nil || len(matches) != 1 {
-		// then try to glob tgz file
-		matches, _ = filepath.Glob(filepath.Join(localPath, "*.tgz"))
-		if matches == nil || len(matches) != 1 {
-			return "", reporter.NewErrorEvent(
-				reporter.InvalidKclPkg,
-				err,
-				fmt.Sprintf("failed to find the kcl package from '%s'.", localPath),
-			)
-		}
+	tarPath, err := utils.FindPkgArchive(localPath)
+	if err != nil {
+		return "", err
 	}
-
-	tarPath := matches[0]
 	if utils.IsTar(tarPath) {
 		err = utils.UnTarDir(tarPath, localPath)
 	} else {

pkg/downloader/downloader.go

@@ -11,6 +11,7 @@ import (
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/otiai10/copy"
 	"kcl-lang.io/kpm/pkg/constants"
+	"kcl-lang.io/kpm/pkg/features"
 	"kcl-lang.io/kpm/pkg/git"
 	"kcl-lang.io/kpm/pkg/oci"
 	"kcl-lang.io/kpm/pkg/reporter"
@@ -291,44 +292,85 @@ func (d *OciDownloader) Download(opts DownloadOptions) error {
 		ociSource.Tag = tagSelected
 	}
 
-	reporter.ReportMsgTo(
-		fmt.Sprintf(
-			"downloading '%s:%s' from '%s/%s:%s'",
-			ociSource.Repo, ociSource.Tag, ociSource.Reg, ociSource.Repo, ociSource.Tag,
-		),
-		opts.LogWriter,
-	)
-
-	err = ociCli.Pull(localPath, ociSource.Tag)
-	if err != nil {
-		return err
-	}
+	if ok, err := features.Enabled(features.SupportNewStorage); err == nil && ok {
+		if opts.EnableCache {
+			hash, err := ociSource.Hash()
+			if err != nil {
+				return err
+			}
+			cacheFullPath := filepath.Join(opts.CachePath, hash)
+			localFullPath := filepath.Join(opts.LocalPath, hash)
+
+			if utils.DirExists(localFullPath) &&
+				utils.DirExists(filepath.Join(localFullPath, constants.KCL_MOD)) {
+				return nil
+			} else {
+				cacheTarPath, err := utils.FindPkgArchive(cacheFullPath)
+				if err != nil && errors.Is(err, utils.PkgArchiveNotFound) {
+					reporter.ReportMsgTo(
+						fmt.Sprintf(
+							"downloading '%s:%s' from '%s/%s:%s'",
+							ociSource.Repo, ociSource.Tag, ociSource.Reg, ociSource.Repo, ociSource.Tag,
+						),
+						opts.LogWriter,
+					)
+
+					err = ociCli.Pull(cacheFullPath, ociSource.Tag)
+					if err != nil {
+						return err
+					}
+					cacheTarPath, err = utils.FindPkgArchive(cacheFullPath)
+					if err != nil {
+						return err
+					}
+				} else if err != nil {
+					return err
+				}
 
-	matches, _ := filepath.Glob(filepath.Join(localPath, "*.tar"))
-	if matches == nil || len(matches) != 1 {
-		// then try to glob tgz file
-		matches, _ = filepath.Glob(filepath.Join(localPath, "*.tgz"))
-		if matches == nil || len(matches) != 1 {
-			return fmt.Errorf("failed to find the downloaded kcl package tar file in '%s'", localPath)
+				if utils.IsTar(cacheTarPath) {
+					err = utils.UnTarDir(cacheTarPath, localFullPath)
+				} else {
+					err = utils.ExtractTarball(cacheTarPath, localFullPath)
+				}
+				if err != nil {
+					return err
+				}
+			}
 		}
-	}
-
-	tarPath := matches[0]
-	if utils.IsTar(tarPath) {
-		err = utils.UnTarDir(tarPath, localPath)
 	} else {
-		err = utils.ExtractTarball(tarPath, localPath)
-	}
-	if err != nil {
-		return fmt.Errorf("failed to untar the kcl package tar from '%s' into '%s'", tarPath, localPath)
-	}
+		reporter.ReportMsgTo(
+			fmt.Sprintf(
+				"downloading '%s:%s' from '%s/%s:%s'",
+				ociSource.Repo, ociSource.Tag, ociSource.Reg, ociSource.Repo, ociSource.Tag,
+			),
+			opts.LogWriter,
+		)
 
-	// After untar the downloaded kcl package tar file, remove the tar file.
-	if utils.DirExists(tarPath) {
-		rmErr := os.Remove(tarPath)
-		if rmErr != nil {
-			return fmt.Errorf("failed to remove the downloaded kcl package tar file '%s'", tarPath)
+		err = ociCli.Pull(localPath, ociSource.Tag)
+		if err != nil {
+			return err
 		}
+		tarPath, err := utils.FindPkgArchive(localPath)
+		if err != nil {
+			return err
+		}
+		if utils.IsTar(tarPath) {
+			err = utils.UnTarDir(tarPath, localPath)
+		} else {
+			err = utils.ExtractTarball(tarPath, localPath)
+		}
+		if err != nil {
+			return fmt.Errorf("failed to untar the kcl package tar from '%s' into '%s'", tarPath, localPath)
+		}
+
+		// After untar the downloaded kcl package tar file, remove the tar file.
+		if utils.DirExists(tarPath) {
+			rmErr := os.Remove(tarPath)
+			if rmErr != nil {
+				return fmt.Errorf("failed to remove the downloaded kcl package tar file '%s'", tarPath)
+			}
+		}
+
 	}
 
 	return err

pkg/downloader/source.go

@@ -548,3 +548,47 @@ func ParseSourceUrlFrom(sourceStr string, settings *settings.Settings) (*url.URL
 	}
 	return nil, fmt.Errorf("invalid source url: %s", sourceStr)
 }
+
+func (s *Source) Hash() (string, error) {
+	if s.Git != nil {
+		return s.Git.Hash()
+	}
+	if s.Oci != nil {
+		return s.Oci.Hash()
+	}
+	if s.Local != nil {
+		return s.Local.Hash()
+	}
+	if s.Registry != nil {
+		return s.Registry.Hash()
+	}
+	return "", nil
+}
+
+func (g *Git) Hash() (string, error) {
+	return utils.ShortHash(g.Url)
+}
+
+func (o *Oci) Hash() (string, error) {
+	var packageFilename string
+	if o.Tag == "" {
+		packageFilename = filepath.Base(o.Repo)
+	} else {
+		packageFilename = fmt.Sprintf("%s_%s", filepath.Base(o.Repo), o.Tag)
+	}
+
+	hash, err := utils.ShortHash(utils.JoinPath(o.Reg, filepath.Dir(o.Repo)))
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(hash, packageFilename), nil
+}
+
+func (l *Local) Hash() (string, error) {
+	return utils.ShortHash(l.Path)
+}
+
+func (r *Registry) Hash() (string, error) {
+	return r.Oci.Hash()
+}

pkg/features/features.go

@@ -10,10 +10,13 @@ import (
 const (
 	// SupportMVS is the feature gate for enabling the support for MVS.
 	SupportMVS = "SupportMVS"
+	// SupportNewStorage is the feature gate for enabling the support for the new storage structure.
+	SupportNewStorage = "SupportNewStorage"
 )
 
 var features = map[string]bool{
-	SupportMVS: false,
+	SupportMVS:        false,
+	SupportNewStorage: false,
 }
 
 func init() {

pkg/utils/utils.go

@@ -6,6 +6,7 @@ import (
 	"compress/gzip"
 	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
 	goerrors "errors"
 	"fmt"
 	"io"
@@ -18,6 +19,8 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/dchest/siphash"
+
 	"github.com/BurntSushi/toml"
 	"github.com/distribution/reference"
 	"github.com/moby/term"
@@ -663,3 +666,31 @@ func MoveOrCopy(src, dest string) error {
 	}
 	return nil
 }
+
+// ShortHash takes a string (in this case, the Git URL) and returns a siphash.
+// The method name references `cargo` which means performs a simple and quick hash
+func ShortHash(input string) (string, error) {
+	// siphash is quick and simple to hash for simple scenarios
+	hasher := siphash.New(make([]byte, 16))
+	_, err := hasher.Write([]byte(input))
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hasher.Sum(nil)), nil
+}
+
+var PkgArchiveNotFound = goerrors.New("failed to find the downloaded kcl package tar file")
+
+// FindPkgArchive will find the full path of the KCL package archive in the 'path' directory.
+func FindPkgArchive(path string) (string, error) {
+	matches, _ := filepath.Glob(filepath.Join(path, "*.tar"))
+	if matches == nil || len(matches) != 1 {
+		// then try to glob tgz file
+		matches, _ = filepath.Glob(filepath.Join(path, "*.tgz"))
+		if matches == nil || len(matches) != 1 {
+			return "", fmt.Errorf("failed to find the downloaded kcl package tar file in '%s': %w", path, PkgArchiveNotFound)
+		}
+	}
+
+	return matches[0], nil
+}

pkg/utils/utils_test.go

@@ -314,3 +314,11 @@ func TestMatchesPackageName(t *testing.T) {
 	address := filepath.Join(getTestDir("test_find_package"), "test_2", "kcl.mod")
 	assert.Equal(t, matchesPackageName(address, "test_find_package"), true)
 }
+
+func TestShortHash(t *testing.T) {
+	hash, err := ShortHash(JoinPath("ghcr.io", "kcl-lang"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, hash, "9ebd0ad063dba405")
+}