From 971f79b8cd3fd75a329e54d0da6255657083ba64 Mon Sep 17 00:00:00 2001 From: GavinPJK Date: Wed, 16 Oct 2024 13:46:45 +0100 Subject: [PATCH 1/2] feat: add function for sparse/partial cloning of cluster repo --- pkg/gitclient/helpers.go | 70 ++++++++++++++++++++++---------- pkg/requirements/requirements.go | 69 +++++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 39 deletions(-) diff --git a/pkg/gitclient/helpers.go b/pkg/gitclient/helpers.go index a4d3f0f..f9e982f 100644 --- a/pkg/gitclient/helpers.go +++ b/pkg/gitclient/helpers.go @@ -331,17 +331,9 @@ func NthTag(g Interface, dir string, n int) (string, string, error) { // CloneToDir clones the git repository to either the given directory or create a temporary func CloneToDir(g Interface, gitURL, dir string) (string, error) { - var err error - if dir != "" { - err = os.MkdirAll(dir, util.DefaultWritePermissions) - if err != nil { - return "", fmt.Errorf("failed to create directory %s: %w", dir, err) - } - } else { - dir, err = os.MkdirTemp("", "jx-git-") - if err != nil { - return "", fmt.Errorf("failed to create temporary directory: %w", err) - } + dir, err := createDir(dir) + if err != nil { + return "", err } log.Logger().Debugf("cloning %s to directory %s", termcolor.ColorInfo(gitURL), termcolor.ColorInfo(dir)) @@ -354,23 +346,39 @@ func CloneToDir(g Interface, gitURL, dir string) (string, error) { return dir, nil } +// PartialCloneToDir Partially clones the git repository to either the given directory or create a temporary one +// Alternative to SparseCloneToDir when git provider does not support sparse-checkout +// sparseCheckoutPatterns not supported +// If shallow is true the clone is made with --depth=1 +func PartialCloneToDir(g Interface, gitURL, dir string, shallow bool) (string, error) { + dir, err := createDir(dir) + if err != nil { + return "", err + } + + log.Logger().Debugf("shallow cloning %s to directory %s", termcolor.ColorInfo(gitURL), termcolor.ColorInfo(dir)) + + parentDir := filepath.Dir(dir) + sparseCloneArgs := []string{"clone", "--filter=blob:none"} + if shallow { + sparseCloneArgs = append(sparseCloneArgs, "--depth=1") + } + _, err = g.Command(parentDir, append(sparseCloneArgs, gitURL, dir)...) + if err != nil { + return "", fmt.Errorf("failed to sparsely clone repository %s to directory: %s: %w", gitURL, dir, err) + } + return dir, nil +} + // SparseCloneToDir clones the git repository sparsely to either the given directory or create a temporary on. // SparseCheckoutPatterns are checked out interpreted as in .gitignore. If no sparseCheckoutPatterns are given the files // directly under the root of the repository are checked out. // NOTE: This functionality is experimental and also the behaviour may vary between different git servers. // If shallow is true the clone is made with --depth=1 func SparseCloneToDir(g Interface, gitURL, dir string, shallow bool, sparseCheckoutPatterns ...string) (string, error) { - var err error - if dir != "" { - err = os.MkdirAll(dir, util.DefaultWritePermissions) - if err != nil { - return "", fmt.Errorf("failed to create directory %s: %w", dir, err) - } - } else { - dir, err = os.MkdirTemp("", "jx-git-") - if err != nil { - return "", fmt.Errorf("failed to create temporary directory: %w", err) - } + dir, err := createDir(dir) + if err != nil { + return "", err } log.Logger().Debugf("cloning %s to directory %s sparsely", termcolor.ColorInfo(gitURL), termcolor.ColorInfo(dir)) @@ -566,6 +574,24 @@ func CheckoutRemoteBranch(g Interface, dir string, branch string) error { return Checkout(g, dir, branch) } +// createDir creates input directory if it does not exist, or creates a temporary directory +// createDir creates the input directory if it does not exist, or creates a temporary directory +func createDir(dir string) (string, error) { + var err error + if dir != "" { + err = os.MkdirAll(dir, util.DefaultWritePermissions) + if err != nil { + return "", fmt.Errorf("failed to create directory %s: %w", dir, err) + } + } else { + dir, err = os.MkdirTemp("", "jx-git-") + if err != nil { + return "", fmt.Errorf("failed to create temporary directory: %w", err) + } + } + return dir, nil +} + // GetLatestCommitMessage returns the latest git commit message func GetLatestCommitMessage(g Interface, dir string) (string, error) { return g.Command(dir, "log", "-1", "--pretty=%B") diff --git a/pkg/requirements/requirements.go b/pkg/requirements/requirements.go index 10f3f56..7c16682 100644 --- a/pkg/requirements/requirements.go +++ b/pkg/requirements/requirements.go @@ -64,23 +64,9 @@ func GetRequirementsAndGit(g gitclient.Interface, gitURL string) (*jxcore.Requir func CloneClusterRepo(g gitclient.Interface, gitURL string) (string, error) { // if we have a kubernetes secret with git auth mounted to the filesystem when running in cluster // we need to turn it into a git credentials file see https://git-scm.com/docs/git-credential-store - secretMountPath := os.Getenv(credentialhelper.GIT_SECRET_MOUNT_PATH) - if secretMountPath != "" { - err := credentialhelper.WriteGitCredentialFromSecretMount() - if err != nil { - return "", fmt.Errorf("failed to write git credentials file for secret %s : %w", secretMountPath, err) - } - - gitURL, err = AddUserPasswordToURLFromDir(gitURL, secretMountPath) - if err != nil { - return "", fmt.Errorf("failed to add username and password to git URL: %w", err) - } - } else { - if kube.IsInCluster() { - log.Logger().Warnf("no $GIT_SECRET_MOUNT_PATH environment variable set") - } else { - log.Logger().Debugf("no $GIT_SECRET_MOUNT_PATH environment variable set") - } + gitURL, err := gitCredsFromCluster(gitURL) + if err != nil { + return "", err } // clone cluster repo to a temp dir and load the requirements @@ -91,6 +77,33 @@ func CloneClusterRepo(g gitclient.Interface, gitURL string) (string, error) { return dir, nil } +// PartialCloneClusterRepo clones the cluster repo to a temporary directory and returns the directory path +// Attempts a sparse clone first, falling back to a partial clone without checkout patterns, then a default clone +func PartialCloneClusterRepo(g gitclient.Interface, gitURL string, shallow bool, sparseCheckoutPatterns ...string) (string, error) { + gitURL, err := gitCredsFromCluster(gitURL) + if err != nil { + return "", err + } + // Attempt sparse clone first + dir, err := gitclient.SparseCloneToDir(g, gitURL, "", shallow, sparseCheckoutPatterns...) + if err != nil { + log.Logger().Warnf("failed sparse clone of cluster git repo %s: %v", gitURL, err) + log.Logger().Warnf("falling back to partial clone without checkout patterns") + // If sparse clone fails, fall back to partial clone + dir, err = gitclient.PartialCloneToDir(g, gitURL, "", shallow) + if err != nil { + log.Logger().Warnf("failed partial clone of cluster git repo %s: %v", gitURL, err) + log.Logger().Warnf("falling back to default clone, without checkout patterns") + dir, err = gitclient.CloneToDir(g, gitURL, "") + if err != nil { + return "", fmt.Errorf("failed to clone cluster git repo %s: %w", gitURL, err) + } + } + return dir, nil + } + return dir, nil +} + // AddUserPasswordToURLFromDir loads the username and password files from the given directory and adds them to the URL if they are found func AddUserPasswordToURLFromDir(gitURL, path string) (string, error) { username, err := loadFile(filepath.Join(path, "username")) @@ -107,6 +120,28 @@ func AddUserPasswordToURLFromDir(gitURL, path string) (string, error) { return gitURL, nil } +func gitCredsFromCluster(gitURL string) (string, error) { + secretMountPath := os.Getenv(credentialhelper.GIT_SECRET_MOUNT_PATH) + if secretMountPath != "" { + err := credentialhelper.WriteGitCredentialFromSecretMount() + if err != nil { + return "", fmt.Errorf("failed to write git credentials file for secret %s : %w", secretMountPath, err) + } + + gitURL, err = AddUserPasswordToURLFromDir(gitURL, secretMountPath) + if err != nil { + return "", fmt.Errorf("failed to add username and password to git URL: %w", err) + } + } else { + if kube.IsInCluster() { + log.Logger().Warnf("no $GIT_SECRET_MOUNT_PATH environment variable set") + } else { + log.Logger().Debugf("no $GIT_SECRET_MOUNT_PATH environment variable set") + } + } + return gitURL, nil +} + func loadFile(path string) (string, error) { exists, err := files.FileExists(path) if err != nil { From 7b66a76df0d193367e4e77ae49390422a58aa622 Mon Sep 17 00:00:00 2001 From: GavinPJK Date: Tue, 22 Oct 2024 16:29:44 +0100 Subject: [PATCH 2/2] chore: rename vars for partial clone --- pkg/gitclient/helpers.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/gitclient/helpers.go b/pkg/gitclient/helpers.go index f9e982f..153ddf7 100644 --- a/pkg/gitclient/helpers.go +++ b/pkg/gitclient/helpers.go @@ -356,16 +356,17 @@ func PartialCloneToDir(g Interface, gitURL, dir string, shallow bool) (string, e return "", err } - log.Logger().Debugf("shallow cloning %s to directory %s", termcolor.ColorInfo(gitURL), termcolor.ColorInfo(dir)) + log.Logger().Debugf("initiating partial clone %s to directory %s", termcolor.ColorInfo(gitURL), termcolor.ColorInfo(dir)) parentDir := filepath.Dir(dir) - sparseCloneArgs := []string{"clone", "--filter=blob:none"} + partialCloneArgs := []string{"clone", "--filter=blob:none"} if shallow { - sparseCloneArgs = append(sparseCloneArgs, "--depth=1") + log.Logger().Debugf("setting clone depth to 1") + partialCloneArgs = append(partialCloneArgs, "--depth=1") } - _, err = g.Command(parentDir, append(sparseCloneArgs, gitURL, dir)...) + _, err = g.Command(parentDir, append(partialCloneArgs, gitURL, dir)...) if err != nil { - return "", fmt.Errorf("failed to sparsely clone repository %s to directory: %s: %w", gitURL, dir, err) + return "", fmt.Errorf("failed to partially clone repository %s to directory: %s: %w", gitURL, dir, err) } return dir, nil }