support common git source

cyclops-ctrl/internal/template/git.go

@@ -18,6 +18,8 @@ import (
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/protocol/packp/capability"
+	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/go-git/go-git/v5/storage/memory"
 	"helm.sh/helm/v3/pkg/chart"
@@ -70,87 +72,119 @@ func (r Repo) LoadTemplate(repoURL, path, commit, resolvedVersion string) (*mode
 		return nil, err
 	}
 
-	// load helm chart metadata
-	chartMetadata, err := fs.Open(path2.Join(path, "Chart.yaml"))
+	// region read files
+	filesFs, err := fs.Chroot(path)
 	if err != nil {
-		return nil, errors.Wrap(err, "could not read 'Chart.yaml' file; it should be placed in the repo/path you provided; make sure you provided the correct path")
+		return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists")
 	}
 
-	var chartMetadataBuffer bytes.Buffer
-	_, err = io.Copy(bufio.NewWriter(&chartMetadataBuffer), chartMetadata)
+	files, err := readFiles("", filesFs)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "failed to read template files")
 	}
+	// endregion
 
-	var metadata *helm.Metadata
-	if err := yaml.Unmarshal(chartMetadataBuffer.Bytes(), &metadata); err != nil {
-		return nil, err
+	if len(files) == 0 {
+		return nil, errors.Errorf("no files found in repo %v on path %v; make sure the repo, path and version are correct", repoURL, path)
 	}
-	// endregion
 
-	// region read templates
-	templatesPath := path2.Join(path, "templates")
+	// region map files to template
+	metadataBytes := []byte{}
+	schemaBytes := []byte{}
+	chartFiles := make([]*chart.File, 0)
 
-	_, err = fs.ReadDir(templatesPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists")
-	}
+	templateFiles := make([]*chart.File, 0)
+	crdFiles := make([]*chart.File, 0)
 
-	manifests, err := concatenateTemplates(templatesPath, fs)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to load template files")
-	}
-	// endregion
+	dependenciesFromChartsDir := make(map[string]map[string][]byte, 0)
 
-	// region read files
-	filesFs, err := fs.Chroot(path)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not find 'templates' dir; it should be placed in the repo/path you provided; make sure 'templates' directory exists")
-	}
+	for _, f := range files {
+		parts := strings.Split(f.Name, "/")
 
-	chartFiles, err := readFiles("", filesFs)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to read template files")
-	}
-	// endregion
+		if len(parts) == 1 && parts[0] == "Chart.yaml" {
+			metadataBytes = f.Data
+			continue
+		}
 
-	// region read schema
-	schemaFile, err := fs.Open(path2.Join(path, "values.schema.json"))
-	if err != nil {
-		return nil, errors.Wrap(err, "could not read 'values.schema.json' file; it should be placed in the repo/path you provided; make sure 'templates' directory exists")
-	}
+		if len(parts) == 1 && parts[0] == "values.schema.json" {
+			schemaBytes = f.Data
+			continue
+		}
 
-	var schemaChartBuffer bytes.Buffer
-	_, err = io.Copy(bufio.NewWriter(&schemaChartBuffer), schemaFile)
-	if err != nil {
-		return nil, err
+		if len(parts) > 1 && parts[0] == "templates" &&
+			(parts[1] != "Notes.txt" && parts[1] != "NOTES.txt" && parts[1] != "tests") {
+			templateFiles = append(templateFiles, f)
+			continue
+		}
+
+		if len(parts) > 1 && parts[0] == "crds" &&
+			(parts[1] != "Notes.txt" && parts[1] != "NOTES.txt" && parts[1] != "tests") {
+			crdFiles = append(crdFiles, f)
+			continue
+		}
+
+		if len(parts) > 2 && parts[0] == "charts" {
+			depName := parts[1]
+			if _, ok := dependenciesFromChartsDir[depName]; !ok {
+				dependenciesFromChartsDir[depName] = make(map[string][]byte)
+			}
+
+			dependenciesFromChartsDir[depName][path2.Join(parts[1:]...)] = f.Data
+			continue
+		}
+
+		chartFiles = append(chartFiles, f)
 	}
 
 	var schema helm.Property
-	if err := json.Unmarshal(schemaChartBuffer.Bytes(), &schema); err != nil {
-		return nil, err
+	// unmarshal values schema only if present
+	if len(schemaBytes) > 0 {
+		if err := json.Unmarshal(schemaBytes, &schema); err != nil {
+			fmt.Println("error on schema bytes", repoURL, path)
+			return &models.Template{}, err
+		}
+	}
+
+	var metadata *helm.Metadata
+	if err := yaml.Unmarshal(metadataBytes, &metadata); err != nil {
+		fmt.Println("error on meta unmarshal", repoURL, path)
+		return &models.Template{}, err
 	}
-	// endregion
 
 	// region load dependencies
 	dependencies, err := r.loadDependencies(metadata)
 	if err != nil {
-		return nil, err
+		return &models.Template{}, err
+	}
+
+	for depName, files := range dependenciesFromChartsDir {
+		if dependencyExists(depName, dependencies) {
+			continue
+		}
+
+		dep, err := r.mapHelmChart(depName, files)
+		if err != nil {
+			return nil, err
+		}
+
+		dependencies = append(dependencies, dep)
 	}
 	// endregion
 
 	template := &models.Template{
-		Name:              "",
-		Manifest:          strings.Join(manifests, "---\n"),
-		RootField:         mapper.HelmSchemaToFields("", schema, schema.Definitions, dependencies),
-		Created:           "",
-		Edited:            "",
-		Version:           commit,
+		Name:              path,
 		ResolvedVersion:   commitSHA,
+		Version:           commit,
+		RootField:         mapper.HelmSchemaToFields("", schema, schema.Definitions, dependencies),
 		Files:             chartFiles,
+		Templates:         templateFiles,
+		CRDs:              crdFiles,
 		Dependencies:      dependencies,
 		HelmChartMetadata: metadata,
+		RawSchema:         schemaBytes,
+		IconURL:           metadata.Icon,
 	}
+	// endregion
 
 	r.cache.SetTemplate(repoURL, path, commitSHA, string(cyclopsv1alpha1.TemplateSourceTypeGit), template)
 
@@ -318,6 +352,12 @@ func readValuesFile(fs billy.Filesystem, path string) ([]byte, error) {
 
 func clone(repoURL, commit string, creds *auth.Credentials) (billy.Filesystem, error) {
 	// region clone from git
+	if gitproviders.IsAzureRepo(repoURL) {
+		transport.UnsupportedCapabilities = []capability.Capability{
+			capability.ThinPack,
+		}
+	}
+
 	repo, err := git.Clone(memory.NewStorage(), memfs.New(), &git.CloneOptions{
 		URL:  repoURL,
 		Tags: git.AllTags,

cyclops-ctrl/internal/template/gitproviders/azure.go

@@ -0,0 +1,12 @@
+package gitproviders
+
+import "net/url"
+
+func IsAzureRepo(repoURL string) bool {
+	host, err := url.Parse(repoURL)
+	if err != nil {
+		return false
+	}
+
+	return host.Host == "dev.azure.com"
+}