feat: implement discovering CRDs in git repositories directly

README.md

@@ -58,6 +58,35 @@ A single file will be created containing all versions in the CRD delimited by `-
 
 Optionally, you can provide the flag `-s` which will output the generated content to `stdout`.
 
+You can also point at a git repository to _discover_ CRDs inside the repository. Simply call `crd` with:
+
+```
+➜ cty generate crd -g https://github.com/Skarlso/crd-bootstrap
+Discovered number of CRDs:  1
+```
+
+The following authentication methods are available:
+- username and password
+```
+./cty generate crd -g https://github.com/Skarlso/crd-bootstrap --username skarlso --password password
+```
+- token
+```
+./cty generate crd -g https://github.com/Skarlso/crd-bootstrap --token token
+```
+- SSH with provided private key
+```
+./cty generate crd -g [email protected]:Skarlso/crd-bootstrap --private-ssh-key-file ~/.ssh/main-key
+```
+- SSH by using the local ssh-agent
+```
+./cty generate crd -g [email protected]:Skarlso/crd-bootstrap --ssh-agent
+```
+
+Notice the URL change in case SSH authentication is provided.
+
+Further certificate bundles can be provided for privately hosted git servers with `--ca-bundle-file`.
+
 ### HTML output
 
 It's possible to generate a pre-rendered HTML based output for self-hosting what the website produces online.

cmd/crd.go

@@ -134,6 +134,17 @@ func constructHandler(args *rootArgs) (Handler, error) {
 		crdHandler = &FolderHandler{location: args.folderLocation}
 	case args.configFileLocation != "":
 		crdHandler = &ConfigHandler{configFileLocation: args.configFileLocation}
+	case args.gitURL != "":
+		crdHandler = &GitHandler{
+			URL:         args.gitURL,
+			Username:    args.username,
+			Password:    args.password,
+			Token:       args.token,
+			Tag:         args.tag,
+			caBundle:    args.caBundle,
+			privSSHKey:  args.privSSHKey,
+			useSSHAgent: args.useSSHAgent,
+		}
 	case args.url != "":
 		crdHandler = &URLHandler{
 			url:      args.url,

cmd/generate.go

@@ -12,6 +12,11 @@ type rootArgs struct {
 	username           string
 	password           string
 	token              string
+	tag                string
+	caBundle           string
+	privSSHKey         string
+	useSSHAgent        bool
+	gitURL             string
 }
 
 var (
@@ -31,8 +36,13 @@ func init() {
 	f.StringVarP(&args.fileLocation, "crd", "c", "", "The CRD file to generate a yaml from.")
 	f.StringVarP(&args.folderLocation, "folder", "r", "", "A folder from which to parse a series of CRDs.")
 	f.StringVarP(&args.url, "url", "u", "", "If provided, will use this URL to fetch CRD YAML content from.")
+	f.StringVarP(&args.gitURL, "git-url", "g", "", "If provided, CRDs will be discovered using a git repository.")
 	f.StringVar(&args.username, "username", "", "Optional username to authenticate a URL.")
 	f.StringVar(&args.password, "password", "", "Optional password to authenticate a URL.")
 	f.StringVar(&args.token, "token", "", "A bearer token to authenticate a URL.")
 	f.StringVar(&args.configFileLocation, "config", "", "An optional configuration file that can define grouping data for various rendered crds.")
+	f.StringVar(&args.tag, "tag", "", "The ref to check out. Default is head.")
+	f.StringVar(&args.caBundle, "ca-bundle-file", "", "Additional certificate bundle to load. Should the name of the file.")
+	f.StringVar(&args.privSSHKey, "private-ssh-key-file", "", "Private key to use for cloning. Should the name of the file.")
+	f.BoolVar(&args.useSSHAgent, "ssh-agent", false, "If set, the configured SSH agent will be used to clone the repository..")
 }

cmd/git_handler.go

@@ -0,0 +1,183 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
+	"github.com/go-git/go-git/v5/storage/memory"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/yaml"
+
+	"github.com/Skarlso/crd-to-sample-yaml/pkg"
+	"github.com/Skarlso/crd-to-sample-yaml/pkg/sanitize"
+)
+
+type GitHandler struct {
+	URL      string
+	Username string
+	Password string
+	Token    string
+	Tag      string
+
+	caBundle    string
+	privSSHKey  string
+	useSSHAgent bool
+}
+
+func (g *GitHandler) CRDs() ([]*pkg.SchemaType, error) {
+	opts, err := g.constructGitOptions()
+	if err != nil {
+		return nil, err
+	}
+
+	r, err := git.Clone(memory.NewStorage(), nil, opts)
+	if err != nil {
+		return nil, fmt.Errorf("error cloning git repository: %w", err)
+	}
+
+	var ref *plumbing.Reference
+	if g.Tag != "" {
+		ref, err = r.Tag(g.Tag)
+	} else {
+		ref, err = r.Head()
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to construct reference: %w", err)
+	}
+
+	crds, err := gatherSchemaTypesForRef(r, ref)
+	if err != nil {
+		return nil, err
+	}
+
+	_, _ = fmt.Fprintln(os.Stderr, "Discovered number of CRDs: ", len(crds))
+
+	return crds, nil
+}
+
+func gatherSchemaTypesForRef(r *git.Repository, ref *plumbing.Reference) ([]*pkg.SchemaType, error) {
+	// Need to resolve the ref first to the right hash otherwise it's not found.
+	hash, err := r.ResolveRevision(plumbing.Revision(ref.Hash().String()))
+	if err != nil {
+		return nil, fmt.Errorf("failed to resolve revision: %w", err)
+	}
+
+	commit, err := r.CommitObject(*hash)
+	if err != nil {
+		return nil, fmt.Errorf("error getting commit object: %w", err)
+	}
+
+	commitTree, err := commit.Tree()
+	if err != nil {
+		return nil, err
+	}
+
+	var crds []*pkg.SchemaType
+	// Tried to make this concurrent, but there was very little gain. It just takes this long to
+	// clone a large repository. It's not the processing OR the rendering that takes long.
+	if err := commitTree.Files().ForEach(func(f *object.File) error {
+		crd, err := processEntry(f)
+		if err != nil {
+			return err
+		}
+
+		if crd != nil {
+			crds = append(crds, crd)
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return crds, nil
+}
+
+func processEntry(f *object.File) (*pkg.SchemaType, error) {
+	for _, path := range strings.Split(f.Name, string(filepath.Separator)) {
+		if path == "test" {
+			return nil, nil
+		}
+	}
+
+	if ext := filepath.Ext(f.Name); ext != ".yaml" {
+		return nil, nil
+	}
+
+	content, err := f.Contents()
+	if err != nil {
+		return nil, err
+	}
+
+	sanitized, err := sanitize.Sanitize([]byte(content))
+	if err != nil {
+		return nil, fmt.Errorf("failed to sanitize content: %w", err)
+	}
+
+	crd := &unstructured.Unstructured{}
+	if err := yaml.Unmarshal(sanitized, crd); err != nil {
+		return nil, nil //nolint:nilerr // intentional
+	}
+
+	schemaType, err := pkg.ExtractSchemaType(crd)
+	if err != nil {
+		return nil, nil //nolint:nilerr // intentional
+	}
+
+	return schemaType, nil
+}
+
+func (g *GitHandler) constructGitOptions() (*git.CloneOptions, error) {
+	opts := &git.CloneOptions{
+		URL:   g.URL,
+		Depth: 1,
+	}
+
+	// trickle down. if ssh key is set, this will be overwritten.
+	if g.Username != "" && g.Password != "" {
+		opts.Auth = &http.BasicAuth{
+			Username: g.Username,
+			Password: g.Password,
+		}
+	}
+	if g.Token != "" {
+		opts.Auth = &http.TokenAuth{
+			Token: g.Token,
+		}
+	}
+	if g.caBundle != "" {
+		opts.CABundle = []byte(g.caBundle)
+	}
+	if g.privSSHKey != "" {
+		if !strings.Contains(g.URL, "@") {
+			return nil, fmt.Errorf("git URL does not contain an ssh address: %s", g.URL)
+		}
+
+		keys, err := ssh.NewPublicKeysFromFile("git", g.privSSHKey, g.Password)
+		if err != nil {
+			return nil, err
+		}
+
+		opts.Auth = keys
+	}
+	if g.useSSHAgent {
+		if !strings.Contains(g.URL, "@") {
+			return nil, fmt.Errorf("git URL does not contain an ssh address: %s", g.URL)
+		}
+
+		authMethod, err := ssh.NewSSHAgentAuth("git")
+		if err != nil {
+			return nil, err
+		}
+		opts.Auth = authMethod
+	}
+
+	return opts, nil
+}

docs/release_notes/v1.1.1.md

@@ -0,0 +1,7 @@
+# Release v1.1.1
+
+- feat: implement discovering CRDs in git repositories directly #155
+
+This feature adds the ability to fetch CRDs through discovery from a git based repository.
+It is only supported through the CLI at the moment. For more information, check out the
+Pull Request description and the README.

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/brianvoe/gofakeit/v6 v6.28.0
 	github.com/fatih/color v1.18.0
 	github.com/fxamacker/cbor/v2 v2.7.0
+	github.com/go-git/go-git/v5 v5.12.0
 	github.com/google/go-cmp v0.6.0
 	github.com/jedib0t/go-pretty/v6 v6.6.5
 	github.com/maxence-charriere/go-app/v10 v10.0.9
@@ -19,57 +20,77 @@ require (
 
 require (
 	cel.dev/expr v0.18.0 // indirect
+	dario.cat/mergo v1.0.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/ProtonMail/go-crypto v1.1.3 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cloudflare/circl v1.5.0 // indirect
+	github.com/cyphar/filepath-securejoin v0.3.5 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.6.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/cel-go v0.22.0 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/onsi/gomega v1.36.1 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
+	github.com/skeema/knownhosts v1.3.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	golang.org/x/crypto v0.30.0 // indirect
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/net v0.32.0 // indirect
+	golang.org/x/oauth2 v0.24.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/time v0.8.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240826202546-f6391c0de4c7 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240826202546-f6391c0de4c7 // indirect
 	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiserver v0.32.0 // indirect
 	k8s.io/component-base v0.32.0 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
-	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
+	k8s.io/utils v0.0.0-20241104163129-6fe5fd82f078 // indirect
 	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect