Feature/SBOM retrieval via GitHub

cmd/transfer.go

@@ -164,7 +164,8 @@ func customUsageFunc(_ *cobra.Command) string {
 		"github": {
 			{"--in-github-url", "URL for input adapter github (required)"},
 			{"--in-github-method", "Method for input adapter github (optional)"},
-			{"--in-github-all-versions", "Fetch all SBOMs for all versions (optional)"},
+			{"--in-github-include-repos", "Comma-separated list of repositories to include (optional)"},
+			{"--in-github-exclude-repos", "Comma-separated list of repositories to exclude (optional)"},
 		},
 	}
 

go.mod

@@ -5,6 +5,7 @@ go 1.22
 toolchain go1.22.12
 
 require (
+	github.com/schollz/progressbar/v3 v3.18.0
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
 	go.uber.org/zap v1.27.0
@@ -23,7 +24,6 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/schollz/progressbar/v3 v3.18.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect

go.sum

@@ -1,3 +1,5 @@
+github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM=
+github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -19,6 +21,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -70,8 +74,6 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=

pkg/iterator/iterator.go

@@ -21,8 +21,10 @@ import (
 
 // SBOM represents a single SBOM file
 type SBOM struct {
-	Path string
-	Data []byte
+	Path    string // File path (empty if stored in memory)
+	Data    []byte // SBOM data stored in memory (nil if using Path)
+	Repo    string // Repository URL (helps track multi-repo processing)
+	Version string // Version of the SBOM (e.g., "latest" or "v1.2.3")
 }
 
 // SBOMIterator provides a way to lazily fetch SBOMs one by one

pkg/source/github/adapter.go

@@ -17,7 +17,9 @@ package github
 import (
 	"context"
 	"fmt"
-	"net/http"
+	"io"
+	"strings"
+	"sync"
 
 	"github.com/interlynk-io/sbommv/pkg/iterator"
 	"github.com/interlynk-io/sbommv/pkg/logger"
@@ -37,11 +39,22 @@ type GitHubAdapter struct {
 	Branch      string
 	Method      string
 	BinaryPath  string
-	client      *http.Client
+	client      *Client
 	GithubToken string
 	Role        types.AdapterRole
+
+	// Comma-separated list (e.g., "repo1,repo2")
+	IncludeRepos []string
+	ExcludeRepos []string
 }
 
+type ProcessingMode string
+
+const (
+	FetchParallel   ProcessingMode = "parallel"
+	FetchSequential ProcessingMode = "sequential"
+)
+
 type GitHubMethod string
 
 const (
@@ -59,74 +72,274 @@ const (
 func (g *GitHubAdapter) AddCommandParams(cmd *cobra.Command) {
 	cmd.Flags().String("in-github-url", "", "GitHub repository URL")
 	cmd.Flags().String("in-github-method", "release", "GitHub method: release, api, or tool")
-	cmd.Flags().Bool("in-github-all-versions", false, "Fetches SBOMs from all version")
+
+	// Updated to StringSlice to support multiple values (comma-separated)
+	cmd.Flags().StringSlice("in-github-include-repos", nil, "Comma-separated list of repositories to include")
+	cmd.Flags().StringSlice("in-github-exclude-repos", nil, "Comma-separated list of repositories to exclude")
+
+	// (Optional) If you plan to fetch **all versions** of a repo
+	// cmd.Flags().Bool("in-github-all-versions", false, "Fetch SBOMs from all versions")
 }
 
 // ParseAndValidateParams validates the GitHub adapter params
 func (g *GitHubAdapter) ParseAndValidateParams(cmd *cobra.Command) error {
-	var urlFlag, methodFlag, allVersionFlag string
+	var urlFlag, methodFlag, includeFlag, excludeFlag string
 
 	if g.Role == types.InputAdapter {
 		urlFlag = "in-github-url"
 		methodFlag = "in-github-method"
-		allVersionFlag = "in-github-all-versions"
+		includeFlag = "in-github-include-repos"
+		excludeFlag = "in-github-exclude-repos"
 	}
 
-	url, _ := cmd.Flags().GetString(urlFlag)
-	if url == "" {
-		return fmt.Errorf("missing or invalid flag: in-github-url")
+	// Extract GitHub URL
+	githubURL, _ := cmd.Flags().GetString(urlFlag)
+	if githubURL == "" {
+		return fmt.Errorf("missing or invalid flag: %s", urlFlag)
 	}
 
 	method, _ := cmd.Flags().GetString(methodFlag)
 	if method != "release" && method != "api" && method != "tool" {
-		return fmt.Errorf("missing or invalid flag: in-github-method")
+		return fmt.Errorf("missing or invalid flag: %s", methodFlag)
 	}
 
-	allVersion, _ := cmd.Flags().GetBool(allVersionFlag)
+	includeRepos, _ := cmd.Flags().GetStringSlice(includeFlag)
+	excludeRepos, _ := cmd.Flags().GetStringSlice(excludeFlag)
+
+	g.IncludeRepos = includeRepos
+	g.ExcludeRepos = excludeRepos
+
+	// Validate that both include & exclude are not used together
+	if len(g.IncludeRepos) > 0 && len(g.ExcludeRepos) > 0 {
+		return fmt.Errorf("cannot use both --in-github-include-repos and --in-github-exclude-repos together")
+	}
 
 	if method == "tool" {
 		binaryPath, err := utils.GetBinaryPath()
 		if err != nil {
 			return fmt.Errorf("failed to get Syft binary: %w", err)
 		}
-		fmt.Println("Binary Path: ", binaryPath)
+
 		g.BinaryPath = binaryPath
 		logger.LogDebug(context.Background(), "Binary Path", "value", g.BinaryPath)
 	}
 
 	token := viper.GetString("GITHUB_TOKEN")
 
-	repoURL, version, err := utils.ParseRepoVersion(url)
+	// Parse URL into owner, repo, and version
+	owner, repo, version, err := utils.ParseGithubURL(githubURL)
 	if err != nil {
-		return fmt.Errorf("falied to parse github repo and version %v", err)
-	}
-	if repoURL == "" {
-		return fmt.Errorf("failed to parse repo URL: %s", url)
+		return fmt.Errorf("invalid GitHub URL format: %w", err)
 	}
 
+	// // Set version to "latest" if empty
 	if version == "" {
 		version = "latest"
 	}
 
-	if allVersion {
-		version = ""
-	}
-
-	g.URL = url
-	g.Method = method
-	g.Repo = repoURL
+	// Assign extracted values to struct
+	g.URL = githubURL
+	g.Owner = owner
+	g.Repo = repo
 	g.Version = version
+	g.Method = method
 	g.GithubToken = token
 
+	// Initialize GitHub client
+	g.client = NewClient(g)
+
+	// Debugging logs for tracking
+	logger.LogDebug(cmd.Context(), "Parsed GitHub parameters",
+		"url", g.URL,
+		"owner", g.Owner,
+		"repo", g.Repo,
+		"version", g.Version,
+		"include_repos", g.IncludeRepos,
+		"exclude_repos", g.ExcludeRepos,
+		"method", g.Method,
+	)
 	return nil
 }
 
 // FetchSBOMs initializes the GitHub SBOM iterator using the unified method
 func (g *GitHubAdapter) FetchSBOMs(ctx *tcontext.TransferMetadata) (iterator.SBOMIterator, error) {
-	return NewGitHubIterator(ctx, g)
+	logger.LogDebug(ctx.Context, "Intializing SBOM fetching process")
+
+	if g.Repo != "" {
+		return NewGitHubIterator(ctx, g)
+	}
+
+	// Org Mode: Fetch all repositories
+	repos, err := g.client.GetAllRepositories(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get repositories: %w", err)
+	}
+
+	// logger.LogDebug(ctx.Context, "Listing repos of organization", "values", repos)
+
+	// filtering to include/exclude repos
+	repos = g.applyRepoFilters(repos)
+
+	if len(repos) == 0 {
+		return nil, fmt.Errorf("no repositories left after applying filters")
+	}
+
+	logger.LogDebug(ctx.Context, "Listing repos of organization after filtering", "values", repos)
+
+	processingMode := "parallel"
+	var sbomIterator iterator.SBOMIterator
+
+	switch ProcessingMode(processingMode) {
+
+	case FetchParallel:
+		sbomIterator, err = g.fetchSBOMsConcurrently(ctx, repos)
+
+	case FetchSequential:
+		sbomIterator, err = g.fetchSBOMsSequentially(ctx, repos)
+
+	default:
+		fmt.Println("Unsupported Processing Mode !!")
+		return nil, fmt.Errorf("Unsupported Processing Mode !!")
+
+	}
+
+	if err != nil {
+		logger.LogError(ctx.Context, err, "Failed to fetch SBOMs via Processing Mode")
+		return nil, err
+	}
+
+	return sbomIterator, err
 }
 
 // OutputSBOMs should return an error since GitHub does not support SBOM uploads
 func (g *GitHubAdapter) UploadSBOMs(ctx *tcontext.TransferMetadata, iterator iterator.SBOMIterator) error {
 	return fmt.Errorf("GitHub adapter does not support SBOM uploading")
 }
+
+// applyRepoFilters filters repositories based on inclusion/exclusion flags
+func (g *GitHubAdapter) applyRepoFilters(repos []string) []string {
+	includedRepos := make(map[string]bool)
+	excludedRepos := make(map[string]bool)
+
+	for _, repo := range g.IncludeRepos {
+		if repo != "" {
+			includedRepos[strings.TrimSpace(repo)] = true
+		}
+	}
+
+	for _, repo := range g.ExcludeRepos {
+		if repo != "" {
+			excludedRepos[strings.TrimSpace(repo)] = true
+		}
+	}
+
+	var filteredRepos []string
+
+	for _, repo := range repos {
+		if _, isExcluded := excludedRepos[repo]; isExcluded {
+			continue // Skip excluded repositories
+		}
+
+		// Include only if in the inclusion list (if provided)
+		if len(includedRepos) > 0 {
+			if _, isIncluded := includedRepos[repo]; !isIncluded {
+				continue // Skip repos that are not in the include list
+			}
+		}
+		// filtered repo are added to the final list
+		filteredRepos = append(filteredRepos, repo)
+	}
+
+	return filteredRepos
+}
+
+// fetchSBOMsConcurrently: fetch SBOMs from repositories concurrently
+func (g *GitHubAdapter) fetchSBOMsConcurrently(ctx *tcontext.TransferMetadata, repos []string) (iterator.SBOMIterator, error) {
+	var wg sync.WaitGroup
+	sbomsChan := make(chan *iterator.SBOM, len(repos))
+
+	for _, repo := range repos {
+		wg.Add(1)
+		go func(repo string) {
+			defer wg.Done()
+			g.Repo = repo
+			g.client.Repo = repo
+			iter, err := NewGitHubIterator(ctx, g)
+			if err != nil {
+				logger.LogError(ctx.Context, err, "Failed to fetch SBOMs for repo", "repo", repo)
+				return
+			}
+			for {
+				fmt.Println("BEFORE")
+				sbom, err := iter.Next(ctx.Context)
+				if err == io.EOF {
+					break
+				}
+				fmt.Println("1. BEFORE")
+
+				if err != nil {
+					logger.LogError(ctx.Context, err, "Error reading SBOM for", "repo", repo)
+					break
+				}
+				fmt.Println("2. BEFORE")
+
+				sbomsChan <- sbom
+				fmt.Println("AFTER")
+
+			}
+		}(repo)
+	}
+
+	wg.Wait()
+	close(sbomsChan)
+
+	// Collect SBOMs from channel
+	var sbomList []*iterator.SBOM
+	for sbom := range sbomsChan {
+		sbomList = append(sbomList, sbom)
+	}
+
+	return &GitHubIterator{
+		sboms: sbomList,
+	}, nil
+}
+
+// fetchSBOMsSequentially: fetch SBOMs from repositories one at a time
+func (g *GitHubAdapter) fetchSBOMsSequentially(ctx *tcontext.TransferMetadata, repos []string) (iterator.SBOMIterator, error) {
+	var sbomList []*iterator.SBOM
+
+	// Iterate over repositories one by one (sequential processing)
+	for _, repo := range repos {
+		g.Repo = repo // Set current repository
+
+		logger.LogInfo(ctx.Context, "Fetching SBOMs sequentially", "repo", repo)
+
+		// Fetch SBOMs for the current repository
+		iter, err := NewGitHubIterator(ctx, g)
+		if err != nil {
+			logger.LogError(ctx.Context, err, "Failed to fetch SBOMs for repo", "repo", repo)
+			continue
+		}
+
+		// use iterator to add the SBOMs to the final sboms list
+		for {
+			sbom, err := iter.Next(ctx.Context)
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				logger.LogError(ctx.Context, err, "Error reading SBOM for", "repo", repo)
+				break
+			}
+			sbomList = append(sbomList, sbom)
+		}
+	}
+
+	if len(sbomList) == 0 {
+		return nil, fmt.Errorf("no SBOMs found for any repository")
+	}
+
+	return &GitHubIterator{
+		sboms: sbomList,
+	}, nil
+}