Skip to content

Commit

Permalink
all repo fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@riteshnoronha
riteshnoronha committed Feb 8, 2025
1 parent d0cde36 commit 65647e1
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 111 deletions.
4 changes: 0 additions & 4 deletions pkg/engine/transfer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,11 @@ func TransferRun(ctx context.Context, cmd *cobra.Command, config mvtypes.Config)
logger.LogDebug(transferCtx.Context, "output adapter instance config", "value", outputAdapterInstance)

// Fetch SBOMs lazily using the iterator
logger.LogDebug(transferCtx.Context, "Fetching SBOMs from input adapter...")
sbomIterator, err := inputAdapterInstance.FetchSBOMs(transferCtx)
if err != nil {
return fmt.Errorf("failed to fetch SBOMs: %w", err)
}

logger.LogDebug(transferCtx.Context, "SBOM fetching started successfully", "sbomIterator", sbomIterator)

// Dry-Run Mode: Display SBOMs Without Uploading
if config.DryRun {
logger.LogDebug(transferCtx.Context, "Dry-run mode enabled: Displaying retrieved SBOMs", "values", config.DryRun)
Expand All @@ -89,7 +86,6 @@ func TransferRun(ctx context.Context, cmd *cobra.Command, config mvtypes.Config)

// Process & Upload SBOMs Sequentially
if err := outputAdapterInstance.UploadSBOMs(transferCtx, sbomIterator); err != nil {
logger.LogError(transferCtx.Context, err, "Failed to output SBOMs")
return fmt.Errorf("failed to output SBOMs: %w", err)
}

Expand Down
10 changes: 3 additions & 7 deletions pkg/source/github/adapter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,17 +173,13 @@ func (g *GitHubAdapter) ParseAndValidateParams(cmd *cobra.Command) error {
func (g *GitHubAdapter) FetchSBOMs(ctx *tcontext.TransferMetadata) (iterator.SBOMIterator, error) {
logger.LogDebug(ctx.Context, "Intializing SBOM fetching process")

if g.Repo != "" {
return NewGitHubIterator(ctx, g)
}

// Org Mode: Fetch all repositories
repos, err := g.client.GetAllRepositories(ctx)
if err != nil {
return nil, fmt.Errorf("failed to get repositories: %w", err)
}

logger.LogDebug(ctx.Context, "Found %d repos", len(repos))
logger.LogDebug(ctx.Context, "Found repos", "number", len(repos))

// filtering to include/exclude repos
repos = g.applyRepoFilters(repos)
Expand Down Expand Up @@ -267,7 +263,7 @@ func (g *GitHubAdapter) fetchSBOMsConcurrently(ctx *tcontext.TransferMetadata, r
defer wg.Done()
g.Repo = repo
g.client.Repo = repo
iter, err := NewGitHubIterator(ctx, g)
iter, err := NewGitHubIterator(ctx, g, repo)
if err != nil {
logger.LogError(ctx.Context, err, "Failed to fetch SBOMs for repo", "repo", repo)
return
Expand Down Expand Up @@ -311,7 +307,7 @@ func (g *GitHubAdapter) fetchSBOMsSequentially(ctx *tcontext.TransferMetadata, r
logger.LogDebug(ctx.Context, "Fetching SBOMs sequentially", "repo", repo)

// Fetch SBOMs for the current repository
iter, err := NewGitHubIterator(ctx, g)
iter, err := NewGitHubIterator(ctx, g, repo)
if err != nil {
logger.LogError(ctx.Context, err, "Failed to fetch SBOMs for repo", "repo", repo)
continue
Expand Down
44 changes: 5 additions & 39 deletions pkg/source/github/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,6 @@ func NewClient(g *GitHubAdapter) *Client {
// filter out the particular provided release asset and
// extract SBOMs from that
func (c *Client) FindSBOMs(ctx *tcontext.TransferMetadata) ([]SBOMAsset, error) {
// owner, repo, err := ParseGitHubURL(c.RepoURL)
// if err != nil {
// return nil, fmt.Errorf("parsing GitHub URL: %w", err)
// }

logger.LogDebug(ctx.Context, "Fetching GitHub releases", "repo_url", c.RepoURL, "owner", c.Owner, "repo", c.Repo)

releases, err := c.GetReleases(ctx, c.Owner, c.Repo)
Expand Down Expand Up @@ -244,39 +239,6 @@ func (c *Client) DownloadAsset(ctx *tcontext.TransferMetadata, downloadURL strin
return resp.Body, nil
}

// // FetchSBOMsFromReleases fetches and downloads SBOMs from GitHub releases
// func (c *Client) FetchSBOMsFromReleases(ctx *tcontext.TransferMetadata) (map[string][]byte, error) {
// logger.LogDebug(ctx.Context, "Fetching SBOMs from GitHub Releases", "repo", c.RepoURL)

// // Step 1: Get All Releases
// sbomAssets, err := c.FindSBOMs(ctx)
// if err != nil {
// return nil, fmt.Errorf("error finding SBOMs in releases: %w", err)
// }

// if len(sbomAssets) == 0 {
// return nil, fmt.Errorf("no SBOMs found in repository")
// }

// logger.LogDebug(ctx.Context, "Total SBOMs found in the repository", "version", c.Version, "total sboms", len(sbomAssets))

// // Step 2: Download Each SBOM
// versionedSBOMs := make(map[string][]byte)
// for _, asset := range sbomAssets {
// sbomData, err := c.DownloadSBOM(ctx, asset)
// if err != nil {
// logger.LogError(ctx.Context, err, "Failed to download SBOM", "file", asset.Name)
// continue
// }

// versionedSBOMs[asset.Release] = sbomData
// logger.LogDebug(ctx.Context, "Downloaded SBOM successfully", "version", asset.Release, "file", asset.Name)
// }

// // Step 3: Return All Downloaded SBOMs
// return versionedSBOMs, nil
// }

// DownloadSBOM fetches an SBOM from its download URL
func (c *Client) DownloadSBOM(ctx *tcontext.TransferMetadata, asset SBOMAsset) ([]byte, error) {
logger.LogDebug(ctx.Context, "Downloading SBOM", "url", asset.DownloadURL)
Expand Down Expand Up @@ -452,6 +414,11 @@ func (c *Client) FetchSBOMFromAPI(ctx *tcontext.TransferMetadata) ([]byte, error
return response.SBOM, nil
}

func (c *Client) updateRepo(repo string) {
c.Repo = repo
c.RepoURL = fmt.Sprintf("https://github.com/%s/%s", c.Owner, repo)
}

func (c *Client) GetAllRepositories(ctx *tcontext.TransferMetadata) ([]string, error) {
logger.LogDebug(ctx.Context, "Fetching all repositories for an organization", "name", c.Owner)

Expand Down Expand Up @@ -494,7 +461,6 @@ func (c *Client) GetAllRepositories(ctx *tcontext.TransferMetadata) ([]string, e
if len(repoNames) == 0 {
return nil, fmt.Errorf("no repositories found for organization %s", c.Owner)
}
logger.LogDebug(ctx.Context, "Total number of", "repositories", "value", len(repoNames), "present are", repoNames, "in an Organization", c.Organization)

return repoNames, nil
}
27 changes: 7 additions & 20 deletions pkg/source/github/iterator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,23 +29,19 @@ import (

// // GitHubIterator iterates over SBOMs fetched from GitHub (API, Release, Tool)
type GitHubIterator struct {
// ctx context.Context
client *Client
sboms []*iterator.SBOM // Stores all fetched SBOMs
position int // Tracks iteration position
binaryPath string
}

// NewGitHubIterator initializes the iterator based on the GitHub method
func NewGitHubIterator(ctx *tcontext.TransferMetadata, g *GitHubAdapter) (*GitHubIterator, error) {
logger.LogDebug(ctx.Context, "Initializing GitHub Iterator", "repo", g.URL, "method", g.Method)
func NewGitHubIterator(ctx *tcontext.TransferMetadata, g *GitHubAdapter, repo string) (*GitHubIterator, error) {
logger.LogDebug(ctx.Context, "Initializing GitHub Iterator", "repo", g.URL, "method", g.Method, "repo", repo)

ctx.WithValue("repo_url", g.URL)
ctx.WithValue("repo_version", g.Version)
g.client.updateRepo(repo)

// client := NewClient(g.URL, g.Version, string(g.Method))
iterator := &GitHubIterator{
// ctx: ctx,
client: g.client,
sboms: []*iterator.SBOM{},
binaryPath: g.BinaryPath,
Expand Down Expand Up @@ -82,7 +78,6 @@ func NewGitHubIterator(ctx *tcontext.TransferMetadata, g *GitHubAdapter) (*GitHu

// Next returns the next SBOM from the stored list
func (it *GitHubIterator) Next(ctx context.Context) (*iterator.SBOM, error) {
logger.LogDebug(ctx, "Iterating via Next")
if it.position >= len(it.sboms) {
return nil, io.EOF // No more SBOMs left
}
Expand All @@ -94,41 +89,33 @@ func (it *GitHubIterator) Next(ctx context.Context) (*iterator.SBOM, error) {

// Fetch SBOM via GitHub API
func (it *GitHubIterator) fetchSBOMFromAPI(ctx *tcontext.TransferMetadata) error {
logger.LogDebug(ctx.Context, "fetchSBOMFromAPI", "repo", it.client.RepoURL)

sbomData, err := it.client.FetchSBOMFromAPI(ctx)
if err != nil {
return err
}

it.sboms = append(it.sboms, &iterator.SBOM{
Path: "",
Data: sbomData,
Repo: it.client.RepoURL,

// github API creates SBOM from the latest version
Path: "",
Data: sbomData,
Repo: fmt.Sprintf("%s/%s", it.client.Owner, it.client.Repo),
Version: "latest",
})
return nil
}

// Fetch SBOMs from GitHub Releases
func (it *GitHubIterator) fetchSBOMFromReleases(ctx *tcontext.TransferMetadata) error {
logger.LogDebug(ctx.Context, "fetchSBOMFromReleases %s", it.client.RepoURL)

sbomFiles, err := it.client.GetSBOMs(ctx)
if err != nil {
return fmt.Errorf("error retrieving SBOMs from releases: %w", err)
}

logger.LogDebug(ctx.Context, "fetchSBOMFromReleases found sboms", len(sbomFiles))

for version, sbomDataList := range sbomFiles {
for _, sbomData := range sbomDataList { // sbomPath is a string (file path)
it.sboms = append(it.sboms, &iterator.SBOM{
Path: "", // No file path, storing in memory
Data: sbomData,
Repo: it.client.RepoURL,
Repo: fmt.Sprintf("%s/%s", it.client.Owner, it.client.Repo),
Version: version,
})
}
Expand Down
74 changes: 39 additions & 35 deletions pkg/target/interlynk/adapter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package interlynk

import (
"errors"
"fmt"
"io"
"net/http"
Expand All @@ -25,7 +24,6 @@ import (
"github.com/interlynk-io/sbommv/pkg/logger"
"github.com/interlynk-io/sbommv/pkg/tcontext"
"github.com/interlynk-io/sbommv/pkg/types"
"github.com/schollz/progressbar/v3"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
Expand Down Expand Up @@ -157,41 +155,42 @@ func (i *InterlynkAdapter) uploadSequential(ctx *tcontext.TransferMetadata, sbom
ProjectID: i.ProjectID,
})

// Retrieve metadata from context
repoURL, _ := ctx.Value("repo_url").(string)
repoVersion, _ := ctx.Value("repo_version").(string)
totalSBOMs, _ := ctx.Value("total_sboms").(int)
// // Retrieve metadata from context
// repoURL, _ := ctx.Value("repo_url").(string)
// repoVersion, _ := ctx.Value("repo_version").(string)
// totalSBOMs, _ := ctx.Value("total_sboms").(int)

if repoVersion == "" {
repoVersion = "all-version"
}
// if repoVersion == "" {
// repoVersion = "all-version"
// }

repoName := sanitizeRepoName(repoURL)
// repoName := sanitizeRepoName(repoURL)

// Create project if needed
if client.ProjectID == "" {
projectName := fmt.Sprintf("%s", repoName)
// // Create project if needed
// if client.ProjectID == "" {
// projectName := fmt.Sprintf("%s", repoName)

projectID, err := client.FindProjectGroup(ctx, projectName, "default")
// projectID, err := client.FindProjectGroup(ctx, projectName, "default")

if err != nil {
logger.LogDebug(ctx.Context, "Project not found, creating new project", "name", projectName)
projectID, err = client.CreateProjectGroup(ctx, projectName, "default")
if err != nil {
return fmt.Errorf("failed to create project: %w", err)
}
}
// if err != nil {
// logger.LogDebug(ctx.Context, "Project not found, creating new project", "name", projectName)
// projectID, err = client.CreateProjectGroup(ctx, projectName, "default")
// if err != nil {
// return fmt.Errorf("failed to create project: %w", err)
// }
// }

logger.LogDebug(ctx.Context, "Found Project", "name", projectName, "ID", projectID)
client.SetProjectID(projectID)
}
// logger.LogDebug(ctx.Context, "Found Project", "name", projectName, "ID", projectID)
// client.SetProjectID(projectID)
// }

if totalSBOMs == 0 {
return errors.New("no SBOMs to upload")
}
// if totalSBOMs == 0 {
// return errors.New("no SBOMs to upload")
// }
//

// Initialize progress bar
bar := progressbar.Default(int64(totalSBOMs), "🚀 Uploading SBOMs")
// bar := progressbar.Default(int64(0), "🚀 Uploading SBOMs")
errorCount := 0
maxRetries := 5

Expand All @@ -212,22 +211,27 @@ func (i *InterlynkAdapter) uploadSequential(ctx *tcontext.TransferMetadata, sbom
}
errorCount = 0 // Reset error counter on successful iteration

logger.LogDebug(ctx.Context, "Uploading SBOM", "repo", sbom.Repo, "version", sbom.Version)
logger.LogDebug(ctx.Context, "Uploading SBOM", "repo", sbom.Repo, "version", sbom.Version, "data size", len(sbom.Data))

projectID, err := client.FindOrCreateProjectGroup(ctx, sbom.Repo, "default")
if err != nil {
logger.LogError(ctx.Context, err, "Failed to create project", "repo", sbom.Repo)
continue
}

// Upload SBOM content (stored in memory)
err = client.UploadSBOM(ctx, sbom.Data)
err = client.UploadSBOM(ctx, projectID, sbom.Data)
if err != nil {
logger.LogDebug(ctx.Context, "Failed to upload SBOM", "repo", sbom.Repo, "version", sbom.Version)
} else {
logger.LogDebug(ctx.Context, "Successfully uploaded SBOM", "repo", sbom.Repo, "version", sbom.Version)
}

// Update progress bar
if err := bar.Add(1); err != nil {
logger.LogError(ctx.Context, err, "Error updating progress bar")
}
// // Update progress bar
// if err := bar.Add(1); err != nil {
// logger.LogError(ctx.Context, err, "Error updating progress bar")
// }
}
logger.LogDebug(ctx.Context, "✅ All SBOMs uploaded successfully!")

return nil
}
Expand Down
26 changes: 20 additions & 6 deletions pkg/target/interlynk/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,28 @@ func (c *Client) SetProjectID(projectID string) {
c.ProjectID = projectID
}

func (c *Client) FindOrCreateProjectGroup(ctx *tcontext.TransferMetadata, repoName, env string) (string, error) {
projectName := fmt.Sprintf("%s", repoName)
projectID, err := c.FindProjectGroup(ctx, projectName, env)

if err != nil {
projectID, err = c.CreateProjectGroup(ctx, projectName, env)
if err != nil {
return "", fmt.Errorf("failed to create project: %w", err)
}
}

return projectID, nil
}

// UploadSBOM uploads a single SBOM from memory to Interlynk
func (c *Client) UploadSBOM(ctx *tcontext.TransferMetadata, sbomData []byte) error {
func (c *Client) UploadSBOM(ctx *tcontext.TransferMetadata, projectID string, sbomData []byte) error {
if len(sbomData) == 0 {
return fmt.Errorf("SBOM data is empty")
}

// Create a context-aware request with appropriate timeout
req, err := c.createUploadRequest(ctx, sbomData)
req, err := c.createUploadRequest(ctx, projectID, sbomData)
if err != nil {
return fmt.Errorf("preparing request: %w", err)
}
Expand All @@ -109,7 +123,7 @@ func (c *Client) UploadSBOM(ctx *tcontext.TransferMetadata, sbomData []byte) err
return c.executeUploadRequest(ctx, req)
}

func (c *Client) createUploadRequest(ctx *tcontext.TransferMetadata, sbomData []byte) (*http.Request, error) {
func (c *Client) createUploadRequest(ctx *tcontext.TransferMetadata, projectID string, sbomData []byte) (*http.Request, error) {
const uploadMutation = `
mutation uploadSbom($doc: Upload!, $projectId: ID!) {
sbomUpload(input: { doc: $doc, projectId: $projectId }) {
Expand All @@ -119,7 +133,7 @@ func (c *Client) createUploadRequest(ctx *tcontext.TransferMetadata, sbomData []
`

// Prepare multipart form data
body, writer, err := c.prepareMultipartForm(sbomData, uploadMutation)
body, writer, err := c.prepareMultipartForm(projectID, sbomData, uploadMutation)
if err != nil {
return nil, err
}
Expand All @@ -139,15 +153,15 @@ func (c *Client) createUploadRequest(ctx *tcontext.TransferMetadata, sbomData []
return req, nil
}

func (c *Client) prepareMultipartForm(sbomData []byte, query string) (*bytes.Buffer, *multipart.Writer, error) {
func (c *Client) prepareMultipartForm(projectID string, sbomData []byte, query string) (*bytes.Buffer, *multipart.Writer, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)

// Add GraphQL operations
operations := map[string]interface{}{
"query": strings.TrimSpace(strings.ReplaceAll(query, "\n", " ")),
"variables": map[string]interface{}{
"projectId": c.ProjectID,
"projectId": projectID,
"doc": nil,
},
}
Expand Down

0 comments on commit 65647e1

Please sign in to comment.