Skip to content

Commit

Permalink
update delete hasSBOM to be more efficient
Browse files Browse the repository at this point in the history 
Signed-off-by: pxp928 <[email protected]>
  • Loading branch information
@pxp928
pxp928 committed Jun 28, 2024
1 parent 8f6cf45 commit 5c10c23
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 194 deletions.
5 changes: 3 additions & 2 deletions pkg/assembler/backends/ent/backend/dependency.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,11 +123,12 @@ func getIsDepObjectWithoutEdges(q *ent.DependencyQuery) *ent.DependencyQuery {
}

// deleteIsDependency is called by hasSBOM to delete the isDependency nodes that are part of the hasSBOM
func (b *EntBackend) deleteIsDependency(ctx context.Context, isDependencyID uuid.UUID) error {
func (b *EntBackend) deleteIsDependency(ctx context.Context, hasSBOMID string) error {
_, txErr := WithinTX(ctx, b.client, func(ctx context.Context) (*string, error) {
tx := ent.TxFromContext(ctx)

if err := tx.Dependency.DeleteOneID(isDependencyID).Exec(ctx); err != nil {
if _, err := tx.Dependency.Delete().Where(dependency.HasIncludedInSbomsWith([]predicate.BillOfMaterials{
optionalPredicate(&hasSBOMID, IDEQ)}...)).Exec(ctx); err != nil {
return nil, errors.Wrap(err, "failed to delete isDependency with error")
}
return nil, nil
Expand Down
9 changes: 6 additions & 3 deletions pkg/assembler/backends/ent/backend/occurrence.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,13 +121,16 @@ func getOccurrenceObjectWithoutEdges(q *ent.OccurrenceQuery) *ent.OccurrenceQuer
}

// deleteIsOccurrences is called by hasSBOM to delete the isOccurrence nodes that are part of the hasSBOM
func (b *EntBackend) deleteIsOccurrences(ctx context.Context, isOccurrenceID uuid.UUID) error {
func (b *EntBackend) deleteIsOccurrences(ctx context.Context, sbomID string) error {
_, txErr := WithinTX(ctx, b.client, func(ctx context.Context) (*string, error) {
tx := ent.TxFromContext(ctx)

if err := tx.Occurrence.DeleteOneID(isOccurrenceID).Exec(ctx); err != nil {
return nil, errors.Wrap(err, "failed to delete isOccurrenceID with error")
if _, err := tx.Occurrence.Delete().Where(occurrence.HasIncludedInSbomsWith([]predicate.BillOfMaterials{
optionalPredicate(&sbomID, IDEQ)}...)).Exec(ctx); err != nil {

return nil, errors.Wrap(err, "failed to delete all occurrences based on the SBOM with error")
}

return nil, nil
})
if txErr != nil {
Expand Down
195 changes: 6 additions & 189 deletions pkg/assembler/backends/ent/backend/sbom.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,202 +373,19 @@ func getSBOMObjectWithIncludes(q *ent.BillOfMaterialsQuery) *ent.BillOfMaterials
})
}

// GetHasSBOMForDeletion is used by the delete function to query for isDependency and isOccurrence IDs to cascade delete
func (b *EntBackend) getHasSBOMForDeletion(ctx context.Context, hasSBOMID *string, after *string, first *int) (*model.HasSBOMConnection, error) {
var afterCursor *entgql.Cursor[uuid.UUID]

if after != nil {
globalID := fromGlobalID(*after)
afterUUID, err := uuid.Parse(globalID.id)
if err != nil {
return nil, err
}
afterCursor = &ent.Cursor{ID: afterUUID}
} else {
afterCursor = nil
}

sbomQuery := b.client.BillOfMaterials.Query().
Where([]predicate.BillOfMaterials{optionalPredicate(hasSBOMID, IDEQ)}...)

hasSBOMConnection, err := getSBOMObjectWithOutIncludes(sbomQuery).Paginate(ctx, afterCursor, first, nil, nil)
if err != nil {
return nil, fmt.Errorf("failed hasSBOM query with error: %w", err)
}

// Large SBOMs (50MB+) hit the postgres parameter issue (HasSBOM: pq: got 97137 parameters but PostgreSQL only supports 65535 parameters).
// To overcome this, we can breakout the "included" pieces of the hasSBOM node into individual queries and reconstruct the node at the end.

reconstructedSBOMs := map[string]*model.HasSbom{}
includedFirst := 60000

type depResult struct {
deps []*ent.Dependency
depErr error
}

type occurResult struct {
occurs []*ent.Occurrence
occurErr error
}

for _, foundSBOM := range hasSBOMConnection.Edges {

var includedDeps []*ent.Dependency
var includedOccurs []*ent.Occurrence

depsChan := make(chan depResult, 1)
occursChan := make(chan occurResult, 1)

sbomID := foundSBOM.Cursor.ID.String()

// query included dependencies but only return IDs
go func(ctx context.Context, b *EntBackend, sbomID string, first int, artChan chan<- depResult) {
var afterCursor *entgql.Cursor[uuid.UUID]
defer close(depsChan)
for {
isDepQuery := b.client.Dependency.Query().
Where(dependency.HasIncludedInSbomsWith([]predicate.BillOfMaterials{
optionalPredicate(&sbomID, IDEQ)}...))

depConnect, err := getIsDepObjectWithoutEdges(isDepQuery).
Paginate(ctx, afterCursor, &first, nil, nil)
if err != nil {
depsChan <- depResult{deps: nil,
depErr: fmt.Errorf("failed included dependency query for hasSBOM with error: %w", err)}
}

var paginatedDeps []*ent.Dependency

for _, edge := range depConnect.Edges {
paginatedDeps = append(paginatedDeps, edge.Node)
}

depsChan <- depResult{deps: paginatedDeps,
depErr: nil}

if !depConnect.PageInfo.HasNextPage {
break
}
afterCursor = depConnect.PageInfo.EndCursor
}
}(ctx, b, sbomID, includedFirst, depsChan)

// query included occurrences
go func(ctx context.Context, b *EntBackend, sbomID string, first int, occursChan chan<- occurResult) {
var afterCursor *entgql.Cursor[uuid.UUID]
defer close(occursChan)
for {
occurQuery := b.client.Occurrence.Query().
Where(occurrence.HasIncludedInSbomsWith([]predicate.BillOfMaterials{
optionalPredicate(&sbomID, IDEQ)}...))

occurConnect, err := getOccurrenceObjectWithoutEdges(occurQuery).
Paginate(ctx, afterCursor, &first, nil, nil)
if err != nil {
occursChan <- occurResult{occurs: nil,
occurErr: fmt.Errorf("failed included occurrence query for hasSBOM with error: %w", err)}
}

var paginatedOccurs []*ent.Occurrence

for _, edge := range occurConnect.Edges {
paginatedOccurs = append(paginatedOccurs, edge.Node)
}

occursChan <- occurResult{occurs: paginatedOccurs,
occurErr: nil}

if !occurConnect.PageInfo.HasNextPage {
break
}
afterCursor = occurConnect.PageInfo.EndCursor
}
}(ctx, b, sbomID, includedFirst, occursChan)

for occur := range occursChan {
if occur.occurErr != nil {
return nil, fmt.Errorf("occurrence channel failure: %w", occur.occurErr)
}
includedOccurs = append(includedOccurs, occur.occurs...)
}

for dep := range depsChan {
if dep.depErr != nil {
return nil, fmt.Errorf("dependency channel failure: %w", dep.depErr)
}
includedDeps = append(includedDeps, dep.deps...)
}
reconstructedSBOM := toModelHasSBOMWithJustIncludedIDs(foundSBOM.Node, includedDeps, includedOccurs)
reconstructedSBOMs[sbomID] = reconstructedSBOM
}

var edges []*model.HasSBOMEdge
for id, edge := range reconstructedSBOMs {
edges = append(edges, &model.HasSBOMEdge{
Cursor: hasSBOMGlobalID(id),
Node: edge,
})
}

if hasSBOMConnection.PageInfo.StartCursor != nil {
return &model.HasSBOMConnection{
TotalCount: hasSBOMConnection.TotalCount,
PageInfo: &model.PageInfo{
HasNextPage: hasSBOMConnection.PageInfo.HasNextPage,
StartCursor: ptrfrom.String(hasSBOMGlobalID(hasSBOMConnection.PageInfo.StartCursor.ID.String())),
EndCursor: ptrfrom.String(hasSBOMGlobalID(hasSBOMConnection.PageInfo.EndCursor.ID.String())),
},
Edges: edges,
}, nil
} else {
// if not found return nil
return nil, nil
}
}

func (b *EntBackend) deleteHasSbom(ctx context.Context, hasSBOMID uuid.UUID) (bool, error) {
_, txErr := WithinTX(ctx, b.client, func(ctx context.Context) (*string, error) {
tx := ent.TxFromContext(ctx)

first := 1
hasSBOMIDString := hasSBOMID.String()
hasSBOMConnection, err := b.getHasSBOMForDeletion(ctx, &hasSBOMIDString, nil, &first)
if err != nil {
return nil, errors.Wrap(err, "failed to query for hasSBOM node for deletion with error")
// first delete isDependency and isOccurrence nodes that are part of the hasSBOM node
if err := b.deleteIsDependency(ctx, hasSBOMID.String()); err != nil {
return nil, fmt.Errorf("failed to delete isDependency with error: %w", err)
}

// first delete isDependency and isOccurrence nodes that are part of the hasSBOM node
for _, hasSBOMNEdge := range hasSBOMConnection.Edges {
for _, isDep := range hasSBOMNEdge.Node.IncludedDependencies {
foundGlobalID := fromGlobalID(isDep.ID)
if foundGlobalID.nodeType == "" {
return nil, fmt.Errorf("failed to parse globalID %s. Missing Node Type", isDep.ID)
}
// return uuid if valid, else error
isDepID, err := uuid.Parse(foundGlobalID.id)
if err != nil {
return nil, fmt.Errorf("isDependency uuid conversion from string failed with error: %w", err)
}
if err := b.deleteIsDependency(ctx, isDepID); err != nil {
return nil, fmt.Errorf("failed to delete isDependency with error: %w", err)
}
}
for _, isOccur := range hasSBOMNEdge.Node.IncludedOccurrences {
foundGlobalID := fromGlobalID(isOccur.ID)
if foundGlobalID.nodeType == "" {
return nil, fmt.Errorf("failed to parse globalID %s. Missing Node Type", isOccur.ID)
}
// return uuid if valid, else error
isOccurID, err := uuid.Parse(foundGlobalID.id)
if err != nil {
return nil, fmt.Errorf("isOccurrence uuid conversion from string failed with error: %w", err)
}
if err := b.deleteIsOccurrences(ctx, isOccurID); err != nil {
return nil, fmt.Errorf("failed to delete isOccurrence with error: %w", err)
}
}
if err := b.deleteIsOccurrences(ctx, hasSBOMID.String()); err != nil {
return nil, fmt.Errorf("failed to delete isOccurrence with error: %w", err)
}

// delete hasSBOM node
if err := tx.BillOfMaterials.DeleteOneID(hasSBOMID).Exec(ctx); err != nil {
return nil, errors.Wrap(err, "failed to delete hasSBOM with error")
Expand Down

0 comments on commit 5c10c23

Please sign in to comment.