ROX-26604: Add layer lineage to RHEL v2 data model (#1720)

Author: dcaravel

api/v1/models.go

@@ -143,7 +143,7 @@ func LayerFromDatabaseModel(db database.Datastore, dbLayer database.Layer, linea
 			layer.Features = append(layer.Features, *feature)
 		}
 		if !uncertifiedRHEL && namespaces.IsRHELNamespace(layer.NamespaceName) {
-			certified, err := addRHELv2Vulns(db, &layer)
+			certified, err := addRHELv2Vulns(db, &layer, lineage)
 			if err != nil {
 				return layer, notes, err
 			}
@@ -212,7 +212,7 @@ func ComponentsFromDatabaseModel(db database.Datastore, dbLayer *database.Layer,
 	if !uncertifiedRHEL && namespaces.IsRHELNamespace(namespaceName) {
 		var certified bool
 		var err error
-		rhelv2PkgEnvs, certified, err = getRHELv2PkgEnvs(db, dbLayer.Name)
+		rhelv2PkgEnvs, certified, err = getRHELv2PkgEnvs(db, dbLayer.Name, lineage)
 		if err != nil {
 			return nil, err
 		}

api/v1/models_rhelv2.go

@@ -29,8 +29,8 @@ const (
 // certified as part of Red Hat's Scanner Certification Program.
 // The returned bool indicates if full certified scanning was performed.
 // This is typically only `false` for images without proper CPE information.
-func addRHELv2Vulns(db database.Datastore, layer *Layer) (bool, error) {
-	pkgEnvs, cpesExist, err := getRHELv2PkgEnvs(db, layer.Name)
+func addRHELv2Vulns(db database.Datastore, layer *Layer, lineage string) (bool, error) {
+	pkgEnvs, cpesExist, err := getRHELv2PkgEnvs(db, layer.Name, lineage)
 	if err != nil {
 		return false, err
 	}
@@ -197,8 +197,8 @@ func shareCPEs(layers []*database.RHELv2Layer) bool {
 }
 
 // getRHELv2PkgEnvs returns a map from package ID to package environment and a bool to indicate CPEs exist in the image.
-func getRHELv2PkgEnvs(db database.Datastore, layerName string) (map[int]*database.RHELv2PackageEnv, bool, error) {
-	layers, err := db.GetRHELv2Layers(layerName)
+func getRHELv2PkgEnvs(db database.Datastore, layerName, layerLineage string) (map[int]*database.RHELv2PackageEnv, bool, error) {
+	layers, err := db.GetRHELv2Layers(layerName, layerLineage)
 	if err != nil {
 		return nil, false, err
 	}

api/v1/models_rhelv2_test.go

@@ -29,7 +29,7 @@ func newMockRHELv2Datastore() *mockRHELv2Datastore {
 		layers: make(map[string][]*database.RHELv2Layer),
 		vulns:  make(map[int][]*database.RHELv2Vulnerability),
 	}
-	db.FctGetRHELv2Layers = func(layer string) ([]*database.RHELv2Layer, error) {
+	db.FctGetRHELv2Layers = func(layer, lineage string) ([]*database.RHELv2Layer, error) {
 		return db.layers[layer], nil
 	}
 	db.FctGetRHELv2Vulnerabilities = func(records []*database.RHELv2Record) (map[int][]*database.RHELv2Vulnerability, error) {

database/database.go

@@ -145,7 +145,7 @@ type Datastore interface {
 	// GetRHELv2Layers retrieves the corresponding layers for the image
 	// represented by the given layer.
 	// The returned slice is sorted in order from base layer to top.
-	GetRHELv2Layers(layer string) ([]*RHELv2Layer, error)
+	GetRHELv2Layers(layer, lineage string) ([]*RHELv2Layer, error)
 
 	// GetRHELv2Vulnerabilities retrieves RHELv2 vulnerabilities based on the given records.
 	// The returned value maps package ID to the related vulnerabilities.

database/mock.go

@@ -31,7 +31,7 @@ type MockDatastore struct {
 	FctFindLayer                           func(name, lineage string, opts *DatastoreOptions) (Layer, error)
 	FctDeleteLayer                         func(name string) error
 	FctInsertRHELv2Layer                   func(*RHELv2Layer) error
-	FctGetRHELv2Layers                     func(layer string) ([]*RHELv2Layer, error)
+	FctGetRHELv2Layers                     func(layer, lineage string) ([]*RHELv2Layer, error)
 	FctGetRHELv2Vulnerabilities            func(records []*RHELv2Record) (map[int][]*RHELv2Vulnerability, error)
 	FctListVulnerabilities                 func(namespaceName string, limit int, page int) ([]Vulnerability, int, error)
 	FctInsertVulnerabilities               func(vulnerabilities []Vulnerability) error
@@ -82,9 +82,9 @@ func (mds *MockDatastore) InsertRHELv2Layer(layer *RHELv2Layer) error {
 	panic("required mock function not implemented")
 }
 
-func (mds *MockDatastore) GetRHELv2Layers(layer string) ([]*RHELv2Layer, error) {
+func (mds *MockDatastore) GetRHELv2Layers(layer, layerLineage string) ([]*RHELv2Layer, error) {
 	if mds.FctGetRHELv2Layers != nil {
-		return mds.FctGetRHELv2Layers(layer)
+		return mds.FctGetRHELv2Layers(layer, layerLineage)
 	}
 	panic("required mock function not implemented")
 }

database/models.go

@@ -205,11 +205,13 @@ func (p *RHELv2Package) GetPackageVersion() string {
 type RHELv2Layer struct {
 	Model
 
-	Hash       string
-	ParentHash string
-	Dist       string
-	Pkgs       []*RHELv2Package
-	CPEs       []string
+	Hash          string
+	ParentHash    string
+	Dist          string
+	Pkgs          []*RHELv2Package
+	CPEs          []string
+	Lineage       string
+	ParentLineage string
 }
 
 // RHELv2Components defines the RHELv2 components found in a layer.

database/pgsql/migrations/00021_rhel_layer_lineage.go

@@ -0,0 +1,24 @@
+package migrations
+
+import "github.com/remind101/migrate"
+
+func init() {
+	RegisterMigration(migrate.Migration{
+		ID: 21,
+		Up: migrate.Queries([]string{
+
+			// The lineage column mimics the existing `layer` table. The parent_lineage column is used
+			// instead of equivalent parent_id column from the 'layer' table to avoid an extra query
+			// on insert (which would be necessary to determine the parent id).
+			`ALTER TABLE rhelv2_layer ADD COLUMN IF NOT EXISTS lineage varchar;`,
+			`ALTER TABLE rhelv2_layer ADD COLUMN IF NOT EXISTS parent_lineage varchar`,
+
+			// Create a new unique constraint that includes lineage (and drop the old constraint)
+			`ALTER TABLE rhelv2_layer ADD CONSTRAINT rhelv2_layer_hash_lineage_key UNIQUE (hash, lineage)`,
+			`ALTER TABLE rhelv2_layer DROP CONSTRAINT IF EXISTS rhelv2_layer_hash_key`,
+
+			// Create additional index to improve performance when recursively traversing parents.
+			`CREATE INDEX IF NOT EXISTS rhelv2_layer_parent_idx on rhelv2_layer (parent_hash, parent_lineage)`,
+		}),
+	})
+}

database/pgsql/queries.go

@@ -264,9 +264,9 @@ const (
 	deleteStaleRHELv2Vulns = `DELETE FROM vuln_v2 WHERE name = ANY($1::text[]) and package_name = ANY($2::text[]) and cpe = ANY($3::text[]) and package_module = $4;`
 
 	insertRHELv2Layer = `
-		INSERT INTO rhelv2_layer (hash, parent_hash, dist, cpes)
-		VALUES ($1, $2, $3, $4)
-		ON CONFLICT (hash) DO NOTHING;`
+		INSERT INTO rhelv2_layer (hash, parent_hash, dist, cpes, lineage, parent_lineage)
+		VALUES ($1, $2, $3, $4, $5, $6)
+		ON CONFLICT (hash, lineage) DO NOTHING;`
 
 	// Inside the `WITH RECURSIVE`, the base case is the top query, and the
 	// recursive case is the bottom query.
@@ -276,15 +276,17 @@ const (
 	// This query looks for all the layers in the given layer's hierarchy.
 	searchRHELv2Layers = `
 		WITH RECURSIVE layers AS (
-			SELECT id, hash, parent_hash, dist, cpes
+			SELECT id, hash, parent_hash, dist, cpes, lineage, parent_lineage
 			FROM rhelv2_layer
-			WHERE hash = $1
-			UNION
-				SELECT l.id, l.hash, l.parent_hash, l.dist, l.cpes
+			WHERE hash = $1 
+			  AND lineage = $2
+			UNION 
+				SELECT l.id, l.hash, l.parent_hash, l.dist, l.cpes, l.lineage, l.parent_lineage
 				FROM layers ll, rhelv2_layer l
 				WHERE ll.parent_hash = l.hash
+				  AND ll.parent_lineage = l.lineage
 		)
-		SELECT id, hash, dist, cpes
+		SELECT id, hash, dist, cpes, lineage
 		FROM layers;`
 
 	insertRHELv2Package = `
@@ -304,7 +306,8 @@ const (
 		layer AS (
 			SELECT id AS layer_id
 			FROM rhelv2_layer
-			WHERE rhelv2_layer.hash = $5
+			WHERE rhelv2_layer.hash = $5 
+			  AND rhelv2_layer.lineage = $6
 		)
 		INSERT
 		INTO rhelv2_package_scanartifact (layer_id, package_id)
@@ -324,7 +327,8 @@ const (
 			rhelv2_package_scanartifact
 			LEFT JOIN rhelv2_package ON
 				rhelv2_package_scanartifact.package_id = rhelv2_package.id
-			JOIN rhelv2_layer ON rhelv2_layer.hash = $1
+			JOIN rhelv2_layer ON rhelv2_layer.hash = $1 
+				AND rhelv2_layer.lineage = $2
 		WHERE
 			rhelv2_package_scanartifact.layer_id = rhelv2_layer.id;`
 

database/pgsql/rhelv2_layer.go

@@ -30,7 +30,7 @@ func (pgSQL *pgSQL) InsertRHELv2Layer(layer *database.RHELv2Layer) error {
 		return err
 	}
 
-	if err := pgSQL.insertRHELv2Packages(tx, layer.Hash, layer.Pkgs); err != nil {
+	if err := pgSQL.insertRHELv2Packages(tx, layer.Hash, layer.Pkgs, layer.Lineage); err != nil {
 		utils.IgnoreError(tx.Rollback)
 		return err
 	}
@@ -46,11 +46,11 @@ func (pgSQL *pgSQL) InsertRHELv2Layer(layer *database.RHELv2Layer) error {
 func (pgSQL *pgSQL) insertRHELv2Layer(tx *sql.Tx, layer *database.RHELv2Layer) error {
 	defer metrics.ObserveQueryTime("insertRHELv2Layer", "layer", time.Now())
 
-	_, err := tx.Exec(insertRHELv2Layer, layer.Hash, layer.ParentHash, layer.Dist, pq.Array(layer.CPEs))
+	_, err := tx.Exec(insertRHELv2Layer, layer.Hash, layer.ParentHash, layer.Dist, pq.Array(layer.CPEs), layer.Lineage, layer.ParentLineage)
 	return err
 }
 
-func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*database.RHELv2Package) error {
+func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*database.RHELv2Package, lineage string) error {
 	// Sort packages to avoid potential deadlock.
 	// Sort by the unique index (name, version, module, arch).
 	sort.SliceStable(pkgs, func(i, j int) bool {
@@ -91,6 +91,7 @@ func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*datab
 			pkg.Module,
 			pkg.Arch,
 			layer,
+			lineage,
 		)
 
 		if err != nil {
@@ -101,7 +102,7 @@ func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*datab
 	return nil
 }
 
-func (pgSQL *pgSQL) GetRHELv2Layers(layerHash string) ([]*database.RHELv2Layer, error) {
+func (pgSQL *pgSQL) GetRHELv2Layers(layerHash, layerLineage string) ([]*database.RHELv2Layer, error) {
 	defer metrics.ObserveQueryTime("getRHELv2Layers", "all", time.Now())
 
 	tx, err := pgSQL.BeginTx(context.Background(), &sql.TxOptions{
@@ -111,7 +112,7 @@ func (pgSQL *pgSQL) GetRHELv2Layers(layerHash string) ([]*database.RHELv2Layer,
 		return nil, handleError("GetRHELv2Layers.Begin()", err)
 	}
 
-	rows, err := tx.Query(searchRHELv2Layers, layerHash)
+	rows, err := tx.Query(searchRHELv2Layers, layerHash, layerLineage)
 	if err != nil {
 		return nil, err
 	}
@@ -124,7 +125,7 @@ func (pgSQL *pgSQL) GetRHELv2Layers(layerHash string) ([]*database.RHELv2Layer,
 			rhelv2Layer database.RHELv2Layer
 			cpes        []string
 		)
-		if err := rows.Scan(&rhelv2Layer.ID, &rhelv2Layer.Hash, &rhelv2Layer.Dist, pq.Array(&cpes)); err != nil {
+		if err := rows.Scan(&rhelv2Layer.ID, &rhelv2Layer.Hash, &rhelv2Layer.Dist, pq.Array(&cpes), &rhelv2Layer.Lineage); err != nil {
 			utils.IgnoreError(tx.Rollback)
 			return nil, err
 		}
@@ -176,7 +177,7 @@ func (pgSQL *pgSQL) populatePackages(tx *sql.Tx, layers []*database.RHELv2Layer)
 func (pgSQL *pgSQL) getPackagesByLayer(tx *sql.Tx, layer *database.RHELv2Layer) error {
 	defer metrics.ObserveQueryTime("getRHELv2Layers", "packagesByLayer", time.Now())
 
-	rows, err := tx.Query(searchRHELv2Package, layer.Hash)
+	rows, err := tx.Query(searchRHELv2Package, layer.Hash, layer.Lineage)
 	if err != nil {
 		return err
 	}