Skip to content

Commit

Permalink
Change base query to use inner joins
Browse files Browse the repository at this point in the history
  • Loading branch information
@jlsherrill
jlsherrill committed Sep 25, 2024
1 parent 42f95bd commit 08b254b
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 29 deletions.
1 change: 1 addition & 0 deletions compose_files/pulp/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ services:
postgres:
condition: service_healthy
command: pulpcore-manager migrate --noinput
restart: on-failure
volumes:
- "./assets/settings.py:/etc/pulp/settings.py:z"
- "./assets/certs:/etc/pulp/certs:z"
Expand Down
21 changes: 12 additions & 9 deletions pkg/tangy/queries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,28 @@ func contentIdsInVersion(repoId string, versionNum int, namedArgs *pgx.NamedArgs
repoIdName := fmt.Sprintf("%v%v", "repoName", ran)
versionNumName := fmt.Sprintf("%v%v", "versionNum", ran)
query := `
SELECT crc.content_id
FROM core_repositorycontent crc
INNER JOIN core_repositoryversion crv ON (crc.version_added_id = crv.pulp_id)
LEFT OUTER JOIN core_repositoryversion crv2 ON (crc.version_removed_id = crv2.pulp_id)
WHERE crv.repository_id = @%v AND crv.number <= @%v AND NOT (crv2.number <= @%v AND crv2.number IS NOT NULL)
(crv.repository_id = @%v AND crv.number <= @%v AND NOT (crv2.number <= @%v AND crv2.number IS NOT NULL))
`
(*namedArgs)[repoIdName] = repoId
(*namedArgs)[versionNumName] = versionNum
return fmt.Sprintf(query, repoIdName, versionNumName, versionNumName)
}

// Creates a sub query (including parenthesis) to lookup the content IDs of a list of repository versions.
// returns part of a query that joins a table to the needed tables to select content units in a given set of versions
//
// Takes in a pointer to Named args in order to add required named arguments for the query. Multiple queries are created
// and UNION'd together
// The return of this functions should be added to a query such as "select ** from TABLE rp" query,
// Where rp has a column 'content_ptr_id', such as rpm_updaterecord, rpm_package, etc.
// Takes in a pointer to Named args in order to add required named arguments for the query.
func contentIdsInVersions(repoVerMap []ParsedRepoVersion, namedArgs *pgx.NamedArgs) string {
mainQuery := `
INNER JOIN core_repositorycontent crc on rp.content_ptr_id = crc.content_id
INNER JOIN core_repositoryversion crv ON (crc.version_added_id = crv.pulp_id)
LEFT OUTER JOIN core_repositoryversion crv2 ON (crc.version_removed_id = crv2.pulp_id)
WHERE
`
queries := []string{}
for _, parsed := range repoVerMap {
queries = append(queries, contentIdsInVersion(parsed.RepositoryUUID, parsed.Version, namedArgs))
}
return fmt.Sprintf("( %v ) ", strings.Join(queries, " UNION "))
return fmt.Sprintf("%v (%v)", mainQuery, strings.Join(queries, " OR "))
}
40 changes: 20 additions & 20 deletions pkg/tangy/rpm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func (t *tangyImpl) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs
innerUnion := contentIdsInVersions(repoVerMap, &args)

query := `SELECT DISTINCT ON (rp.name) rp.name, rp.summary
FROM rpm_package rp WHERE rp.content_ptr_id IN `
FROM rpm_package rp `

rows, err := conn.Query(context.Background(), query+innerUnion+" AND rp.name ILIKE CONCAT( '%', @nameFilter::text, '%') ORDER BY rp.name LIMIT @limit", args)
if err != nil {
Expand Down Expand Up @@ -140,7 +140,7 @@ func (t *tangyImpl) RpmRepositoryVersionPackageGroupSearch(ctx context.Context,
innerUnion := contentIdsInVersions(repoVerMap, &args)

query := `SELECT DISTINCT ON (rp.name, rp.id, rp.packages) rp.name, rp.id, rp.description, rp.packages
FROM rpm_packagegroup rp WHERE rp.content_ptr_id IN
FROM rpm_packagegroup rp
`

rows, err := conn.Query(ctx, query+innerUnion+"AND rp.name ILIKE CONCAT( '%', @nameFilter::text, '%') ORDER BY rp.name", args)
Expand Down Expand Up @@ -221,7 +221,7 @@ func (t *tangyImpl) RpmRepositoryVersionEnvironmentSearch(ctx context.Context, h
innerUnion := contentIdsInVersions(repoVerMap, &args)

query := `SELECT DISTINCT ON (rp.name, rp.id) rp.name, rp.id, rp.description
FROM rpm_packageenvironment rp WHERE rp.content_ptr_id IN
FROM rpm_packageenvironment rp
`

rows, err := conn.Query(ctx, query+innerUnion+" AND rp.name ILIKE CONCAT( '%', @nameFilter::text, '%') ORDER BY rp.name LIMIT @limit", args)
Expand Down Expand Up @@ -258,7 +258,7 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []
return nil, 0, fmt.Errorf("error parsing repository version hrefs: %w", err)
}

countQueryOpen := "select count(*) as total FROM rpm_updaterecord re WHERE re.content_ptr_id IN "
countQueryOpen := "select count(distinct rp.content_ptr_id) as total FROM rpm_updaterecord rp "

args := pgx.NamedArgs{
"searchFilter": filterOpts.Search,
Expand All @@ -270,16 +270,16 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []

var concatFilter strings.Builder
if filterOpts.Search != "" {
concatFilter.WriteString(" AND (re.id ILIKE CONCAT( '%', @searchFilter::text, '%') OR re.summary ILIKE CONCAT( '%', @searchFilter::text, '%'))")
concatFilter.WriteString(" AND (rp.id ILIKE CONCAT( '%', @searchFilter::text, '%') OR rp.summary ILIKE CONCAT( '%', @searchFilter::text, '%'))")
}
if filterOpts.Type != nil {
if strings.Contains(filterOpts.Type[0], ",") {
filterOpts.Type = strings.Split(filterOpts.Type[0], ",")
}
args["typeFilter"] = filterOpts.Type
concatFilter.WriteString(" AND (re.type = ANY(@typeFilter)")
concatFilter.WriteString(" AND (rp.type = ANY(@typeFilter)")
if containsString(filterOpts.Type, "other") {
concatFilter.WriteString(" OR NOT (re.type = ANY(@typeList))")
concatFilter.WriteString(" OR NOT (rp.type = ANY(@typeList))")
}
concatFilter.WriteString(")")
}
Expand All @@ -288,9 +288,9 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []
filterOpts.Severity = strings.Split(filterOpts.Severity[0], ",")
}
args["severityFilter"] = filterOpts.Severity
concatFilter.WriteString(" AND (re.severity = ANY(@severityFilter)")
concatFilter.WriteString(" AND (rp.severity = ANY(@severityFilter)")
if containsString(filterOpts.Severity, "Unknown") {
concatFilter.WriteString(" OR NOT (re.severity = ANY(@severityList))")
concatFilter.WriteString(" OR NOT (rp.severity = ANY(@severityList))")
}
concatFilter.WriteString(")")
}
Expand All @@ -306,12 +306,12 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []
return nil, 0, err
}

queryOpen := `SELECT re.content_ptr_id as id, re.id as ErrataId, re.title, re.summary, re.description, re.issued_date as IssuedDate, re.updated_date as UpdatedDate, re.type, re.severity, re.reboot_suggested as RebootSuggested,
queryOpen := `SELECT distinct rp.content_ptr_id as id, rp.id as ErrataId, rp.title, rp.summary, rp.description, rp.issued_date as IssuedDate, rp.updated_date as UpdatedDate, rp.type, rp.severity, rp.reboot_suggested as RebootSuggested,
(SELECT ARRAY_AGG(ru.ref_id)
FROM rpm_updatereference ru
WHERE ru.update_record_id = re.content_ptr_id
WHERE ru.update_record_id = rp.content_ptr_id
AND ru.ref_type = 'cve') AS CVEs
FROM rpm_updaterecord re WHERE re.content_ptr_id IN `
FROM rpm_updaterecord rp `

args["limit"] = pageOpts.Limit
args["offset"] = pageOpts.Offset
Expand All @@ -321,15 +321,15 @@ func (t *tangyImpl) RpmRepositoryVersionErrataList(ctx context.Context, hrefs []
sortField := strings.Split(pageOpts.SortBy, ":")[0]
switch sortField {
case "issued_date":
orderBy = "re.issued_date"
orderBy = "rp.issued_date"
case "updated_date":
orderBy = "re.updated_date"
orderBy = "rp.updated_date"
case "type":
orderBy = "re.type"
orderBy = "rp.type"
case "severity":
orderBy = "re.severity"
orderBy = "rp.severity"
default:
orderBy = "re.issued_date"
orderBy = "rp.issued_date"
}

if strings.Contains(pageOpts.SortBy, "asc") {
Expand Down Expand Up @@ -374,7 +374,7 @@ func (t *tangyImpl) RpmRepositoryVersionPackageList(ctx context.Context, hrefs [
return nil, 0, fmt.Errorf("error parsing repository version hrefs: %w", err)
}

countQueryOpen := "select count(*) as total FROM rpm_package rp WHERE rp.content_ptr_id IN "
countQueryOpen := "select count(distinct(rp.content_ptr_id)) as total FROM rpm_package rp "
args := pgx.NamedArgs{"nameFilter": "%" + filterOpts.Name + "%"}
innerUnion := contentIdsInVersions(repoVerMap, &args)

Expand All @@ -384,8 +384,8 @@ func (t *tangyImpl) RpmRepositoryVersionPackageList(ctx context.Context, hrefs [
return nil, 0, err
}

queryOpen := `SELECT rp.content_ptr_id as id, rp.name, rp.version, rp.arch, rp.release, rp.epoch, rp.summary
FROM rpm_package rp WHERE rp.content_ptr_id IN `
queryOpen := `SELECT distinct rp.content_ptr_id as id, rp.name, rp.version, rp.arch, rp.release, rp.epoch, rp.summary
FROM rpm_package rp `

args["limit"] = pageOpts.Limit
args["offset"] = pageOpts.Offset
Expand Down

0 comments on commit 08b254b

Please sign in to comment.