From c1d9d4fdb07020c3dace478bf4b40d4e1eed0c64 Mon Sep 17 00:00:00 2001 From: rverdile Date: Wed, 20 Dec 2023 15:01:35 -0500 Subject: [PATCH] update config.yaml.example fix query add limit add close method add new test cases add mock Co-authored-by: Justin Sherrill --- README.md | 10 ++-- configs/config.yaml.example | 2 +- example.go | 7 +-- go.mod | 2 + go.sum | 3 ++ internal/test/integration/rpm_test.go | 68 +++++++++++++++++++++--- internal/zestwrapper/rpm.go | 11 ++++ pkg/tangy/interface.go | 9 +++- pkg/tangy/rpm.go | 74 +++++++++++++++++++++------ pkg/tangy/tangy_mock.go | 59 +++++++++++++++++++++ 10 files changed, 215 insertions(+), 30 deletions(-) create mode 100644 pkg/tangy/tangy_mock.go diff --git a/README.md b/README.md index a0da58a..90f156f 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,11 @@ t, err := tangy.New(dbConfig, tangy.Logger{Enabled: false}) if err != nil { return err } +defer t.Close() -// Use Tangy to search for RPMs, by name, that are associated to a specific repository version +// Use Tangy to search for RPMs, by name, that are associated to a specific repository version, returning up to the first 100 results versionHref := "/pulp/e1c6bee3/api/v3/repositories/rpm/rpm/018c1c95-4281-76eb-b277-842cbad524f4/versions/1/" -rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja") +rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja", 100) if err != nil { return err } @@ -56,4 +57,7 @@ The default values provided in config.yaml.example will work with this server. `make compose-down` #### Clean container volumes -`make compose-clean` \ No newline at end of file +`make compose-clean` + +### Mocking +Tangy also exports a mock interface you can regenerate using the [mockery](https://github.com/vektra/mockery) tool. \ No newline at end of file diff --git a/configs/config.yaml.example b/configs/config.yaml.example index 4d5ba87..11f8c4a 100644 --- a/configs/config.yaml.example +++ b/configs/config.yaml.example @@ -9,7 +9,7 @@ database: # Configuration options for the pulp server server: url: "http://localhost:8087" - username: "pulp" + username: "admin" password: "password" storage_type: "local" download_policy: "on_demand" \ No newline at end of file diff --git a/example.go b/example.go index 48282b1..cd8ba27 100644 --- a/example.go +++ b/example.go @@ -27,6 +27,7 @@ func main() { fmt.Println(err) return } + defer t.Close() // Call helper function that creates and syncs a repository versionHref, err := CreateRepositoryVersion() @@ -35,8 +36,8 @@ func main() { return } - // Use Tangy to search for RPMs, by name, that are associated to a specific repository version - rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja") + // Use Tangy to search for RPMs, by name, that are associated to a specific repository version, returning up to the first 100 results + rows, err := t.RpmRepositoryVersionPackageSearch(context.Background(), []string{versionHref}, "ninja", 100) if err != nil { fmt.Println(err) return @@ -58,7 +59,7 @@ func CreateRepositoryVersion() (versionHref string, err error) { DownloadPolicy: "on_demand", }) - domainName := "example_domain" + domainName := "example-domain" // Create domain and repository, then sync repository, to create a new repository version with rpm packages _, err = rpmZest.LookupOrCreateDomain(domainName) diff --git a/go.mod b/go.mod index ddb7b41..91b96c8 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/content-services/zest/release/v2023 v2023.11.1701177874 + github.com/google/uuid v1.4.0 github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb github.com/jackc/pgx/v5 v5.5.1 github.com/rs/zerolog v1.31.0 @@ -32,6 +33,7 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect diff --git a/go.sum b/go.sum index 5c6211a..ec1e90e 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -61,6 +63,7 @@ github.com/spf13/viper v1.18.1 h1:rmuU42rScKWlhhJDyXZRKJQHXFX02chSVW1IvkPGiVM= github.com/spf13/viper v1.18.1/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/internal/test/integration/rpm_test.go b/internal/test/integration/rpm_test.go index c1be33b..f0dd035 100644 --- a/internal/test/integration/rpm_test.go +++ b/internal/test/integration/rpm_test.go @@ -18,10 +18,13 @@ type RpmSuite struct { client *zestwrapper.RpmZest tangy tangy.Tangy domainName string + remoteHref string + repoHref string } const testRepoName = "rpm modular" -const testRepoURL = "https://fixtures.pulpproject.org/rpm-modular/" +const testRepoURL = "https://jlsherrill.fedorapeople.org/fake-repos/revision/one/" +const testRepoURLTwo = "https://jlsherrill.fedorapeople.org/fake-repos/revision/two/" func (r *RpmSuite) CreateTestRepository(t *testing.T) { domainName := RandStringBytes(10) @@ -33,6 +36,9 @@ func (r *RpmSuite) CreateTestRepository(t *testing.T) { repoHref, remoteHref, err := r.client.CreateRepository(domainName, testRepoName, testRepoURL) require.NoError(t, err) + r.repoHref = repoHref + r.remoteHref = remoteHref + syncTask, err := r.client.SyncRpmRepository(repoHref, remoteHref) require.NoError(t, err) @@ -40,6 +46,17 @@ func (r *RpmSuite) CreateTestRepository(t *testing.T) { require.NoError(t, err) } +func (r *RpmSuite) UpdateTestRepository(t *testing.T, url string) { + err := r.client.UpdateRemote(r.remoteHref, url) + require.NoError(t, err) + + syncTask, err := r.client.SyncRpmRepository(r.repoHref, r.remoteHref) + require.NoError(t, err) + + _, err = r.client.PollTask(syncTask) + require.NoError(t, err) +} + func TestRpmSuite(t *testing.T) { s := config.Get().Server rpmZest := zestwrapper.NewRpmZest(context.Background(), s) @@ -51,7 +68,7 @@ func TestRpmSuite(t *testing.T) { Port: dbConfig.Port, User: dbConfig.User, Password: dbConfig.Password, - }, tangy.Logger{Enabled: false}) + }, tangy.Logger{}) require.NoError(t, err) r := RpmSuite{} @@ -64,12 +81,51 @@ func TestRpmSuite(t *testing.T) { func (r *RpmSuite) TestRpmRepositoryVersionPackageSearch() { resp, err := r.client.GetRpmRepositoryByName(r.domainName, testRepoName) require.NoError(r.T(), err) - versionHref := resp.LatestVersionHref - require.NotNil(r.T(), versionHref) + firstVersionHref := resp.LatestVersionHref + require.NotNil(r.T(), firstVersionHref) + + // Search first repository version + search, err := r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "bea", 100) + assert.NoError(r.T(), err) + assert.Equal(r.T(), search[0].Name, "bear") + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "cam", 100) + assert.NoError(r.T(), err) + assert.Empty(r.T(), search) + + // Create second repository version + r.UpdateTestRepository(r.T(), testRepoURLTwo) + resp, err = r.client.GetRpmRepositoryByName(r.domainName, testRepoName) + require.NoError(r.T(), err) + secondVersionHref := resp.LatestVersionHref + require.NotNil(r.T(), secondVersionHref) + + // Search second repository version, should have new package + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*secondVersionHref}, "bea", 100) + assert.NoError(r.T(), err) + assert.Equal(r.T(), search[0].Name, "bear") + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*secondVersionHref}, "cam", 100) + assert.NoError(r.T(), err) + assert.Equal(r.T(), search[0].Name, "camel") + + // Re-search the first version, should be the same + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "bea", 100) + assert.NoError(r.T(), err) + assert.Equal(r.T(), search[0].Name, "bear") + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref}, "cam", 100) + assert.NoError(r.T(), err) + assert.Empty(r.T(), search) + + // Search both versions + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*firstVersionHref, *secondVersionHref}, "a", 100) + assert.NoError(r.T(), err) + assert.Len(r.T(), search, 2) + assert.Equal(r.T(), search[0].Name, "bear") + assert.Equal(r.T(), search[1].Name, "camel") - search, err := r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*versionHref}, "ninja") + // Test search limit + search, err = r.tangy.RpmRepositoryVersionPackageSearch(context.Background(), []string{*secondVersionHref}, "a", 1) assert.NoError(r.T(), err) - assert.Equal(r.T(), search[0].Name, "ninja-build") + assert.Len(r.T(), search, 1) } func RandStringBytes(n int) string { diff --git a/internal/zestwrapper/rpm.go b/internal/zestwrapper/rpm.go index 7c70509..8d9cef6 100644 --- a/internal/zestwrapper/rpm.go +++ b/internal/zestwrapper/rpm.go @@ -116,6 +116,17 @@ func (r *RpmZest) CreateRepository(domain, name, url string) (repoHref string, r return *resp.PulpHref, *remoteResponse.PulpHref, nil } +func (r *RpmZest) UpdateRemote(remoteHref string, url string) error { + _, httpResp, err := r.client.RemotesRpmAPI.RemotesRpmRpmPartialUpdate(r.ctx, remoteHref).PatchedrpmRpmRemote(zest.PatchedrpmRpmRemote{Url: &url}).Execute() + if httpResp != nil { + defer httpResp.Body.Close() + } + if err != nil { + return err + } + return nil +} + func (r *RpmZest) SyncRpmRepository(rpmRpmRepositoryHref string, remoteHref string) (string, error) { rpmRepositoryHref := *zest.NewRpmRepositorySyncURL() rpmRepositoryHref.SetRemote(remoteHref) diff --git a/pkg/tangy/interface.go b/pkg/tangy/interface.go index 2b47c85..f57f0f6 100644 --- a/pkg/tangy/interface.go +++ b/pkg/tangy/interface.go @@ -50,6 +50,13 @@ type tangyImpl struct { logger Logger } +//go:generate mockery --name Tangy --filename tangy_mock.go --inpackage type Tangy interface { - RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string) ([]RpmPackageSearch, error) + RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string, limit int) ([]RpmPackageSearch, error) + Close() +} + +// Close closes the DB connection pool +func (t *tangyImpl) Close() { + t.pool.Close() } diff --git a/pkg/tangy/rpm.go b/pkg/tangy/rpm.go index c71f6f2..6be16fd 100644 --- a/pkg/tangy/rpm.go +++ b/pkg/tangy/rpm.go @@ -2,36 +2,64 @@ package tangy import ( "context" + "fmt" + "strconv" "strings" + "github.com/google/uuid" "github.com/jackc/pgx/v5" ) +const DefaultLimit = 500 + type RpmPackageSearch struct { Name string Summary string } -func (t *tangyImpl) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string) ([]RpmPackageSearch, error) { +// RpmRepositoryVersionPackageSearch search for RPMs, by name, associated to repository hrefs, returning an amount up to limit +func (t *tangyImpl) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string, limit int) ([]RpmPackageSearch, error) { conn, err := t.pool.Acquire(ctx) if err != nil { return nil, err } defer conn.Release() - query := ` - SELECT DISTINCT ON (rp.name) rp.name, rp.summary - FROM core_repositoryversion crv - INNER JOIN core_repository cr on crv.repository_id = cr.pulp_id - INNER JOIN core_repositorycontent crc on cr.pulp_id = crc.repository_id - INNER JOIN core_content cc on crc.content_id = cc.pulp_id - INNER JOIN rpm_package rp on cc.pulp_id = rp.content_ptr_id - WHERE CONCAT(crv.repository_id, '/', crv.number) = ANY ($1) - AND rp.name ILIKE CONCAT( '%', $2::text, '%') - ORDER BY rp.name ASC - ` - concatenatedIdAndVersion := parseRepositoryVersionHrefs(hrefs) - rows, err := conn.Query(context.Background(), query, concatenatedIdAndVersion, search) + if limit == 0 { + limit = DefaultLimit + } + + repositoryIDs, versions, err := parseRepositoryVersionHrefs(hrefs) + if err != nil { + return nil, fmt.Errorf("error parsing repository version hrefs: %w", err) + } + + var query string + for i := 0; i < len(repositoryIDs); i++ { + id := repositoryIDs[i] + ver := versions[i] + + query += fmt.Sprintf(` + SELECT DISTINCT ON (rp.name) rp.name, rp.summary + FROM rpm_package rp + WHERE rp.content_ptr_id IN ( + SELECT crc.content_id + FROM core_repositorycontent crc + INNER JOIN core_repositoryversion crv ON (crc.version_added_id = crv.pulp_id) + LEFT OUTER JOIN core_repositoryversion crv2 ON (crc.version_removed_id = crv2.pulp_id) + WHERE crv.repository_id = '%v' AND crv.number <= %v AND NOT (crv2.number <= %v AND crv2.number IS NOT NULL) + AND rp.name ILIKE CONCAT( '%%', '%v'::text, '%%') ORDER BY rp.name ASC LIMIT %v + ) + `, id, ver, ver, search, limit) + + if i == len(repositoryIDs)-1 { + query += ";" + break + } + + query += "UNION" + } + rows, err := conn.Query(context.Background(), query) if err != nil { return nil, err } @@ -42,11 +70,25 @@ func (t *tangyImpl) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs return rpms, nil } -func parseRepositoryVersionHrefs(hrefs []string) (concatenatedIdAndVersion []string) { +func parseRepositoryVersionHrefs(hrefs []string) (repositoryIDs []string, versions []int, err error) { // /pulp/e1c6bee3/api/v3/repositories/rpm/rpm/018c1c95-4281-76eb-b277-842cbad524f4/versions/1/ for _, href := range hrefs { splitHref := strings.Split(href, "/") - concatenatedIdAndVersion = append(concatenatedIdAndVersion, splitHref[8]+"/"+splitHref[10]) + id := splitHref[8] + num := splitHref[10] + + _, err = uuid.Parse(id) + if err != nil { + return nil, nil, fmt.Errorf("%v is not a valid uuid", id) + } + + ver, err := strconv.Atoi(num) + if err != nil { + return nil, nil, fmt.Errorf("%v is not a valid integer", num) + } + + repositoryIDs = append(repositoryIDs, id) + versions = append(versions, ver) } return } diff --git a/pkg/tangy/tangy_mock.go b/pkg/tangy/tangy_mock.go new file mode 100644 index 0000000..c00bbeb --- /dev/null +++ b/pkg/tangy/tangy_mock.go @@ -0,0 +1,59 @@ +// Code generated by mockery v2.32.0. DO NOT EDIT. + +package tangy + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" +) + +// MockTangy is an autogenerated mock type for the Tangy type +type MockTangy struct { + mock.Mock +} + +// Close provides a mock function with given fields: +func (_m *MockTangy) Close() { + _m.Called() +} + +// RpmRepositoryVersionPackageSearch provides a mock function with given fields: ctx, hrefs, search, limit +func (_m *MockTangy) RpmRepositoryVersionPackageSearch(ctx context.Context, hrefs []string, search string, limit int) ([]RpmPackageSearch, error) { + ret := _m.Called(ctx, hrefs, search, limit) + + var r0 []RpmPackageSearch + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []string, string, int) ([]RpmPackageSearch, error)); ok { + return rf(ctx, hrefs, search, limit) + } + if rf, ok := ret.Get(0).(func(context.Context, []string, string, int) []RpmPackageSearch); ok { + r0 = rf(ctx, hrefs, search, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]RpmPackageSearch) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []string, string, int) error); ok { + r1 = rf(ctx, hrefs, search, limit) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewMockTangy creates a new instance of MockTangy. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockTangy(t interface { + mock.TestingT + Cleanup(func()) +}) *MockTangy { + mock := &MockTangy{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +}