Merge pull request #174 from idoru/paginate-repositories

Paginate repository list for an exhaustive search
genevieve · Apr 8, 2024 · 1ecf83c · 1ecf83c
2 parents 5569345 + 3e214e7
commit 1ecf83c
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 21 deletions.
diff --git a/gcp/artifacts/client.go b/gcp/artifacts/client.go
@@ -2,6 +2,7 @@ package artifacts
 
 import (
 	"fmt"
+	"time"
 
 	gcpartifact "google.golang.org/api/artifactregistry/v1"
 	"google.golang.org/api/googleapi"
@@ -25,9 +26,28 @@ func NewClient(project string, service *gcpartifact.Service, logger logger) clie
 	}
 }
 
-func (c client) ListRepositories(region string) (*gcpartifact.ListRepositoriesResponse, error) {
+func (c client) ListRepositories(region string) ([]*gcpartifact.Repository, error) {
 	parent := fmt.Sprintf("projects/%v/locations/%v", c.project, region)
-	return c.repositories.List(parent).Do()
+	var token string
+	var list []*gcpartifact.Repository
+
+	for {
+		resp, err := c.repositories.List(parent).PageToken(token).Do()
+		if err != nil {
+			return nil, err
+		}
+
+		list = append(list, resp.Repositories...)
+
+		token = resp.NextPageToken
+		if token == "" {
+			break
+		}
+
+		time.Sleep(time.Second)
+	}
+
+	return list, nil
 }
 
 func (c client) DeleteRepository(name string) error {

diff --git a/gcp/artifacts/fakes/repositories_client.go b/gcp/artifacts/fakes/repositories_client.go
@@ -25,10 +25,10 @@ type RepositoriesClient struct {
 			Region string
 		}
 		Returns struct {
-			ListRepositoriesResponse *artifactregistry.ListRepositoriesResponse
+			ListRepositoriesResponse []*artifactregistry.Repository
 			Error                    error
 		}
-		Stub func(string) (*artifactregistry.ListRepositoriesResponse, error)
+		Stub func(string) ([]*artifactregistry.Repository, error)
 	}
 }
 
@@ -42,7 +42,7 @@ func (f *RepositoriesClient) DeleteRepository(param1 string) error {
 	}
 	return f.DeleteRepositoryCall.Returns.Error
 }
-func (f *RepositoriesClient) ListRepositories(param1 string) (*artifactregistry.ListRepositoriesResponse, error) {
+func (f *RepositoriesClient) ListRepositories(param1 string) ([]*artifactregistry.Repository, error) {
 	f.ListRepositoriesCall.mutex.Lock()
 	defer f.ListRepositoriesCall.mutex.Unlock()
 	f.ListRepositoriesCall.CallCount++

diff --git a/gcp/artifacts/repositories.go b/gcp/artifacts/repositories.go
@@ -2,10 +2,9 @@ package artifacts
 
 import (
 	"fmt"
-	"strings"
-
 	"github.com/genevieve/leftovers/common"
 	gcpartifact "google.golang.org/api/artifactregistry/v1"
+	"strings"
 )
 
 type Repositories struct {
@@ -16,7 +15,7 @@ type Repositories struct {
 
 //go:generate faux --interface repositoriesClient --output fakes/repositories_client.go
 type repositoriesClient interface {
-	ListRepositories(region string) (*gcpartifact.ListRepositoriesResponse, error)
+	ListRepositories(region string) ([]*gcpartifact.Repository, error)
 	DeleteRepository(cluster string) error
 }
 
@@ -31,13 +30,12 @@ func NewRepositories(client repositoriesClient, logger logger, regions map[strin
 func (c Repositories) List(filter string) ([]common.Deletable, error) {
 	repositories := []*gcpartifact.Repository{}
 	for _, region := range c.regions {
-		c.logger.Debugf("Listing Repositories for region %s...\n", region)
-		l, err := c.client.ListRepositories(region)
+		repositoryList, err := c.client.ListRepositories(region)
 		if err != nil {
 			return nil, fmt.Errorf("list repositories for region %v: %w", region, err)
 		}
 
-		repositories = append(repositories, l.Repositories...)
+		repositories = append(repositories, repositoryList...)
 	}
 
 	deletables := []common.Deletable{}

diff --git a/gcp/artifacts/repositories_test.go b/gcp/artifacts/repositories_test.go
@@ -32,11 +32,9 @@ var _ = Describe("Clusters", func() {
 
 	Describe("List", func() {
 		BeforeEach(func() {
-			client.ListRepositoriesCall.Returns.ListRepositoriesResponse = &gcpartifact.ListRepositoriesResponse{
-				Repositories: []*gcpartifact.Repository{{
-					Name: "banana-repository",
-				}},
-			}
+			client.ListRepositoriesCall.Returns.ListRepositoriesResponse = []*gcpartifact.Repository{{
+				Name: "banana-repository",
+			}}
 		})
 
 		It("returns a list of clusters to delete", func() {
@@ -64,11 +62,9 @@ var _ = Describe("Clusters", func() {
 
 		Context("when the resource name does not contain the filter", func() {
 			BeforeEach(func() {
-				client.ListRepositoriesCall.Returns.ListRepositoriesResponse = &gcpartifact.ListRepositoriesResponse{
-					Repositories: []*gcpartifact.Repository{{
-						Name: "kiwi-repository",
-					}},
-				}
+				client.ListRepositoriesCall.Returns.ListRepositoriesResponse = []*gcpartifact.Repository{{
+					Name: "kiwi-repository",
+				}}
 			})
 
 			It("does not return the resource in the list", func() {