From 14bfb9922c30595f95f393c7a29f8ee42c092763 Mon Sep 17 00:00:00 2001
From: Sam Starling <42478+samstarling@users.noreply.github.com>
Date: Fri, 22 Nov 2024 15:04:36 +0000
Subject: [PATCH] Support configuring the page size for listing catalog entries
---
cmd/catalog-importer/cmd/sync.go | 31 ++++++++++++++++++-------------
reconcile/entries.go | 14 +++++++-------
2 files changed, 25 insertions(+), 20 deletions(-)
diff --git a/cmd/catalog-importer/cmd/sync.go b/cmd/catalog-importer/cmd/sync.go
index 4a36aac..e784953 100644
--- a/cmd/catalog-importer/cmd/sync.go
+++ b/cmd/catalog-importer/cmd/sync.go
@@ -23,15 +23,16 @@ import (
)
type SyncOptions struct {
- ConfigFile string
- APIEndpoint string
- APIKey string
- Targets []string
- SampleLength int
- DryRun bool
- Prune bool
- AllowDeleteAll bool
- SourceRepoUrl string
+ ConfigFile string
+ APIEndpoint string
+ APIKey string
+ Targets []string
+ SampleLength int
+ DryRun bool
+ Prune bool
+ AllowDeleteAll bool
+ SourceRepoUrl string
+ CatalogEntriesAPIPageSize int
}
func (opt *SyncOptions) Bind(cmd *kingpin.CmdClause) *SyncOptions {
@@ -59,6 +60,10 @@ func (opt *SyncOptions) Bind(cmd *kingpin.CmdClause) *SyncOptions {
BoolVar(&opt.Prune)
cmd.Flag("allow-delete-all", "Allow removing all entries from a catalog entry").
BoolVar(&opt.AllowDeleteAll)
+ cmd.Flag("catalog-entries-api-page-size", "The page size to use when listing catalog entries from the API").
+ Envar("CATALOG_ENTRIES_API_PAGE_SIZE").
+ Default("250").
+ IntVar(&opt.CatalogEntriesAPIPageSize)
return opt
}
@@ -473,7 +478,7 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf
logger.Log("msg", "reconciling catalog entries", "output", outputType.TypeName)
catalogType := catalogTypesByOutput[outputType.TypeName]
- err = reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, entryModels, newEntriesProgress(!opt.DryRun))
+ err = reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, entryModels, newEntriesProgress(!opt.DryRun), opt.CatalogEntriesAPIPageSize)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("outputs (type_name = '%s'): reconciling catalog entries", outputType.TypeName))
}
@@ -509,7 +514,7 @@ func (opt *SyncOptions) Run(ctx context.Context, logger kitlog.Logger, cfg *conf
OUT("\n ↻ %s (enum)", enumModel.TypeName)
catalogType := catalogTypesByOutput[enumModel.TypeName]
- err := reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, enumModels, newEntriesProgress(!opt.DryRun))
+ err := reconcile.Entries(ctx, logger, entriesClient, outputType, catalogType, enumModels, newEntriesProgress(!opt.DryRun), opt.CatalogEntriesAPIPageSize)
if err != nil {
return errors.Wrap(err,
fmt.Sprintf("outputs (type_name = '%s'): enum for attribute (id = '%s'): %s: reconciling catalog entries",
@@ -530,7 +535,7 @@ func newEntriesClient(cl *client.ClientWithResponses, existingCatalogTypes []cli
}
return reconcile.EntriesClient{
- GetEntries: func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
+ GetEntries: func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
// We're in dry-run and this catalog type is yet to be created. We can't ask the API
// for the entries of a type that doesn't exist, so we return the type we faked from
// the dry-run create and an empty list of entries.
@@ -545,7 +550,7 @@ func newEntriesClient(cl *client.ClientWithResponses, existingCatalogTypes []cli
}
// We're just a normal catalog type, use the real client.
- return reconcile.GetEntries(ctx, cl, catalogTypeID)
+ return reconcile.GetEntries(ctx, cl, catalogTypeID, pageSize)
},
Delete: func(ctx context.Context, entry *client.CatalogEntryV2) error {
DIFF(" ", *entry, client.CatalogEntryV2{})
diff --git a/reconcile/entries.go b/reconcile/entries.go
index 827cbcd..5d905d0 100644
--- a/reconcile/entries.go
+++ b/reconcile/entries.go
@@ -15,7 +15,7 @@ import (
)
type EntriesClient struct {
- GetEntries func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error)
+ GetEntries func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error)
Delete func(ctx context.Context, entry *client.CatalogEntryV2) error
Create func(ctx context.Context, payload client.CreateEntryRequestBody) (*client.CatalogEntryV2, error)
Update func(ctx context.Context, entry *client.CatalogEntryV2, payload client.UpdateEntryRequestBody) (*client.CatalogEntryV2, error)
@@ -26,8 +26,8 @@ type EntriesClient struct {
// actually perform updates.
func EntriesClientFromClient(cl *client.ClientWithResponses) EntriesClient {
return EntriesClient{
- GetEntries: func(ctx context.Context, catalogTypeID string) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
- return GetEntries(ctx, cl, catalogTypeID)
+ GetEntries: func(ctx context.Context, catalogTypeID string, pageSize int) (*client.CatalogTypeV2, []client.CatalogEntryV2, error) {
+ return GetEntries(ctx, cl, catalogTypeID, pageSize)
},
Delete: func(ctx context.Context, entry *client.CatalogEntryV2) error {
_, err := cl.CatalogV2DestroyEntryWithResponse(ctx, entry.Id)
@@ -65,7 +65,7 @@ type EntriesProgress struct {
OnUpdateProgress func()
}
-func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, outputType *output.Output, catalogType *client.CatalogTypeV2, entryModels []*output.CatalogEntryModel, progress *EntriesProgress) error {
+func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, outputType *output.Output, catalogType *client.CatalogTypeV2, entryModels []*output.CatalogEntryModel, progress *EntriesProgress, pageSize int) error {
logger = kitlog.With(logger,
"catalog_type_id", catalogType.Id,
"catalog_type_name", catalogType.TypeName,
@@ -77,7 +77,7 @@ func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, output
}
logger.Log("msg", "listing existing entries")
- catalogType, entries, err := cl.GetEntries(ctx, catalogType.Id)
+ catalogType, entries, err := cl.GetEntries(ctx, catalogType.Id, pageSize)
if err != nil {
return errors.Wrap(err, "listing entries")
}
@@ -307,7 +307,7 @@ func Entries(ctx context.Context, logger kitlog.Logger, cl EntriesClient, output
}
// GetEntries paginates through all catalog entries for the given type.
-func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogTypeID string) (catalogType *client.CatalogTypeV2, entries []client.CatalogEntryV2, err error) {
+func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogTypeID string, pageSize int) (catalogType *client.CatalogTypeV2, entries []client.CatalogEntryV2, err error) {
var (
after *string
)
@@ -315,7 +315,7 @@ func GetEntries(ctx context.Context, cl *client.ClientWithResponses, catalogType
for {
result, err := cl.CatalogV2ListEntriesWithResponse(ctx, &client.CatalogV2ListEntriesParams{
CatalogTypeId: catalogTypeID,
- PageSize: lo.ToPtr(int64(250)),
+ PageSize: lo.ToPtr(int64(pageSize)),
After: after,
})
if err != nil {