From 017b53de8a30d946c8faac01742c5fea72ac03cb Mon Sep 17 00:00:00 2001 From: Luca Burgazzoli Date: Wed, 12 Feb 2025 11:48:41 +0100 Subject: [PATCH 1/4] Improve gc service/action --- pkg/controller/actions/gc/action_gc.go | 43 +++-- .../actions/gc/action_gc_support.go | 7 +- pkg/controller/actions/gc/action_gc_test.go | 148 ++++++++++++++- pkg/services/gc/gc.go | 179 +++++++++++------- 4 files changed, 294 insertions(+), 83 deletions(-) diff --git a/pkg/controller/actions/gc/action_gc.go b/pkg/controller/actions/gc/action_gc.go index b39dc705bd2..fdaac7bca10 100644 --- a/pkg/controller/actions/gc/action_gc.go +++ b/pkg/controller/actions/gc/action_gc.go @@ -17,15 +17,17 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/pkg/services/gc" ) -type PredicateFn func(*odhTypes.ReconciliationRequest, unstructured.Unstructured) (bool, error) +type ObjectPredicateFn func(*odhTypes.ReconciliationRequest, unstructured.Unstructured) (bool, error) +type TypePredicateFn func(*odhTypes.ReconciliationRequest, schema.GroupVersionKind) (bool, error) type ActionOpts func(*Action) type Action struct { - labels map[string]string - selector labels.Selector - unremovables []schema.GroupVersionKind - gc *gc.GC - predicateFn PredicateFn + labels map[string]string + selector labels.Selector + unremovables []schema.GroupVersionKind + gc *gc.GC + objectPredicateFn ObjectPredicateFn + typePredicateFn TypePredicateFn } func WithLabel(name string, value string) ActionOpts { @@ -56,13 +58,22 @@ func WithUnremovables(items ...schema.GroupVersionKind) ActionOpts { } } -func WithPredicate(value PredicateFn) ActionOpts { +func WithObjectPredicate(value ObjectPredicateFn) ActionOpts { return func(action *Action) { if value == nil { return } - action.predicateFn = value + action.objectPredicateFn = value + } +} +func WithTypePredicate(value TypePredicateFn) ActionOpts { + return func(action *Action) { + if value == nil { + return + } + + action.typePredicateFn = value } } @@ -102,13 +113,20 @@ func (a *Action) run(ctx context.Context, rr *odhTypes.ReconciliationRequest) er deleted, err := a.gc.Run( ctx, selector, - func(ctx context.Context, obj unstructured.Unstructured) (bool, error) { + gc.WitTypeFilter(func(ctx context.Context, kind schema.GroupVersionKind) (bool, error) { + if slices.Contains(a.unremovables, kind) { + return false, nil + } + + return a.typePredicateFn(rr, kind) + }), + gc.WitObjectFilter(func(ctx context.Context, obj unstructured.Unstructured) (bool, error) { if slices.Contains(a.unremovables, obj.GroupVersionKind()) { return false, nil } - return a.predicateFn(rr, obj) - }, + return a.objectPredicateFn(rr, obj) + }), ) if err != nil { @@ -124,7 +142,8 @@ func (a *Action) run(ctx context.Context, rr *odhTypes.ReconciliationRequest) er func NewAction(opts ...ActionOpts) actions.Fn { action := Action{} - action.predicateFn = DefaultPredicate + action.objectPredicateFn = DefaultObjectPredicate + action.typePredicateFn = DefaultTypePredicate action.unremovables = make([]schema.GroupVersionKind, 0) for _, opt := range opts { diff --git a/pkg/controller/actions/gc/action_gc_support.go b/pkg/controller/actions/gc/action_gc_support.go index 6ee3da9ebff..ab8babbcfa0 100644 --- a/pkg/controller/actions/gc/action_gc_support.go +++ b/pkg/controller/actions/gc/action_gc_support.go @@ -5,13 +5,14 @@ import ( "strconv" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" odhTypes "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/types" odhAnnotations "github.com/opendatahub-io/opendatahub-operator/v2/pkg/metadata/annotations" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/resources" ) -func DefaultPredicate(rr *odhTypes.ReconciliationRequest, obj unstructured.Unstructured) (bool, error) { +func DefaultObjectPredicate(rr *odhTypes.ReconciliationRequest, obj unstructured.Unstructured) (bool, error) { if obj.GetAnnotations() == nil { return false, nil } @@ -44,3 +45,7 @@ func DefaultPredicate(rr *odhTypes.ReconciliationRequest, obj unstructured.Unstr return rr.Instance.GetGeneration() != int64(g), nil } + +func DefaultTypePredicate(_ *odhTypes.ReconciliationRequest, _ schema.GroupVersionKind) (bool, error) { + return true, nil +} diff --git a/pkg/controller/actions/gc/action_gc_test.go b/pkg/controller/actions/gc/action_gc_test.go index 27ef6c11811..94efe54fe7a 100644 --- a/pkg/controller/actions/gc/action_gc_test.go +++ b/pkg/controller/actions/gc/action_gc_test.go @@ -13,6 +13,7 @@ import ( appsv1 "k8s.io/api/apps/v1" authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -129,7 +130,7 @@ func TestGcAction(t *testing.T) { generated: true, matcher: Not(HaveOccurred()), metricsMatcher: BeNumerically("==", 1), - options: []gc.ActionOpts{gc.WithPredicate( + options: []gc.ActionOpts{gc.WithObjectPredicate( func(request *types.ReconciliationRequest, unstructured unstructured.Unstructured) (bool, error) { return unstructured.GroupVersionKind() != gvk.ConfigMap, nil }, @@ -254,3 +255,148 @@ func TestGcAction(t *testing.T) { }) } } + +func TestGcActionCluster(t *testing.T) { + g := NewWithT(t) + + s := runtime.NewScheme() + ctx := context.Background() + id := xid.New().String() + nsn := xid.New().String() + + utilruntime.Must(corev1.AddToScheme(s)) + utilruntime.Must(appsv1.AddToScheme(s)) + utilruntime.Must(apiextensionsv1.AddToScheme(s)) + utilruntime.Must(authorizationv1.AddToScheme(s)) + utilruntime.Must(rbacv1.AddToScheme(s)) + + envTest := &envtest.Environment{ + CRDInstallOptions: envtest.CRDInstallOptions{ + Scheme: s, + CleanUpAfterUse: true, + }, + } + + t.Cleanup(func() { + _ = envTest.Stop() + }) + + cfg, err := envTest.Start() + g.Expect(err).NotTo(HaveOccurred()) + + envTestClient, err := ctrlCli.New(cfg, ctrlCli.Options{Scheme: s}) + g.Expect(err).NotTo(HaveOccurred()) + + cli, err := client.NewFromConfig(cfg, envTestClient) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cli).NotTo(BeNil()) + + gci := gcSvc.New( + cli, + nsn, + // This is required as there are no kubernetes controller running + // with the envtest, hence we can't use the foreground deletion + // policy (default) + gcSvc.WithPropagationPolicy(metav1.DeletePropagationBackground), + ) + + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsn, + }, + } + + g.Expect(cli.Create(ctx, &ns)). + NotTo(HaveOccurred()) + g.Expect(gci.Start(ctx)). + NotTo(HaveOccurred()) + + rr := types.ReconciliationRequest{ + Client: cli, + DSCI: &dsciv1.DSCInitialization{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + }, + Instance: &componentApi.Dashboard{ + TypeMeta: metav1.TypeMeta{ + APIVersion: componentApi.GroupVersion.String(), + Kind: componentApi.DashboardKind, + }, + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + UID: apytypes.UID(id), + }, + }, + Release: common.Release{ + Name: cluster.OpenDataHub, + Version: version.OperatorVersion{ + Version: semver.Version{Major: 0, Minor: 2, Patch: 0}, + }, + }, + Generated: true, + } + + om := metav1.ObjectMeta{ + Namespace: nsn, + Annotations: map[string]string{ + annotations.InstanceGeneration: "1", + annotations.InstanceUID: id, + annotations.PlatformType: string(cluster.OpenDataHub), + }, + Labels: map[string]string{ + labels.PlatformPartOf: strings.ToLower(componentApi.DashboardKind), + }, + } + + cm1 := corev1.ConfigMap{ObjectMeta: *om.DeepCopy()} + cm1.Name = xid.New().String() + cm1.Annotations[annotations.PlatformVersion] = "0.1.0" + + cm2 := corev1.ConfigMap{ObjectMeta: *om.DeepCopy()} + cm2.Name = xid.New().String() + cm2.Annotations[annotations.PlatformVersion] = rr.Release.Version.String() + + cr1 := rbacv1.ClusterRole{ObjectMeta: *om.DeepCopy()} + cr1.Name = xid.New().String() + cr1.Annotations[annotations.PlatformVersion] = "0.1.0" + + cr2 := rbacv1.ClusterRole{ObjectMeta: *om.DeepCopy()} + cr2.Name = xid.New().String() + cr2.Annotations[annotations.PlatformVersion] = rr.Release.Version.String() + + g.Expect(cli.Create(ctx, &cm1)). + NotTo(HaveOccurred()) + + g.Expect(cli.Create(ctx, &cm2)). + NotTo(HaveOccurred()) + + g.Expect(cli.Create(ctx, &cr1)). + NotTo(HaveOccurred()) + + g.Expect(cli.Create(ctx, &cr2)). + NotTo(HaveOccurred()) + + a := gc.NewAction(gc.WithGC(gci)) + + gc.DeletedTotal.Reset() + gc.DeletedTotal.WithLabelValues("dashboard").Add(0) + + err = a(ctx, &rr) + g.Expect(err).NotTo(HaveOccurred()) + + err = cli.Get(ctx, ctrlCli.ObjectKeyFromObject(&cm1), &corev1.ConfigMap{}) + g.Expect(err).To(MatchError(k8serr.IsNotFound, "IsNotFound")) + + err = cli.Get(ctx, ctrlCli.ObjectKeyFromObject(&cm2), &corev1.ConfigMap{}) + g.Expect(err).ToNot(HaveOccurred()) + + err = cli.Get(ctx, ctrlCli.ObjectKeyFromObject(&cr1), &rbacv1.ClusterRole{}) + g.Expect(err).To(MatchError(k8serr.IsNotFound, "IsNotFound")) + + err = cli.Get(ctx, ctrlCli.ObjectKeyFromObject(&cr2), &rbacv1.ClusterRole{}) + g.Expect(err).ToNot(HaveOccurred()) + + ct := testutil.ToFloat64(gc.DeletedTotal) + g.Expect(ct).Should(BeNumerically("==", 2)) +} diff --git a/pkg/services/gc/gc.go b/pkg/services/gc/gc.go index 284a2da3dd7..71197a162f0 100644 --- a/pkg/services/gc/gc.go +++ b/pkg/services/gc/gc.go @@ -22,13 +22,12 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/client" ) -// Instance a global instance of the GC service. -// // TODO: since the GC service is quite heavy, as it has to discover -// -// resources that can be subject to GC, we share single global -// instance, however as long term, we should find a better way -// to let consumer of the service to access it. +// resources that can be subject to GC, we share single global +// instance, however as long term, we should find a better way +// to let consumer of the service to access it. + +// Instance a global instance of the GC service. var Instance *GC const ( @@ -39,14 +38,16 @@ const ( type options struct { propagationPolicy ctrlCli.PropagationPolicy - unremovables []schema.GroupVersionKind + unremovables map[schema.GroupVersionKind]struct{} } type OptsFn func(*options) func WithUnremovables(items ...schema.GroupVersionKind) OptsFn { return func(o *options) { - o.unremovables = append(o.unremovables, items...) + for _, i := range items { + o.unremovables[i] = struct{}{} + } } } @@ -62,7 +63,7 @@ func New(cli *client.Client, ns string, opts ...OptsFn) *GC { ns: ns, options: options{ propagationPolicy: ctrlCli.PropagationPolicy(metav1.DeletePropagationForeground), - unremovables: make([]schema.GroupVersionKind, 0), + unremovables: make(map[schema.GroupVersionKind]struct{}), }, resources: Resources{ @@ -100,48 +101,98 @@ func (gc *GC) Start(ctx context.Context) error { return nil } +type runOptions struct { + typePredicate func(context.Context, schema.GroupVersionKind) (bool, error) + objectPredicate func(context.Context, unstructured.Unstructured) (bool, error) +} + +type RunOptionsFn func(*runOptions) + +func WitObjectFilter(fn func(context.Context, unstructured.Unstructured) (bool, error)) RunOptionsFn { + return func(o *runOptions) { + o.objectPredicate = fn + } +} +func WitTypeFilter(fn func(context.Context, schema.GroupVersionKind) (bool, error)) RunOptionsFn { + return func(o *runOptions) { + o.typePredicate = fn + } +} + +func (gc *GC) listResources( + ctx context.Context, + res Resource, + opts metav1.ListOptions, +) ([]unstructured.Unstructured, error) { + items, err := gc.client.Dynamic().Resource(res.GroupVersionResource()).Namespace("").List(ctx, opts) + if err != nil { + if k8serr.IsForbidden(err) || k8serr.IsMethodNotSupported(err) { + gc.log(ctx).V(3).Info( + "cannot list resource", + "reason", err.Error(), + "gvk", res.GroupVersionKind(), + ) + + return nil, nil + } + + return nil, err + } + + return items.Items, nil +} + func (gc *GC) Run( ctx context.Context, selector labels.Selector, - predicate func(context.Context, unstructured.Unstructured) (bool, error), + opts ...RunOptionsFn, ) (int, error) { - l := gc.log(ctx) + ro := runOptions{ + typePredicate: func(_ context.Context, _ schema.GroupVersionKind) (bool, error) { + return true, nil + }, + objectPredicate: func(_ context.Context, _ unstructured.Unstructured) (bool, error) { + return true, nil + }, + } + + for _, opt := range opts { + opt(&ro) + } deleted := 0 resources := gc.resources.Get() - - dc := gc.client.Dynamic() lo := metav1.ListOptions{LabelSelector: selector.String()} - for r := range resources { - items, err := dc.Resource(resources[r].GroupVersionResource()).Namespace("").List(ctx, lo) + for _, res := range resources { + canBeDeleted, err := ro.typePredicate(ctx, res.GroupVersionKind()) if err != nil { - if k8serr.IsForbidden(err) { - l.V(3).Info( - "cannot list resource", - "reason", err.Error(), - "gvk", resources[r].GroupVersionKind(), - ) - - continue - } + return 0, fmt.Errorf("cannot determine if resource %s can be deleted: %w", res.String(), err) + } - if k8serr.IsNotFound(err) { - continue - } + if !canBeDeleted { + continue + } - return 0, fmt.Errorf("cannot list child resources %s: %w", resources[r].String(), err) + items, err := gc.listResources(ctx, res, lo) + if err != nil { + return 0, fmt.Errorf("cannot list child resources %s: %w", res.String(), err) } - for i := range items.Items { - ok, err := gc.delete(ctx, items.Items[i], predicate) + for i := range items { + canBeDeleted, err = ro.objectPredicate(ctx, items[i]) if err != nil { return 0, err } + if !canBeDeleted { + continue + } - if ok { - deleted++ + if err := gc.delete(ctx, items[i]); err != nil { + return 0, err } + + deleted++ } } @@ -151,21 +202,7 @@ func (gc *GC) Run( func (gc *GC) delete( ctx context.Context, resource unstructured.Unstructured, - predicate func(context.Context, unstructured.Unstructured) (bool, error), -) (bool, error) { - if slices.Contains(gc.options.unremovables, resource.GroupVersionKind()) { - return false, nil - } - - canBeDeleted, err := predicate(ctx, resource) - if err != nil { - return false, err - } - - if !canBeDeleted { - return false, err - } - +) error { gc.log(ctx).Info( "delete", "gvk", resource.GroupVersionKind(), @@ -173,14 +210,9 @@ func (gc *GC) delete( "name", resource.GetName(), ) - err = gc.client.Delete(ctx, &resource, gc.options.propagationPolicy) - if err != nil { - // The resource may have already been deleted - if k8serr.IsNotFound(err) { - return true, nil - } - - return false, fmt.Errorf( + err := gc.client.Delete(ctx, &resource, gc.options.propagationPolicy) + if err != nil && !k8serr.IsNotFound(err) { + return fmt.Errorf( "cannot delete resources gvk:%s, namespace: %s, name: %s, reason: %w", resource.GroupVersionKind().String(), resource.GetNamespace(), @@ -189,22 +221,31 @@ func (gc *GC) delete( ) } - return true, nil + return nil } -func (gc *GC) computeDeletableTypes( - ctx context.Context, -) ([]Resource, error) { - // We rely on the discovery API to retrieve all the resources GVK, - // that results in an unbounded set that can impact garbage collection - // latency when scaling up. - items, err := gc.client.Discovery().ServerPreferredNamespacedResources() +func (gc *GC) discoverResources() ([]*metav1.APIResourceList, error) { + // We rely on the discovery API to retrieve all the resources GVK, that + // results in an unbounded set that can impact garbage collection latency + // when scaling up. + items, err := gc.client.Discovery().ServerPreferredResources() - // Swallow group discovery errors, e.g., Knative serving exposes - // an aggregated API for custom.metrics.k8s.io that requires special + // Swallow group discovery errors, e.g., Knative serving exposes an + // aggregated API for custom.metrics.k8s.io that requires special // authentication scheme while discovering preferred resources. if err != nil && !discovery.IsGroupDiscoveryFailedError(err) { - return nil, fmt.Errorf("failure retrieving supported namespaced resources: %w", err) + return nil, fmt.Errorf("failure retrieving supported resources: %w", err) + } + + return items, nil +} + +func (gc *GC) computeDeletableTypes( + ctx context.Context, +) ([]Resource, error) { + items, err := gc.discoverResources() + if err != nil { + return nil, fmt.Errorf("failure discovering resources: %w", err) } // We only take types that support the "delete" verb, @@ -220,13 +261,13 @@ func (gc *GC) computeDeletableTypes( // Get the permissions of the service account in the specified namespace. rules, err := gc.retrieveResourceRules(ctx) if err != nil { - return nil, fmt.Errorf("failure retiring resource rules: %w", err) + return nil, fmt.Errorf("failure retrieving resource rules: %w", err) } // Collect deletable resources. resources, err := gc.collectDeletableResources(apiResourceLists, rules) if err != nil { - return nil, fmt.Errorf("failure retiring deletable resources: %w", err) + return nil, fmt.Errorf("failure retrieving deletable resources: %w", err) } return resources, nil @@ -320,7 +361,7 @@ func (gc *GC) collectDeletableResources( gvr.Scope = meta.RESTScopeRoot } - if slices.Contains(gc.options.unremovables, gvr.GroupVersionKind()) { + if _, ok := gc.options.unremovables[gvr.GroupVersionKind()]; ok { continue } From 3ffa781df6b740bcc2baae35d5ce5669288e0eb0 Mon Sep 17 00:00:00 2001 From: Luca Burgazzoli Date: Wed, 12 Feb 2025 15:12:09 +0100 Subject: [PATCH 2/4] Remove useless lock --- pkg/services/gc/gc_support.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/services/gc/gc_support.go b/pkg/services/gc/gc_support.go index dc445b1452d..a11bb8128ca 100644 --- a/pkg/services/gc/gc_support.go +++ b/pkg/services/gc/gc_support.go @@ -2,7 +2,6 @@ package gc import ( "slices" - "sync" authorizationv1 "k8s.io/api/authorization/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -37,20 +36,13 @@ func (r Resource) IsNamespaced() bool { // We may want to introduce iterators (https://pkg.go.dev/iter) once moved to go 1.23 type Resources struct { - lock sync.RWMutex items []Resource } func (r *Resources) Set(resources []Resource) { - r.lock.Lock() - defer r.lock.Unlock() - r.items = resources } func (r *Resources) Get() []Resource { - r.lock.RLock() - defer r.lock.RUnlock() - return slices.Clone(r.items) } From bf42cf45ab0ebf7e0993589d485fdf31a259acfa Mon Sep 17 00:00:00 2001 From: Luca Burgazzoli Date: Wed, 12 Feb 2025 15:13:50 +0100 Subject: [PATCH 3/4] Fix typos --- pkg/controller/actions/gc/action_gc.go | 4 ++-- pkg/services/gc/gc.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/controller/actions/gc/action_gc.go b/pkg/controller/actions/gc/action_gc.go index fdaac7bca10..081323b49ed 100644 --- a/pkg/controller/actions/gc/action_gc.go +++ b/pkg/controller/actions/gc/action_gc.go @@ -113,14 +113,14 @@ func (a *Action) run(ctx context.Context, rr *odhTypes.ReconciliationRequest) er deleted, err := a.gc.Run( ctx, selector, - gc.WitTypeFilter(func(ctx context.Context, kind schema.GroupVersionKind) (bool, error) { + gc.WithTypeFilter(func(ctx context.Context, kind schema.GroupVersionKind) (bool, error) { if slices.Contains(a.unremovables, kind) { return false, nil } return a.typePredicateFn(rr, kind) }), - gc.WitObjectFilter(func(ctx context.Context, obj unstructured.Unstructured) (bool, error) { + gc.WithObjectFilter(func(ctx context.Context, obj unstructured.Unstructured) (bool, error) { if slices.Contains(a.unremovables, obj.GroupVersionKind()) { return false, nil } diff --git a/pkg/services/gc/gc.go b/pkg/services/gc/gc.go index 71197a162f0..3b7807006c6 100644 --- a/pkg/services/gc/gc.go +++ b/pkg/services/gc/gc.go @@ -108,12 +108,12 @@ type runOptions struct { type RunOptionsFn func(*runOptions) -func WitObjectFilter(fn func(context.Context, unstructured.Unstructured) (bool, error)) RunOptionsFn { +func WithObjectFilter(fn func(context.Context, unstructured.Unstructured) (bool, error)) RunOptionsFn { return func(o *runOptions) { o.objectPredicate = fn } } -func WitTypeFilter(fn func(context.Context, schema.GroupVersionKind) (bool, error)) RunOptionsFn { +func WithTypeFilter(fn func(context.Context, schema.GroupVersionKind) (bool, error)) RunOptionsFn { return func(o *runOptions) { o.typePredicate = fn } From 478641ac74b00e5406d66e4a9c45c8b192d292e7 Mon Sep 17 00:00:00 2001 From: Luca Burgazzoli Date: Wed, 12 Feb 2025 16:47:46 +0100 Subject: [PATCH 4/4] Improve error handling --- pkg/services/gc/gc.go | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pkg/services/gc/gc.go b/pkg/services/gc/gc.go index 3b7807006c6..de80a46f49c 100644 --- a/pkg/services/gc/gc.go +++ b/pkg/services/gc/gc.go @@ -125,21 +125,20 @@ func (gc *GC) listResources( opts metav1.ListOptions, ) ([]unstructured.Unstructured, error) { items, err := gc.client.Dynamic().Resource(res.GroupVersionResource()).Namespace("").List(ctx, opts) - if err != nil { - if k8serr.IsForbidden(err) || k8serr.IsMethodNotSupported(err) { - gc.log(ctx).V(3).Info( - "cannot list resource", - "reason", err.Error(), - "gvk", res.GroupVersionKind(), - ) - - return nil, nil - } + switch { + case k8serr.IsForbidden(err) || k8serr.IsMethodNotSupported(err): + gc.log(ctx).V(3).Info( + "cannot list resource", + "reason", err.Error(), + "gvk", res.GroupVersionKind(), + ) + return nil, nil + case err != nil: return nil, err + default: + return items.Items, nil } - - return items.Items, nil } func (gc *GC) Run(