Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: improve pool deletion #21

Merged
merged 1 commit into from
Oct 25, 2023
Merged

feat: improve pool deletion #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions api/v1alpha1/image_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,8 @@ func (i *Image) attachedPools(ctx context.Context) ([]Pool, error) {
}

for _, pool := range pools.Items {
// we do not care about pools that are already deleted
if pool.GetDeletionTimestamp() == nil {
if pool.Spec.ImageName == i.Name {
result = append(result, pool)
}
if pool.Spec.ImageName == i.Name {
result = append(result, pool)
}
}

Expand Down
2 changes: 1 addition & 1 deletion api/v1alpha1/pool_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func (r *Pool) ValidateCreate() (admission.Warnings, error) {
if len(poolList.Items) > 0 {
existing := poolList.Items[0]
return nil, apierrors.NewBadRequest(
fmt.Sprintf("can not create pool, pool=%s with same image=%s, flavor=%s and provider=%s already exists for specified GitHubScope=%s", existing.Name, existing.Spec.ImageName, existing.Spec.Flavor, existing.Spec.ProviderName, existing.Spec.GitHubScopeRef.Name))
fmt.Sprintf("can not create pool, pool=%s with same image=%s , flavor=%s and provider=%s already exists for specified GitHubScope=%s", existing.Name, existing.Spec.ImageName, existing.Spec.Flavor, existing.Spec.ProviderName, existing.Spec.GitHubScopeRef.Name))
}

return nil, nil
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.21.3

require (
github.com/cloudbase/garm v0.1.3
github.com/cloudbase/garm-provider-common v0.0.0-20230724114054-7aa0a3dfbce0
github.com/cloudbase/garm-provider-common v0.1.0
github.com/go-openapi/runtime v0.26.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.16.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ github.com/cloudbase/garm v0.1.3 h1:/8F7Rnk/tKfK9G3z1mm3IyvizbaZ5WbNFWNA1kpviHE=
github.com/cloudbase/garm v0.1.3/go.mod h1:R+EGVGriGx/t9TNUwfIQFnu/58rh1Inka08fsV6IB/c=
github.com/cloudbase/garm-provider-common v0.0.0-20230724114054-7aa0a3dfbce0 h1:5ScMXea/ZIcUbw1aXAgN8xTqSG84AOf5Maf5hBC82wQ=
github.com/cloudbase/garm-provider-common v0.0.0-20230724114054-7aa0a3dfbce0/go.mod h1:RKzgL0MXkNeGfloQpE2swz/y4LWJr5+2Wd45bSXPB0k=
github.com/cloudbase/garm-provider-common v0.1.0 h1:gc2n8nsLjt7G3InAfqZ+75iZjSIUkIx86d6/DFA2+jc=
github.com/cloudbase/garm-provider-common v0.1.0/go.mod h1:igxJRT3OlykERYc6ssdRQXcb+BCaeSfnucg6I0OSoDc=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
Expand Down
35 changes: 33 additions & 2 deletions internal/controller/pool_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"fmt"
"time"

garmProviderParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/client/enterprises"
"github.com/cloudbase/garm/client/instances"
"github.com/cloudbase/garm/client/organizations"
Expand Down Expand Up @@ -294,8 +295,21 @@ func (r *PoolReconciler) reconcileDelete(ctx context.Context, garmClient garmCli

if controllerutil.ContainsFinalizer(pool, key.PoolFinalizerName) && pool.Spec.MinIdleRunners != 0 {
pool.Spec.MinIdleRunners = 0
err := r.Update(ctx, pool)
pool.Spec.Enabled = false

image, err := r.getImage(ctx, pool)
if err != nil {
log.Error(err, "error getting image")
return r.handleUpdateError(ctx, pool, err)
}

_, err = updatePool(garmClient, pool, image)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this line is esential and new.

in the past, we've just updated the pool-cr and then the pool got scaled down.
With this implementation we immediately scaling the pool down to zero and disable the pool as well.

And after that we try to delete all the corresponding runners.

if err != nil {
log.Error(err, "error updating pool")
return r.handleUpdateError(ctx, pool, err)
}

if err := r.Update(ctx, pool); err != nil {
return r.handleUpdateError(ctx, pool, err)
}
log.Info("scaling pool down before deleting")
Expand All @@ -306,10 +320,27 @@ func (r *PoolReconciler) reconcileDelete(ctx context.Context, garmClient garmCli
runners, err := instanceClient.ListPoolInstances(
instances.NewListPoolInstancesParams().WithPoolID(pool.Status.ID))
if err != nil {
return r.handleUpdateError(ctx, pool, err)
return r.handleUpdateError(ctx, pool, fmt.Errorf("error deleting pool %s: %w", pool.Name, err))
}

if len(runners.GetPayload()) > 0 {
for _, runner := range runners.GetPayload() {
switch runner.Status {
case garmProviderParams.InstanceRunning, garmProviderParams.InstanceError:
if runner.RunnerStatus != params.RunnerActive {
err := instanceClient.DeleteInstance(instances.NewDeleteInstanceParams().WithInstanceName(runner.Name))
if err != nil {
log.Error(err, "unable to delete runner", "runner", runner.Name)
}
} else {
log.Info("Runner has an active run that does not allow deletion", "runner", runner.Name, "state", runner.Status, "runner state", runner.RunnerStatus)
}
default:
log.Info("Runner is in state that does not allow deletion", "runner", runner.Name, "state", runner.Status)
}
}

log.Info("Not all runners could be deleted or are still in deleting phase, reconcile in 1 minute again.")
return ctrl.Result{Requeue: true, RequeueAfter: 1 * time.Minute}, nil
}

Expand Down
66 changes: 64 additions & 2 deletions internal/controller/pool_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -729,7 +729,7 @@ func TestPoolController_ReconcileDelete(t *testing.T) {
OSType: "linux",
OSArch: "arm64",
Tags: []string{"kubernetes", "linux", "arm64", "ubuntu"},
Enabled: true,
Enabled: false,
RunnerBootstrapTimeout: 20,
ExtraSpecs: "",
GitHubRunnerGroup: "",
Expand Down Expand Up @@ -781,7 +781,69 @@ func TestPoolController_ReconcileDelete(t *testing.T) {
},
},
},
expectGarmRequest: func(m *mock.MockPoolClientMockRecorder, instanceClient *mock.MockInstanceClientMockRecorder) {},
expectGarmRequest: func(m *mock.MockPoolClientMockRecorder, instanceClient *mock.MockInstanceClientMockRecorder) {
maxRunners := uint(5)
minIdleRunners := uint(0)
enabled := false
runnerBootstrapTimeout := uint(20)
extraSpecs := json.RawMessage([]byte{})
gitHubRunnerGroup := ""

m.UpdatePool(pools.NewUpdatePoolParams().WithPoolID(poolID).WithBody(params.UpdatePoolParams{
RunnerPrefix: params.RunnerPrefix{
Prefix: "",
},
MaxRunners: &maxRunners,
MinIdleRunners: &minIdleRunners,
Image: "linux-ubuntu-22.04-arm64",
Flavor: "medium",
OSType: "linux",
OSArch: "arm64",
Tags: []string{"kubernetes", "linux", "arm64", "ubuntu"},
Enabled: &enabled,
RunnerBootstrapTimeout: &runnerBootstrapTimeout,
ExtraSpecs: extraSpecs,
GitHubRunnerGroup: &gitHubRunnerGroup,
})).Return(&pools.UpdatePoolOK{Payload: params.Pool{
RunnerPrefix: params.RunnerPrefix{
Prefix: "",
},
ID: poolID,
ProviderName: "kubernetes_external",
MaxRunners: 5,
MinIdleRunners: 0,
Image: "linux-ubuntu-22.04-arm64",
Flavor: "medium",
OSType: "linux",
OSArch: "arm64",
Tags: []params.Tag{
{
ID: "b3ea9882-a25c-4eb1-94ba-6c70b9abb6da",
Name: "kubernetes",
},
{
ID: "b3ea9882-a25c-4eb1-94ba-6c70b9abb6db",
Name: "linux",
},
{
ID: "b3ea9882-a25c-4eb1-94ba-6c70b9abb6dc",
Name: "arm64",
},
{
ID: "b3ea9882-a25c-4eb1-94ba-6c70b9abb6dd",
Name: "ubuntu",
},
},
Enabled: false,
Instances: []params.Instance{},
RepoID: "",
RepoName: "",
OrgID: "",
OrgName: "",
EnterpriseID: enterpriseID,
EnterpriseName: enterpriseName,
}}, nil)
},
},
{
name: "delete pool - deleting garm resource",
Expand Down
11 changes: 11 additions & 0 deletions pkg/client/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ type InstanceClient interface {
GetInstance(params *instances.GetInstanceParams) (*instances.GetInstanceOK, error)
ListInstances(params *instances.ListInstancesParams) (*instances.ListInstancesOK, error)
ListPoolInstances(params *instances.ListPoolInstancesParams) (*instances.ListPoolInstancesOK, error)
DeleteInstance(params *instances.DeleteInstanceParams) error
}

type instanceClient struct {
Expand Down Expand Up @@ -68,3 +69,13 @@ func (i *instanceClient) ListPoolInstances(params *instances.ListPoolInstancesPa
}
return instances, nil
}

func (i *instanceClient) DeleteInstance(params *instances.DeleteInstanceParams) error {
metrics.TotalGarmCalls.WithLabelValues("instances.Delete").Inc()
err := i.client.Instances.DeleteInstance(params, i.token)
if err != nil {
metrics.GarmCallErrors.WithLabelValues("instances.ListPool").Inc()
return err
}
return nil
}
14 changes: 14 additions & 0 deletions pkg/client/mock/instance.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.