Skip to content

Commit

Permalink
fix remaining resources (#348)
Browse files Browse the repository at this point in the history
* add warden gc work

* update changelog
  • Loading branch information
tiancandevloper committed Oct 25, 2023
1 parent 77d029c commit 55cb1ed
Show file tree
Hide file tree
Showing 7 changed files with 163 additions and 10 deletions.
3 changes: 2 additions & 1 deletion docs/changelog.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# v1.8.10

## BugFix
- add list cube resource quota [#345](https://github.com/kubecube-io/KubeCube/pull/336)
- fix remaining resources [#348](https://github.com/kubecube-io/KubeCube/pull/348)
- add list cube resource quota [#345](https://github.com/kubecube-io/KubeCube/pull/345)

## Dependencies

Expand Down
2 changes: 2 additions & 0 deletions pkg/utils/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ const (
// ApiPathRoot the root api route
ApiPathRoot = "/api/v1/cube"

ApiK8sProxyPath = "/api/v1/cube/kubernetes"

// LocalCluster the internal cluster where program stand with
LocalCluster = "_local_cluster"

Expand Down
88 changes: 88 additions & 0 deletions pkg/warden/syncmgr/gc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
Copyright 2023 KubeCube Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package syncmgr

import (
"context"
"strings"

"github.com/kubecube-io/kubecube/pkg/clog"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type Gc struct {
client.Client
cfg *rest.Config
}

func NewGc(cfg *rest.Config, client client.Client) *Gc {
return &Gc{
cfg: cfg,
Client: client,
}
}

// GcWork gc work
// If a resource exists on the current cluster but not on the control cluster, the resource is a residual resource and needs to be deleted
func (g *Gc) GcWork() {
pivotClient, err := client.New(g.cfg, client.Options{Scheme: scheme})
if err != nil {
clog.Fatal("error new pivot client: %s", err.Error())
}
for _, r := range syncListResources {
err := g.List(context.Background(), r)
if err != nil {
clog.Warn("error list resource: %s", err.Error())
continue
}
list := &unstructured.UnstructuredList{}
err = scheme.Convert(r, list, nil)
if err != nil {
clog.Warn("error convert resource: %s", err.Error())
continue
}
for _, item := range list.Items {
if !isSyncResource(&item) {
continue
}
u := unstructured.Unstructured{}
u.SetNamespace(item.GetNamespace())
u.SetName(item.GetName())
kinds, _, err := scheme.ObjectKinds(r)
if err != nil {
clog.Warn("error get object kinds: %s", err.Error())
continue
}
u.SetAPIVersion(kinds[0].GroupVersion().String())
kind := strings.TrimSuffix(kinds[0].Kind, "List")
u.SetKind(kind)

err = pivotClient.Get(context.Background(), client.ObjectKeyFromObject(&u), &u)
if err != nil {
if errors.IsNotFound(err) {
clog.Info("the resource %s/%s/%s/%s is not found on the pivot cluster, delete it", u.GetAPIVersion(), u.GetKind(), u.GetNamespace(), u.GetName())
err = g.Delete(context.Background(), &u)
if err != nil {
clog.Warn("error delete resource: %s", err.Error())
}
} else {
clog.Warn("error get resource: %s", err.Error())
}
}
}
}
}
8 changes: 5 additions & 3 deletions pkg/warden/syncmgr/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ import (
"github.com/kubecube-io/kubecube/pkg/clog"
"github.com/kubecube-io/kubecube/pkg/utils/exit"
"github.com/kubecube-io/kubecube/pkg/warden/reporter"
"github.com/kubecube-io/kubecube/pkg/warden/utils"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd"
)

const healthProbeAddr = "0.0.0.0:9777"
Expand All @@ -58,12 +58,13 @@ type SyncManager struct {
ctrl.Manager
LocalClient client.Client
PivotClusterKubeConfig string
PivotCubeHost string
}

func (s *SyncManager) Initialize() error {
log = clog.WithName("syncmgr")

cfg, err := clientcmd.BuildConfigFromFlags("", s.PivotClusterKubeConfig)
cfg, err := utils.GetPivotConfig(s.PivotClusterKubeConfig, s.PivotCubeHost)
if err != nil {
return fmt.Errorf("error building kubeconfig: %s", err.Error())
}
Expand All @@ -84,7 +85,8 @@ func (s *SyncManager) Initialize() error {
return err
}
}

gc := NewGc(cfg, s.LocalClient)
go gc.GcWork()
err = s.Manager.AddReadyzCheck("readyz", healthz.Ping)
if err != nil {
return err
Expand Down
16 changes: 16 additions & 0 deletions pkg/warden/syncmgr/resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,22 @@ var syncResources = []client.Object{
&quota.CubeResourceQuota{},
}

var syncListResources = []client.ObjectList{
// k8s resources
&v1.RoleBindingList{},
&v1.ClusterRoleBindingList{},
&v1.RoleList{},
&v1.ClusterRoleList{},

// kubecube resources
&hotplug.HotplugList{},
&tenant.TenantList{},
&tenant.ProjectList{},
&user.UserList{},
&extension.ExternalResourceList{},
&quota.CubeResourceQuotaList{},
}

type GenericObjFunc func(obj client.Object) (client.Object, error)

// newGenericObj new a struct point implemented client.Object
Expand Down
46 changes: 46 additions & 0 deletions pkg/warden/utils/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,52 @@ limitations under the License.

package utils

import (
"strings"

"k8s.io/api/authentication/v1beta1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"

"github.com/kubecube-io/kubecube/pkg/authentication/authenticators/jwt"
"github.com/kubecube-io/kubecube/pkg/utils/constants"
)

// Cluster local cluster name
// deprecated: global v is not good
var Cluster string

func GetPivotConfig(pivotClusterKubeConfig, pivotCubeHost string) (*restclient.Config, error) {
if len(pivotClusterKubeConfig) != 0 {
cfg, err := clientcmd.BuildConfigFromFlags("", pivotClusterKubeConfig)
if err != nil {
return nil, err
}
return cfg, nil
}
authJwtImpl := jwt.GetAuthJwtImpl()
token, errInfo := authJwtImpl.GenerateTokenWithExpired(&v1beta1.UserInfo{Username: "admin"}, 100*365*24*3600)
if errInfo != nil {
return nil, errInfo
}

host := pivotCubeHost
if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") {
// default, use https as scheme
host = "https://" + host
}
if strings.HasSuffix(host, "/") {
host = strings.TrimSuffix(host, "/")
}
host += constants.ApiK8sProxyPath

cfg := &restclient.Config{
Host: host,
BearerToken: token,
TLSClientConfig: restclient.TLSClientConfig{
Insecure: true,
},
}
return cfg, nil

}
10 changes: 4 additions & 6 deletions pkg/warden/warden.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ package warden
import (
"context"

"k8s.io/client-go/tools/clientcmd"

"github.com/kubecube-io/kubecube/pkg/clog"
multiclient "github.com/kubecube-io/kubecube/pkg/multicluster/client"
"github.com/kubecube-io/kubecube/pkg/warden/localmgr"
Expand All @@ -43,7 +41,7 @@ type Warden struct {
}

func NewWardenWithOpts(opts *Config) *Warden {
pivotClient, err := makePivotClient(opts.PivotClusterKubeConfig)
pivotClient, err := makePivotClient(opts)
if err != nil {
clog.Fatal("init pivot client failed: %v", err)
}
Expand Down Expand Up @@ -90,6 +88,7 @@ func NewWardenWithOpts(opts *Config) *Warden {
if opts.InMemberCluster {
w.SyncCtrl = &syncmgr.SyncManager{
PivotClusterKubeConfig: opts.PivotClusterKubeConfig,
PivotCubeHost: opts.PivotCubeHost,
}
}

Expand Down Expand Up @@ -137,12 +136,11 @@ func (w *Warden) Run(stop <-chan struct{}) {
}

// makePivotClient make client for pivot client
func makePivotClient(kubeconfig string) (multiclient.Client, error) {
cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
func makePivotClient(opts *Config) (multiclient.Client, error) {
cfg, err := utils.GetPivotConfig(opts.PivotClusterKubeConfig, opts.PivotCubeHost)
if err != nil {
return nil, err
}

cli, err := multiclient.NewClientFor(context.Background(), cfg)
if err != nil {
return nil, err
Expand Down

0 comments on commit 55cb1ed

Please sign in to comment.