Merge pull request #532 from l1b0k/feat/eip

Support migrate eip to ack-extend-network-controller

daemon/daemon.go

@@ -12,10 +12,15 @@ import (
 	"sync"
 	"time"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
+
 	"github.com/AliyunContainerService/terway/deviceplugin"
 	"github.com/AliyunContainerService/terway/pkg/aliyun"
 	"github.com/AliyunContainerService/terway/pkg/aliyun/client"
 	"github.com/AliyunContainerService/terway/pkg/aliyun/credential"
+	"github.com/AliyunContainerService/terway/pkg/apis/alibabacloud.com/v1beta1"
+	"github.com/AliyunContainerService/terway/pkg/apis/crds"
 	podENITypes "github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1"
 	"github.com/AliyunContainerService/terway/pkg/backoff"
 	vswpool "github.com/AliyunContainerService/terway/pkg/controller/vswitch"
@@ -34,6 +39,7 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	k8sErr "k8s.io/apimachinery/pkg/api/errors"
+	k8stypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/flowcontrol"
 )
@@ -1099,6 +1105,108 @@ func (n *networkService) exclusiveENIFromCRD(podInfo *types.PodInfo, waitReady b
 	return netConf, nil
 }
 
+func (n *networkService) migrateEIP(ctx context.Context, objs []interface{}) error {
+	once := sync.Once{}
+
+	for _, resObj := range objs {
+		podRes, ok := resObj.(types.PodResources)
+		if !ok {
+			continue
+		}
+		if podRes.PodInfo == nil || !podRes.PodInfo.EipInfo.PodEip {
+			continue
+		}
+		for _, eipRes := range podRes.Resources {
+			if eipRes.Type != types.ResourceTypeEIP {
+				continue
+			}
+			allocType := v1beta1.IPAllocTypeAuto
+			if podRes.PodInfo.EipInfo.PodEipID != "" {
+				allocType = v1beta1.IPAllocTypeStatic
+			}
+			releaseStrategy := v1beta1.ReleaseStrategyFollow
+			releaseAfter := ""
+
+			if podRes.PodInfo.IPStickTime > 0 {
+				releaseStrategy = v1beta1.ReleaseStrategyTTL
+				releaseAfter = podRes.PodInfo.IPStickTime.String()
+			}
+
+			var err error
+			once.Do(func() {
+				err = crds.RegisterCRD([]string{crds.CRDPodEIP})
+			})
+			if err != nil {
+				return err
+			}
+
+			ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+
+			l := serviceLog.WithField("name", fmt.Sprintf("%s/%s", podRes.PodInfo.Namespace, podRes.PodInfo.Name))
+
+			c := n.k8s.GetClient()
+			podEIP := &v1beta1.PodEIP{}
+			err = c.Get(ctx, k8stypes.NamespacedName{Namespace: podRes.PodInfo.Namespace, Name: podRes.PodInfo.Name}, podEIP)
+			if err == nil {
+				cancel()
+				l.Info("skip create podEIP, already exist")
+				continue
+			}
+			if !k8sErr.IsNotFound(err) {
+				cancel()
+				return err
+			}
+
+			err = retry.OnError(wait.Backoff{
+				Steps:    4,
+				Duration: 200 * time.Millisecond,
+				Factor:   5.0,
+				Jitter:   0.1,
+			}, func(err error) bool {
+				if k8sErr.IsTooManyRequests(err) {
+					return true
+				}
+				if k8sErr.IsInternalError(err) {
+					return true
+				}
+				return false
+			}, func() error {
+				podEIP = &v1beta1.PodEIP{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:        podRes.PodInfo.Name,
+						Namespace:   podRes.PodInfo.Namespace,
+						Annotations: map[string]string{},
+						Finalizers:  []string{"podeip-controller.alibabacloud.com/finalizer"},
+					},
+					Spec: v1beta1.PodEIPSpec{
+						AllocationID:       eipRes.ID,
+						BandwidthPackageID: podRes.PodInfo.EipInfo.PodEipBandwidthPackageID,
+						AllocationType: v1beta1.AllocationType{
+							Type:            allocType,
+							ReleaseStrategy: releaseStrategy,
+							ReleaseAfter:    releaseAfter,
+						},
+					},
+				}
+
+				l.Infof("create podEIP for %v", podRes)
+
+				err := c.Create(ctx, podEIP)
+				if k8sErr.IsAlreadyExists(err) {
+					return nil
+				}
+				return err
+			})
+			cancel()
+
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // tracing
 func (n *networkService) Config() []tracing.MapKeyValueEntry {
 	// name, daemon_mode, configFilePath, kubeconfig, master
@@ -1344,6 +1452,10 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (
 		}
 	}
 
+	if limit.TrunkPod() <= 0 {
+		config.EnableENITrunking = false
+	}
+
 	ecs := aliyun.NewAliyunImpl(aliyunClient, config.EnableENITrunking && !config.WaitTrunkENI, ipFamily, config.ENITagFilter)
 
 	netSrv.resourceDB, err = storage.NewDiskStorage(
@@ -1476,6 +1588,14 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (
 		}
 	}
 
+	if config.EnableEIPMigrate {
+		err = netSrv.migrateEIP(ctx, resObjList)
+		if err != nil {
+			return nil, err
+		}
+		serviceLog.Infof("eip migrate finished")
+	}
+
 	resStr, err := json.Marshal(localResource)
 	if err != nil {
 		return nil, err
@@ -1511,25 +1631,26 @@ func newNetworkService(ctx context.Context, configFilePath, daemonMode string) (
 		if err != nil {
 			return nil, errors.Wrapf(err, "error init ENI ip resource manager")
 		}
-		if config.EnableEIPPool == conditionTrue {
-			netSrv.eipResMgr = newEipResourceManager(ecs, netSrv.k8s, config.AllowEIPRob == conditionTrue)
-		}
 		netSrv.mgrForResource = map[string]ResourceManager{
 			types.ResourceTypeENIIP: netSrv.eniIPResMgr,
-			types.ResourceTypeEIP:   netSrv.eipResMgr,
 		}
+		if config.EnableEIPPool == conditionTrue && !config.EnableEIPMigrate {
+			netSrv.eipResMgr = newEipResourceManager(ecs, netSrv.k8s, config.AllowEIPRob == conditionTrue)
+			netSrv.mgrForResource[types.ResourceTypeEIP] = netSrv.eipResMgr
+		}
+
 	case daemonModeENIOnly:
 		//init eni
 		netSrv.eniResMgr, err = newENIResourceManager(poolConfig, ecs, localResource[types.ResourceTypeENI], ipFamily, netSrv.k8s, netSrv.ipamType)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error init eni resource manager")
 		}
-		if config.EnableEIPPool == conditionTrue && !config.EnableENITrunking {
-			netSrv.eipResMgr = newEipResourceManager(ecs, netSrv.k8s, config.AllowEIPRob == conditionTrue)
-		}
 		netSrv.mgrForResource = map[string]ResourceManager{
-			types.ResourceTypeENI: netSrv.eniResMgr,
-			types.ResourceTypeEIP: netSrv.eipResMgr,
+			types.ResourceTypeENIIP: netSrv.eniIPResMgr,
+		}
+		if config.EnableEIPPool == conditionTrue && !config.EnableENITrunking && !config.EnableEIPMigrate {
+			netSrv.eipResMgr = newEipResourceManager(ecs, netSrv.k8s, config.AllowEIPRob == conditionTrue)
+			netSrv.mgrForResource[types.ResourceTypeEIP] = netSrv.eipResMgr
 		}
 	default:
 		panic("unsupported daemon mode" + daemonMode)

docs/eip.md

@@ -0,0 +1,70 @@
+# eip
+
+## Feature Description
+
+This document aims to explain to users that the EIP (Elastic IP) functionality in Terway is deprecated and no longer maintained. Users can use the EIP functionality in ACK Extend Network Controller as an alternative.
+
+Why the EIP Functionality is Deprecated
+Due to technological advancements and evolving business requirements, the Terway team has decided to optimize and upgrade the EIP functionality. In order to provide better network capabilities and enhance user experience, the Terway team has separated the EIP functionality from Terway and integrated it into [ACK Extend Network Controller](https://help.aliyun.com/zh/ack/product-overview/ack-extend-network-controller).
+
+Using the EIP Functionality in ACK Extend Network Controller
+After deprecation of the EIP functionality in Terway, users can utilize the EIP functionality in ACK Extend Network Controller. ACK Extend Network Controller is an advanced container network plugin provided by Alibaba Cloud for extending Kubernetes network capabilities. It offers the Elastic IP (EIP) functionality and seamlessly integrates with other network functionalities in Terway.
+
+## How migration works
+
+ACK Extend Network Controller usr CRD to trace the lifecycle of EIPs.
+When enable the EIP migration feature in Terway, Terway will create a `podEIP` object for each EIP in the cluster.
+So ACK Extend Network Controller can get the EIPs information from the `podEIP` object and manage the EIPs. Also Terway will no longer manage the EIPs.
+
+### pod with specific EIP id
+
+For pod specific EIP id, the EIP is provided by user. The `spec.allocationType.type` of `podEIP` cr would be `Static` .
+For pod not specific EIP id, the EIP is create by terway , and the `spec.allocationType.type` of `podEIP` cr would be `Auto` .
+
+### non-stateful workload
+
+For non-stateful workload, the `spec.allocationType.releaseStrategy` of `podEIP` cr would be `Follow` .
+
+### stateful workload
+
+For stateful workload, the `spec.allocationType.releaseStrategy` of `podEIP` cr would be `TTL` .
+
+## Migrate to ACK Extend Network Controller
+
+1. Make sure ACK Extend Network Controller is **NOT** installed in your cluster. If ACK Extend Network Controller is already installed, please uninstall it first.
+2. Upgrade Terway to the latest version.
+3. Modify the Terway configuration file to enable the EIP migration functionality in Terway.
+   - Set the value of the `enable_eip_migrate` parameter to `true` .
+   - Keep the value of the `enable_eip_pool` parameter to `"true"` .
+
+    ```yaml
+    apiVersion: v1  
+    data:
+      10-terway.conf: |
+        {
+          "type": "terway"
+        }
+      eni_conf: |
+        {
+          "version": "1",
+    
+          "enable_eip_pool": "true",
+          "enable_eip_migrate": true,
+          "vswitch_selection_policy": "ordered"
+        }
+    kind: ConfigMap
+    metadata:
+      name: eni-config
+      namespace: kube-system
+    ```
+
+4. Make sure to restart the Terway daemonset after modifying the Terway configuration file.
+5. After the Terway daemonset is restarted, Terway will automatically migrate the EIPs in your cluster to ACK Extend Network Controller. You can check the migration status in the Terway log.
+6. Check the `podEIP` CRD object in your cluster to see if the config is successfully created. The CRD object has the same name and namespace as the Pod.
+7. Start the [ACK Extend Network Controller](https://www.alibabacloud.com/help/en/ack/ack-managed-and-ack-dedicated/user-guide/associate-an-eip-with-a-pod-1) with eip function.
+8. Turn off terway eip function.
+   - Set the value of the `enable_eip_migrate` parameter to `false` or delete it.
+   - Set the value of the `enable_eip_pool` parameter to `"false"` or delete it.
+   - Restart the Terway daemonset.
+
+By following the steps mentioned above, you can successfully replace the deprecated EIP functionality in Terway with the EIP functionality in ACK Extend Network Controller. If you have any questions or need further assistance, please feel free to contact us.

pkg/apis/alibabacloud.com/v1beta1/groupversion_info.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2022 l1b0k.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v1beta1 contains API Schema definitions for the network v1beta1 API group
+// +kubebuilder:object:generate=true
+// +groupName=alibabacloud.com
+package v1beta1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "alibabacloud.com", Version: "v1beta1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)

pkg/apis/alibabacloud.com/v1beta1/podeip_types.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2022 l1b0k.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// PodEIPSpec defines the desired state of PodEIP
+type PodEIPSpec struct {
+	// +kubebuilder:validation:Required
+	AllocationID string `json:"allocationID"`
+
+	BandwidthPackageID string `json:"bandwidthPackageID,omitempty"`
+
+	// +kubebuilder:validation:Required
+	AllocationType AllocationType `json:"allocationType"`
+}
+
+// AllocationType ip type and release strategy
+type AllocationType struct {
+	// +kubebuilder:default:=Auto
+	// +kubebuilder:validation:Required
+	Type IPAllocType `json:"type"`
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:Enum=Follow;TTL;Never
+	ReleaseStrategy ReleaseStrategy `json:"releaseStrategy"`
+	ReleaseAfter    string          `json:"releaseAfter,omitempty"` // go type 5m0s
+}
+
+// +kubebuilder:validation:Enum=Auto;Static
+
+// IPAllocType is the type for eip alloc strategy
+type IPAllocType string
+
+// IPAllocType
+const (
+	IPAllocTypeAuto   IPAllocType = "Auto"
+	IPAllocTypeStatic IPAllocType = "Static"
+)
+
+// +kubebuilder:validation:Enum=Follow;TTL;Never
+
+// ReleaseStrategy is the type for eip release strategy
+type ReleaseStrategy string
+
+// ReleaseStrategy
+const (
+	ReleaseStrategyFollow ReleaseStrategy = "Follow" // default policy
+	ReleaseStrategyTTL    ReleaseStrategy = "TTL"
+	ReleaseStrategyNever  ReleaseStrategy = "Never"
+)
+
+// PodEIPStatus defines the observed state of PodEIP
+type PodEIPStatus struct {
+	// eni
+	NetworkInterfaceID string `json:"networkInterfaceID,omitempty"`
+	PrivateIPAddress   string `json:"privateIPAddress,omitempty"`
+	// eip
+	EipAddress            string `json:"eipAddress,omitempty"`
+	ISP                   string `json:"isp,omitempty"`
+	InternetChargeType    string `json:"internetChargeType,omitempty"`
+	ResourceGroupID       string `json:"resourceGroupID,omitempty"`
+	Name                  string `json:"name,omitempty"`
+	PublicIpAddressPoolID string `json:"publicIpAddressPoolID,omitempty"` // nolint
+	Status                string `json:"status,omitempty"`
+
+	// BandwidthPackageID
+	BandwidthPackageID string `json:"bandwidthPackageID,omitempty"`
+
+	// PodLastSeen is the timestamp when pod resource last seen
+	PodLastSeen metav1.Time `json:"podLastSeen,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+
+// PodEIP is the Schema for the podeips API
+type PodEIP struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   PodEIPSpec   `json:"spec,omitempty"`
+	Status PodEIPStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// PodEIPList contains a list of PodEIP
+type PodEIPList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []PodEIP `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&PodEIP{}, &PodEIPList{})
+}

pkg/apis/crds/alibabacloud.com_podeips.yaml

@@ -0,0 +1,113 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.9.2
+  creationTimestamp: null
+  name: podeips.alibabacloud.com
+spec:
+  group: alibabacloud.com
+  names:
+    kind: PodEIP
+    listKind: PodEIPList
+    plural: podeips
+    singular: podeip
+  scope: Namespaced
+  versions:
+  - name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: PodEIP is the Schema for the podeips API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: PodEIPSpec defines the desired state of PodEIP
+            properties:
+              allocationID:
+                type: string
+              allocationType:
+                description: AllocationType ip type and release strategy
+                properties:
+                  releaseAfter:
+                    type: string
+                  releaseStrategy:
+                    allOf:
+                    - enum:
+                      - Follow
+                      - TTL
+                      - Never
+                    - enum:
+                      - Follow
+                      - TTL
+                      - Never
+                    description: ReleaseStrategy is the type for eip release strategy
+                    type: string
+                  type:
+                    default: Auto
+                    description: IPAllocType is the type for eip alloc strategy
+                    enum:
+                    - Auto
+                    - Static
+                    type: string
+                required:
+                - releaseStrategy
+                - type
+                type: object
+              bandwidthPackageID:
+                type: string
+            required:
+            - allocationID
+            - allocationType
+            type: object
+          status:
+            description: PodEIPStatus defines the observed state of PodEIP
+            properties:
+              bandwidth:
+                type: string
+              bandwidthPackageID:
+                description: BandwidthPackageID
+                type: string
+              eipAddress:
+                description: eip
+                type: string
+              internetChargeType:
+                type: string
+              isp:
+                type: string
+              name:
+                type: string
+              networkInterfaceID:
+                description: eni
+                type: string
+              period:
+                type: integer
+              podLastSeen:
+                description: PodLastSeen is the timestamp when pod resource last seen
+                format: date-time
+                type: string
+              privateIPAddress:
+                type: string
+              publicIpAddressPoolID:
+                type: string
+              resourceGroupID:
+                type: string
+              status:
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

pkg/apis/crds/register.go

@@ -5,9 +5,10 @@ import (
 	_ "embed"
 	"fmt"
 
-	"github.com/AliyunContainerService/terway/pkg/utils"
 	"k8s.io/apimachinery/pkg/api/errors"
 
+	"github.com/AliyunContainerService/terway/pkg/utils"
+
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -22,6 +23,8 @@ const (
 	CRDPodENI        = "podenis.network.alibabacloud.com"
 	CRDPodNetworking = "podnetworkings.network.alibabacloud.com"
 
+	CRDPodEIP = "podeips.alibabacloud.com"
+
 	crdVersionKey = "crd.network.alibabacloud.com/version"
 )
 
@@ -31,6 +34,9 @@ var (
 
 	//go:embed network.alibabacloud.com_podnetworkings.yaml
 	crdsPodNetworking []byte
+
+	//go:embed alibabacloud.com_podeips.yaml
+	crdsPodEIP []byte
 )
 
 func getCRD(name string) apiextensionsv1.CustomResourceDefinition {
@@ -40,6 +46,8 @@ func getCRD(name string) apiextensionsv1.CustomResourceDefinition {
 		crdBytes = crdsPodENI
 	case CRDPodNetworking:
 		crdBytes = crdsPodNetworking
+	case CRDPodEIP:
+		crdBytes = crdsPodEIP
 	default:
 		panic(fmt.Sprintf("crd %s name not exist", name))
 	}
@@ -52,6 +60,21 @@ func getCRD(name string) apiextensionsv1.CustomResourceDefinition {
 	return ciliumCRD
 }
 
+func createCRD(cs apiextensionsclient.Interface, name string) error {
+	log.Info("creating", "crd", name)
+	client := cs.ApiextensionsV1().CustomResourceDefinitions()
+	crd := getCRD(name)
+	_, err := client.Create(context.TODO(), &crd, metav1.CreateOptions{})
+	if err == nil {
+		return nil
+	}
+	if errors.IsAlreadyExists(err) {
+		log.Info("already exist", "name", name)
+		return nil
+	}
+	return err
+}
+
 func createOrUpdateCRD(cs apiextensionsclient.Interface, name string) error {
 	log.Info("syncing", "crd", name)
 	client := cs.ApiextensionsV1().CustomResourceDefinitions()
@@ -95,3 +118,14 @@ func RegisterCRDs() error {
 	log.Info("syncing crd success")
 	return nil
 }
+
+func RegisterCRD(crds []string) error {
+	for _, crd := range crds {
+		err := createCRD(utils.APIExtensionsClient, crd)
+		if err != nil {
+			return err
+		}
+	}
+	log.Info("create crd success")
+	return nil
+}

types/daemon/config.go

@@ -38,6 +38,7 @@ type Config struct {
 	EniCapShift            int                 `yaml:"eni_cap_shift" json:"eni_cap_shift"`
 	VSwitchSelectionPolicy string              `yaml:"vswitch_selection_policy" json:"vswitch_selection_policy" mod:"default=random"`
 	EnableEIPPool          string              `yaml:"enable_eip_pool" json:"enable_eip_pool"`
+	EnableEIPMigrate       bool                `yaml:"enable_eip_migrate" json:"enable_eip_migrate"`
 	IPStack                string              `yaml:"ip_stack" json:"ip_stack" validate:"oneof=ipv4 ipv6 dual" mod:"default=ipv4"` // default ipv4 , support ipv4 dual
 	// rob the eip instance even the eip already bound to other resource
 	AllowEIPRob                 string                  `yaml:"allow_eip_rob" json:"allow_eip_rob"`

types/scheme.go

@@ -0,0 +1,87 @@
+package types
+
+import (
+	corev1 "k8s.io/api/core/v1"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+
+	aliv1beta1 "github.com/AliyunContainerService/terway/pkg/apis/alibabacloud.com/v1beta1"
+	"github.com/AliyunContainerService/terway/pkg/apis/network.alibabacloud.com/v1beta1"
+)
+
+var Scheme = runtime.NewScheme()
+
+func init() {
+	utilruntime.Must(corev1.AddToScheme(Scheme))
+	utilruntime.Must(aliv1beta1.AddToScheme(Scheme))
+	utilruntime.Must(v1beta1.AddToScheme(Scheme))
+}
+
+func NewRESTMapper() meta.RESTMapper {
+	mapper := meta.NewDefaultRESTMapper(Scheme.PreferredVersionAllGroups())
+	// enumerate all supported versions, get the kinds, and register with the mapper how to address
+	// our resources.
+	for _, gv := range Scheme.PreferredVersionAllGroups() {
+		for kind := range Scheme.KnownTypes(gv) {
+			scope := meta.RESTScopeNamespace
+			if rootScopedKinds[gv.WithKind(kind).GroupKind()] {
+				scope = meta.RESTScopeRoot
+			}
+			mapper.Add(gv.WithKind(kind), scope)
+		}
+	}
+	return mapper
+}
+
+// hardcoded is good enough for the test we're running
+var rootScopedKinds = map[schema.GroupKind]bool{
+	{Group: "admission.k8s.io", Kind: "AdmissionReview"}: true,
+
+	{Group: "admissionregistration.k8s.io", Kind: "ValidatingWebhookConfiguration"}: true,
+	{Group: "admissionregistration.k8s.io", Kind: "MutatingWebhookConfiguration"}:   true,
+
+	{Group: "authentication.k8s.io", Kind: "TokenReview"}: true,
+
+	{Group: "authorization.k8s.io", Kind: "SubjectAccessReview"}:     true,
+	{Group: "authorization.k8s.io", Kind: "SelfSubjectAccessReview"}: true,
+	{Group: "authorization.k8s.io", Kind: "SelfSubjectRulesReview"}:  true,
+
+	{Group: "certificates.k8s.io", Kind: "CertificateSigningRequest"}: true,
+
+	{Group: "", Kind: "Node"}:             true,
+	{Group: "", Kind: "Namespace"}:        true,
+	{Group: "", Kind: "PersistentVolume"}: true,
+	{Group: "", Kind: "ComponentStatus"}:  true,
+
+	{Group: "extensions", Kind: "PodSecurityPolicy"}: true,
+
+	{Group: "policy", Kind: "PodSecurityPolicy"}: true,
+
+	{Group: "extensions", Kind: "PodSecurityPolicy"}: true,
+
+	{Group: "rbac.authorization.k8s.io", Kind: "ClusterRole"}:        true,
+	{Group: "rbac.authorization.k8s.io", Kind: "ClusterRoleBinding"}: true,
+
+	{Group: "scheduling.k8s.io", Kind: "PriorityClass"}: true,
+
+	{Group: "storage.k8s.io", Kind: "StorageClass"}:     true,
+	{Group: "storage.k8s.io", Kind: "VolumeAttachment"}: true,
+
+	{Group: "apiextensions.k8s.io", Kind: "CustomResourceDefinition"}: true,
+
+	{Group: "apiserver.k8s.io", Kind: "AdmissionConfiguration"}: true,
+
+	{Group: "audit.k8s.io", Kind: "Event"}:  true,
+	{Group: "audit.k8s.io", Kind: "Policy"}: true,
+
+	{Group: "apiregistration.k8s.io", Kind: "APIService"}: true,
+
+	{Group: "metrics.k8s.io", Kind: "NodeMetrics"}: true,
+
+	{Group: "wardle.example.com", Kind: "Fischer"}: true,
+
+	{Group: "network.alibabacloud.com", Kind: "PodNetworking"}: true,
+}