From 4b0a308ad4ffb0f4b689cb5527771c7434425f2a Mon Sep 17 00:00:00 2001 From: Yang Le Date: Mon, 25 Nov 2024 14:50:26 +0800 Subject: [PATCH] use internal endpoint for local-cluster (#425) Signed-off-by: Yang Le --- ...r-management.io_klusterletconfigs.crd.yaml | 134 ++++++++++++++++-- pkg/bootstrap/bootstrapkubeconfig.go | 70 +++++++-- pkg/bootstrap/bootstrapkubeconfig_test.go | 122 ++++++++++++++-- pkg/controller/agentregistration/server.go | 6 +- pkg/controller/importconfig/cluster_info.go | 30 ++-- .../importconfig/cluster_info_test.go | 10 ++ .../importconfig_controller_test.go | 101 ++++++++++++- test/e2e/e2e_suite_test.go | 130 ++++++++++++----- test/e2e/klusterletconfig_test.go | 124 +++++++++++++++- test/e2e/selfmanagedcluster_test.go | 4 + 10 files changed, 650 insertions(+), 81 deletions(-) diff --git a/deploy/base/config.open-cluster-management.io_klusterletconfigs.crd.yaml b/deploy/base/config.open-cluster-management.io_klusterletconfigs.crd.yaml index faf65d04..7e1367b4 100644 --- a/deploy/base/config.open-cluster-management.io_klusterletconfigs.crd.yaml +++ b/deploy/base/config.open-cluster-management.io_klusterletconfigs.crd.yaml @@ -46,20 +46,136 @@ spec: cluster. pattern: ^([0-9]+(s|m|h))+$|^INFINITE$ type: string + bootstrapKubeConfigs: + description: BootstrapKubeConfigSecrets is the list of secrets that + reflects the Klusterlet.Spec.RegistrationConfiguration.BootstrapKubeConfigs. + properties: + localSecretsConfig: + description: LocalSecretsConfig include a list of secrets that + contains the kubeconfigs for ordered bootstrap kubeconifigs. + The secrets must be in the same namespace where the agent controller + runs. + properties: + hubConnectionTimeoutSeconds: + default: 600 + description: HubConnectionTimeoutSeconds is used to set the + timeout of connecting to the hub cluster. When agent loses + the connection to the hub over the timeout seconds, the + agent do a rebootstrap. By default is 10 mins. + format: int32 + minimum: 180 + type: integer + kubeConfigSecrets: + description: KubeConfigSecrets is a list of secret names. + The secrets are in the same namespace where the agent controller + runs. + items: + properties: + name: + description: Name is the name of the secret. + type: string + type: object + type: array + type: object + type: + default: None + description: Type specifies the type of priority bootstrap kubeconfigs. + By default, it is set to None, representing no priority bootstrap + kubeconfigs are set. + enum: + - None + - LocalSecrets + type: string + type: object hubKubeAPIServerCABundle: - description: 'HubKubeAPIServerCABundle is the CA bundle to verify + description: "HubKubeAPIServerCABundle is the CA bundle to verify the server certificate of the hub kube API against. If not present, CA bundle will be determined with the logic below: 1). Use the certificate of the named certificate configured in APIServer/cluster if FQDN matches; 2). Otherwise use the CA certificates from kube-root-ca.crt - ConfigMap in the cluster namespace;' + ConfigMap in the cluster namespace; \n Deprecated and maintained + for backward compatibility, use HubKubeAPIServerConfig.ServerVarificationStrategy + and HubKubeAPIServerConfig.TrustedCABundles instead" format: byte type: string + hubKubeAPIServerConfig: + description: 'HubKubeAPIServerConfig specifies the settings required + for connecting to the hub Kube API server. If this field is present, + the below deprecated fields will be ignored: - HubKubeAPIServerProxyConfig + - HubKubeAPIServerURL - HubKubeAPIServerCABundle' + properties: + proxyURL: + description: ProxyURL is the URL to the proxy to be used for all + requests made by client If an HTTPS proxy server is configured, + you may also need to add the necessary CA certificates to TrustedCABundles. + type: string + serverVerificationStrategy: + description: "ServerVerificationStrategy is the strategy used + for verifying the server certification; The value could be \"UseSystemTruststore\", + \"UseAutoDetectedCABundle\", \"UseCustomCABundles\", empty. + \n When this strategy is not set or value is empty; if there + is only one klusterletConfig configured for a cluster, the strategy + is eaual to \"UseAutoDetectedCABundle\", if there are more than + one klusterletConfigs, the empty strategy will be overrided + by other non-empty strategies." + enum: + - UseSystemTruststore + - UseAutoDetectedCABundle + - UseCustomCABundles + type: string + trustedCABundles: + description: TrustedCABundles refers to a collection of user-provided + CA bundles used for verifying the server certificate of the + hub Kubernetes API If the ServerVerificationStrategy is set + to "UseSystemTruststore", this field will be ignored. Otherwise, + the CA certificates from the configured bundles will be appended + to the klusterlet CA bundle. + items: + description: CABundle is a user-provided CA bundle + properties: + caBundle: + description: CABundle refers to a ConfigMap with label "import.open-cluster-management.io/ca-bundle" + containing the user-provided CA bundle The key of the + CA data could be "ca-bundle.crt", "ca.crt", or "tls.crt". + properties: + name: + description: name is the metadata.name of the referenced + config map + type: string + namespace: + description: name is the metadata.namespace of the referenced + config map + type: string + required: + - name + - namespace + type: object + name: + description: Name is the identifier used to reference the + CA bundle; Do not use "auto-detected" as the name since + it is the reserved name for the auto-detected CA bundle. + type: string + required: + - caBundle + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + url: + description: URL is the endpoint of the hub Kube API server. If + not present, the .status.apiServerURL of Infrastructure/cluster + will be used as the default value. e.g. `oc get infrastructure + cluster -o jsonpath='{.status.apiServerURL}'` + type: string + type: object hubKubeAPIServerProxyConfig: - description: HubKubeAPIServerProxyConfig holds proxy settings for + description: "HubKubeAPIServerProxyConfig holds proxy settings for connections between klusterlet/add-on agents on the managed cluster and the kube-apiserver on the hub cluster. Empty means no proxy - settings is available. + settings is available. \n Deprecated and maintained for backward + compatibility, use HubKubeAPIServerConfig.ProxyURL instead" properties: caBundle: description: CABundle is a CA certificate bundle to verify the @@ -78,10 +194,11 @@ spec: type: string type: object hubKubeAPIServerURL: - description: HubKubeAPIServerURL is the URL of the hub Kube API server. + description: "HubKubeAPIServerURL is the URL of the hub Kube API server. If not present, the .status.apiServerURL of Infrastructure/cluster will be used as the default value. e.g. `oc get infrastructure cluster - -o jsonpath='{.status.apiServerURL}'` + -o jsonpath='{.status.apiServerURL}'` \n Deprecated and maintained + for backward compatibility, use HubKubeAPIServerConfig.URL instead" type: string installMode: description: InstallMode is the mode to install the klusterlet @@ -120,8 +237,8 @@ spec: on. The default is an empty list. type: object tolerations: - description: Tolerations is attached by pods to tolerate any taint - that matches the triple using the matching + description: Tolerations are attached by pods to tolerate any + taint that matches the triple using the matching operator . The default is an empty list. items: description: The pod this Toleration is attached to tolerates @@ -199,6 +316,7 @@ spec: description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string type: object + x-kubernetes-map-type: atomic registries: description: Registries includes the mirror and source registries. The source registry will be replaced by the Mirror. diff --git a/pkg/bootstrap/bootstrapkubeconfig.go b/pkg/bootstrap/bootstrapkubeconfig.go index ad8f4602..03ff79cd 100644 --- a/pkg/bootstrap/bootstrapkubeconfig.go +++ b/pkg/bootstrap/bootstrapkubeconfig.go @@ -39,12 +39,19 @@ import ( ) const ( - bootstrapSASuffix = "bootstrap-sa" + bootstrapSASuffix = "bootstrap-sa" + apiServerInternalEndpoint = "https://kubernetes.default.svc:443" + apiServerInternalEndpointCA = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" ) // create kubeconfig for bootstrap func CreateBootstrapKubeConfig(ctxClusterName string, - kubeAPIServer, proxyURL string, caData, token []byte) ([]byte, error) { + kubeAPIServer, proxyURL, ca string, caData, token []byte) ([]byte, error) { + + // CA file and CA data cannot be set simultaneously + if len(caData) > 0 { + ca = "" + } bootstrapConfig := clientcmdapi.Config{ // Define a cluster stanza based on the bootstrap kubeconfig. @@ -52,6 +59,7 @@ func CreateBootstrapKubeConfig(ctxClusterName string, ctxClusterName: { Server: kubeAPIServer, InsecureSkipTLSVerify: false, + CertificateAuthority: ca, CertificateAuthorityData: caData, ProxyURL: proxyURL, }}, @@ -76,21 +84,61 @@ func CreateBootstrapKubeConfig(ctxClusterName string, return boostrapConfigData, err } +// GetKubeAPIServerConfig returns the expected apiserver url, proxy url, ca file and ca data +// for cluster registration. func GetKubeAPIServerConfig(ctx context.Context, clientHolder *helpers.ClientHolder, ns string, - klusterletConfig *klusterletconfigv1alpha1.KlusterletConfig) (string, string, []byte, error) { + klusterletConfig *klusterletconfigv1alpha1.KlusterletConfig, selfManaged bool) (string, string, + string, []byte, error) { + // the proxy settings in the klusterletConfig will be ignored when the internal endpoint + // is used for the self managed cluster + if selfManaged && !hasCustomServerURLOrStrategy(klusterletConfig) { + return apiServerInternalEndpoint, "", apiServerInternalEndpointCA, nil, nil + } + + // get the proxy settings proxy, _ := GetProxySettings(klusterletConfig) + // get the apiserver address url, err := GetKubeAPIServerAddress(ctx, clientHolder.RuntimeClient, klusterletConfig) if err != nil { - return "", "", nil, err + return "", "", "", nil, err } + // get the ca data caData, err := GetBootstrapCAData(ctx, clientHolder, url, ns, klusterletConfig) if err != nil { - return "", "", nil, err + return "", "", "", nil, err } - return url, proxy, caData, err + return url, proxy, "", caData, err +} + +// Return true if the managed cluster has a custom URL or its server verification strategy +// is not `UseAutoDetectedCABundle`. +func hasCustomServerURLOrStrategy(klusterletConfig *klusterletconfigv1alpha1.KlusterletConfig) bool { + if klusterletConfig == nil { + return false + } + + if klusterletConfig.Spec.HubKubeAPIServerConfig != nil { + if len(klusterletConfig.Spec.HubKubeAPIServerConfig.URL) > 0 { + return true + } + if len(klusterletConfig.Spec.HubKubeAPIServerConfig.ServerVerificationStrategy) > 0 && + klusterletConfig.Spec.HubKubeAPIServerConfig.ServerVerificationStrategy != + klusterletconfigv1alpha1.ServerVerificationStrategyUseAutoDetectedCABundle { + return true + } + } else { + if len(klusterletConfig.Spec.HubKubeAPIServerURL) > 0 { + return true + } + if len(klusterletConfig.Spec.HubKubeAPIServerCABundle) > 0 { + return true + } + } + + return false } func GetBootstrapSAName(clusterName string) string { @@ -561,8 +609,8 @@ func mergeCertificateData(caBundles ...[]byte) ([]byte, error) { // - the proxy url // - the context cluster name func ValidateBootstrapKubeconfig(clusterName string, - kubeAPIServer, proxyURL string, caData []byte, ctxClusterName string, - requiredKubeAPIServer, requiredProxyURL string, requiredCAData []byte, + kubeAPIServer, proxyURL, ca string, caData []byte, ctxClusterName string, + requiredKubeAPIServer, requiredProxyURL, requiredCA string, requiredCAData []byte, requiredCtxClusterName string) bool { // validate kube api server endpoint if kubeAPIServer != requiredKubeAPIServer { @@ -570,6 +618,12 @@ func ValidateBootstrapKubeconfig(clusterName string, return false } + // validate kube api server CA file path + if ca != requiredCA { + klog.Infof("CA is invalid for the managed cluster %s: %s", clusterName, ca) + return false + } + // validate kube api server CA data if !bytes.Equal(caData, requiredCAData) { klog.Infof("CAdata is invalid for the managed cluster %s", clusterName) diff --git a/pkg/bootstrap/bootstrapkubeconfig_test.go b/pkg/bootstrap/bootstrapkubeconfig_test.go index 032af85f..47aadf46 100644 --- a/pkg/bootstrap/bootstrapkubeconfig_test.go +++ b/pkg/bootstrap/bootstrapkubeconfig_test.go @@ -12,7 +12,6 @@ import ( "reflect" "testing" - configv1 "github.com/openshift/api/config/v1" hivev1 "github.com/openshift/hive/apis/hive/v1" klusterletconfigv1alpha1 "github.com/stolostron/cluster-lifecycle-api/klusterletconfig/v1alpha1" "github.com/stolostron/managedcluster-import-controller/pkg/helpers" @@ -41,8 +40,8 @@ var testscheme = scheme.Scheme func init() { testscheme.AddKnownTypes(clusterv1.SchemeGroupVersion, &clusterv1.ManagedCluster{}) testscheme.AddKnownTypes(hivev1.SchemeGroupVersion, &hivev1.ClusterDeployment{}) - testscheme.AddKnownTypes(hivev1.SchemeGroupVersion, &configv1.Infrastructure{}) - testscheme.AddKnownTypes(hivev1.SchemeGroupVersion, &configv1.APIServer{}) + testscheme.AddKnownTypes(hivev1.SchemeGroupVersion, &ocinfrav1.Infrastructure{}) + testscheme.AddKnownTypes(hivev1.SchemeGroupVersion, &ocinfrav1.APIServer{}) } func TestGetKubeAPIServerConfig(t *testing.T) { @@ -180,6 +179,7 @@ func TestGetKubeAPIServerConfig(t *testing.T) { clientObjs []client.Object runtimeObjs []runtime.Object klusterletConfig *klusterletconfigv1alpha1.KlusterletConfig + selfManaged bool want wantData wantErr bool }{ @@ -387,6 +387,70 @@ func TestGetKubeAPIServerConfig(t *testing.T) { }, wantErr: false, }, + { + name: "self managed cluster", + clientObjs: []client.Object{testInfraConfigIP}, + runtimeObjs: []runtime.Object{cm, proxyCAcm}, + selfManaged: true, + want: wantData{ + serverURL: apiServerInternalEndpoint, + ca: apiServerInternalEndpointCA, + }, + wantErr: false, + }, + { + name: "self managed cluster with proxy settings", + clientObjs: []client.Object{testInfraConfigIP}, + runtimeObjs: []runtime.Object{cm, proxyCAcm}, + klusterletConfig: &klusterletconfigv1alpha1.KlusterletConfig{ + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + HubKubeAPIServerConfig: &klusterletconfigv1alpha1.KubeAPIServerConfig{ + ProxyURL: "https://127.0.0.1:3129", + }, + }, + }, + selfManaged: true, + want: wantData{ + serverURL: apiServerInternalEndpoint, + ca: apiServerInternalEndpointCA, + }, + wantErr: false, + }, + { + name: "self managed cluster with custom server address", + clientObjs: []client.Object{testInfraConfigIP}, + runtimeObjs: []runtime.Object{cm, proxyCAcm}, + klusterletConfig: &klusterletconfigv1alpha1.KlusterletConfig{ + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + HubKubeAPIServerConfig: &klusterletconfigv1alpha1.KubeAPIServerConfig{ + URL: "http://internal.com", + }, + }, + }, + selfManaged: true, + want: wantData{ + serverURL: "http://internal.com", + certData: rootCACertData, + }, + wantErr: false, + }, + { + name: "self managed cluster with custom strategy", + clientObjs: []client.Object{testInfraConfigIP}, + runtimeObjs: []runtime.Object{cm, proxyCAcm}, + klusterletConfig: &klusterletconfigv1alpha1.KlusterletConfig{ + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + HubKubeAPIServerConfig: &klusterletconfigv1alpha1.KubeAPIServerConfig{ + ServerVerificationStrategy: klusterletconfigv1alpha1.ServerVerificationStrategyUseSystemTruststore, + }, + }, + }, + selfManaged: true, + want: wantData{ + serverURL: "http://127.0.0.1:6443", + }, + wantErr: false, + }, } for _, tt := range testcases { @@ -403,8 +467,8 @@ func TestGetKubeAPIServerConfig(t *testing.T) { KubeClient: fakeKubeClinet, } - kubeAPIServer, proxyURL, caData, err := GetKubeAPIServerConfig( - context.Background(), clientHolder, cluster.Name, tt.klusterletConfig) + kubeAPIServer, proxyURL, ca, caData, err := GetKubeAPIServerConfig( + context.Background(), clientHolder, cluster.Name, tt.klusterletConfig, tt.selfManaged) if (err != nil) != tt.wantErr { t.Errorf("GetKubeAPIServerConfig() error = %v, wantErr %v", err, tt.wantErr) @@ -423,6 +487,14 @@ func TestGetKubeAPIServerConfig(t *testing.T) { ) } + if ca != tt.want.ca { + t.Errorf( + "GetKubeAPIServerConfig() returns wrong ca. want %v, got %v", + tt.want.ca, + ca, + ) + } + if !reflect.DeepEqual(caData, tt.want.certData) { t.Errorf( "GetKubeAPIServerConfig() returns wrong cert. want %v, got %v", @@ -451,6 +523,7 @@ func TestCreateBootstrapKubeConfig(t *testing.T) { type wantData struct { serverURL string + ca string certData []byte token string proxyURL string @@ -460,6 +533,7 @@ func TestCreateBootstrapKubeConfig(t *testing.T) { name string serverURL string + ca string certData []byte token string proxyURL string @@ -483,10 +557,12 @@ func TestCreateBootstrapKubeConfig(t *testing.T) { { name: "with CA file", serverURL: "http://127.0.0.1:6443", + ca: "/etc/ca.crt", ctxClusterName: "cluster1", token: "fake-token", want: wantData{ serverURL: "http://127.0.0.1:6443", + ca: "/etc/ca.crt", ctxClusterName: "cluster1", token: "fake-token", }, @@ -494,6 +570,7 @@ func TestCreateBootstrapKubeConfig(t *testing.T) { { name: "with both CA file and CA data", serverURL: "http://127.0.0.1:6443", + ca: "/etc/ca.crt", certData: rootCACertData, ctxClusterName: "cluster1", token: "fake-token", @@ -509,7 +586,7 @@ func TestCreateBootstrapKubeConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Logf("Test name: %s", tt.name) - kubeconfigData, err := CreateBootstrapKubeConfig(tt.ctxClusterName, tt.serverURL, tt.proxyURL, tt.certData, []byte(tt.token)) + kubeconfigData, err := CreateBootstrapKubeConfig(tt.ctxClusterName, tt.serverURL, tt.proxyURL, tt.ca, tt.certData, []byte(tt.token)) if err != nil { t.Errorf("CreateBootstrapKubeConfig() error = %v", err) return @@ -544,6 +621,14 @@ func TestCreateBootstrapKubeConfig(t *testing.T) { ) } + if clusterConfig.CertificateAuthority != tt.want.ca { + t.Errorf( + "createKubeconfigData() returns wrong ca. want %v, got %v", + tt.want.ca, + clusterConfig.CertificateAuthority, + ) + } + if !reflect.DeepEqual(clusterConfig.CertificateAuthorityData, tt.want.certData) { t.Errorf( "createKubeconfigData() returns wrong cert. want %v, got %v", @@ -1250,11 +1335,13 @@ func TestValidateBootstrapKubeconfig(t *testing.T) { name string kubeAPIServer string proxyURL string + ca string caData []byte ctxClusterName string requiredKubeAPIServer string requiredProxyURL string + requiredCA string requiredCAData []byte requiredCtxClusterName string @@ -1287,6 +1374,25 @@ func TestValidateBootstrapKubeconfig(t *testing.T) { caData: certData1, requiredCAData: certData2, }, + { + name: "replace ca data with ca file", + caData: certData1, + requiredCA: "/etc/ca.crt", + }, + { + name: "replace ca file with ca data", + requiredCA: "/etc/ca.crt", + requiredCAData: certData1, + }, + { + name: "ca file is empty", + requiredCA: "/etc/ca.crt", + }, + { + name: "ca file changed", + ca: "/etc/ca.crt", + requiredCA: "/etc/new-ca.crt", + }, { name: "all valid", kubeAPIServer: "https://api.my-cluster.example.com:6443", @@ -1302,8 +1408,8 @@ func TestValidateBootstrapKubeconfig(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { t.Logf("Test name: %s", c.name) - valid := ValidateBootstrapKubeconfig("cluster1", c.kubeAPIServer, c.proxyURL, c.caData, c.ctxClusterName, - c.requiredKubeAPIServer, c.requiredProxyURL, c.requiredCAData, c.requiredCtxClusterName) + valid := ValidateBootstrapKubeconfig("cluster1", c.kubeAPIServer, c.proxyURL, c.ca, c.caData, c.ctxClusterName, + c.requiredKubeAPIServer, c.requiredProxyURL, c.requiredCA, c.requiredCAData, c.requiredCtxClusterName) if valid != c.valid { t.Errorf("expected %v, but got %v", c.valid, valid) } diff --git a/pkg/controller/agentregistration/server.go b/pkg/controller/agentregistration/server.go index 1833478c..50623e90 100644 --- a/pkg/controller/agentregistration/server.go +++ b/pkg/controller/agentregistration/server.go @@ -119,8 +119,8 @@ func RunAgentRegistrationServer(ctx context.Context, port int, clientHolder *hel } // get the latest kube apiserver configuration - kubeAPIServer, proxyURL, caData, err := bootstrap.GetKubeAPIServerConfig( - ctx, clientHolder, ns, mergedKlusterletConfig) + kubeAPIServer, proxyURL, ca, caData, err := bootstrap.GetKubeAPIServerConfig( + ctx, clientHolder, ns, mergedKlusterletConfig, false) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -131,7 +131,7 @@ func RunAgentRegistrationServer(ctx context.Context, port int, clientHolder *hel return } - bootstrapkubeconfig, err := bootstrap.CreateBootstrapKubeConfig(ctxClusterName, kubeAPIServer, proxyURL, caData, token) + bootstrapkubeconfig, err := bootstrap.CreateBootstrapKubeConfig(ctxClusterName, kubeAPIServer, proxyURL, ca, caData, token) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/pkg/controller/importconfig/cluster_info.go b/pkg/controller/importconfig/cluster_info.go index 05d8b8b7..5b404250 100644 --- a/pkg/controller/importconfig/cluster_info.go +++ b/pkg/controller/importconfig/cluster_info.go @@ -6,6 +6,7 @@ package importconfig import ( "context" "fmt" + "strings" "time" klusterletconfigv1alpha1 "github.com/stolostron/cluster-lifecycle-api/klusterletconfig/v1alpha1" @@ -50,22 +51,23 @@ func extractBootstrapKubeConfigDataFromImportSecret(importSecret *corev1.Secret) } func parseKubeConfigData(kubeConfigData []byte) ( - kubeAPIServer, proxyURL string, caData []byte, token string, ctxClusterName string, err error) { + kubeAPIServer, proxyURL, ca string, caData []byte, token string, ctxClusterName string, err error) { config, err := clientcmd.Load(kubeConfigData) if err != nil { // kubeconfig data is invalid - return "", "", nil, "", "", err + return "", "", "", nil, "", "", err } context := config.Contexts[config.CurrentContext] if context == nil { - return "", "", nil, "", "", fmt.Errorf("failed to get current context") + return "", "", "", nil, "", "", fmt.Errorf("failed to get current context") } if cluster, ok := config.Clusters[context.Cluster]; ok { ctxClusterName = context.Cluster kubeAPIServer = cluster.Server + ca = cluster.CertificateAuthority caData = cluster.CertificateAuthorityData proxyURL = cluster.ProxyURL } @@ -120,8 +122,8 @@ func buildBootstrapKubeconfigData(ctx context.Context, clientHolder *helpers.Cli } // get the latest kube apiserver configuration - requiredKubeAPIServer, requiredProxyURL, requiredCAData, err := bootstrap.GetKubeAPIServerConfig( - ctx, clientHolder, managedCluster.Name, klusterletConfig) + requiredKubeAPIServer, requiredProxyURL, requiredCA, requiredCAData, err := bootstrap.GetKubeAPIServerConfig( + ctx, clientHolder, managedCluster.Name, klusterletConfig, isSelfManaged(managedCluster)) if err != nil { return nil, nil, nil, err } @@ -138,7 +140,7 @@ func buildBootstrapKubeconfigData(ctx context.Context, clientHolder *helpers.Cli // check if the bootstrap kubeconfig and token in the import secret are still valid if kubeconfigData := extractBootstrapKubeConfigDataFromImportSecret(importSecret); len(kubeconfigData) > 0 { - kubeAPIServer, proxyURL, caData, tokenString, ctxClusterName, err := parseKubeConfigData(kubeconfigData) + kubeAPIServer, proxyURL, ca, caData, tokenString, ctxClusterName, err := parseKubeConfigData(kubeconfigData) if err != nil { klog.Infof("failed to parse the bootstrap hub kubeconfig in the import.yaml. Recreation is required: %v", err) } else { @@ -156,8 +158,8 @@ func buildBootstrapKubeconfigData(ctx context.Context, clientHolder *helpers.Cli // use the kubeconfig if it is still valid if valid := bootstrap.ValidateBootstrapKubeconfig(managedCluster.Name, - kubeAPIServer, proxyURL, caData, ctxClusterName, - requiredKubeAPIServer, requiredProxyURL, requiredCAData, requiredCtxClusterName); valid { + kubeAPIServer, proxyURL, ca, caData, ctxClusterName, + requiredKubeAPIServer, requiredProxyURL, requiredCA, requiredCAData, requiredCtxClusterName); valid { bootstrapKubeconfigData = kubeconfigData } } @@ -181,7 +183,7 @@ func buildBootstrapKubeconfigData(ctx context.Context, clientHolder *helpers.Cli if len(bootstrapKubeconfigData) == 0 { klog.Infof("create a new bootstrap kubeconfig for the managed cluster %s", managedCluster.Name) bootstrapKubeconfigData, err = bootstrap.CreateBootstrapKubeConfig(requiredCtxClusterName, - requiredKubeAPIServer, requiredProxyURL, requiredCAData, tokenData) + requiredKubeAPIServer, requiredProxyURL, requiredCA, requiredCAData, tokenData) if err != nil { return nil, nil, nil, err } @@ -190,6 +192,16 @@ func buildBootstrapKubeconfigData(ctx context.Context, clientHolder *helpers.Cli return bootstrapKubeconfigData, tokenCreation, tokenExpiration, nil } +func isSelfManaged(managedCluster *clusterv1.ManagedCluster) bool { + if managedCluster == nil { + return false + } + if value := managedCluster.Labels[constants.SelfManagedLabel]; strings.EqualFold(value, "true") { + return true + } + return false +} + func buildImportSecret(ctx context.Context, clientHolder *helpers.ClientHolder, managedCluster *clusterv1.ManagedCluster, mode operatorv1.InstallMode, klusterletConfig *klusterletconfigv1alpha1.KlusterletConfig, bootstrapKubeconfigData, tokenCreation, tokenExpiration []byte) (*corev1.Secret, error) { diff --git a/pkg/controller/importconfig/cluster_info_test.go b/pkg/controller/importconfig/cluster_info_test.go index 95242f82..003b3ddb 100644 --- a/pkg/controller/importconfig/cluster_info_test.go +++ b/pkg/controller/importconfig/cluster_info_test.go @@ -260,6 +260,16 @@ func TestBuildBootstrapKubeconfigData(t *testing.T) { token: "mock-token", }, }, + { + name: "self managed cluster without import secret", + selfManaged: true, + wantErr: false, + want: &wantData{ + serverURL: "https://kubernetes.default.svc:443", + ca: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + token: "fake-token", + }, + }, } for _, tt := range tests { diff --git a/pkg/controller/importconfig/importconfig_controller_test.go b/pkg/controller/importconfig/importconfig_controller_test.go index 07501081..0e6d869d 100644 --- a/pkg/controller/importconfig/importconfig_controller_test.go +++ b/pkg/controller/importconfig/importconfig_controller_test.go @@ -913,7 +913,7 @@ func TestReconcile(t *testing.T) { t.Errorf("invalid bootstrap hub kubeconfig") } - _, proxyURL, caData, _, _, err := parseKubeConfigData(kubeConfigData) + _, proxyURL, _, caData, _, _, err := parseKubeConfigData(kubeConfigData) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -931,6 +931,105 @@ func TestReconcile(t *testing.T) { } }, }, + { + name: "self managed cluster", + clientObjs: []runtimeclient.Object{ + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{ + constants.SelfManagedLabel: "true", + }, + }, + }, + &configv1.Infrastructure{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + }, + }, + runtimeObjs: []runtime.Object{ + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bootstrap-sa", + Namespace: "test", + }, + Secrets: []corev1.ObjectReference{ + { + Name: "test-bootstrap-sa-token-5pw5c", + Namespace: "test", + }, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-bootstrap-sa-token-5pw5c", + Namespace: "test", + }, + Data: map[string][]byte{ + "token": []byte("fake-token"), + }, + Type: corev1.SecretTypeServiceAccountToken, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: os.Getenv("DEFAULT_IMAGE_PULL_SECRET"), + Namespace: os.Getenv("POD_NAMESPACE"), + }, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte("fake-token"), + }, + Type: corev1.SecretTypeDockerConfigJson, + }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-root-ca.crt", + Namespace: "test", + }, + Data: map[string]string{ + "ca.crt": string(rootCACertData), + }, + }, + }, + request: reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "test", + }, + }, + validateFunc: func(t *testing.T, client runtimeclient.Client, kubeClient kubernetes.Interface) { + importSecret, err := kubeClient.CoreV1().Secrets("test").Get(context.TODO(), "test-import", metav1.GetOptions{}) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + kubeConfigData := extractBootstrapKubeConfigDataFromImportSecret(importSecret) + if len(kubeConfigData) == 0 { + t.Errorf("invalid bootstrap hub kubeconfig") + } + + kubeAPIServer, _, ca, caData, _, _, err := parseKubeConfigData(kubeConfigData) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if kubeAPIServer != "https://kubernetes.default.svc:443" { + t.Errorf("expected apiserver address https://kubernetes.default.svc:443, bug got %s", kubeAPIServer) + } + + if ca != "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" { + t.Errorf("expected ca file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt, bug got %s", ca) + } + + if len(caData) > 0 { + t.Errorf("expected empty ca data, bug got %s", string(caData)) + } + }, + }, } for _, c := range cases { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 2d40a072..d424646a 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -341,6 +341,91 @@ func assertManagedClusterPriorityClass(managedClusterName string) { }) } +func assertBootstrapKubeconfig(serverURL, proxyURL, ca string, caData []byte, verifyHubKubeconfig bool) { + start := time.Now() + defer func() { + util.Logf("assert kubeconfig spending time: %.2f seconds", time.Since(start).Seconds()) + }() + ginkgo.By("Should have the expected bootstrap kubeconfig", func() { + gomega.Eventually(func() error { + err := assertKubeconfig("bootstrap-hub-kubeconfig", serverURL, proxyURL, ca, caData) + if err != nil { + return err + } + + if verifyHubKubeconfig { + return assertKubeconfig("hub-kubeconfig-secret", serverURL, proxyURL, ca, caData) + } + return nil + }, 120*time.Second, 1*time.Second).Should(gomega.Succeed()) + }) +} + +func assertBootstrapKubeconfigConsistently(serverURL, proxyURL, ca string, caData []byte, verifyHubKubeconfig bool, duration time.Duration) { + start := time.Now() + defer func() { + util.Logf("assert kubeconfig with internal endpoint consistently spending time: %.2f seconds", time.Since(start).Seconds()) + }() + ginkgo.By("Should use the internal endpoint", func() { + gomega.Consistently(func() error { + err := assertKubeconfig("bootstrap-hub-kubeconfig", serverURL, proxyURL, ca, caData) + if err != nil { + return err + } + + if verifyHubKubeconfig { + return assertKubeconfig("hub-kubeconfig-secret", serverURL, proxyURL, ca, caData) + } + return nil + }, duration, 1*time.Second).Should(gomega.Succeed()) + }) +} + +func assertKubeconfig(secretName, serverURL, proxyURL, ca string, caData []byte) error { + namespace := "open-cluster-management-agent" + secret, err := hubKubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return err + } + kubeconfigData, ok := secret.Data["kubeconfig"] + if !ok { + return fmt.Errorf("secret %s/%s has no kubeconfig", namespace, secretName) + } + + config, err := clientcmd.Load(kubeconfigData) + if err != nil { + return err + } + + context, ok := config.Contexts[config.CurrentContext] + if !ok { + return fmt.Errorf("kubeconfig in secret %s/%s has no context %q", namespace, secretName, config.CurrentContext) + } + + cluster, ok := config.Clusters[context.Cluster] + if !ok { + return fmt.Errorf("kubeconfig in secret %s/%s has no cluster %q", namespace, secretName, context.Cluster) + } + + if cluster.Server != serverURL { + return fmt.Errorf("kubeconfig in secret %s/%s expects server %q but got: %s", namespace, secretName, serverURL, cluster.Server) + } + + if cluster.CertificateAuthority != ca { + return fmt.Errorf("kubeconfig in secret %s/%s expects ca %q but got: %s", namespace, secretName, ca, cluster.CertificateAuthority) + } + + if cluster.ProxyURL != proxyURL { + return fmt.Errorf("kubeconfig in secret %s/%s expects proxy %q but got: %s", namespace, secretName, proxyURL, cluster.ProxyURL) + } + + if !reflect.DeepEqual(cluster.CertificateAuthorityData, caData) { + return fmt.Errorf("kubeconfig in secret %s/%s expects ca data %q but got: %s", namespace, secretName, string(caData), string(cluster.CertificateAuthorityData)) + } + + return nil +} + func assertManagedClusterPriorityClassHosted(managedClusterName string) { start := time.Now() defer func() { @@ -812,6 +897,13 @@ func assertBootstrapKubeconfigWithProxyConfig(proxyURL string, caDataIncluded, c return fmt.Errorf("expected proxy url %q but got: %s", proxyURL, cluster.ProxyURL) } + if len(cluster.CertificateAuthorityData) == 0 { + if len(caDataIncluded) == 0 { + return nil + } + return fmt.Errorf("kubeconfig has no ca bundle specified") + } + caCerts, err := certutil.ParseCertsPEM(cluster.CertificateAuthorityData) if err != nil { return err @@ -850,44 +942,6 @@ func assertBootstrapKubeconfigWithProxyConfig(proxyURL string, caDataIncluded, c }) } -func assertBootstrapKubeconfigServerURLAndCABundle(serverURL string, caData []byte) { - ginkgo.By("Klusterlet should have bootstrap kubeconfig with expected serverURL & CA bundle", func() { - var bootstrapKubeconfigSecret *corev1.Secret - gomega.Eventually(func() error { - var err error - bootstrapKubeconfigSecret, err = hubKubeClient.CoreV1().Secrets("open-cluster-management-agent").Get(context.TODO(), "bootstrap-hub-kubeconfig", metav1.GetOptions{}) - if err != nil { - return err - } - - config, err := clientcmd.Load(bootstrapKubeconfigSecret.Data["kubeconfig"]) - if err != nil { - return err - } - - // check server url - context, ok := config.Contexts[config.CurrentContext] - if !ok { - return fmt.Errorf("current context %s not found", config.CurrentContext) - } - cluster, ok := config.Clusters[context.Cluster] - if !ok { - return fmt.Errorf("cluster %s not found", context.Cluster) - } - if cluster.Server != serverURL { - return fmt.Errorf("expected server url %q but got: %s", serverURL, cluster.Server) - } - - // check ca data - if !reflect.DeepEqual(cluster.CertificateAuthorityData, caData) { - return fmt.Errorf("unexpected CA bundle is included in the bootstrap kubeconfig: open-cluster-management-agent/bootstrap-hub-kubeconfig") - } - - return nil - }, 60*time.Second, 1*time.Second).Should(gomega.Succeed()) - }) -} - func AssertKlusterletNamespace(clusterName, name, namespace string) { ginkgo.By(fmt.Sprintf("Klusterlet %s should be deployed in the namespace %s", name, namespace), func() { gomega.Eventually(func() error { diff --git a/test/e2e/klusterletconfig_test.go b/test/e2e/klusterletconfig_test.go index e7f18d51..cec868c1 100644 --- a/test/e2e/klusterletconfig_test.go +++ b/test/e2e/klusterletconfig_test.go @@ -16,6 +16,7 @@ import ( . "github.com/onsi/gomega" klusterletconfigv1alpha1 "github.com/stolostron/cluster-lifecycle-api/klusterletconfig/v1alpha1" "github.com/stolostron/managedcluster-import-controller/pkg/bootstrap" + "github.com/stolostron/managedcluster-import-controller/pkg/constants" "github.com/stolostron/managedcluster-import-controller/pkg/helpers" "github.com/stolostron/managedcluster-import-controller/test/e2e/util" corev1 "k8s.io/api/core/v1" @@ -136,8 +137,18 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( managedClusterName, map[string]string{ "agent.open-cluster-management.io/klusterlet-config": klusterletConfigName, - }, - util.NewLable("local-cluster", "true")) + }) + Expect(err).ToNot(HaveOccurred()) + }) + + By(fmt.Sprintf("Create auto-import-secret for managed cluster %s with kubeconfig", managedClusterName), func() { + secret, err := util.NewAutoImportSecret(hubKubeClient, managedClusterName) + Expect(err).ToNot(HaveOccurred()) + secret.Annotations = map[string]string{ + constants.AnnotationKeepingAutoImportSecret: "true", + } + + _, err = hubKubeClient.CoreV1().Secrets(managedClusterName).Create(context.TODO(), secret, metav1.CreateOptions{}) Expect(err).ToNot(HaveOccurred()) }) @@ -207,7 +218,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( assertManagedClusterAvailable(managedClusterName) }) - It("Should deploy the klusterlet with custom server URL and CA bundle", func() { + It("Should ignore the proxy config for self managed cluster", func() { By("Create managed cluster", func() { _, err := util.CreateManagedClusterWithShortLeaseDuration( hubClusterClient, @@ -219,6 +230,50 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( Expect(err).ToNot(HaveOccurred()) }) + // klusterletconfig is missing and it will be ignored + assertBootstrapKubeconfigWithProxyConfig("", nil, nil) + assertManagedClusterAvailable(managedClusterName) + + By("Create KlusterletConfig with http proxy", func() { + _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: klusterletConfigName, + }, + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + HubKubeAPIServerProxyConfig: klusterletconfigv1alpha1.KubeAPIServerProxyConfig{ + HTTPSProxy: "http://127.0.0.1:3128", + }, + }, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + assertBootstrapKubeconfigConsistently("https://kubernetes.default.svc:443", "", + "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", nil, true, 30*time.Second) + }) + + It("Should deploy the klusterlet with custom server URL and CA bundle", func() { + By("Create managed cluster", func() { + _, err := util.CreateManagedClusterWithShortLeaseDuration( + hubClusterClient, + managedClusterName, + map[string]string{ + "agent.open-cluster-management.io/klusterlet-config": klusterletConfigName, + }) + Expect(err).ToNot(HaveOccurred()) + }) + + By(fmt.Sprintf("Create auto-import-secret for managed cluster %s with kubeconfig", managedClusterName), func() { + secret, err := util.NewAutoImportSecret(hubKubeClient, managedClusterName) + Expect(err).ToNot(HaveOccurred()) + secret.Annotations = map[string]string{ + constants.AnnotationKeepingAutoImportSecret: "true", + } + + _, err = hubKubeClient.CoreV1().Secrets(managedClusterName).Create(context.TODO(), secret, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + // klusterletconfig is missing and it will be ignored defaultServerUrl, err := bootstrap.GetKubeAPIServerAddress(context.TODO(), hubRuntimeClient, nil) Expect(err).ToNot(HaveOccurred()) @@ -227,7 +282,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( RuntimeClient: hubRuntimeClient, }, defaultServerUrl, managedClusterName, nil) Expect(err).ToNot(HaveOccurred()) - assertBootstrapKubeconfigServerURLAndCABundle(defaultServerUrl, defaultCABundle) + assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false) assertManagedClusterAvailable(managedClusterName) customServerURL := "https://invalid.server.url:6443" @@ -247,7 +302,7 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( Expect(err).ToNot(HaveOccurred()) }) - assertBootstrapKubeconfigServerURLAndCABundle(customServerURL, customCAData) + assertBootstrapKubeconfig(customServerURL, "", "", customCAData, false) // here to restart agent pods to trigger bootstrap secret update to save time. restartAgentPods() @@ -259,7 +314,64 @@ var _ = Describe("Use KlusterletConfig to customize klusterlet manifests", func( Expect(err).ToNot(HaveOccurred()) }) - assertBootstrapKubeconfigServerURLAndCABundle(defaultServerUrl, defaultCABundle) + assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false) + + // here to restart agent pods to trigger bootstrap secret update to save time. + restartAgentPods() + // cluster should become available because custom server URL and CA bundle is removed + assertManagedClusterAvailable(managedClusterName) + }) + + It("Should deploy the klusterlet with custom server URL for self managed cluster", func() { + By("Create managed cluster", func() { + _, err := util.CreateManagedClusterWithShortLeaseDuration( + hubClusterClient, + managedClusterName, + map[string]string{ + "agent.open-cluster-management.io/klusterlet-config": klusterletConfigName, + }, + util.NewLable("local-cluster", "true")) + Expect(err).ToNot(HaveOccurred()) + }) + + // klusterletconfig is missing and it will be ignored + assertBootstrapKubeconfig("https://kubernetes.default.svc:443", "", + "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", nil, false) + assertManagedClusterAvailable(managedClusterName) + + defaultServerUrl, err := bootstrap.GetKubeAPIServerAddress(context.TODO(), hubRuntimeClient, nil) + Expect(err).ToNot(HaveOccurred()) + + By("Create KlusterletConfig with custom server URL", func() { + _, err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Create(context.TODO(), &klusterletconfigv1alpha1.KlusterletConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: klusterletConfigName, + }, + Spec: klusterletconfigv1alpha1.KlusterletConfigSpec{ + HubKubeAPIServerURL: defaultServerUrl, + }, + }, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + defaultCABundle, err := bootstrap.GetBootstrapCAData(context.TODO(), &helpers.ClientHolder{ + KubeClient: hubKubeClient, + RuntimeClient: hubRuntimeClient, + }, defaultServerUrl, managedClusterName, nil) + Expect(err).ToNot(HaveOccurred()) + assertBootstrapKubeconfig(defaultServerUrl, "", "", defaultCABundle, false) + + // here to restart agent pods to trigger bootstrap secret update to save time. + restartAgentPods() + assertManagedClusterAvailable(managedClusterName) + + By("Delete Klusterletconfig", func() { + err := klusterletconfigClient.ConfigV1alpha1().KlusterletConfigs().Delete(context.TODO(), klusterletConfigName, metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + }) + + assertBootstrapKubeconfig("https://kubernetes.default.svc:443", "", + "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", nil, false) // here to restart agent pods to trigger bootstrap secret update to save time. restartAgentPods() diff --git a/test/e2e/selfmanagedcluster_test.go b/test/e2e/selfmanagedcluster_test.go index d4824b43..07a84b79 100644 --- a/test/e2e/selfmanagedcluster_test.go +++ b/test/e2e/selfmanagedcluster_test.go @@ -38,6 +38,8 @@ var _ = ginkgo.Describe("Importing a self managed cluster", func() { assertManagedClusterAvailable(localClusterName) assertManagedClusterManifestWorksAvailable(localClusterName) assertManagedClusterPriorityClass(localClusterName) + assertBootstrapKubeconfig("https://kubernetes.default.svc:443", "", + "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", nil, true) }) }) @@ -83,6 +85,8 @@ var _ = ginkgo.Describe("Importing a self managed cluster", func() { assertManagedClusterAvailable(managedClusterName) assertManagedClusterManifestWorksAvailable(managedClusterName) assertManagedClusterPriorityClass(managedClusterName) + assertBootstrapKubeconfig("https://kubernetes.default.svc:443", "", + "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", nil, true) }) }) })