Skip to content

Commit

Permalink
use internal endpoint for local-cluster (#425)
Browse files Browse the repository at this point in the history 
Signed-off-by: Yang Le <[email protected]>
  • Loading branch information
@elgnay
elgnay authored Nov 25, 2024
1 parent 0f1f6b3 commit 4b0a308
Show file tree
Hide file tree
Showing 10 changed files with 650 additions and 81 deletions.
134 changes: 126 additions & 8 deletions deploy/base/config.open-cluster-management.io_klusterletconfigs.crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,20 +46,136 @@ spec:
cluster.
pattern: ^([0-9]+(s|m|h))+$|^INFINITE$
type: string
bootstrapKubeConfigs:
description: BootstrapKubeConfigSecrets is the list of secrets that
reflects the Klusterlet.Spec.RegistrationConfiguration.BootstrapKubeConfigs.
properties:
localSecretsConfig:
description: LocalSecretsConfig include a list of secrets that
contains the kubeconfigs for ordered bootstrap kubeconifigs.
The secrets must be in the same namespace where the agent controller
runs.
properties:
hubConnectionTimeoutSeconds:
default: 600
description: HubConnectionTimeoutSeconds is used to set the
timeout of connecting to the hub cluster. When agent loses
the connection to the hub over the timeout seconds, the
agent do a rebootstrap. By default is 10 mins.
format: int32
minimum: 180
type: integer
kubeConfigSecrets:
description: KubeConfigSecrets is a list of secret names.
The secrets are in the same namespace where the agent controller
runs.
items:
properties:
name:
description: Name is the name of the secret.
type: string
type: object
type: array
type: object
type:
default: None
description: Type specifies the type of priority bootstrap kubeconfigs.
By default, it is set to None, representing no priority bootstrap
kubeconfigs are set.
enum:
- None
- LocalSecrets
type: string
type: object
hubKubeAPIServerCABundle:
description: 'HubKubeAPIServerCABundle is the CA bundle to verify
description: "HubKubeAPIServerCABundle is the CA bundle to verify
the server certificate of the hub kube API against. If not present,
CA bundle will be determined with the logic below: 1). Use the certificate
of the named certificate configured in APIServer/cluster if FQDN
matches; 2). Otherwise use the CA certificates from kube-root-ca.crt
ConfigMap in the cluster namespace;'
ConfigMap in the cluster namespace; \n Deprecated and maintained
for backward compatibility, use HubKubeAPIServerConfig.ServerVarificationStrategy
and HubKubeAPIServerConfig.TrustedCABundles instead"
format: byte
type: string
hubKubeAPIServerConfig:
description: 'HubKubeAPIServerConfig specifies the settings required
for connecting to the hub Kube API server. If this field is present,
the below deprecated fields will be ignored: - HubKubeAPIServerProxyConfig
- HubKubeAPIServerURL - HubKubeAPIServerCABundle'
properties:
proxyURL:
description: ProxyURL is the URL to the proxy to be used for all
requests made by client If an HTTPS proxy server is configured,
you may also need to add the necessary CA certificates to TrustedCABundles.
type: string
serverVerificationStrategy:
description: "ServerVerificationStrategy is the strategy used
for verifying the server certification; The value could be \"UseSystemTruststore\",
\"UseAutoDetectedCABundle\", \"UseCustomCABundles\", empty.
\n When this strategy is not set or value is empty; if there
is only one klusterletConfig configured for a cluster, the strategy
is eaual to \"UseAutoDetectedCABundle\", if there are more than
one klusterletConfigs, the empty strategy will be overrided
by other non-empty strategies."
enum:
- UseSystemTruststore
- UseAutoDetectedCABundle
- UseCustomCABundles
type: string
trustedCABundles:
description: TrustedCABundles refers to a collection of user-provided
CA bundles used for verifying the server certificate of the
hub Kubernetes API If the ServerVerificationStrategy is set
to "UseSystemTruststore", this field will be ignored. Otherwise,
the CA certificates from the configured bundles will be appended
to the klusterlet CA bundle.
items:
description: CABundle is a user-provided CA bundle
properties:
caBundle:
description: CABundle refers to a ConfigMap with label "import.open-cluster-management.io/ca-bundle"
containing the user-provided CA bundle The key of the
CA data could be "ca-bundle.crt", "ca.crt", or "tls.crt".
properties:
name:
description: name is the metadata.name of the referenced
config map
type: string
namespace:
description: name is the metadata.namespace of the referenced
config map
type: string
required:
- name
- namespace
type: object
name:
description: Name is the identifier used to reference the
CA bundle; Do not use "auto-detected" as the name since
it is the reserved name for the auto-detected CA bundle.
type: string
required:
- caBundle
- name
type: object
type: array
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
url:
description: URL is the endpoint of the hub Kube API server. If
not present, the .status.apiServerURL of Infrastructure/cluster
will be used as the default value. e.g. `oc get infrastructure
cluster -o jsonpath='{.status.apiServerURL}'`
type: string
type: object
hubKubeAPIServerProxyConfig:
description: HubKubeAPIServerProxyConfig holds proxy settings for
description: "HubKubeAPIServerProxyConfig holds proxy settings for
connections between klusterlet/add-on agents on the managed cluster
and the kube-apiserver on the hub cluster. Empty means no proxy
settings is available.
settings is available. \n Deprecated and maintained for backward
compatibility, use HubKubeAPIServerConfig.ProxyURL instead"
properties:
caBundle:
description: CABundle is a CA certificate bundle to verify the
Expand All @@ -78,10 +194,11 @@ spec:
type: string
type: object
hubKubeAPIServerURL:
description: HubKubeAPIServerURL is the URL of the hub Kube API server.
description: "HubKubeAPIServerURL is the URL of the hub Kube API server.
If not present, the .status.apiServerURL of Infrastructure/cluster
will be used as the default value. e.g. `oc get infrastructure cluster
-o jsonpath='{.status.apiServerURL}'`
-o jsonpath='{.status.apiServerURL}'` \n Deprecated and maintained
for backward compatibility, use HubKubeAPIServerConfig.URL instead"
type: string
installMode:
description: InstallMode is the mode to install the klusterlet
Expand Down Expand Up @@ -120,8 +237,8 @@ spec:
on. The default is an empty list.
type: object
tolerations:
description: Tolerations is attached by pods to tolerate any taint
that matches the triple <key,value,effect> using the matching
description: Tolerations are attached by pods to tolerate any
taint that matches the triple <key,value,effect> using the matching
operator <operator>. The default is an empty list.
items:
description: The pod this Toleration is attached to tolerates
Expand Down Expand Up @@ -199,6 +316,7 @@ spec:
description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
type: string
type: object
x-kubernetes-map-type: atomic
registries:
description: Registries includes the mirror and source registries.
The source registry will be replaced by the Mirror.
Expand Down
70 changes: 62 additions & 8 deletions pkg/bootstrap/bootstrapkubeconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,19 +39,27 @@ import (
)

const (
bootstrapSASuffix = "bootstrap-sa"
bootstrapSASuffix = "bootstrap-sa"
apiServerInternalEndpoint = "https://kubernetes.default.svc:443"
apiServerInternalEndpointCA = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
)

// create kubeconfig for bootstrap
func CreateBootstrapKubeConfig(ctxClusterName string,
kubeAPIServer, proxyURL string, caData, token []byte) ([]byte, error) {
kubeAPIServer, proxyURL, ca string, caData, token []byte) ([]byte, error) {

// CA file and CA data cannot be set simultaneously
if len(caData) > 0 {
ca = ""
}

bootstrapConfig := clientcmdapi.Config{
// Define a cluster stanza based on the bootstrap kubeconfig.
Clusters: map[string]*clientcmdapi.Cluster{
ctxClusterName: {
Server: kubeAPIServer,
InsecureSkipTLSVerify: false,
CertificateAuthority: ca,
CertificateAuthorityData: caData,
ProxyURL: proxyURL,
}},
Expand All @@ -76,21 +84,61 @@ func CreateBootstrapKubeConfig(ctxClusterName string,
return boostrapConfigData, err
}

// GetKubeAPIServerConfig returns the expected apiserver url, proxy url, ca file and ca data
// for cluster registration.
func GetKubeAPIServerConfig(ctx context.Context, clientHolder *helpers.ClientHolder, ns string,
klusterletConfig *klusterletconfigv1alpha1.KlusterletConfig) (string, string, []byte, error) {
klusterletConfig *klusterletconfigv1alpha1.KlusterletConfig, selfManaged bool) (string, string,
string, []byte, error) {
// the proxy settings in the klusterletConfig will be ignored when the internal endpoint
// is used for the self managed cluster
if selfManaged && !hasCustomServerURLOrStrategy(klusterletConfig) {
return apiServerInternalEndpoint, "", apiServerInternalEndpointCA, nil, nil
}

// get the proxy settings
proxy, _ := GetProxySettings(klusterletConfig)

// get the apiserver address
url, err := GetKubeAPIServerAddress(ctx, clientHolder.RuntimeClient, klusterletConfig)
if err != nil {
return "", "", nil, err
return "", "", "", nil, err
}

// get the ca data
caData, err := GetBootstrapCAData(ctx, clientHolder, url, ns, klusterletConfig)
if err != nil {
return "", "", nil, err
return "", "", "", nil, err
}

return url, proxy, caData, err
return url, proxy, "", caData, err
}

// Return true if the managed cluster has a custom URL or its server verification strategy
// is not `UseAutoDetectedCABundle`.
func hasCustomServerURLOrStrategy(klusterletConfig *klusterletconfigv1alpha1.KlusterletConfig) bool {
if klusterletConfig == nil {
return false
}

if klusterletConfig.Spec.HubKubeAPIServerConfig != nil {
if len(klusterletConfig.Spec.HubKubeAPIServerConfig.URL) > 0 {
return true
}
if len(klusterletConfig.Spec.HubKubeAPIServerConfig.ServerVerificationStrategy) > 0 &&
klusterletConfig.Spec.HubKubeAPIServerConfig.ServerVerificationStrategy !=
klusterletconfigv1alpha1.ServerVerificationStrategyUseAutoDetectedCABundle {
return true
}
} else {
if len(klusterletConfig.Spec.HubKubeAPIServerURL) > 0 {
return true
}
if len(klusterletConfig.Spec.HubKubeAPIServerCABundle) > 0 {
return true
}
}

return false
}

func GetBootstrapSAName(clusterName string) string {
Expand Down Expand Up @@ -561,15 +609,21 @@ func mergeCertificateData(caBundles ...[]byte) ([]byte, error) {
// - the proxy url
// - the context cluster name
func ValidateBootstrapKubeconfig(clusterName string,
kubeAPIServer, proxyURL string, caData []byte, ctxClusterName string,
requiredKubeAPIServer, requiredProxyURL string, requiredCAData []byte,
kubeAPIServer, proxyURL, ca string, caData []byte, ctxClusterName string,
requiredKubeAPIServer, requiredProxyURL, requiredCA string, requiredCAData []byte,
requiredCtxClusterName string) bool {
// validate kube api server endpoint
if kubeAPIServer != requiredKubeAPIServer {
klog.Infof("KubeAPIServer invalid for the managed cluster %s: %s", clusterName, kubeAPIServer)
return false
}

// validate kube api server CA file path
if ca != requiredCA {
klog.Infof("CA is invalid for the managed cluster %s: %s", clusterName, ca)
return false
}

// validate kube api server CA data
if !bytes.Equal(caData, requiredCAData) {
klog.Infof("CAdata is invalid for the managed cluster %s", clusterName)
Expand Down
Loading

0 comments on commit 4b0a308

Please sign in to comment.