Skip to content

Commit

Permalink
Kubernetes App Auto Discovery: improve protocol detection
Browse files Browse the repository at this point in the history
Kubernetes App Auto Discovery iterates over all Services and tries to
auto enroll them as Teleport Applications.

During this process, it tries to guess the Service's port protocol to
ensure we add the application only if it's either an HTTP or HTTPS
capable Service.

When there's not annotation configuration (which are teleport specific),
we try to infer from the Service's ports.

When that doesn't work out, the teleport-agent issues an HTTP HEAD
request against the port. This way we detect whether the service can
answer HTTP or HTTPS.

This PR changes the way teleport infers the protocol using the Service's
Port.
It was checking for HTTPS (checking for port number and port name), then
it did a HTTP HEAD request and then it was checking for HTTP (checking
port number and port name).

This PR changes 4 things:
- checks the port, the node port and the target port against well known
  ports (443, 80, 8080)
- checks the name of the port in bother Port.Name and Port.TargetPort
- tries to do HTTPS and HTTP checks before trying an HTTP request
- decreases the HTTP request timeout from 5s to 500ms

With a demo cluster with 2700+ Services, the reconciliation time
decreased from 2m to something very close to 0s.
  • Loading branch information
marcoandredinis authored and github-actions committed Dec 16, 2024
1 parent ff40022 commit 8336a39
Showing 1 changed file with 28 additions and 21 deletions.
49 changes: 28 additions & 21 deletions lib/srv/discovery/fetchers/kube_services.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func isInternalKubeService(s v1.Service) bool {
s.GetNamespace() == metav1.NamespacePublic
}

func (f *KubeAppFetcher) getServices(ctx context.Context) ([]v1.Service, error) {
func (f *KubeAppFetcher) getServices(ctx context.Context, discoveryType string) ([]v1.Service, error) {
var result []v1.Service
nextToken := ""
namespaceFilter := func(ns string) bool {
Expand All @@ -125,6 +125,17 @@ func (f *KubeAppFetcher) getServices(ctx context.Context) ([]v1.Service, error)
// Namespace is not in the list of namespaces to fetch or it's an internal service
continue
}

// Skip service if it has type annotation and it's not the expected type.
if v, ok := s.GetAnnotations()[types.DiscoveryTypeLabel]; ok && v != discoveryType {
continue
}

// If the service is marked with the ignore annotation, skip it.
if v := s.GetAnnotations()[types.DiscoveryAppIgnore]; v == "true" {
continue
}

match, _, err := services.MatchLabels(f.FilterLabels, s.Labels)
if err != nil {
return nil, trace.Wrap(err)
Expand All @@ -150,7 +161,7 @@ const (

// Get fetches Kubernetes apps from the cluster
func (f *KubeAppFetcher) Get(ctx context.Context) (types.ResourcesWithLabels, error) {
kubeServices, err := f.getServices(ctx)
kubeServices, err := f.getServices(ctx, types.KubernetesMatchersApp)
if err != nil {
return nil, trace.Wrap(err)
}
Expand All @@ -159,7 +170,7 @@ func (f *KubeAppFetcher) Get(ctx context.Context) (types.ResourcesWithLabels, er
// Both services and ports inside services are processed in parallel to minimize time.
// We also set limit to prevent potential spike load on a cluster in case there are a lot of services.
g, _ := errgroup.WithContext(ctx)
g.SetLimit(10)
g.SetLimit(20)

// Convert services to resources
var (
Expand All @@ -168,17 +179,6 @@ func (f *KubeAppFetcher) Get(ctx context.Context) (types.ResourcesWithLabels, er
)
for _, service := range kubeServices {
service := service

// Skip service if it has type annotation and it's not 'app'
if v, ok := service.GetAnnotations()[types.DiscoveryTypeLabel]; ok && v != types.KubernetesMatchersApp {
continue
}

// If the service is marked with the ignore annotation, skip it.
if v := service.GetAnnotations()[types.DiscoveryAppIgnore]; v == "true" {
continue
}

g.Go(func() error {
protocolAnnotation := service.GetAnnotations()[types.DiscoveryProtocolLabel]

Expand Down Expand Up @@ -256,9 +256,9 @@ func (f *KubeAppFetcher) String() string {
// by protocol checker. It is used when no explicit annotation for port's protocol was provided.
// - If port's AppProtocol specifies `http` or `https` we return it
// - If port's name is `https` or number is 443 we return `https`
// - If port's name is `http` or number is 80 or 8080, we return `http`
// - If protocol checker is available it will perform HTTP request to the service fqdn trying to find out protocol. If it
// gives us result `http` or `https` we return it
// - If port's name is `http` or number is 80 or 8080, we return `http`
func autoProtocolDetection(serviceFQDN string, port v1.ServicePort, pc ProtocolChecker) string {
if port.AppProtocol != nil {
switch p := strings.ToLower(*port.AppProtocol); p {
Expand All @@ -267,21 +267,26 @@ func autoProtocolDetection(serviceFQDN string, port v1.ServicePort, pc ProtocolC
}
}

if port.Port == 443 || strings.EqualFold(port.Name, protoHTTPS) {
if strings.EqualFold(port.Name, protoHTTPS) || strings.EqualFold(port.TargetPort.StrVal, protoHTTPS) ||
port.Port == 443 || port.NodePort == 443 || port.TargetPort.IntVal == 443 {

return protoHTTPS
}

if strings.EqualFold(port.Name, protoHTTP) || strings.EqualFold(port.TargetPort.StrVal, protoHTTP) ||
port.Port == 80 || port.NodePort == 80 || port.TargetPort.IntVal == 80 ||
port.Port == 8080 || port.NodePort == 8080 || port.TargetPort.IntVal == 8080 {

return protoHTTP
}

if pc != nil {
result := pc.CheckProtocol(fmt.Sprintf("%s:%d", serviceFQDN, port.Port))
if result != protoTCP {
return result
}
}

if port.Port == 80 || port.Port == 8080 || strings.EqualFold(port.Name, protoHTTP) {
return protoHTTP
}

return protoTCP
}

Expand Down Expand Up @@ -327,7 +332,9 @@ func NewProtoChecker(insecureSkipVerify bool) *ProtoChecker {
p := &ProtoChecker{
InsecureSkipVerify: insecureSkipVerify,
client: &http.Client{
Timeout: 5 * time.Second,
// This is a best-effort scenario, where teleport tries to guess which protocol is being used.
// Ideally it should either be inferred by the Service's ports or explicitly configured by using annotations on the service.
Timeout: 500 * time.Millisecond,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: insecureSkipVerify,
Expand Down

0 comments on commit 8336a39

Please sign in to comment.