Kubernetes App Auto Discovery: improve protocol detection

Kubernetes App Auto Discovery iterates over all Services and tries to
auto enroll them as Teleport Applications.

During this process, it tries to guess the Service's port protocol to
ensure we add the application only if it's either an HTTP or HTTPS
capable Service.

When there's not annotation configuration (which are teleport specific),
we try to infer from the Service's ports.

When that doesn't work out, the teleport-agent issues an HTTP HEAD
request against the port. This way we detect whether the service can
answer HTTP or HTTPS.

This PR changes the way teleport infers the protocol using the Service's
Port.
It was checking for HTTPS (checking for port number and port name), then
it did a HTTP HEAD request and then it was checking for HTTP (checking
port number and port name).

This PR changes 4 things:
- checks the port, the node port and the target port against well known
  ports (443, 80, 8080)
- checks the name of the port in bother Port.Name and Port.TargetPort
- tries to do HTTPS and HTTP checks before trying an HTTP request
- decreases the HTTP request timeout from 5s to 500ms

With a demo cluster with 2700+ Services, the reconciliation time
decreased from 2m to something very close to 0s.

lib/srv/discovery/fetchers/kube_services.go

@@ -102,7 +102,7 @@ func isInternalKubeService(s v1.Service) bool {
 		s.GetNamespace() == metav1.NamespacePublic
 }
 
-func (f *KubeAppFetcher) getServices(ctx context.Context) ([]v1.Service, error) {
+func (f *KubeAppFetcher) getServices(ctx context.Context, discoveryType string) ([]v1.Service, error) {
 	var result []v1.Service
 	nextToken := ""
 	namespaceFilter := func(ns string) bool {
@@ -125,6 +125,17 @@ func (f *KubeAppFetcher) getServices(ctx context.Context) ([]v1.Service, error)
 				// Namespace is not in the list of namespaces to fetch or it's an internal service
 				continue
 			}
+
+			// Skip service if it has type annotation and it's not the expected type.
+			if v, ok := s.GetAnnotations()[types.DiscoveryTypeLabel]; ok && v != discoveryType {
+				continue
+			}
+
+			// If the service is marked with the ignore annotation, skip it.
+			if v := s.GetAnnotations()[types.DiscoveryAppIgnore]; v == "true" {
+				continue
+			}
+
 			match, _, err := services.MatchLabels(f.FilterLabels, s.Labels)
 			if err != nil {
 				return nil, trace.Wrap(err)
@@ -150,7 +161,7 @@ const (
 
 // Get fetches Kubernetes apps from the cluster
 func (f *KubeAppFetcher) Get(ctx context.Context) (types.ResourcesWithLabels, error) {
-	kubeServices, err := f.getServices(ctx)
+	kubeServices, err := f.getServices(ctx, types.KubernetesMatchersApp)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -159,7 +170,7 @@ func (f *KubeAppFetcher) Get(ctx context.Context) (types.ResourcesWithLabels, er
 	// Both services and ports inside services are processed in parallel to minimize time.
 	// We also set limit to prevent potential spike load on a cluster in case there are a lot of services.
 	g, _ := errgroup.WithContext(ctx)
-	g.SetLimit(10)
+	g.SetLimit(20)
 
 	// Convert services to resources
 	var (
@@ -168,17 +179,6 @@ func (f *KubeAppFetcher) Get(ctx context.Context) (types.ResourcesWithLabels, er
 	)
 	for _, service := range kubeServices {
 		service := service
-
-		// Skip service if it has type annotation and it's not 'app'
-		if v, ok := service.GetAnnotations()[types.DiscoveryTypeLabel]; ok && v != types.KubernetesMatchersApp {
-			continue
-		}
-
-		// If the service is marked with the ignore annotation, skip it.
-		if v := service.GetAnnotations()[types.DiscoveryAppIgnore]; v == "true" {
-			continue
-		}
-
 		g.Go(func() error {
 			protocolAnnotation := service.GetAnnotations()[types.DiscoveryProtocolLabel]
 
@@ -256,9 +256,9 @@ func (f *KubeAppFetcher) String() string {
 // by protocol checker. It is used when no explicit annotation for port's protocol was provided.
 //   - If port's AppProtocol specifies `http` or `https` we return it
 //   - If port's name is `https` or number is 443 we return `https`
+//   - If port's name is `http` or number is 80 or 8080, we return `http`
 //   - If protocol checker is available it will perform HTTP request to the service fqdn trying to find out protocol. If it
 //     gives us result `http` or `https` we return it
-//   - If port's name is `http` or number is 80 or 8080, we return `http`
 func autoProtocolDetection(serviceFQDN string, port v1.ServicePort, pc ProtocolChecker) string {
 	if port.AppProtocol != nil {
 		switch p := strings.ToLower(*port.AppProtocol); p {
@@ -267,21 +267,26 @@ func autoProtocolDetection(serviceFQDN string, port v1.ServicePort, pc ProtocolC
 		}
 	}
 
-	if port.Port == 443 || strings.EqualFold(port.Name, protoHTTPS) {
+	if strings.EqualFold(port.Name, protoHTTPS) || strings.EqualFold(port.TargetPort.StrVal, protoHTTPS) ||
+		port.Port == 443 || port.NodePort == 443 || port.TargetPort.IntVal == 443 {
+
 		return protoHTTPS
 	}
 
+	if strings.EqualFold(port.Name, protoHTTP) || strings.EqualFold(port.TargetPort.StrVal, protoHTTP) ||
+		port.Port == 80 || port.NodePort == 80 || port.TargetPort.IntVal == 80 ||
+		port.Port == 8080 || port.NodePort == 8080 || port.TargetPort.IntVal == 8080 {
+
+		return protoHTTP
+	}
+
 	if pc != nil {
 		result := pc.CheckProtocol(fmt.Sprintf("%s:%d", serviceFQDN, port.Port))
 		if result != protoTCP {
 			return result
 		}
 	}
 
-	if port.Port == 80 || port.Port == 8080 || strings.EqualFold(port.Name, protoHTTP) {
-		return protoHTTP
-	}
-
 	return protoTCP
 }
 
@@ -327,7 +332,9 @@ func NewProtoChecker(insecureSkipVerify bool) *ProtoChecker {
 	p := &ProtoChecker{
 		InsecureSkipVerify: insecureSkipVerify,
 		client: &http.Client{
-			Timeout: 5 * time.Second,
+			// This is a best-effort scenario, where teleport tries to guess which protocol is being used.
+			// Ideally it should either be inferred by the Service's ports or explicitly configured by using annotations on the service.
+			Timeout: 500 * time.Millisecond,
 			Transport: &http.Transport{
 				TLSClientConfig: &tls.Config{
 					InsecureSkipVerify: insecureSkipVerify,