Configure Alloy to be use as a self-service tool for logs (source podlogs)

CHANGELOG.md

@@ -10,6 +10,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Add manual e2e testing procedure and script.
+- [Alloy] Add capability to dynamically configure log targets using the `giantswarm.io/logging` label [#3618](https://github.com/giantswarm/roadmap/issues/3518)
+  - Changes Alloy config to use PodLogs for Kubernetes pods discovery.
+  - There is a performance impact on Kubernetes API
+  - Available from observability-bundle v17.0.0
+
+### Changed
+
+- [Alloy] Use PodLogs for Kubernetes pods discovery.
+- [Alloy] Enable clustering
 - Expose healthcheck port for kube-linter.
 
 ### Removed

go.mod

@@ -1,6 +1,6 @@
 module github.com/giantswarm/logging-operator
 
-go 1.22.0
+go 1.23
 
 toolchain go1.23.2
 
@@ -9,6 +9,7 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/giantswarm/apiextensions-application v0.6.2
 	github.com/giantswarm/grafana-multi-tenant-proxy v0.8.0
+	github.com/google/go-cmp v0.6.0
 	github.com/onsi/ginkgo/v2 v2.20.2
 	github.com/onsi/gomega v1.34.2
 	github.com/pkg/errors v0.9.1
@@ -46,7 +47,6 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -84,7 +84,7 @@ require (
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiextensions-apiserver v0.31.0 // indirect
+	k8s.io/apiextensions-apiserver v0.31.1 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect
 	k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect

go.sum

@@ -263,18 +263,18 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU=
 k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI=
-k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk=
-k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk=
+k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40=
+k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ=
 k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U=
 k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
-k8s.io/apiserver v0.31.0 h1:p+2dgJjy+bk+B1Csz+mc2wl5gHwvNkC9QJV+w55LVrY=
-k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk=
+k8s.io/apiserver v0.31.1 h1:Sars5ejQDCRBY5f7R3QFHdqN3s61nhkpaX8/k1iEw1c=
+k8s.io/apiserver v0.31.1/go.mod h1:lzDhpeToamVZJmmFlaLwdYZwd7zB+WYRYIboqA1kGxM=
 k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0=
 k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg=
 k8s.io/cluster-bootstrap v0.30.3 h1:MgxyxMkpaC6mu0BKWJ8985XCOnKU+eH3Iy+biwtDXRk=
 k8s.io/cluster-bootstrap v0.30.3/go.mod h1:h8BoLDfdD7XEEIXy7Bx9FcMzxHwz29jsYYi34bM5DKU=
-k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs=
-k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo=
+k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8=
+k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo=

helm/logging-operator/templates/deployment.yaml

@@ -39,6 +39,7 @@ spec:
           - -insecure-ca={{ .Values.managementCluster.insecureCA }}
           - -installation-name={{ .Values.managementCluster.name }}
           - -logging-agent={{ .Values.loggingOperator.loggingAgent }}
+          - -default-namespaces={{ .Values.loggingOperator.defaultNamespaces }}
         livenessProbe:
           httpGet:
             path: /healthz

helm/logging-operator/values.yaml

@@ -38,6 +38,7 @@ networkPolicy:
   flavor: cilium
 
 loggingOperator:
+  defaultNamespaces: "kube-system,giantswarm"
   vintageMode: true
   loggingEnabled: true
   loggingAgent: promtail

main.go

@@ -20,6 +20,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
@@ -65,7 +66,19 @@ func init() {
 	//+kubebuilder:scaffold:scheme
 }
 
+type StringSliceVar []string
+
+func (s StringSliceVar) String() string {
+	return strings.Join(s, ",")
+}
+
+func (s *StringSliceVar) Set(value string) error {
+	*s = strings.Split(value, ",")
+	return nil
+}
+
 func main() {
+	var defaultNamespaces StringSliceVar
 	var enableLeaderElection bool
 	var enableLogging bool
 	var loggingAgent string
@@ -75,6 +88,7 @@ func main() {
 	var profilesAddr string
 	var probeAddr string
 	var vintageMode bool
+	flag.Var(&defaultNamespaces, "default-namespaces", "List of namespaces to collect logs from by default on workload clusters")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -147,7 +161,8 @@ func main() {
 	}
 
 	loggingConfig := loggingconfig.Reconciler{
-		Client: mgr.GetClient(),
+		Client:                           mgr.GetClient(),
+		DefaultWorkloadClusterNamespaces: defaultNamespaces,
 	}
 
 	grafanaAgentSecret := grafanaagentsecret.Reconciler{

pkg/resource/logging-config/alloy-logging-config.go

@@ -6,8 +6,10 @@ import (
 	"text/template"
 
 	"github.com/Masterminds/sprig/v3"
+	"github.com/blang/semver"
 
 	"github.com/giantswarm/logging-operator/pkg/common"
+	"github.com/giantswarm/logging-operator/pkg/key"
 	loggedcluster "github.com/giantswarm/logging-operator/pkg/logged-cluster"
 	loggingsecret "github.com/giantswarm/logging-operator/pkg/resource/logging-secret"
 )
@@ -20,6 +22,8 @@ var (
 	//go:embed alloy/logging-config.alloy.yaml.template
 	alloyLoggingConfig         string
 	alloyLoggingConfigTemplate *template.Template
+
+	supportPodLogs = semver.MustParse("1.7.0")
 )
 
 func init() {
@@ -29,20 +33,29 @@ func init() {
 
 // GenerateAlloyLoggingConfig returns a configmap for
 // the logging extra-config
-func GenerateAlloyLoggingConfig(lc loggedcluster.Interface) (string, error) {
+func GenerateAlloyLoggingConfig(lc loggedcluster.Interface, observabilityBundleVersion semver.Version, defaultNamespaces []string) (string, error) {
 	var values bytes.Buffer
 
-	alloyConfig, err := generateAlloyConfig(lc)
+	alloyConfig, err := generateAlloyConfig(lc, observabilityBundleVersion)
 	if err != nil {
 		return "", err
 	}
 
 	data := struct {
-		AlloyConfig string
-		SecretName  string
+		AlloyConfig                      string
+		DefaultWorkloadClusterNamespaces []string
+		IsWorkloadCluster                bool
+		LoggingLabel                     string
+		SecretName                       string
+		SupportPodLogs                   bool
 	}{
-		AlloyConfig: alloyConfig,
-		SecretName:  common.AlloyLogAgentAppName,
+		AlloyConfig:                      alloyConfig,
+		DefaultWorkloadClusterNamespaces: defaultNamespaces,
+		IsWorkloadCluster:                common.IsWorkloadCluster(lc),
+		LoggingLabel:                     key.LoggingLabel,
+		SecretName:                       common.AlloyLogAgentAppName,
+		// Observability bundle in older versions do not support PodLogs
+		SupportPodLogs: observabilityBundleVersion.GE(supportPodLogs),
 	}
 
 	err = alloyLoggingConfigTemplate.Execute(&values, data)
@@ -53,7 +66,7 @@ func GenerateAlloyLoggingConfig(lc loggedcluster.Interface) (string, error) {
 	return values.String(), nil
 }
 
-func generateAlloyConfig(lc loggedcluster.Interface) (string, error) {
+func generateAlloyConfig(lc loggedcluster.Interface, observabilityBundleVersion semver.Version) (string, error) {
 	var values bytes.Buffer
 
 	clusterName := lc.GetClusterName()
@@ -67,6 +80,7 @@ func generateAlloyConfig(lc loggedcluster.Interface) (string, error) {
 		TenantIDEnvVarName          string
 		BasicAuthUsernameEnvVarName string
 		BasicAuthPasswordEnvVarName string
+		SupportPodLogs              bool
 	}{
 		ClusterID:                   clusterName,
 		Installation:                lc.GetInstallationName(),
@@ -76,6 +90,8 @@ func generateAlloyConfig(lc loggedcluster.Interface) (string, error) {
 		TenantIDEnvVarName:          loggingsecret.AlloyTenantIDEnvVarName,
 		BasicAuthUsernameEnvVarName: loggingsecret.AlloyBasicAuthUsernameEnvVarName,
 		BasicAuthPasswordEnvVarName: loggingsecret.AlloyBasicAuthPasswordEnvVarName,
+		// Observability bundle in older versions do not support PodLogs
+		SupportPodLogs: observabilityBundleVersion.GE(supportPodLogs),
 	}
 
 	err := alloyLoggingTemplate.Execute(&values, data)

pkg/resource/logging-config/alloy-logging-config_test.go

@@ -0,0 +1,114 @@
+package loggingconfig
+
+import (
+	_ "embed"
+	"flag"
+	"os"
+	"path/filepath"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	capi "sigs.k8s.io/cluster-api/api/v1beta1"
+
+	"github.com/blang/semver"
+	"github.com/google/go-cmp/cmp"
+
+	loggedcluster "github.com/giantswarm/logging-operator/pkg/logged-cluster"
+	"github.com/giantswarm/logging-operator/pkg/logged-cluster/capicluster"
+)
+
+var (
+	update = flag.Bool("update", false, "update .golden files")
+)
+
+func TestGenerateAlloyLoggingConfig(t *testing.T) {
+	testCases := []struct {
+		goldenFile                 string
+		observabilityBundleVersion string
+		defaultNamespaces          []string
+		installationName           string
+		clusterName                string
+	}{
+		{
+			goldenFile:                 "alloy/test/logging-config.alloy.162_MC.yaml",
+			observabilityBundleVersion: "1.6.2",
+			installationName:           "test-installation",
+			clusterName:                "test-installation",
+		},
+		{
+			goldenFile:                 "alloy/test/logging-config.alloy.162_WC.yaml",
+			observabilityBundleVersion: "1.6.2",
+			installationName:           "test-installation",
+			clusterName:                "test-cluster",
+		},
+		{
+			goldenFile:                 "alloy/test/logging-config.alloy.170_MC.yaml",
+			observabilityBundleVersion: "1.7.0",
+			defaultNamespaces:          []string{"test-selector"},
+			installationName:           "test-installation",
+			clusterName:                "test-installation",
+		},
+		{
+			goldenFile:                 "alloy/test/logging-config.alloy.170_WC.yaml",
+			observabilityBundleVersion: "1.7.0",
+			defaultNamespaces:          []string{"test-selector"},
+			installationName:           "test-installation",
+			clusterName:                "test-cluster",
+		},
+		{
+			goldenFile:                 "alloy/test/logging-config.alloy.170_WC_default_namespaces_nil.yaml",
+			observabilityBundleVersion: "1.7.0",
+			defaultNamespaces:          nil,
+			installationName:           "test-installation",
+			clusterName:                "test-cluster",
+		},
+		{
+			goldenFile:                 "alloy/test/logging-config.alloy.170_WC_default_namespaces_empty.yaml",
+			observabilityBundleVersion: "1.7.0",
+			defaultNamespaces:          []string{""},
+			installationName:           "test-installation",
+			clusterName:                "test-cluster",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(filepath.Base(tc.goldenFile), func(t *testing.T) {
+			observabilityBundleVersion, err := semver.Parse(tc.observabilityBundleVersion)
+			if err != nil {
+				t.Fatalf("Failed to parse observability bundle version: %v", err)
+			}
+			golden, err := os.ReadFile(tc.goldenFile)
+			if err != nil {
+				t.Fatalf("Failed to read golden file: %v", err)
+			}
+
+			loggedCluster := &capicluster.Object{
+				Object: &capi.Cluster{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: tc.clusterName,
+					},
+				},
+				Options: loggedcluster.Options{
+					InstallationName: tc.installationName,
+				},
+			}
+
+			config, err := GenerateAlloyLoggingConfig(loggedCluster, observabilityBundleVersion, tc.defaultNamespaces)
+			if err != nil {
+				t.Fatalf("Failed to generate alloy config: %v", err)
+			}
+
+			//fmt.Printf("=>> config\n%s", config)
+			//fmt.Printf("=>> config_v170_MC\n%s", config_v170_MC)
+			if !cmp.Equal(string(golden), config) {
+				t.Logf("Generated config differs from %s, diff:\n%s", tc.goldenFile, cmp.Diff(string(golden), config))
+				if *update {
+					//nolint:gosec
+					if err := os.WriteFile(tc.goldenFile, []byte(config), 0644); err != nil {
+						t.Fatalf("Failed to update golden file: %v", err)
+					}
+				}
+			}
+		})
+	}
+}