separate clients - introduce cache (run-int-tests) (gardener#948)

charts/container-deployer/templates/clusterrole.yaml

@@ -56,27 +56,24 @@ rules:
   - patch
   - update
 
-- apiGroups:
-  - ""
-  resources:
-  - "secrets"
-  verbs:
-  - "*"
-
 - apiGroups:
   - ""
   resources:
   - "pods"
   - "pods/status"
+  - "secrets"
+  - "serviceaccounts"
   verbs:
   - "*"
 
 - apiGroups:
   - ""
   resources:
-  - "serviceaccounts"
+  - namespaces
   verbs:
-  - "*"
+  - get
+  - list
+  - watch
 
 - apiGroups:
   - "rbac.authorization.k8s.io"

charts/helm-deployer/templates/clusterrole.yaml

@@ -37,7 +37,8 @@ rules:
 - apiGroups:
   - ""
   resources:
-  - "pods"
+  - namespaces
+  - pods
   verbs:
   - get
   - watch

charts/manifest-deployer/templates/clusterrole.yaml

@@ -58,7 +58,8 @@ rules:
 - apiGroups:
     - ""
   resources:
-    - "pods"
+    - namespaces
+    - pods
   verbs:
     - get
     - watch

charts/mock-deployer/templates/clusterrole.yaml

@@ -47,7 +47,8 @@ rules:
 - apiGroups:
     - ""
   resources:
-    - "pods"
+    - namespaces
+    - pods
   verbs:
     - get
     - watch

cmd/container-deployer/container-deployer-controller/app/app.go

@@ -37,7 +37,10 @@ func NewContainerDeployerControllerCommand(ctx context.Context) *cobra.Command {
 func (o *options) run(ctx context.Context) error {
 	o.DeployerOptions.Log.Info("Starting Container Deployer", lc.KeyVersion, version.Get().GitVersion)
 
-	gc, err := containerctlr.AddControllerToManager(o.DeployerOptions.Log,
+	gc, err := containerctlr.AddControllerToManager(
+		o.DeployerOptions.LsUncachedClient, o.DeployerOptions.LsCachedClient, o.DeployerOptions.HostUncachedClient, o.DeployerOptions.HostCachedClient,
+		o.DeployerOptions.FinishedObjectCache,
+		o.DeployerOptions.Log,
 		o.DeployerOptions.HostMgr,
 		o.DeployerOptions.LsMgr,
 		o.Config,
@@ -46,6 +49,8 @@ func (o *options) run(ctx context.Context) error {
 		return fmt.Errorf("unable to setup container controller")
 	}
 
+	o.DeployerOptions.Log.Info("Starting container deployer manager")
+
 	if gc == nil {
 		return o.DeployerOptions.StartManagers(ctx)
 	} else {

cmd/helm-deployer-controller/app/app.go

@@ -37,9 +37,14 @@ func NewHelmDeployerControllerCommand(ctx context.Context) *cobra.Command {
 
 func (o *options) run(ctx context.Context) error {
 	o.DeployerOptions.Log.Info("Starting helm deployer", lc.KeyVersion, version.Get().GitVersion)
-	if err := helmctrl.AddDeployerToManager(o.DeployerOptions.Log, o.DeployerOptions.LsMgr, o.DeployerOptions.HostMgr,
+	if err := helmctrl.AddDeployerToManager(
+		o.DeployerOptions.LsUncachedClient, o.DeployerOptions.LsCachedClient, o.DeployerOptions.HostUncachedClient, o.DeployerOptions.HostCachedClient,
+		o.DeployerOptions.FinishedObjectCache,
+		o.DeployerOptions.Log, o.DeployerOptions.LsMgr, o.DeployerOptions.HostMgr,
 		o.Config, "helm"); err != nil {
 		return fmt.Errorf("unable to setup helm controller")
 	}
+
+	o.DeployerOptions.Log.Info("Starting helm deployer manager")
 	return o.DeployerOptions.StartManagers(ctx)
 }

cmd/landscaper-agent/app/app.go

@@ -14,7 +14,7 @@ import (
 
 	lc "github.com/gardener/landscaper/controller-utils/pkg/logging/constants"
 	"github.com/gardener/landscaper/pkg/agent"
-
+	lsutils "github.com/gardener/landscaper/pkg/utils"
 	"github.com/gardener/landscaper/pkg/version"
 )
 
@@ -43,7 +43,13 @@ func NewLandscaperAgentCommand(ctx context.Context) *cobra.Command {
 func (o *options) run(ctx context.Context) error {
 	o.log.Info("Starting Landscaper Agent", lc.KeyVersion, version.Get().String())
 
-	if err := agent.AddToManager(ctx, o.log, o.LsMgr, o.HostMgr, o.config, "agent-helm"); err != nil {
+	lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient, err := lsutils.ClientsFromManagers(o.LsMgr, o.HostMgr)
+	if err != nil {
+		return err
+	}
+
+	if err := agent.AddToManager(ctx, lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient,
+		o.log, o.LsMgr, o.HostMgr, o.config, "agent-helm"); err != nil {
 		return fmt.Errorf("unable to setup default agent: %w", err)
 	}
 

cmd/landscaper-agent/app/options.go

@@ -10,9 +10,6 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/gardener/landscaper/controller-utils/pkg/logging"
-	lsutils "github.com/gardener/landscaper/pkg/utils"
-
 	flag "github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/client-go/tools/clientcmd"
@@ -21,7 +18,9 @@ import (
 
 	"github.com/gardener/landscaper/apis/config"
 	lsinstall "github.com/gardener/landscaper/apis/core/install"
+	"github.com/gardener/landscaper/controller-utils/pkg/logging"
 	"github.com/gardener/landscaper/pkg/api"
+	lsutils "github.com/gardener/landscaper/pkg/utils"
 )
 
 type options struct {
@@ -69,31 +68,30 @@ func (o *options) Complete() error {
 		LeaderElection:     false,
 		Port:               9443,
 		MetricsBindAddress: "0",
-		NewClient:          lsutils.NewUncachedClient(lsutils.LsHostClientBurstDefault, lsutils.LsHostClientQpsDefault),
 	}
+
 	hostRestConfig, err := ctrl.GetConfig()
 	if err != nil {
 		return fmt.Errorf("unable to get host kubeconfig: %w", err)
 	}
+	hostRestConfig = lsutils.RestConfigWithModifiedClientRequestRestrictions(log, hostRestConfig, lsutils.LsHostClientBurstDefault, lsutils.LsHostClientQpsDefault)
+
 	o.HostMgr, err = ctrl.NewManager(hostRestConfig, opts)
 	if err != nil {
-		return fmt.Errorf("unable to setup manager")
+		return fmt.Errorf("unable to setup host manager")
 	}
 
 	data, err := os.ReadFile(o.landscaperKubeconfigPath)
 	if err != nil {
 		return fmt.Errorf("unable to read landscaper kubeconfig from %s: %w", o.landscaperKubeconfigPath, err)
 	}
-	client, err := clientcmd.NewClientConfigFromBytes(data)
-	if err != nil {
-		return fmt.Errorf("unable to build landscaper cluster client from %s: %w", o.landscaperKubeconfigPath, err)
-	}
-	lsRestConfig, err := client.ClientConfig()
+
+	lsRestConfig, err := clientcmd.RESTConfigFromKubeConfig(data)
 	if err != nil {
-		return fmt.Errorf("unable to build landscaper cluster rest client from %s: %w", o.landscaperKubeconfigPath, err)
+		return fmt.Errorf("unable to build landscaper cluster rest client for agent: %w", err)
 	}
+	lsRestConfig = lsutils.RestConfigWithModifiedClientRequestRestrictions(log, lsRestConfig, lsutils.LsResourceClientBurstDefault, lsutils.LsResourceClientQpsDefault)
 
-	opts.NewClient = lsutils.NewUncachedClient(lsutils.LsResourceClientBurstDefault, lsutils.LsResourceClientQpsDefault)
 	o.LsMgr, err = ctrl.NewManager(lsRestConfig, opts)
 	if err != nil {
 		return fmt.Errorf("unable to setup manager")

cmd/landscaper-controller/app/app.go

@@ -85,7 +85,6 @@ func (o *Options) run(ctx context.Context) error {
 		LeaderElection:     false,
 		Port:               9443,
 		MetricsBindAddress: "0",
-		NewClient:          lsutils.NewUncachedClient(burst, qps),
 	}
 
 	//TODO: investigate whether this is used with an uncached client
@@ -97,7 +96,10 @@ func (o *Options) run(ctx context.Context) error {
 		opts.MetricsBindAddress = fmt.Sprintf(":%d", o.Config.Metrics.Port)
 	}
 
-	hostMgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), opts)
+	hostRestConfig := ctrl.GetConfigOrDie()
+	hostRestConfig = lsutils.RestConfigWithModifiedClientRequestRestrictions(setupLogger, hostRestConfig, burst, qps)
+
+	hostMgr, err := ctrl.NewManager(hostRestConfig, opts)
 	if err != nil {
 		return fmt.Errorf("unable to setup manager: %w", err)
 	}
@@ -108,20 +110,15 @@ func (o *Options) run(ctx context.Context) error {
 		if err != nil {
 			return fmt.Errorf("unable to read landscaper kubeconfig from %s: %w", o.landscaperKubeconfigPath, err)
 		}
-		client, err := clientcmd.NewClientConfigFromBytes(data)
-		if err != nil {
-			return fmt.Errorf("unable to build landscaper cluster client from %s: %w", o.landscaperKubeconfigPath, err)
-		}
-		lsConfig, err := client.ClientConfig()
+
+		lsRestConfig, err := clientcmd.RESTConfigFromKubeConfig(data)
 		if err != nil {
-			return fmt.Errorf("unable to build landscaper cluster rest client from %s: %w", o.landscaperKubeconfigPath, err)
+			return fmt.Errorf("unable to build landscaper cluster rest client: %w", err)
 		}
-
-		opts.MetricsBindAddress = "0"
 		burst, qps = lsutils.GetResourceClientRequestRestrictions(setupLogger)
-		opts.NewClient = lsutils.NewUncachedClient(burst, qps)
+		lsRestConfig = lsutils.RestConfigWithModifiedClientRequestRestrictions(setupLogger, lsRestConfig, burst, qps)
 
-		lsMgr, err = ctrl.NewManager(lsConfig, opts)
+		lsMgr, err = ctrl.NewManager(lsRestConfig, opts)
 		if err != nil {
 			return fmt.Errorf("unable to setup landscaper cluster manager from %s: %w", o.landscaperKubeconfigPath, err)
 		}
@@ -130,29 +127,47 @@ func (o *Options) run(ctx context.Context) error {
 	metrics.RegisterMetrics(controllerruntimeMetrics.Registry)
 	ctrlLogger := o.Log.WithName("controllers")
 
+	if err := o.ensureCRDs(ctx, lsMgr); err != nil {
+		return err
+	}
+	if lsMgr != hostMgr {
+		if err := o.ensureCRDs(ctx, hostMgr); err != nil {
+			return err
+		}
+	}
+	install.Install(lsMgr.GetScheme())
+
+	lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient, err := lsutils.ClientsFromManagers(lsMgr, hostMgr)
+	if err != nil {
+		return err
+	}
+
 	if os.Getenv("LANDSCAPER_MODE") == "central-landscaper" {
-		return o.startCentralLandscaper(ctx, lsMgr, hostMgr, ctrlLogger, setupLogger)
+		return o.startCentralLandscaper(ctx, lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient,
+			lsMgr, hostMgr, ctrlLogger, setupLogger)
 	} else {
-		return o.startMainController(ctx, lsMgr, hostMgr, ctrlLogger, setupLogger)
+		return o.startMainController(ctx, lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient,
+			lsMgr, hostMgr, ctrlLogger, setupLogger)
 	}
-
 }
 
-func (o *Options) startMainController(ctx context.Context, lsMgr, hostMgr manager.Manager,
-	ctrlLogger, setupLogger logging.Logger) error {
-	install.Install(lsMgr.GetScheme())
+func (o *Options) startMainController(ctx context.Context,
+	lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient client.Client,
+	lsMgr, hostMgr manager.Manager, ctrlLogger, setupLogger logging.Logger) error {
 
 	store, err := blueprint.NewStore(o.Log.WithName("blueprintStore"), osfs.New(), o.Config.BlueprintStore)
 	if err != nil {
 		return fmt.Errorf("unable to setup blueprint store: %w", err)
 	}
 	blueprint.SetStore(store)
 
-	if err := installationsctrl.AddControllerToManager(ctrlLogger, lsMgr, hostMgr, o.Config, "installations"); err != nil {
+	if err := installationsctrl.AddControllerToManager(lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient,
+		ctrlLogger, lsMgr, o.Config, "installations"); err != nil {
 		return fmt.Errorf("unable to setup installation controller: %w", err)
 	}
 
-	if err := executionactrl.AddControllerToManager(ctrlLogger, lsMgr, hostMgr, o.Config); err != nil {
+	if err := executionactrl.AddControllerToManager(lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient,
+		ctrlLogger, lsMgr, hostMgr, o.Config); err != nil {
 		return fmt.Errorf("unable to setup execution controller: %w", err)
 	}
 
@@ -181,27 +196,16 @@ func (o *Options) startMainController(ctx context.Context, lsMgr, hostMgr manage
 	return eg.Wait()
 }
 
-func (o *Options) startCentralLandscaper(ctx context.Context, lsMgr, hostMgr manager.Manager,
-	ctrlLogger, setupLogger logging.Logger) error {
-
-	if err := o.ensureCRDs(ctx, lsMgr); err != nil {
-		return err
-	}
-
-	if lsMgr != hostMgr {
-		if err := o.ensureCRDs(ctx, hostMgr); err != nil {
-			return err
-		}
-	}
-
-	install.Install(lsMgr.GetScheme())
+func (o *Options) startCentralLandscaper(ctx context.Context,
+	lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient client.Client,
+	lsMgr, hostMgr manager.Manager, ctrlLogger, setupLogger logging.Logger) error {
 
-	if err := contextctrl.AddControllerToManager(ctrlLogger, lsMgr, o.Config); err != nil {
+	if err := contextctrl.AddControllerToManager(lsUncachedClient, lsCachedClient, ctrlLogger, lsMgr, o.Config); err != nil {
 		return fmt.Errorf("unable to setup context controller: %w", err)
 	}
 
 	if !o.Config.DeployerManagement.Disable {
-		if err := deployers.AddControllersToManager(ctrlLogger, lsMgr, o.Config); err != nil {
+		if err := deployers.AddControllersToManager(lsUncachedClient, lsCachedClient, ctrlLogger, lsMgr, o.Config); err != nil {
 			return fmt.Errorf("unable to setup deployer controllers: %w", err)
 		}
 		if !o.Config.DeployerManagement.Agent.Disable {
@@ -217,11 +221,12 @@ func (o *Options) startCentralLandscaper(ctx context.Context, lsMgr, hostMgr man
 					},
 				},
 			)
-			if err := agent.AddToManager(ctx, o.Log, lsMgr, hostMgr, agentConfig, "landscaper-helm"); err != nil {
+			if err := agent.AddToManager(ctx, lsUncachedClient, lsCachedClient, hostUncachedClient, hostCachedClient,
+				o.Log, lsMgr, hostMgr, agentConfig, "landscaper-helm"); err != nil {
 				return fmt.Errorf("unable to setup default agent: %w", err)
 			}
 		}
-		if err := o.DeployInternalDeployers(ctx, lsMgr); err != nil {
+		if err := o.DeployInternalDeployers(ctx, lsUncachedClient); err != nil {
 			return err
 		}
 
@@ -236,35 +241,36 @@ func (o *Options) startCentralLandscaper(ctx context.Context, lsMgr, hostMgr man
 		}
 	}
 
-	if err := deployitemctrl.AddControllerToManager(ctrlLogger,
+	if err := deployitemctrl.AddControllerToManager(lsUncachedClient, lsCachedClient,
+		ctrlLogger,
 		lsMgr,
 		o.Config.Controllers.DeployItems,
 		o.Config.DeployItemTimeouts.Pickup); err != nil {
 		return fmt.Errorf("unable to setup deployitem controller: %w", err)
 	}
 
-	if err := targetsync.AddControllerToManagerForTargetSyncs(ctrlLogger, lsMgr); err != nil {
+	if err := targetsync.AddControllerToManagerForTargetSyncs(lsUncachedClient, lsCachedClient, ctrlLogger, lsMgr); err != nil {
 		return fmt.Errorf("unable to register target sync controller: %w", err)
 	}
 
 	eg, ctx := errgroup.WithContext(ctx)
 
 	eg.Go(func() error {
-		healthChecker := healthcheck.NewHealthChecker(o.Config.LsDeployments, hostMgr.GetClient())
+		healthChecker := healthcheck.NewHealthChecker(o.Config.LsDeployments, hostUncachedClient)
 		if err := healthChecker.StartPeriodicalHealthCheck(ctx, ctrlLogger); err != nil {
 			return err
 		}
 		return nil
 	})
 
 	eg.Go(func() error {
-		lockCleaner := lock.NewLockCleaner(lsMgr.GetClient())
+		lockCleaner := lock.NewLockCleaner(lsUncachedClient)
 		lockCleaner.StartPeriodicalSyncObjectCleanup(ctx, ctrlLogger)
 		return nil
 	})
 
 	eg.Go(func() error {
-		monitor := monitoring.NewMonitor(lsutils.GetCurrentPodNamespace(), hostMgr.GetClient())
+		monitor := monitoring.NewMonitor(lsutils.GetCurrentPodNamespace(), hostUncachedClient)
 		monitor.StartMonitoring(ctx, ctrlLogger)
 		return nil
 	})
@@ -293,15 +299,9 @@ func (o *Options) startCentralLandscaper(ctx context.Context, lsMgr, hostMgr man
 }
 
 // DeployInternalDeployers automatically deploys configured deployers using the new Deployer registrations.
-func (o *Options) DeployInternalDeployers(ctx context.Context, mgr manager.Manager) error {
-	directClient, err := client.New(mgr.GetConfig(), client.Options{
-		Scheme: mgr.GetScheme(),
-	})
-	if err != nil {
-		return fmt.Errorf("unable to create direct client: %q", err)
-	}
+func (o *Options) DeployInternalDeployers(ctx context.Context, lsUncachedClient client.Client) error {
 	ctx = logging.NewContext(ctx, logging.Wrap(ctrl.Log.WithName("deployerManagement")))
-	return o.Deployer.DeployInternalDeployers(ctx, directClient, o.Config)
+	return o.Deployer.DeployInternalDeployers(ctx, lsUncachedClient, o.Config)
 }
 
 func (o *Options) ensureCRDs(ctx context.Context, mgr manager.Manager) error {