✨ Add pprof option for global hub components (stolostron#907)

* Add pprof option for global hub components Signed-off-by: myan <[email protected]> * make operator pprof server options Signed-off-by: myan <[email protected]> * format code Signed-off-by: Meng Yan <[email protected]> * ut Signed-off-by: Meng Yan <[email protected]> * fix the ut Signed-off-by: myan <[email protected]> * reply review Signed-off-by: myan <[email protected]> * fix the sonar Signed-off-by: myan <[email protected]> --------- Signed-off-by: myan <[email protected]> Signed-off-by: Meng Yan <[email protected]>
yanmxa · May 9, 2024 · a98da42 · a98da42
1 parent 79b2128
commit a98da42
Show file tree

Hide file tree

Showing 17 changed files with 101 additions and 59 deletions.
diff --git a/agent/cmd/agent/main.go b/agent/cmd/agent/main.go
@@ -90,6 +90,10 @@ func doMain(ctx context.Context, restConfig *rest.Config, agentConfig *config.Ag
 		return 1
 	}
 
+	if agentConfig.EnablePprof {
+		go utils.StartDefaultPprofServer()
+	}
+
 	mgr, err := createManager(ctx, restConfig, agentConfig)
 	if err != nil {
 		setupLog.Error(err, "failed to create manager")
@@ -171,6 +175,7 @@ func parseFlags() *config.AgentConfig {
 		"QPS for the multicluster global hub agent")
 	pflag.IntVar(&agentConfig.Burst, "burst", 300,
 		"Burst for the multicluster global hub agent")
+	pflag.BoolVar(&agentConfig.EnablePprof, "enable-pprof", false, "Enable the pprof tool.")
 	pflag.Parse()
 
 	// set zap logger

diff --git a/agent/pkg/config/agent_config.go b/agent/pkg/config/agent_config.go
@@ -18,4 +18,5 @@ type AgentConfig struct {
 	EnableGlobalResource         bool
 	QPS                          float32
 	Burst                        int
+	EnablePprof                  bool
 }
diff --git a/manager/cmd/manager/main.go b/manager/cmd/manager/main.go
@@ -167,7 +167,7 @@ func parseFlags() *managerconfig.ManagerConfig {
 		"data retention indicates how many months the expired data will kept in the database")
 	pflag.BoolVar(&managerConfig.EnableGlobalResource, "enable-global-resource", false,
 		"enable the global resource feature.")
-
+	pflag.BoolVar(&managerConfig.EnablePprof, "enable-pprof", false, "Enable the pprof tool.")
 	pflag.Parse()
 	// set zap logger
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
@@ -299,6 +299,11 @@ func doMain(ctx context.Context, restConfig *rest.Config) int {
 		setupLog.Error(err, "failed to complete configuration")
 		return 1
 	}
+
+	if managerConfig.EnablePprof {
+		go utils.StartDefaultPprofServer()
+	}
+
 	utils.PrintVersion(setupLog)
 	databaseConfig := &database.DatabaseConfig{
 		URL:        managerConfig.DatabaseConfig.ProcessDatabaseURL,

diff --git a/manager/pkg/config/manager_config.go b/manager/pkg/config/manager_config.go
@@ -24,6 +24,7 @@ type ManagerConfig struct {
 	ElectionConfig        *commonobjects.LeaderElectionConfig
 	EnableGlobalResource  bool
 	LaunchJobNames        string
+	EnablePprof           bool
 }
 
 type SyncerConfig struct {

diff --git a/operator/main.go b/operator/main.go
@@ -46,6 +46,10 @@ func doMain(ctx context.Context, cfg *rest.Config) int {
 	operatorConfig := parseFlags()
 	utils.PrintVersion(setupLog)
 
+	if operatorConfig.EnablePprof {
+		go utils.StartDefaultPprofServer()
+	}
+
 	kubeClient, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		setupLog.Error(err, "failed to create kube client")
@@ -106,6 +110,7 @@ func parseFlags() *config.OperatorConfig {
 		"Enable leader election for controller manager. ")
 	pflag.BoolVar(&config.GlobalResourceEnabled, "global-resource-enabled", false,
 		"Enable the global resource. It is expermental feature. Do not support upgrade.")
+	pflag.BoolVar(&config.EnablePprof, "enable-pprof", false, "Enable the pprof tool.")
 	pflag.Parse()
 
 	config.LogLevel = "info"

diff --git a/operator/pkg/config/operator_config.go b/operator/pkg/config/operator_config.go
@@ -6,5 +6,6 @@ type OperatorConfig struct {
 	PodNamespace          string
 	LeaderElection        bool
 	GlobalResourceEnabled bool
+	EnablePprof           bool
 	LogLevel              string
 }
diff --git a/operator/pkg/controllers/addon/addon_controller.go b/operator/pkg/controllers/addon/addon_controller.go
@@ -41,12 +41,11 @@ import (
 // +kubebuilder:rbac:groups=packages.operators.coreos.com,resources=packagemanifests,verbs=get;list;watch
 
 type AddonController struct {
-	addonManager         addonmanager.AddonManager
-	kubeConfig           *rest.Config
-	client               client.Client
-	log                  logr.Logger
-	EnableGlobalResource bool
-	LogLevel             string
+	addonManager   addonmanager.AddonManager
+	kubeConfig     *rest.Config
+	client         client.Client
+	log            logr.Logger
+	operatorConfig *config.OperatorConfig
 }
 
 // used to create addon manager
@@ -59,12 +58,11 @@ func NewAddonController(kubeConfig *rest.Config, client client.Client, operatorC
 		return nil, err
 	}
 	return &AddonController{
-		kubeConfig:           kubeConfig,
-		client:               client,
-		log:                  log,
-		addonManager:         addonMgr,
-		EnableGlobalResource: operatorConfig.GlobalResourceEnabled,
-		LogLevel:             operatorConfig.LogLevel,
+		kubeConfig:     kubeConfig,
+		client:         client,
+		log:            log,
+		addonManager:   addonMgr,
+		operatorConfig: operatorConfig,
 	}, nil
 }
 
@@ -93,13 +91,12 @@ func (a *AddonController) Start(ctx context.Context) error {
 	}
 
 	hohAgentAddon := HohAgentAddon{
-		ctx:                  ctx,
-		kubeClient:           kubeClient,
-		client:               a.client,
-		dynamicClient:        dynamicClient,
-		log:                  a.log.WithName("values"),
-		EnableGlobalResource: a.EnableGlobalResource,
-		LogLevel:             a.LogLevel,
+		ctx:            ctx,
+		kubeClient:     kubeClient,
+		client:         a.client,
+		dynamicClient:  dynamicClient,
+		log:            a.log.WithName("values"),
+		operatorConfig: a.operatorConfig,
 	}
 	_, err = utils.WaitGlobalHubReady(ctx, a.client, 5*time.Second)
 	if err != nil {

diff --git a/operator/pkg/controllers/addon/addon_controller_manifests.go b/operator/pkg/controllers/addon/addon_controller_manifests.go
@@ -68,6 +68,7 @@ type ManifestsConfig struct {
 	AgentQPS               float32
 	AgentBurst             int
 	LogLevel               string
+	EnablePprof            bool
 	// cannot use *corev1.ResourceRequirements, addonfactory.StructToValues removes the real value
 	Resources *Resources
 }
@@ -81,13 +82,12 @@ type Resources struct {
 }
 
 type HohAgentAddon struct {
-	ctx                  context.Context
-	client               client.Client
-	kubeClient           kubernetes.Interface
-	dynamicClient        dynamic.Interface
-	log                  logr.Logger
-	EnableGlobalResource bool
-	LogLevel             string
+	ctx            context.Context
+	client         client.Client
+	kubeClient     kubernetes.Interface
+	dynamicClient  dynamic.Interface
+	log            logr.Logger
+	operatorConfig *config.OperatorConfig
 }
 
 func (a *HohAgentAddon) getMulticlusterGlobalHub() (*globalhubv1alpha4.MulticlusterGlobalHub, error) {
@@ -227,10 +227,11 @@ func (a *HohAgentAddon) GetValues(cluster *clusterv1.ManagedCluster,
 		RetryPeriod:            strconv.Itoa(electionConfig.RetryPeriod),
 		KlusterletNamespace:    "open-cluster-management-agent",
 		KlusterletWorkSA:       "klusterlet-work-sa",
-		EnableGlobalResource:   a.EnableGlobalResource,
+		EnableGlobalResource:   a.operatorConfig.GlobalResourceEnabled,
 		AgentQPS:               agentQPS,
 		AgentBurst:             agentBurst,
-		LogLevel:               a.LogLevel,
+		LogLevel:               a.operatorConfig.LogLevel,
+		EnablePprof:            a.operatorConfig.EnablePprof,
 		Resources:              agentRes,
 	}
 

diff --git a/operator/pkg/controllers/addon/addon_suite_test.go b/operator/pkg/controllers/addon/addon_suite_test.go
@@ -114,6 +114,7 @@ var _ = BeforeSuite(func() {
 	addonController, err := addon.NewAddonController(k8sManager.GetConfig(), k8sClient, &config.OperatorConfig{
 		GlobalResourceEnabled: true,
 		LogLevel:              "info",
+		EnablePprof:           false,
 	})
 	Expect(err).ToNot(HaveOccurred())
 	err = k8sManager.Add(addonController)

diff --git a/...controllers/addon/manifests/templates/agent/multicluster-global-hub-agent-deployment.yaml b/...controllers/addon/manifests/templates/agent/multicluster-global-hub-agent-deployment.yaml
@@ -55,6 +55,7 @@ spec:
             - --enable-global-resource={{.EnableGlobalResource}}
             - --qps={{.AgentQPS}}
             - --burst={{.AgentBurst}}
+            - --enable-pprof={{.EnablePprof}}
           env:
             - name: POD_NAMESPACE
               valueFrom:

diff --git a/operator/pkg/controllers/hubofhubs/globalhub_controller.go b/operator/pkg/controllers/hubofhubs/globalhub_controller.go
@@ -85,31 +85,29 @@ var watchedConfigmap = sets.NewString(
 type MulticlusterGlobalHubReconciler struct {
 	manager.Manager
 	client.Client
-	Scheme               *runtime.Scheme
-	addonMgr             addonmanager.AddonManager
-	KubeClient           kubernetes.Interface
-	Log                  logr.Logger
-	LogLevel             string
-	MiddlewareConfig     *MiddlewareConfig
-	EnableGlobalResource bool
-	upgradeOnce          sync.Once
-	KafkaInit            bool
-	KafkaController      *KafkaController
+	Scheme           *runtime.Scheme
+	addonMgr         addonmanager.AddonManager
+	KubeClient       kubernetes.Interface
+	Log              logr.Logger
+	MiddlewareConfig *MiddlewareConfig
+	OperatorConfig   *config.OperatorConfig
+	upgradeOnce      sync.Once
+	KafkaInit        bool
+	KafkaController  *KafkaController
 }
 
 func NewMulticlusterGlobalHubReconciler(mgr ctrl.Manager, addonMgr addonmanager.AddonManager,
 	kubeClient kubernetes.Interface, operatorConfig *config.OperatorConfig,
 ) *MulticlusterGlobalHubReconciler {
 	return &MulticlusterGlobalHubReconciler{
-		Manager:              mgr,
-		Client:               mgr.GetClient(),
-		addonMgr:             addonMgr,
-		KubeClient:           kubeClient,
-		Scheme:               mgr.GetScheme(),
-		Log:                  ctrl.Log.WithName("global-hub-reconciler"),
-		MiddlewareConfig:     &MiddlewareConfig{},
-		EnableGlobalResource: operatorConfig.GlobalResourceEnabled,
-		LogLevel:             operatorConfig.LogLevel,
+		Manager:          mgr,
+		Client:           mgr.GetClient(),
+		addonMgr:         addonMgr,
+		KubeClient:       kubeClient,
+		Scheme:           mgr.GetScheme(),
+		Log:              ctrl.Log.WithName("global-hub-reconciler"),
+		MiddlewareConfig: &MiddlewareConfig{},
+		OperatorConfig:   operatorConfig,
 	}
 }
 

diff --git a/operator/pkg/controllers/hubofhubs/globalhub_database.go b/operator/pkg/controllers/hubofhubs/globalhub_database.go
@@ -97,7 +97,7 @@ func (r *MulticlusterGlobalHubReconciler) ReconcileDatabase(ctx context.Context,
 		return err
 	}
 
-	if r.EnableGlobalResource {
+	if r.OperatorConfig.GlobalResourceEnabled {
 		if err := applySQL(ctx, conn, databaseOldFS, "database.old", readonlyUsername); err != nil {
 			return err
 		}

diff --git a/operator/pkg/controllers/hubofhubs/globalhub_manager.go b/operator/pkg/controllers/hubofhubs/globalhub_manager.go
@@ -140,8 +140,9 @@ func (r *MulticlusterGlobalHubReconciler) reconcileManager(ctx context.Context,
 			Tolerations:            mgh.Spec.Tolerations,
 			RetentionMonth:         months,
 			StatisticLogInterval:   config.GetStatisticLogInterval(),
-			EnableGlobalResource:   r.EnableGlobalResource,
-			LogLevel:               r.LogLevel,
+			EnableGlobalResource:   r.OperatorConfig.GlobalResourceEnabled,
+			EnablePprof:            r.OperatorConfig.EnablePprof,
+			LogLevel:               r.OperatorConfig.LogLevel,
 			Resources:              utils.GetResources(operatorconstants.Manager, mgh.Spec.AdvancedConfig),
 		}, nil
 	})
@@ -257,6 +258,7 @@ type ManagerVariables struct {
 	RetentionMonth         int
 	StatisticLogInterval   string
 	EnableGlobalResource   bool
+	EnablePprof            bool
 	LogLevel               string
 	Resources              *corev1.ResourceRequirements
 }
diff --git a/operator/pkg/controllers/hubofhubs/integration_test.go b/operator/pkg/controllers/hubofhubs/integration_test.go
@@ -378,6 +378,7 @@ var _ = Describe("MulticlusterGlobalHub controller", Ordered, func() {
 					StatisticLogInterval: config.GetStatisticLogInterval(),
 					EnableGlobalResource: true,
 					LaunchJobNames:       config.GetLaunchJobNames(mgh),
+					EnablePprof:          false,
 					LogLevel:             "info",
 					Resources:            operatorutils.GetResources(operatorconstants.Manager, mgh.Spec.AdvancedConfig),
 				}, nil

diff --git a/operator/pkg/controllers/hubofhubs/manifests/manager/deployment.yaml b/operator/pkg/controllers/hubofhubs/manifests/manager/deployment.yaml
@@ -59,6 +59,7 @@ spec:
             {{- end}}
             - --data-retention={{.RetentionMonth}}
             - --statistics-log-interval={{.StatisticLogInterval}}
+            - --enable-pprof={{.EnablePprof}}
             {{- if eq .SkipAuth true}}
             - --cluster-api-url=
             {{- end}}

diff --git a/operator/pkg/controllers/hubofhubs/suite_test.go b/operator/pkg/controllers/hubofhubs/suite_test.go
@@ -157,14 +157,17 @@ var _ = BeforeSuite(func() {
 	Expect(err).Should(Succeed())
 
 	mghReconciler = &hubofhubscontroller.MulticlusterGlobalHubReconciler{
-		Manager:              k8sManager,
-		Client:               k8sManager.GetClient(),
-		KubeClient:           kubeClient,
-		Scheme:               k8sManager.GetScheme(),
-		Log:                  ctrl.Log.WithName("multicluster-global-hub-reconciler"),
-		LogLevel:             "info",
-		MiddlewareConfig:     &hubofhubscontroller.MiddlewareConfig{},
-		EnableGlobalResource: true,
+		Manager:          k8sManager,
+		Client:           k8sManager.GetClient(),
+		KubeClient:       kubeClient,
+		Scheme:           k8sManager.GetScheme(),
+		Log:              ctrl.Log.WithName("multicluster-global-hub-reconciler"),
+		MiddlewareConfig: &hubofhubscontroller.MiddlewareConfig{},
+		OperatorConfig: &config.OperatorConfig{
+			LogLevel:              "info",
+			EnablePprof:           false,
+			GlobalResourceEnabled: true,
+		},
 	}
 	Expect(mghReconciler.SetupWithManager(k8sManager)).ToNot(HaveOccurred())
 

diff --git a/pkg/utils/pprof.go b/pkg/utils/pprof.go
@@ -0,0 +1,19 @@
+package utils
+
+import (
+	"net/http"
+	_ "net/http/pprof" // #nosec G108
+	"time"
+
+	"k8s.io/klog"
+)
+
+func StartDefaultPprofServer() {
+	server := &http.Server{
+		Addr:              ":6060",
+		ReadHeaderTimeout: 10 * time.Second,
+	}
+	if err := server.ListenAndServe(); err != nil {
+		klog.Error(err, "failed to start the pprof server")
+	}
+}