feat: enable profile in agents (#54)

Author: britaniar

charts/hub-agent/templates/deployment.yaml

@@ -33,6 +33,8 @@ spec:
             - --enable-cluster-inventory-apis={{ .Values.enableClusterInventoryAPI }}
             - --enable-staged-update-run-apis={{ .Values.enableStagedUpdateRunAPIs }}
             - --enable-eviction-apis={{ .Values.enableEvictionAPIs}}
+            - --enable-pprof={{ .Values.enablePprof }}
+            - --pprof-port={{ .Values.pprofPort }}
             - --max-concurrent-cluster-placement={{ .Values.MaxConcurrentClusterPlacement }}
             - --concurrent-resource-change-syncs={{ .Values.ConcurrentResourceChangeSyncs }}
             - --log_file_max_size={{ .Values.logFileMaxSize }}
@@ -48,6 +50,11 @@ spec:
             - name: healthz
               containerPort: 8081
               protocol: TCP
+            {{- if .Values.enablePprof }}
+            - containerPort: {{ .Values.pprofPort }}
+              name: pprof
+              protocol: TCP
+            {{- end }}
           livenessProbe:
             httpGet:
               path: /healthz

charts/hub-agent/values.yaml

@@ -39,6 +39,9 @@ enableClusterInventoryAPI: true
 enableStagedUpdateRunAPIs: true
 enableEvictionAPIs: true
 
+enablePprof: true
+pprofPort: 6065
+
 hubAPIQPS: 250
 hubAPIBurst: 1000
 MaxConcurrentClusterPlacement: 100

charts/member-agent/templates/deployment.yaml

@@ -34,6 +34,9 @@ spec:
             - -add_dir_header
             - --enable-v1alpha1-apis={{ .Values.enableV1Alpha1APIs }}
             - --enable-v1beta1-apis={{ .Values.enableV1Beta1APIs }}
+            - --enable-pprof={{ .Values.enablePprof }}
+            - --pprof-port={{ .Values.pprofPort }}
+            - --hub-pprof-port={{ .Values.hubPprofPort }}
             {{- if .Values.propertyProvider }}
             - --property-provider={{ .Values.propertyProvider }}
             {{- end }}
@@ -75,6 +78,14 @@ spec:
             - containerPort: 8091
               name: memberhealthz
               protocol: TCP
+            {{- if .Values.enablePprof }}
+            - containerPort: {{ .Values.pprofPort }}
+              name: memberpprof
+              protocol: TCP
+            - containerPort: {{ .Values.hubPprofPort }}
+              name: hubpprof
+              protocol: TCP
+            {{- end }}
           livenessProbe:
             httpGet:
               path: /healthz

charts/member-agent/values.yaml

@@ -61,3 +61,7 @@ useCAAuth: false
 
 enableV1Alpha1APIs: true
 enableV1Beta1APIs: false
+
+enablePprof: true
+pprofPort: 6065
+hubPprofPort: 6066

cmd/hubagent/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"math"
 	"os"
 	"strings"
@@ -113,7 +114,7 @@ func main() {
 	config := ctrl.GetConfigOrDie()
 	config.QPS, config.Burst = float32(opts.HubQPS), opts.HubBurst
 
-	mgr, err := ctrl.NewManager(config, ctrl.Options{
+	mgrOpts := ctrl.Options{
 		Scheme: scheme,
 		Cache: cache.Options{
 			SyncPeriod: &opts.ResyncPeriod.Duration,
@@ -130,7 +131,11 @@ func main() {
 			Port:    FleetWebhookPort,
 			CertDir: FleetWebhookCertDir,
 		}),
-	})
+	}
+	if opts.EnablePprof {
+		mgrOpts.PprofBindAddress = fmt.Sprintf(":%d", opts.PprofPort)
+	}
+	mgr, err := ctrl.NewManager(config, mgrOpts)
 	if err != nil {
 		klog.ErrorS(err, "unable to start controller manager.")
 		exitWithErrorFunc()

cmd/hubagent/options/options.go

@@ -98,6 +98,10 @@ type Options struct {
 	EnableStagedUpdateRunAPIs bool
 	// EnableEvictionAPIs enables to agents to watch the eviction and placement disruption budget CRs.
 	EnableEvictionAPIs bool
+	// EnablePprof enables the pprof profiling.
+	EnablePprof bool
+	// PprofPort is the port for pprof profiling.
+	PprofPort int
 	// DenyModifyMemberClusterLabels indicates if the member cluster labels cannot be modified by groups (excluding system:masters)
 	DenyModifyMemberClusterLabels bool
 }
@@ -117,6 +121,8 @@ func NewOptions() *Options {
 		EnableV1Alpha1APIs:            false,
 		EnableClusterInventoryAPIs:    true,
 		EnableStagedUpdateRunAPIs:     true,
+		EnablePprof:                   false,
+		PprofPort:                     6065,
 	}
 }
 
@@ -160,6 +166,8 @@ func (o *Options) AddFlags(flags *flag.FlagSet) {
 	flags.DurationVar(&o.ForceDeleteWaitTime.Duration, "force-delete-wait-time", 15*time.Minute, "The duration the hub agent waits before force deleting a member cluster.")
 	flags.BoolVar(&o.EnableStagedUpdateRunAPIs, "enable-staged-update-run-apis", true, "If set, the agents will watch for the ClusterStagedUpdateRun APIs.")
 	flags.BoolVar(&o.EnableEvictionAPIs, "enable-eviction-apis", true, "If set, the agents will watch for the Eviction and PlacementDisruptionBudget APIs.")
+	flags.BoolVar(&o.EnablePprof, "enable-pprof", false, "If set, the pprof profiling is enabled.")
+	flags.IntVar(&o.PprofPort, "pprof-port", 6065, "The port for pprof profiling.")
 	flags.BoolVar(&o.DenyModifyMemberClusterLabels, "deny-modify-member-cluster-labels", false, "If set, users not in the system:masters cannot modify member cluster labels.")
 
 	o.RateLimiterOpts.AddFlags(flags)

cmd/memberagent/main.go

@@ -91,6 +91,9 @@ var (
 	driftDetectionInterval       = flag.Int("drift-detection-interval", 15, "The interval in seconds between attempts to detect configuration drifts in the cluster.")
 	watchWorkWithPriorityQueue   = flag.Bool("enable-watch-work-with-priority-queue", false, "If set, the apply_work controller will watch/reconcile work objects that are created new or have recent updates")
 	watchWorkReconcileAgeMinutes = flag.Int("watch-work-reconcile-age", 60, "maximum age (in minutes) of work objects for apply_work controller to watch/reconcile")
+	enablePprof                  = flag.Bool("enable-pprof", false, "enable pprof profiling")
+	pprofPort                    = flag.Int("pprof-port", 6065, "port for pprof profiling")
+	hubPprofPort                 = flag.Int("hub-pprof-port", 6066, "port for hub pprof profiling")
 )
 
 func init() {
@@ -185,6 +188,10 @@ func main() {
 		LeaderElectionID:        "136224848560.member.fleet.azure.com",
 	}
 	//+kubebuilder:scaffold:builder
+	if *enablePprof {
+		memberOpts.PprofBindAddress = fmt.Sprintf(":%d", *pprofPort)
+		hubOpts.PprofBindAddress = fmt.Sprintf(":%d", *hubPprofPort)
+	}
 
 	if err := Start(ctrl.SetupSignalHandler(), hubConfig, memberConfig, hubOpts, memberOpts); err != nil {
 		klog.ErrorS(err, "Failed to start the controllers for the member agent")