manage external cert

kangclzjc · kangclzjc · commit fc4f28aba824 · 2025-11-04T10:38:08.000+08:00
Signed-off-by: kangclzjc &lt;kangz@nvidia.com&gt;
diff --git a/docs/api-reference/operator-api.md b/docs/api-reference/operator-api.md
@@ -801,6 +801,24 @@ _Appears in:_
 | `metrics` _[Server](#server)_ | Metrics is the configuration for serving the metrics endpoint. |  |  |
 
 
+#### WebhookCertManagement
+
+
+
+WebhookCertManagement defines how webhook certificates are managed.
+
+
+
+_Appears in:_
+- [WebhookServer](#webhookserver)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `secretName` _string_ | SecretName is the name of the secret containing the webhook server certificate.<br />Default: grove-webhook-server-cert |  |  |
+| `certManagerEnabled` _boolean_ | CertManagerEnabled indicates whether to use cert-manager for certificate management.<br />When true, the operator waits for cert-manager to provide certificates.<br />When false, behavior depends on AutoProvision.<br />This requires cert-manager to be installed in the cluster when set to true.<br />Default: false |  |  |
+| `autoProvision` _boolean_ | AutoProvision indicates whether to use cert-controller for automatic certificate generation.<br />When true, cert-controller generates and manages certificates automatically.<br />When false, certificates are expected to be provided externally (e.g., via Helm chart or manual Secret creation).<br />This field is ignored when CertManagerEnabled is true.<br />Default: true |  |  |
+
+
 #### WebhookServer
 
 
@@ -817,5 +835,6 @@ _Appears in:_
 | `bindAddress` _string_ | BindAddress is the IP address on which to listen for the specified port. |  |  |
 | `port` _integer_ | Port is the port on which to serve requests. |  |  |
 | `serverCertDir` _string_ | ServerCertDir is the directory containing the server certificate and key. |  |  |
+| `certManagement` _[WebhookCertManagement](#webhookcertmanagement)_ | CertManagement defines the certificate management configuration. |  |  |
 
 
diff --git a/operator/api/config/v1alpha1/defaults.go b/operator/api/config/v1alpha1/defaults.go
@@ -79,6 +79,19 @@ func SetDefaults_ServerConfiguration(serverConfig *ServerConfiguration) {
 		serverConfig.Webhooks.ServerCertDir = defaultWebhookServerTLSServerCertDir
 	}
 
+	if serverConfig.Webhooks.CertManagement == nil {
+		serverConfig.Webhooks.CertManagement = &WebhookCertManagement{}
+	}
+	if serverConfig.Webhooks.CertManagement.CertManagerEnabled == nil {
+		serverConfig.Webhooks.CertManagement.CertManagerEnabled = ptr.To(false)
+	}
+	if serverConfig.Webhooks.CertManagement.AutoProvision == nil {
+		serverConfig.Webhooks.CertManagement.AutoProvision = ptr.To(true)
+	}
+	if serverConfig.Webhooks.CertManagement.SecretName == "" {
+		serverConfig.Webhooks.CertManagement.SecretName = "grove-webhook-server-cert"
+	}
+
 	if serverConfig.HealthProbes == nil {
 		serverConfig.HealthProbes = &Server{}
 	}
diff --git a/operator/api/config/v1alpha1/types.go b/operator/api/config/v1alpha1/types.go
@@ -135,6 +135,31 @@ type WebhookServer struct {
 	Server `json:",inline"`
 	// ServerCertDir is the directory containing the server certificate and key.
 	ServerCertDir string `json:"serverCertDir"`
+	// CertManagement defines the certificate management configuration.
+	// +optional
+	CertManagement *WebhookCertManagement `json:"certManagement,omitempty"`
+}
+
+// WebhookCertManagement defines how webhook certificates are managed.
+type WebhookCertManagement struct {
+	// SecretName is the name of the secret containing the webhook server certificate.
+	// Default: grove-webhook-server-cert
+	// +optional
+	SecretName string `json:"secretName,omitempty"`
+	// CertManagerEnabled indicates whether to use cert-manager for certificate management.
+	// When true, the operator waits for cert-manager to provide certificates.
+	// When false, behavior depends on AutoProvision.
+	// This requires cert-manager to be installed in the cluster when set to true.
+	// Default: false
+	// +optional
+	CertManagerEnabled *bool `json:"certManagerEnabled,omitempty"`
+	// AutoProvision indicates whether to use cert-controller for automatic certificate generation.
+	// When true, cert-controller generates and manages certificates automatically.
+	// When false, certificates are expected to be provided externally (e.g., via Helm chart or manual Secret creation).
+	// This field is ignored when CertManagerEnabled is true.
+	// Default: true
+	// +optional
+	AutoProvision *bool `json:"autoProvision,omitempty"`
 }
 
 // Server contains information for HTTP(S) server configuration.
diff --git a/operator/api/config/v1alpha1/zz_generated.deepcopy.go b/operator/api/config/v1alpha1/zz_generated.deepcopy.go
diff --git a/operator/charts/templates/_helpers.tpl b/operator/charts/templates/_helpers.tpl
@@ -16,6 +16,11 @@ config.yaml: |
   server:
     webhooks:
       port: {{ .Values.config.server.webhooks.port }}
+      serverCertDir: {{ .Values.config.server.webhooks.serverCertDir }}
+      certManagement:
+        secretName: {{ .Values.config.server.webhooks.certManagement.secretName }}
+        certManagerEnabled: {{ .Values.config.server.webhooks.certManagement.certManagerEnabled }}
+        autoProvision: {{ if or .Values.config.server.webhooks.certManagement.certManagerEnabled .Values.config.server.webhooks.certManagement.certFilesPath }}false{{ else }}true{{ end }}
     healthProbes:
       port: {{ .Values.config.server.healthProbes.port }}
     metrics:
diff --git a/operator/charts/templates/authorizer-webhook-config.yaml b/operator/charts/templates/authorizer-webhook-config.yaml
@@ -5,12 +5,20 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: authorizer-webhook
   namespace: {{ .Release.Namespace }}
+  {{- if .Values.config.server.webhooks.certManagement.certManagerEnabled }}
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Values.config.server.webhooks.certManagement.secretName }}
+  {{- end }}
   labels:
 {{- include "operator.authorizer.webhook.labels" . | nindent 4 }}
 webhooks:
   - admissionReviewVersions:
       - v1
     clientConfig:
+      {{- if and .Values.config.server.webhooks.certManagement.certFilesPath (not .Values.config.server.webhooks.certManagement.certManagerEnabled) }}
+      # CA Bundle from chart files
+      caBundle: {{ .Files.Get (printf "%s/ca.crt" .Values.config.server.webhooks.certManagement.certFilesPath) | b64enc }}
+      {{- end }}
       service:
         name: {{ required ".Values.service.name is required" .Values.service.name }}
         namespace: {{ .Release.Namespace }}
diff --git a/operator/charts/templates/deployment.yaml b/operator/charts/templates/deployment.yaml
@@ -60,7 +60,7 @@ spec:
               mountPath: /var/run/secrets/kubernetes.io/serviceaccount
               readOnly: true
             - name: grove-webhook-server-cert
-              mountPath: /etc/grove-operator/webhook-certs
+              mountPath: {{ .Values.config.server.webhooks.serverCertDir }}
               readOnly: true
           env:
             - name: GROVE_OPERATOR_SERVICE_ACCOUNT_NAME
@@ -91,6 +91,7 @@ spec:
           configMap:
             name: {{ include "operator.config.name" . }}
         - name: grove-webhook-server-cert
+          # Always mount the Secret volume (cert-controller manages it in auto-provision mode)
           secret:
-            secretName: grove-webhook-server-cert
+            secretName: {{ .Values.config.server.webhooks.certManagement.secretName }}
             defaultMode: 420
diff --git a/operator/charts/templates/pcs-defaulting-webhook-config.yaml b/operator/charts/templates/pcs-defaulting-webhook-config.yaml
@@ -4,12 +4,20 @@ kind: MutatingWebhookConfiguration
 metadata:
   name: podcliqueset-defaulting-webhook
   namespace: {{ .Release.Namespace }}
+  {{- if .Values.config.server.webhooks.certManagement.certManagerEnabled }}
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Values.config.server.webhooks.certManagement.secretName }}
+  {{- end }}
   labels:
 {{- include "operator.pcs.defaulting.webhook.labels" . | nindent 4 }}
 webhooks:
   - admissionReviewVersions:
       - v1
     clientConfig:
+      {{- if and .Values.config.server.webhooks.certManagement.certFilesPath (not .Values.config.server.webhooks.certManagement.certManagerEnabled) }}
+      # CA Bundle from chart files
+      caBundle: {{ .Files.Get (printf "%s/ca.crt" .Values.config.server.webhooks.certManagement.certFilesPath) | b64enc }}
+      {{- end }}
       service:
         name: {{ required ".Values.service.name is required" .Values.service.name }}
         namespace: {{ .Release.Namespace }}
diff --git a/operator/charts/templates/pcs-validating-webhook-config.yaml b/operator/charts/templates/pcs-validating-webhook-config.yaml
@@ -4,12 +4,20 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: podcliqueset-validating-webhook
   namespace: {{ .Release.Namespace }}
+  {{- if .Values.config.server.webhooks.certManagement.certManagerEnabled }}
+  annotations:
+    cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Values.config.server.webhooks.certManagement.secretName }}
+  {{- end }}
   labels:
 {{- include "operator.pcs.validating.webhook.labels" . | nindent 4 }}
 webhooks:
   - admissionReviewVersions:
       - v1
     clientConfig:
+      {{- if and .Values.config.server.webhooks.certManagement.certFilesPath (not .Values.config.server.webhooks.certManagement.certManagerEnabled) }}
+      # CA Bundle from chart files
+      caBundle: {{ .Files.Get (printf "%s/ca.crt" .Values.config.server.webhooks.certManagement.certFilesPath) | b64enc }}
+      {{- end }}
       service:
         name: {{ required ".Values.service.name is required" .Values.service.name }}
         namespace: {{ .Release.Namespace }}
diff --git a/operator/charts/templates/webhook-server-cert-secret.yaml b/operator/charts/templates/webhook-server-cert-secret.yaml
@@ -1,11 +1,24 @@
+{{- if not .Values.config.server.webhooks.certManagement.certManagerEnabled }}
+---
 apiVersion: v1
 kind: Secret
 metadata:
-  name: grove-webhook-server-cert
+  name: {{ .Values.config.server.webhooks.certManagement.secretName }}
   namespace: {{ .Release.Namespace }}
   labels:
 {{- include "operator.server.secret.labels" . | nindent 4 }}
 type: kubernetes.io/tls
 data:
+  {{- if .Values.config.server.webhooks.certManagement.certFilesPath }}
+  # Certificate files provided: read from chart directory
+  tls.crt: {{ .Files.Get (printf "%s/server.crt" .Values.config.server.webhooks.certManagement.certFilesPath) | b64enc }}
+  tls.key: {{ .Files.Get (printf "%s/server.key" .Values.config.server.webhooks.certManagement.certFilesPath) | b64enc }}
+  {{- if .Files.Get (printf "%s/ca.crt" .Values.config.server.webhooks.certManagement.certFilesPath) }}
+  ca.crt: {{ .Files.Get (printf "%s/ca.crt" .Values.config.server.webhooks.certManagement.certFilesPath) | b64enc }}
+  {{- end }}
+  {{- else }}
+  # Auto-provision mode: create empty Secret for operator to populate
   tls.crt: ""
   tls.key: ""
+  {{- end }}
+{{- end }}
diff --git a/operator/charts/values.yaml b/operator/charts/values.yaml
@@ -46,6 +46,37 @@ config:
   server:
     webhooks:
       port: 9443
+      # serverCertDir is the directory path where certificate files are located.
+      # The operator will look for tls.crt and tls.key in this directory.
+      # Default: /etc/grove-operator/webhook-certs
+      serverCertDir: /etc/grove-operator/webhook-certs
+      certManagement:
+        # secretName is the name of the Kubernetes Secret containing webhook certificates.
+        # The Secret must contain tls.crt and tls.key (and optionally ca.crt).
+        # Default: "grove-webhook-server-cert"
+        secretName: grove-webhook-server-cert
+        
+        # certManagerEnabled controls cert-manager integration.
+        # - true: Use cert-manager for certificate management and CA injection
+        #         Chart will NOT create the Secret (cert-manager manages it)
+        #         Adds cert-manager.io/inject-ca-from annotation to webhook configs
+        # - false: Chart manages certificates directly (no cert-manager)
+        # Default: false
+        certManagerEnabled: false
+        
+        # certFilesPath specifies the directory containing certificate files within the chart.
+        # When set, Helm reads certificate files and creates the Secret.
+        # Expected files: server.crt, server.key, ca.crt
+        # Example: "pki-resources" reads from charts/pki-resources/
+        # 
+        # Behavior:
+        # - If certFilesPath is set: Chart creates Secret from files
+        # - If certFilesPath is empty and certManagerEnabled=false: Operator auto-generates certificates
+        # - If certManagerEnabled=true: certFilesPath is ignored (cert-manager manages Secret)
+        # 
+        # Set to empty string ("") to use auto-provision or external Secret.
+        # Default: "pki-resources"
+        certFilesPath: ""
     healthProbes:
       enable: false
       port: 9444
diff --git a/operator/cmd/main.go b/operator/cmd/main.go
@@ -62,8 +62,17 @@ func main() {
 	}
 
 	webhookCertsReadyCh := make(chan struct{})
-	if err = cert.ManageWebhookCerts(mgr, operatorCfg.Server.Webhooks.ServerCertDir, operatorCfg.Authorizer.Enabled, webhookCertsReadyCh); err != nil {
-		logger.Error(err, "failed to setup cert rotation")
+	certMgmt := operatorCfg.Server.Webhooks.CertManagement
+	if err = cert.ManageWebhookCerts(
+		mgr,
+		operatorCfg.Server.Webhooks.ServerCertDir,
+		certMgmt.SecretName,
+		operatorCfg.Authorizer.Enabled,
+		*certMgmt.CertManagerEnabled,
+		*certMgmt.AutoProvision,
+		webhookCertsReadyCh,
+	); err != nil {
+		logger.Error(err, "failed to setup certificate management")
 		os.Exit(1)
 	}
 
diff --git a/operator/internal/controller/cert/cert.go b/operator/internal/controller/cert/cert.go
@@ -39,16 +39,42 @@ const (
 )
 
 // ManageWebhookCerts registers the cert-controller with the manager which will be used to manage
-// webhook certificates.
-func ManageWebhookCerts(mgr ctrl.Manager, certDir string, authorizerEnabled bool, certsReadyCh chan struct{}) error {
+// webhook certificates. The behavior depends on the configuration:
+// - If certManagerEnabled=true: waits for cert-manager to provide certificates
+// - If autoProvision=true: uses cert-controller for automatic certificate generation and management
+// - If autoProvision=false: waits for externally provided certificates (e.g., from Helm chart)
+func ManageWebhookCerts(mgr ctrl.Manager, certDir string, secretName string, authorizerEnabled bool, certManagerEnabled bool, autoProvision bool, certsReadyCh chan struct{}) error {
 	namespace, err := getOperatorNamespace()
 	if err != nil {
 		return err
 	}
+
+	logger := ctrl.Log.WithName("cert-management")
+
+	if certManagerEnabled {
+		logger.Info("Using cert-manager for certificate management",
+			"secretName", secretName, "certDir", certDir)
+		return nil
+	}
+
+	if !autoProvision {
+		logger.Info("Using externally provided certificates",
+			"certDir", certDir, "secretName", secretName)
+		return nil
+	}
+
+	// Auto-provision mode: Use cert-controller for certificate management
+	// cert-controller handles:
+	// - Generating certificates if they don't exist
+	// - Rotating certificates before expiry
+	// - Injecting CA bundle into webhook configurations
+	// - Monitoring certificate files and Secret for changes
+	logger.Info("Auto-provisioning certificates using cert-controller",
+		"secretName", secretName, "certDir", certDir)
 	rotator := &cert.CertRotator{
 		SecretKey: types.NamespacedName{
 			Namespace: namespace,
-			Name:      "grove-webhook-server-cert",
+			Name:      secretName,
 		},
 		CertDir:        certDir,
 		CAName:         certificateAuthorityName,
