CNTRLPLANE-1551: feat(nodepool): add HAProxy image override via annotation

api/hypershift/v1beta1/hostedcluster_types.go

@@ -50,6 +50,10 @@ const (
 	// KonnectivityAgentImageAnnotation is a temporary annotation that allows the specification of the konnectivity agent image.
 	// This will be removed when Konnectivity is added to the Openshift release payload
 	KonnectivityAgentImageAnnotation = "hypershift.openshift.io/konnectivity-agent-image"
+	// HAProxyImageAnnotation can be set on a NodePool to override the HAProxy image
+	// used for worker node API server proxy. This takes precedence over the environment
+	// variable IMAGE_SHARED_INGRESS_HAPROXY and the default shared ingress image.
+	HAProxyImageAnnotation = "hypershift.openshift.io/haproxy-image"
 	// ControlPlaneOperatorImageAnnotation is an annotation that allows the specification of the control plane operator image.
 	// This is used for development and e2e workflows
 	ControlPlaneOperatorImageAnnotation = "hypershift.openshift.io/control-plane-operator-image"

docs/content/how-to/automated-machine-management/haproxy-image-override.md

@@ -0,0 +1,126 @@
+# Customize Worker Node HAProxy Image
+
+This guide explains how to customize the HAProxy image used for worker node API server proxy on a per-NodePool basis.
+
+## Overview
+
+Worker nodes in HyperShift use HAProxy to proxy connections to the hosted control plane API server. By default, the HAProxy image comes from the OpenShift release payload. However, you can override this image using either:
+
+1. **NodePool annotation** (recommended for per-NodePool customization)
+2. **Environment variable** (for global override when shared ingress is enabled)
+
+## Image Resolution Priority
+
+The HAProxy image is resolved in the following priority order (highest to lowest):
+
+1. **NodePool annotation** `hypershift.openshift.io/haproxy-image` (highest priority)
+2. **Environment variable** `IMAGE_SHARED_INGRESS_HAPROXY` (when shared ingress is enabled)
+3. **Hardcoded default** (when shared ingress is enabled)
+4. **Release payload** (default when shared ingress is disabled)
+
+## Per-NodePool Customization
+
+### Use Case
+
+Use the NodePool annotation when you want to:
+- Test a newer HAProxy version on a specific NodePool
+- Use different HAProxy images for different workload types
+- Gradually roll out HAProxy updates across NodePools
+- Use a custom HAProxy image with specific patches or configurations
+
+### Configuration
+
+To override the HAProxy image for a specific NodePool, add the `hypershift.openshift.io/haproxy-image` annotation:
+
+```yaml
+apiVersion: hypershift.openshift.io/v1beta1
+kind: NodePool
+metadata:
+  name: my-nodepool
+  namespace: clusters
+  annotations:
+    hypershift.openshift.io/haproxy-image: "quay.io/my-org/haproxy:custom-v2.9.1"
+spec:
+  clusterName: my-cluster
+  replicas: 3
+  # ... rest of spec
+```
+
+### Applying the Annotation
+
+You can add the annotation to an existing NodePool using `kubectl annotate`:
+
+```bash
+kubectl annotate nodepool my-nodepool \
+  -n clusters \
+  hypershift.openshift.io/haproxy-image="quay.io/my-org/haproxy:custom-v2.9.1"
+```
+
+### Removing the Override
+
+To remove the override and revert to the default behavior:
+
+```bash
+kubectl annotate nodepool my-nodepool \
+  -n clusters \
+  hypershift.openshift.io/haproxy-image-
+```
+
+## Global Override (Shared Ingress Only)
+
+When shared ingress is enabled, you can set a global HAProxy image override using the `IMAGE_SHARED_INGRESS_HAPROXY` environment variable on the HyperShift operator. This affects all NodePools that don't have the annotation set.
+
+**Note**: The NodePool annotation always takes precedence over the environment variable.
+
+## Verification
+
+After applying the annotation, new worker nodes will use the specified HAProxy image. To verify:
+
+1. Check the NodePool's token secret generation to ensure the new configuration is picked up
+2. Verify the ignition configuration contains the correct image
+3. On a worker node, check the static pod manifest:
+
+```bash
+# On a worker node
+cat /etc/kubernetes/manifests/kube-apiserver-proxy.yaml | grep image:
+```
+
+## Rollout Behavior
+
+The HAProxy image change triggers a NodePool rollout:
+- New ignition configuration is generated with the updated image
+- Worker nodes are replaced according to the NodePool's upgrade strategy
+- The rollout respects `maxUnavailable` settings
+
+## Important Notes
+
+1. **Image Availability**: Ensure the custom HAProxy image is accessible from worker nodes
+2. **Pull Secrets**: The worker nodes must have credentials to pull the custom image
+3. **Compatibility**: The custom HAProxy image must be compatible with HyperShift's configuration expectations
+4. **Shared Ingress**: When shared ingress is enabled, ensure the custom image supports proxy protocol v2 with TLV (requires HAProxy v2.9+)
+5. **Multiple NodePools**: Each NodePool can have a different HAProxy image override
+
+
+## Troubleshooting
+
+### Image Pull Errors
+
+If worker nodes fail to pull the custom image:
+1. Verify the image exists and is accessible
+2. Check that the global pull secret includes credentials for the image registry
+3. Verify network connectivity from worker nodes to the image registry
+
+### Wrong Image in Use
+
+If the expected image is not being used:
+1. Check the annotation is correctly set on the NodePool
+2. Verify the NodePool has reconciled (check status conditions)
+3. Inspect the ignition configuration in the NodePool's token secret
+
+### Rollout Issues
+
+If the rollout doesn't complete:
+1. Check NodePool conditions for errors
+2. Verify the custom image is compatible
+3. Check worker node logs for HAProxy startup failures
+4. Ensure the image supports the required features (e.g., proxy protocol for shared ingress)

hypershift-operator/controllers/nodepool/apiserver-haproxy/haproxy.go

@@ -18,7 +18,6 @@ import (
 	sharedingress "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress"
 	api "github.com/openshift/hypershift/support/api"
 	"github.com/openshift/hypershift/support/config"
-	"github.com/openshift/hypershift/support/images"
 	"github.com/openshift/hypershift/support/releaseinfo"
 	"github.com/openshift/hypershift/support/util"
 
@@ -255,11 +254,6 @@ func apiServerProxyConfig(haProxyImage, cpoImage, clusterID,
 		livenessProbeEndpoint = "/livez?exclude=etcd&exclude=log"
 	}
 
-	if sharedingress.UseSharedIngress() {
-		// proxy protocol v2 with TLV support (custom proxy protocol header) requires haproxy v2.9+, see: https://www.haproxy.com/blog/announcing-haproxy-2-9#proxy-protocol-tlv-fields
-		haProxyImage = images.GetSharedIngressHAProxyImage()
-	}
-
 	filesToAdd := []fileToAdd{
 		{
 			template: setupAPIServerIPScriptTemplate,

hypershift-operator/controllers/nodepool/apiserver-haproxy/testdata/zz_fixture_TestAPIServerHAProxyConfig_when_use_shared_router_it_should_use_proxy_protocol.yaml

@@ -18,7 +18,7 @@ storage:
     overwrite: true
     path: /etc/kubernetes/apiserver-proxy-config/haproxy.cfg
   - contents:
-      source: data:text/plain;charset=utf-8;base64,YXBpVmVyc2lvbjogdjEKa2luZDogUG9kCm1ldGFkYXRhOgogIGNyZWF0aW9uVGltZXN0YW1wOiBudWxsCiAgbGFiZWxzOgogICAgazhzLWFwcDoga3ViZS1hcGlzZXJ2ZXItcHJveHkKICBuYW1lOiBrdWJlLWFwaXNlcnZlci1wcm94eQogIG5hbWVzcGFjZToga3ViZS1zeXN0ZW0Kc3BlYzoKICBjb250YWluZXJzOgogIC0gY29tbWFuZDoKICAgIC0gaGFwcm94eQogICAgLSAtZgogICAgLSAvdXNyL2xvY2FsL2V0Yy9oYXByb3h5CiAgICBpbWFnZTogcXVheS5pby9yZWRoYXQtdXNlci13b3JrbG9hZHMvY3J0LXJlZGhhdC1hY20tdGVuYW50L2h5cGVyc2hpZnQtc2hhcmVkLWluZ3Jlc3MtbWFpbkBzaGEyNTY6MWFmNTliN2EyOTQzMjMxNGJkZTU0ZTg5NzdmYTQ1ZmE5MmRjNDg4ODVlZmJmMGRmNjAxNDE4ZWMwOTEyZjQ3MgogICAgbGl2ZW5lc3NQcm9iZToKICAgICAgZmFpbHVyZVRocmVzaG9sZDogMwogICAgICBodHRwR2V0OgogICAgICAgIGhvc3Q6IGNsdXN0ZXIuaW50ZXJuYWwuZXhhbXBsZS5jb20KICAgICAgICBwYXRoOiAvdmVyc2lvbgogICAgICAgIHBvcnQ6IDg0NDMKICAgICAgICBzY2hlbWU6IEhUVFBTCiAgICAgIGluaXRpYWxEZWxheVNlY29uZHM6IDEyMAogICAgICBwZXJpb2RTZWNvbmRzOiAxMjAKICAgICAgc3VjY2Vzc1RocmVzaG9sZDogMQogICAgbmFtZTogaGFwcm94eQogICAgcG9ydHM6CiAgICAtIGNvbnRhaW5lclBvcnQ6IDg0NDMKICAgICAgaG9zdFBvcnQ6IDg0NDMKICAgICAgbmFtZTogYXBpc2VydmVyCiAgICAgIHByb3RvY29sOiBUQ1AKICAgIHJlc291cmNlczoKICAgICAgcmVxdWVzdHM6CiAgICAgICAgY3B1OiAxM20KICAgICAgICBtZW1vcnk6IDE2TWkKICAgIHNlY3VyaXR5Q29udGV4dDoKICAgICAgcnVuQXNVc2VyOiAxMDAxCiAgICB2b2x1bWVNb3VudHM6CiAgICAtIG1vdW50UGF0aDogL3Vzci9sb2NhbC9ldGMvaGFwcm94eQogICAgICBuYW1lOiBjb25maWcKICBob3N0TmV0d29yazogdHJ1ZQogIHByaW9yaXR5Q2xhc3NOYW1lOiBzeXN0ZW0tbm9kZS1jcml0aWNhbAogIHZvbHVtZXM6CiAgLSBob3N0UGF0aDoKICAgICAgcGF0aDogL2V0Yy9rdWJlcm5ldGVzL2FwaXNlcnZlci1wcm94eS1jb25maWcKICAgIG5hbWU6IGNvbmZpZwpzdGF0dXM6IHt9Cg==
+      source: data:text/plain;charset=utf-8;base64,YXBpVmVyc2lvbjogdjEKa2luZDogUG9kCm1ldGFkYXRhOgogIGNyZWF0aW9uVGltZXN0YW1wOiBudWxsCiAgbGFiZWxzOgogICAgazhzLWFwcDoga3ViZS1hcGlzZXJ2ZXItcHJveHkKICBuYW1lOiBrdWJlLWFwaXNlcnZlci1wcm94eQogIG5hbWVzcGFjZToga3ViZS1zeXN0ZW0Kc3BlYzoKICBjb250YWluZXJzOgogIC0gY29tbWFuZDoKICAgIC0gaGFwcm94eQogICAgLSAtZgogICAgLSAvdXNyL2xvY2FsL2V0Yy9oYXByb3h5CiAgICBpbWFnZTogaGEtcHJveHktaW1hZ2U6bGF0ZXN0CiAgICBsaXZlbmVzc1Byb2JlOgogICAgICBmYWlsdXJlVGhyZXNob2xkOiAzCiAgICAgIGh0dHBHZXQ6CiAgICAgICAgaG9zdDogY2x1c3Rlci5pbnRlcm5hbC5leGFtcGxlLmNvbQogICAgICAgIHBhdGg6IC92ZXJzaW9uCiAgICAgICAgcG9ydDogODQ0MwogICAgICAgIHNjaGVtZTogSFRUUFMKICAgICAgaW5pdGlhbERlbGF5U2Vjb25kczogMTIwCiAgICAgIHBlcmlvZFNlY29uZHM6IDEyMAogICAgICBzdWNjZXNzVGhyZXNob2xkOiAxCiAgICBuYW1lOiBoYXByb3h5CiAgICBwb3J0czoKICAgIC0gY29udGFpbmVyUG9ydDogODQ0MwogICAgICBob3N0UG9ydDogODQ0MwogICAgICBuYW1lOiBhcGlzZXJ2ZXIKICAgICAgcHJvdG9jb2w6IFRDUAogICAgcmVzb3VyY2VzOgogICAgICByZXF1ZXN0czoKICAgICAgICBjcHU6IDEzbQogICAgICAgIG1lbW9yeTogMTZNaQogICAgc2VjdXJpdHlDb250ZXh0OgogICAgICBydW5Bc1VzZXI6IDEwMDEKICAgIHZvbHVtZU1vdW50czoKICAgIC0gbW91bnRQYXRoOiAvdXNyL2xvY2FsL2V0Yy9oYXByb3h5CiAgICAgIG5hbWU6IGNvbmZpZwogIGhvc3ROZXR3b3JrOiB0cnVlCiAgcHJpb3JpdHlDbGFzc05hbWU6IHN5c3RlbS1ub2RlLWNyaXRpY2FsCiAgdm9sdW1lczoKICAtIGhvc3RQYXRoOgogICAgICBwYXRoOiAvZXRjL2t1YmVybmV0ZXMvYXBpc2VydmVyLXByb3h5LWNvbmZpZwogICAgbmFtZTogY29uZmlnCnN0YXR1czoge30K
     mode: 420
     overwrite: true
     path: /etc/kubernetes/manifests/kube-apiserver-proxy.yaml

hypershift-operator/controllers/nodepool/conditions.go

@@ -325,7 +325,7 @@ func (r *NodePoolReconciler) validMachineConfigCondition(ctx context.Context, no
 		return &ctrl.Result{}, nil
 	}
 
-	haproxyRawConfig, err := r.generateHAProxyRawConfig(ctx, hcluster, releaseImage)
+	haproxyRawConfig, err := r.generateHAProxyRawConfig(ctx, nodePool, hcluster, releaseImage)
 	if err != nil {
 		return &ctrl.Result{}, fmt.Errorf("failed to generate HAProxy raw config: %w", err)
 	}

hypershift-operator/controllers/nodepool/nodepool_controller.go

@@ -13,8 +13,10 @@ import (
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests"
 	haproxy "github.com/openshift/hypershift/hypershift-operator/controllers/nodepool/apiserver-haproxy"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/nodepool/kubevirt"
+	"github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress"
 	kvinfra "github.com/openshift/hypershift/kubevirtexternalinfra"
 	"github.com/openshift/hypershift/support/capabilities"
+	"github.com/openshift/hypershift/support/images"
 	"github.com/openshift/hypershift/support/releaseinfo"
 	"github.com/openshift/hypershift/support/supportedversion"
 	"github.com/openshift/hypershift/support/upsert"
@@ -336,7 +338,7 @@ func (r *NodePoolReconciler) reconcile(ctx context.Context, hcluster *hyperv1.Ho
 		return ctrl.Result{}, nil
 	}
 
-	haproxyRawConfig, err := r.generateHAProxyRawConfig(ctx, hcluster, releaseImage)
+	haproxyRawConfig, err := r.generateHAProxyRawConfig(ctx, nodePool, hcluster, releaseImage)
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to generate HAProxy raw config: %w", err)
 	}
@@ -413,7 +415,7 @@ func (r *NodePoolReconciler) token(ctx context.Context, hcluster *hyperv1.Hosted
 		return nil, fmt.Errorf("failed to look up release image metadata: %w", err)
 	}
 
-	haproxyRawConfig, err := r.generateHAProxyRawConfig(ctx, hcluster, releaseImage)
+	haproxyRawConfig, err := r.generateHAProxyRawConfig(ctx, nodePool, hcluster, releaseImage)
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate HAProxy raw config: %w", err)
 	}
@@ -974,11 +976,35 @@ func (r *NodePoolReconciler) getAdditionalTrustBundle(ctx context.Context, hoste
 	return additionalTrustBundle, nil
 }
 
-func (r *NodePoolReconciler) generateHAProxyRawConfig(ctx context.Context, hcluster *hyperv1.HostedCluster, releaseImage *releaseinfo.ReleaseImage) (string, error) {
+// resolveHAProxyImage determines which HAProxy image to use based on priority:
+// 1. NodePool annotation (highest priority)
+// 2. Environment variable override (when shared ingress enabled)
+// 3. Hardcoded default (when shared ingress enabled)
+// 4. Release payload (default)
+func resolveHAProxyImage(nodePool *hyperv1.NodePool, releaseImage *releaseinfo.ReleaseImage) (string, error) {
+	// Check NodePool annotation first (highest priority)
+	if annotationImage := strings.TrimSpace(nodePool.Annotations[hyperv1.HAProxyImageAnnotation]); annotationImage != "" {
+		return annotationImage, nil
+	}
+
+	// Check if shared ingress is enabled
+	if sharedingress.UseSharedIngress() {
+		return images.GetSharedIngressHAProxyImage(), nil
+	}
+
+	// Fall back to release payload image
 	haProxyImage, ok := releaseImage.ComponentImages()[haproxy.HAProxyRouterImageName]
 	if !ok {
 		return "", fmt.Errorf("release image doesn't have a %s image", haproxy.HAProxyRouterImageName)
 	}
+	return haProxyImage, nil
+}
+
+func (r *NodePoolReconciler) generateHAProxyRawConfig(ctx context.Context, nodePool *hyperv1.NodePool, hcluster *hyperv1.HostedCluster, releaseImage *releaseinfo.ReleaseImage) (string, error) {
+	haProxyImage, err := resolveHAProxyImage(nodePool, releaseImage)
+	if err != nil {
+		return "", err
+	}
 
 	haProxy := haproxy.HAProxy{
 		Client:                  r.Client,

hypershift-operator/controllers/nodepool/nodepool_controller_test.go

@@ -10,6 +10,7 @@ import (
 
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	"github.com/openshift/hypershift/api/util/ipnet"
+	haproxy "github.com/openshift/hypershift/hypershift-operator/controllers/nodepool/apiserver-haproxy"
 	ignserver "github.com/openshift/hypershift/ignition-server/controllers"
 	kvinfra "github.com/openshift/hypershift/kubevirtexternalinfra"
 	"github.com/openshift/hypershift/support/api"
@@ -1475,3 +1476,139 @@ func Test_validateHCPayloadSupportsNodePoolCPUArch(t *testing.T) {
 		})
 	}
 }
+func TestResolveHAProxyImage(t *testing.T) {
+	const (
+		testAnnotationImage    = "quay.io/test/haproxy:custom"
+		testSharedIngressImage = "quay.io/test/haproxy:shared-ingress"
+		testReleaseImage       = "registry.test.io/openshift/haproxy-router:v4.16"
+	)
+
+	testCases := []struct {
+		name                string
+		nodePoolAnnotations map[string]string
+		useSharedIngress    bool
+		envVarImage         string
+		expectedImage       string
+	}{
+		{
+			name: "When NodePool annotation is set it should use annotation image",
+			nodePoolAnnotations: map[string]string{
+				hyperv1.HAProxyImageAnnotation: testAnnotationImage,
+			},
+			useSharedIngress: false,
+			expectedImage:    testAnnotationImage,
+		},
+		{
+			name: "When NodePool annotation is set it should override shared ingress image",
+			nodePoolAnnotations: map[string]string{
+				hyperv1.HAProxyImageAnnotation: testAnnotationImage,
+			},
+			useSharedIngress: true,
+			envVarImage:      testSharedIngressImage,
+			expectedImage:    testAnnotationImage,
+		},
+		{
+			name: "When NodePool annotation is empty it should use shared ingress image",
+			nodePoolAnnotations: map[string]string{
+				hyperv1.HAProxyImageAnnotation: "",
+			},
+			useSharedIngress: true,
+			envVarImage:      testSharedIngressImage,
+			expectedImage:    testSharedIngressImage,
+		},
+		{
+			name:             "When no annotation and shared ingress enabled it should use shared ingress image",
+			useSharedIngress: true,
+			envVarImage:      testSharedIngressImage,
+			expectedImage:    testSharedIngressImage,
+		},
+		{
+			name:             "When no annotation and shared ingress disabled it should use release payload image",
+			useSharedIngress: false,
+			expectedImage:    testReleaseImage,
+		},
+		{
+			name: "When annotation is empty and shared ingress disabled it should use release payload image",
+			nodePoolAnnotations: map[string]string{
+				hyperv1.HAProxyImageAnnotation: "",
+			},
+			useSharedIngress: false,
+			expectedImage:    testReleaseImage,
+		},
+		{
+			name: "When annotation is whitespace and shared ingress disabled it should use release payload image",
+			nodePoolAnnotations: map[string]string{
+				hyperv1.HAProxyImageAnnotation: "  ",
+			},
+			useSharedIngress: false,
+			expectedImage:    testReleaseImage,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			// Set up environment variables for shared ingress
+			if tc.useSharedIngress {
+				t.Setenv("MANAGED_SERVICE", hyperv1.AroHCP)
+				if tc.envVarImage != "" {
+					t.Setenv("IMAGE_SHARED_INGRESS_HAPROXY", tc.envVarImage)
+				}
+			}
+
+			// Create test NodePool
+			nodePool := &hyperv1.NodePool{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        "test-nodepool",
+					Namespace:   "clusters",
+					Annotations: tc.nodePoolAnnotations,
+				},
+			}
+
+			// Create fake pull secret
+			pullSecret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "pull-secret",
+				},
+				Data: map[string][]byte{
+					corev1.DockerConfigJsonKey: []byte(`{"auths":{"test":{"auth":"dGVzdDp0ZXN0"}}}`),
+				},
+			}
+
+			// Create fake client
+			c := fake.NewClientBuilder().WithObjects(pullSecret).Build()
+
+			// Create fake release provider with component images
+			releaseProvider := &fakereleaseprovider.FakeReleaseProvider{
+				Components: map[string]string{
+					haproxy.HAProxyRouterImageName: testReleaseImage,
+				},
+			}
+
+			// Create test HostedCluster
+			hc := &hyperv1.HostedCluster{
+				Spec: hyperv1.HostedClusterSpec{
+					PullSecret: corev1.LocalObjectReference{
+						Name: "pull-secret",
+					},
+					Release: hyperv1.Release{
+						Image: "test-release:latest",
+					},
+				},
+			}
+
+			// Get release image using the fake provider
+			ctx := t.Context()
+			releaseImage := fakereleaseprovider.GetReleaseImage(ctx, hc, c, releaseProvider)
+			g.Expect(releaseImage).ToNot(BeNil())
+
+			// Call resolveHAProxyImage
+			image, err := resolveHAProxyImage(nodePool, releaseImage)
+
+			// Verify no error and correct image
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(image).To(Equal(tc.expectedImage))
+		})
+	}
+}

hypershift-operator/controllers/nodepool/secret_janitor.go

@@ -93,7 +93,7 @@ func (r *secretJanitor) Reconcile(ctx context.Context, req reconcile.Request) (r
 	if err != nil {
 		return ctrl.Result{}, err
 	}
-	haproxyRawConfig, err := r.generateHAProxyRawConfig(ctx, hcluster, releaseImage)
+	haproxyRawConfig, err := r.generateHAProxyRawConfig(ctx, nodePool, hcluster, releaseImage)
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to generate HAProxy raw config: %w", err)
 	}