[KEP] FlavorFungability: replace FlavorFungibilityImplicitPreferenceDefault feature gate with API

* preference API field

Author: vladikkuzn

apis/kueue/v1beta1/clusterqueue_types.go

@@ -389,6 +389,13 @@ const (
 	TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor"
 )
 
+type FlavorAssignmentPreference string
+
+const (
+	PreferenceBorrowing  FlavorAssignmentPreference = "Borrowing"
+	PreferencePreempting FlavorAssignmentPreference = "Preempting"
+)
+
 // FlavorFungibility determines whether a workload should try the next flavor
 // before borrowing or preempting in current flavor.
 type FlavorFungibility struct {
@@ -415,6 +422,13 @@ type FlavorFungibility struct {
 	// +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor,Preempt}
 	// +kubebuilder:default="TryNextFlavor"
 	WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"`
+	// preference selects the order between borrowing-first and preemption-first
+	// when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+	// If unset, the default preference is borrowing-first.
+	// Preference must be unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+	//
+	// +kubebuilder:validation:Enum={Borrowing,Preempting}
+	Preference *FlavorAssignmentPreference `json:"preference,omitempty"`
 }
 
 // ClusterQueuePreemption contains policies to preempt Workloads from this

apis/kueue/v1beta1/zz_generated.deepcopy.go

@@ -370,6 +370,13 @@ const (
 	TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor"
 )
 
+type FlavorAssignmentPreference string
+
+const (
+	PreferenceBorrowing  FlavorAssignmentPreference = "Borrowing"
+	PreferencePreempting FlavorAssignmentPreference = "Preempting"
+)
+
 // FlavorFungibility determines whether a workload should try the next flavor
 // before borrowing or preempting in current flavor.
 type FlavorFungibility struct {
@@ -396,6 +403,13 @@ type FlavorFungibility struct {
 	// +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor,Preempt}
 	// +kubebuilder:default="TryNextFlavor"
 	WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"`
+	// preference selects the order between borrowing-first and preemption-first
+	// when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+	// If unset, the default preference is borrowing-first. Preference must be
+	// unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+	//
+	// +kubebuilder:validation:Enum={Borrowing,Preempting}
+	Preference *FlavorAssignmentPreference `json:"preference,omitempty"`
 }
 
 // ClusterQueuePreemption contains policies to preempt Workloads from this

apis/kueue/v1beta2/clusterqueue_types.go

@@ -192,6 +192,16 @@ spec:
                     flavorFungibility defines whether a workload should try the next flavor
                     before borrowing or preempting in the flavor being evaluated.
                   properties:
+                    preference:
+                      description: |-
+                        preference selects the order between borrowing-first and preemption-first
+                        when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+                        If unset, the default preference is borrowing-first.
+                        Preference must be unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+                      enum:
+                        - Borrowing
+                        - Preempting
+                      type: string
                     whenCanBorrow:
                       default: MayStopSearch
                       description: |-
@@ -948,6 +958,16 @@ spec:
                     flavorFungibility defines whether a workload should try the next flavor
                     before borrowing or preempting in the flavor being evaluated.
                   properties:
+                    preference:
+                      description: |-
+                        preference selects the order between borrowing-first and preemption-first
+                        when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+                        If unset, the default preference is borrowing-first. Preference must be
+                        unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+                      enum:
+                        - Borrowing
+                        - Preempting
+                      type: string
                     whenCanBorrow:
                       default: MayStopSearch
                       description: |-

apis/kueue/v1beta2/zz_generated.deepcopy.go

@@ -163,6 +163,16 @@ spec:
                   flavorFungibility defines whether a workload should try the next flavor
                   before borrowing or preempting in the flavor being evaluated.
                 properties:
+                  preference:
+                    description: |-
+                      preference selects the order between borrowing-first and preemption-first
+                      when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+                      If unset, the default preference is borrowing-first.
+                      Preference must be unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+                    enum:
+                    - Borrowing
+                    - Preempting
+                    type: string
                   whenCanBorrow:
                     default: MayStopSearch
                     description: |-
@@ -936,6 +946,16 @@ spec:
                   flavorFungibility defines whether a workload should try the next flavor
                   before borrowing or preempting in the flavor being evaluated.
                 properties:
+                  preference:
+                    description: |-
+                      preference selects the order between borrowing-first and preemption-first
+                      when both WhenCanBorrow and WhenCanPreempt are set to `TryNextFlavor`.
+                      If unset, the default preference is borrowing-first. Preference must be
+                      unset when either WhenCanBorrow or WhenCanPreempt is not `TryNextFlavor`.
+                    enum:
+                    - Borrowing
+                    - Preempting
+                    type: string
                   whenCanBorrow:
                     default: MayStopSearch
                     description: |-

charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml

@@ -157,13 +157,6 @@ const (
 	// matches managedJobsNamespaceSelector.
 	ManagedJobsNamespaceSelectorAlwaysRespected featuregate.Feature = "ManagedJobsNamespaceSelectorAlwaysRespected"
 
-	// owner: @pajakd
-	// kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/582-preempt-based-on-flavor-order
-	//
-	// In flavor fungibility, the preference whether to preempt or borrow is inferred from flavor fungibility policy
-	// This feature gate is going to be replaced by an API before graduation or deprecation.
-	FlavorFungibilityImplicitPreferenceDefault featuregate.Feature = "FlavorFungibilityImplicitPreferenceDefault"
-
 	// owner: @alaypatel07
 	// kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/2941-DRA
 	//
@@ -288,9 +281,6 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned
 	ManagedJobsNamespaceSelectorAlwaysRespected: {
 		{Version: version.MustParse("0.13"), Default: false, PreRelease: featuregate.Alpha},
 	},
-	FlavorFungibilityImplicitPreferenceDefault: {
-		{Version: version.MustParse("0.13"), Default: false, PreRelease: featuregate.Alpha},
-	},
 	DynamicResourceAllocation: {
 		{Version: version.MustParse("0.14"), Default: false, PreRelease: featuregate.Alpha},
 	},

client-go/applyconfiguration/kueue/v1beta1/flavorfungibility.go

@@ -350,6 +350,60 @@ type granularMode struct {
 	borrowingLevel borrowingLevel
 }
 
+func effectiveAssignmentPreference(cfg kueue.FlavorFungibility) kueue.FlavorAssignmentPreference {
+	if cfg.WhenCanBorrow == kueue.TryNextFlavor && cfg.WhenCanPreempt == kueue.TryNextFlavor {
+		if cfg.Preference != nil {
+			return *cfg.Preference
+		}
+		return kueue.PreferenceBorrowing
+	}
+	if cfg.WhenCanBorrow == kueue.TryNextFlavor && cfg.WhenCanPreempt != kueue.TryNextFlavor {
+		return kueue.PreferencePreempting
+	}
+	return kueue.PreferenceBorrowing
+}
+
+// preferenceCategory maps a granularMode into one of four categories
+// Interpreting those buckets:
+// Category 0 – Fits without preemption and without borrowing.
+// This is the best possible outcome in every preference mode.
+// Category 1
+// With PreferenceBorrowing: fits without needing to preempt, but does require borrowing.
+// With PreferencePreempting: requires preemption but can do so without borrowing (i.e., reclaiming local quota).
+// Category 2
+// With PreferenceBorrowing: requires preemption, but no borrowing is necessary.
+// With PreferencePreempting: fits without preemption but does need borrowing.
+// Category 3 – Requires both preemption and borrowing; it is the least desirable outcome in either mode.
+func preferenceCategory(mode granularMode, pref kueue.FlavorAssignmentPreference) int {
+	borrowOptimal := mode.borrowingLevel.optimal()
+	requiresPreemption := mode.preemptionMode != fit
+	switch pref {
+	case kueue.PreferencePreempting:
+		switch {
+		case !requiresPreemption && borrowOptimal:
+			return 0
+		case requiresPreemption && borrowOptimal:
+			return 1
+		case !requiresPreemption:
+			return 2
+		default:
+			return 3
+		}
+	// kueue.PreferenceBorrowing
+	default:
+		switch {
+		case !requiresPreemption && borrowOptimal:
+			return 0
+		case !requiresPreemption:
+			return 1
+		case requiresPreemption && borrowOptimal:
+			return 2
+		default:
+			return 3
+		}
+	}
+}
+
 func worstGranularMode() granularMode {
 	return granularMode{preemptionMode: noFit, borrowingLevel: math.MaxInt}
 }
@@ -371,6 +425,9 @@ const (
 )
 
 // isPreferred returns true if mode a is better than b according to the selected policy
+// Preference is driven solely by ClusterQueue's flavorFungibility configuration.
+// If WhenCanBorrow is TryNextFlavor, we prioritize assignments that avoid borrowing,
+// otherwise we prioritize assignments that avoid preemption.
 func isPreferred(a, b granularMode, fungibilityConfig kueue.FlavorFungibility) bool {
 	if a.preemptionMode == noFit {
 		return false
@@ -379,25 +436,19 @@ func isPreferred(a, b granularMode, fungibilityConfig kueue.FlavorFungibility) b
 		return true
 	}
 
-	if !features.Enabled(features.FlavorFungibilityImplicitPreferenceDefault) {
-		if a.preemptionMode != b.preemptionMode {
-			return a.preemptionMode > b.preemptionMode
-		} else {
-			return a.borrowingLevel.betterThan(b.borrowingLevel)
-		}
+	pref := effectiveAssignmentPreference(fungibilityConfig)
+	catA := preferenceCategory(a, pref)
+	catB := preferenceCategory(b, pref)
+	if catA != catB {
+		return catA < catB
 	}
-
-	if fungibilityConfig.WhenCanBorrow == kueue.TryNextFlavor {
-		if a.borrowingLevel != b.borrowingLevel {
-			return a.borrowingLevel.betterThan(b.borrowingLevel)
-		}
+	if a.preemptionMode != b.preemptionMode {
 		return a.preemptionMode > b.preemptionMode
-	} else {
-		if a.preemptionMode != b.preemptionMode {
-			return a.preemptionMode > b.preemptionMode
-		}
+	}
+	if a.borrowingLevel != b.borrowingLevel {
 		return a.borrowingLevel.betterThan(b.borrowingLevel)
 	}
+	return false
 }
 
 func fromPreemptionPossibility(preemptionPossibility preemptioncommon.PreemptionPossibility) preemptionMode {