diff --git a/apis/kueue/v1beta1/clusterqueue_types.go b/apis/kueue/v1beta1/clusterqueue_types.go index 71e48be8609..74231178e2b 100644 --- a/apis/kueue/v1beta1/clusterqueue_types.go +++ b/apis/kueue/v1beta1/clusterqueue_types.go @@ -388,8 +388,16 @@ const ( TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor" ) +type FlavorFungibilityPreference string + +const ( + BorrowingOverPreemption FlavorFungibilityPreference = "BorrowingOverPreemption" + PreemptionOverBorrowing FlavorFungibilityPreference = "PreemptionOverBorrowing" +) + // FlavorFungibility determines whether a workload should try the next flavor // before borrowing or preempting in current flavor. +// +kubebuilder:validation:XValidation:rule="!has(self.preference) || (self.whenCanBorrow == 'TryNextFlavor' && self.whenCanPreempt == 'TryNextFlavor')",message="preference can only be set when both whenCanBorrow and whenCanPreempt are TryNextFlavor" type FlavorFungibility struct { // whenCanBorrow determines whether a workload should try the next flavor // before borrowing in current flavor. The possible values are: @@ -414,6 +422,20 @@ type FlavorFungibility struct { // +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor,Preempt} // +kubebuilder:default="TryNextFlavor" WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` + // preference guides the choosing of the flavor for admission in case all candidate flavors + // require either preemption, borrowing, or both. The possible values are: + // - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption + // when such a choice is possible. More technically it minimizes the borrowing distance + // in the cohort tree, and solves tie-breaks by preferring better preemption mode + // (reclaim over preemption within ClusterQueue). + // - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing + // when such a choice is possible. More technically it optimizes the preemption mode + // (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing + // the borrowing distance in the cohort tree. + // + // +kubebuilder:validation:Enum={BorrowingOverPreemption,PreemptionOverBorrowing} + // +optional + Preference *FlavorFungibilityPreference `json:"preference,omitempty"` } // ClusterQueuePreemption contains policies to preempt Workloads from this diff --git a/apis/kueue/v1beta1/zz_generated.conversion.go b/apis/kueue/v1beta1/zz_generated.conversion.go index 3ae4e022069..0d407f64be8 100644 --- a/apis/kueue/v1beta1/zz_generated.conversion.go +++ b/apis/kueue/v1beta1/zz_generated.conversion.go @@ -1381,6 +1381,7 @@ func Convert_v1beta2_FairSharingStatus_To_v1beta1_FairSharingStatus(in *v1beta2. func autoConvert_v1beta1_FlavorFungibility_To_v1beta2_FlavorFungibility(in *FlavorFungibility, out *v1beta2.FlavorFungibility, s conversion.Scope) error { out.WhenCanBorrow = v1beta2.FlavorFungibilityPolicy(in.WhenCanBorrow) out.WhenCanPreempt = v1beta2.FlavorFungibilityPolicy(in.WhenCanPreempt) + out.Preference = (*v1beta2.FlavorFungibilityPreference)(unsafe.Pointer(in.Preference)) return nil } @@ -1392,6 +1393,7 @@ func Convert_v1beta1_FlavorFungibility_To_v1beta2_FlavorFungibility(in *FlavorFu func autoConvert_v1beta2_FlavorFungibility_To_v1beta1_FlavorFungibility(in *v1beta2.FlavorFungibility, out *FlavorFungibility, s conversion.Scope) error { out.WhenCanBorrow = FlavorFungibilityPolicy(in.WhenCanBorrow) out.WhenCanPreempt = FlavorFungibilityPolicy(in.WhenCanPreempt) + out.Preference = (*FlavorFungibilityPreference)(unsafe.Pointer(in.Preference)) return nil } diff --git a/apis/kueue/v1beta1/zz_generated.deepcopy.go b/apis/kueue/v1beta1/zz_generated.deepcopy.go index 7277dd8b717..9961a7c8fe7 100644 --- a/apis/kueue/v1beta1/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta1/zz_generated.deepcopy.go @@ -425,7 +425,7 @@ func (in *ClusterQueueSpec) DeepCopyInto(out *ClusterQueueSpec) { if in.FlavorFungibility != nil { in, out := &in.FlavorFungibility, &out.FlavorFungibility *out = new(FlavorFungibility) - **out = **in + (*in).DeepCopyInto(*out) } if in.Preemption != nil { in, out := &in.Preemption, &out.Preemption @@ -664,6 +664,11 @@ func (in *FairSharingStatus) DeepCopy() *FairSharingStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FlavorFungibility) DeepCopyInto(out *FlavorFungibility) { *out = *in + if in.Preference != nil { + in, out := &in.Preference, &out.Preference + *out = new(FlavorFungibilityPreference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlavorFungibility. diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index 63549156ad6..f0d904a1dd1 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -380,8 +380,16 @@ const ( TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor" ) +type FlavorFungibilityPreference string + +const ( + BorrowingOverPreemption FlavorFungibilityPreference = "BorrowingOverPreemption" + PreemptionOverBorrowing FlavorFungibilityPreference = "PreemptionOverBorrowing" +) + // FlavorFungibility determines whether a workload should try the next flavor // before borrowing or preempting in current flavor. +// +kubebuilder:validation:XValidation:rule="!has(self.preference) || (self.whenCanBorrow == 'TryNextFlavor' && self.whenCanPreempt == 'TryNextFlavor')",message="preference can only be set when both whenCanBorrow and whenCanPreempt are TryNextFlavor" type FlavorFungibility struct { // whenCanBorrow determines whether a workload should try the next flavor // before borrowing in current flavor. The possible values are: @@ -406,6 +414,20 @@ type FlavorFungibility struct { // +kubebuilder:default="TryNextFlavor" // +optional WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` + // preference guides the choosing of the flavor for admission in case all candidate flavors + // require either preemption, borrowing, or both. The possible values are: + // - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption + // when such a choice is possible. More technically it minimizes the borrowing distance + // in the cohort tree, and solves tie-breaks by preferring better preemption mode + // (reclaim over preemption within ClusterQueue). + // - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing + // when such a choice is possible. More technically it optimizes the preemption mode + // (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing + // the borrowing distance in the cohort tree. + // + // +kubebuilder:validation:Enum={BorrowingOverPreemption,PreemptionOverBorrowing} + // +optional + Preference *FlavorFungibilityPreference `json:"preference,omitempty"` } // ClusterQueuePreemption contains policies to preempt Workloads from this diff --git a/apis/kueue/v1beta2/zz_generated.deepcopy.go b/apis/kueue/v1beta2/zz_generated.deepcopy.go index 77a57cd69d5..20f0c1d2251 100644 --- a/apis/kueue/v1beta2/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta2/zz_generated.deepcopy.go @@ -384,7 +384,7 @@ func (in *ClusterQueueSpec) DeepCopyInto(out *ClusterQueueSpec) { if in.FlavorFungibility != nil { in, out := &in.FlavorFungibility, &out.FlavorFungibility *out = new(FlavorFungibility) - **out = **in + (*in).DeepCopyInto(*out) } if in.Preemption != nil { in, out := &in.Preemption, &out.Preemption @@ -613,6 +613,11 @@ func (in *FairSharingStatus) DeepCopy() *FairSharingStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FlavorFungibility) DeepCopyInto(out *FlavorFungibility) { *out = *in + if in.Preference != nil { + in, out := &in.Preference, &out.Preference + *out = new(FlavorFungibilityPreference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FlavorFungibility. diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml index 8bc12f60d28..36657c77d7f 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml @@ -192,6 +192,22 @@ spec: flavorFungibility defines whether a workload should try the next flavor before borrowing or preempting in the flavor being evaluated. properties: + preference: + description: |- + preference guides the choosing of the flavor for admission in case all candidate flavors + require either preemption, borrowing, or both. The possible values are: + - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption + when such a choice is possible. More technically it minimizes the borrowing distance + in the cohort tree, and solves tie-breaks by preferring better preemption mode + (reclaim over preemption within ClusterQueue). + - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing + when such a choice is possible. More technically it optimizes the preemption mode + (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing + the borrowing distance in the cohort tree. + enum: + - BorrowingOverPreemption + - PreemptionOverBorrowing + type: string whenCanBorrow: default: MayStopSearch description: |- @@ -224,6 +240,9 @@ spec: - Preempt type: string type: object + x-kubernetes-validations: + - message: preference can only be set when both whenCanBorrow and whenCanPreempt are TryNextFlavor + rule: '!has(self.preference) || (self.whenCanBorrow == ''TryNextFlavor'' && self.whenCanPreempt == ''TryNextFlavor'')' namespaceSelector: description: |- namespaceSelector defines which namespaces are allowed to submit workloads to @@ -940,6 +959,22 @@ spec: flavorFungibility defines whether a workload should try the next flavor before borrowing or preempting in the flavor being evaluated. properties: + preference: + description: |- + preference guides the choosing of the flavor for admission in case all candidate flavors + require either preemption, borrowing, or both. The possible values are: + - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption + when such a choice is possible. More technically it minimizes the borrowing distance + in the cohort tree, and solves tie-breaks by preferring better preemption mode + (reclaim over preemption within ClusterQueue). + - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing + when such a choice is possible. More technically it optimizes the preemption mode + (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing + the borrowing distance in the cohort tree. + enum: + - BorrowingOverPreemption + - PreemptionOverBorrowing + type: string whenCanBorrow: default: MayStopSearch description: |- @@ -968,6 +1003,9 @@ spec: - TryNextFlavor type: string type: object + x-kubernetes-validations: + - message: preference can only be set when both whenCanBorrow and whenCanPreempt are TryNextFlavor + rule: '!has(self.preference) || (self.whenCanBorrow == ''TryNextFlavor'' && self.whenCanPreempt == ''TryNextFlavor'')' namespaceSelector: description: |- namespaceSelector defines which namespaces are allowed to submit workloads to diff --git a/client-go/applyconfiguration/kueue/v1beta1/flavorfungibility.go b/client-go/applyconfiguration/kueue/v1beta1/flavorfungibility.go index 7f4caca2006..01eca24b8d2 100644 --- a/client-go/applyconfiguration/kueue/v1beta1/flavorfungibility.go +++ b/client-go/applyconfiguration/kueue/v1beta1/flavorfungibility.go @@ -24,8 +24,9 @@ import ( // FlavorFungibilityApplyConfiguration represents a declarative configuration of the FlavorFungibility type for use // with apply. type FlavorFungibilityApplyConfiguration struct { - WhenCanBorrow *kueuev1beta1.FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"` - WhenCanPreempt *kueuev1beta1.FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` + WhenCanBorrow *kueuev1beta1.FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"` + WhenCanPreempt *kueuev1beta1.FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` + Preference *kueuev1beta1.FlavorFungibilityPreference `json:"preference,omitempty"` } // FlavorFungibilityApplyConfiguration constructs a declarative configuration of the FlavorFungibility type for use with @@ -49,3 +50,11 @@ func (b *FlavorFungibilityApplyConfiguration) WithWhenCanPreempt(value kueuev1be b.WhenCanPreempt = &value return b } + +// WithPreference sets the Preference field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Preference field is set to the value of the last call. +func (b *FlavorFungibilityApplyConfiguration) WithPreference(value kueuev1beta1.FlavorFungibilityPreference) *FlavorFungibilityApplyConfiguration { + b.Preference = &value + return b +} diff --git a/client-go/applyconfiguration/kueue/v1beta2/flavorfungibility.go b/client-go/applyconfiguration/kueue/v1beta2/flavorfungibility.go index 7f29a9b573a..5a98200a37e 100644 --- a/client-go/applyconfiguration/kueue/v1beta2/flavorfungibility.go +++ b/client-go/applyconfiguration/kueue/v1beta2/flavorfungibility.go @@ -24,8 +24,9 @@ import ( // FlavorFungibilityApplyConfiguration represents a declarative configuration of the FlavorFungibility type for use // with apply. type FlavorFungibilityApplyConfiguration struct { - WhenCanBorrow *kueuev1beta2.FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"` - WhenCanPreempt *kueuev1beta2.FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` + WhenCanBorrow *kueuev1beta2.FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"` + WhenCanPreempt *kueuev1beta2.FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` + Preference *kueuev1beta2.FlavorFungibilityPreference `json:"preference,omitempty"` } // FlavorFungibilityApplyConfiguration constructs a declarative configuration of the FlavorFungibility type for use with @@ -49,3 +50,11 @@ func (b *FlavorFungibilityApplyConfiguration) WithWhenCanPreempt(value kueuev1be b.WhenCanPreempt = &value return b } + +// WithPreference sets the Preference field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Preference field is set to the value of the last call. +func (b *FlavorFungibilityApplyConfiguration) WithPreference(value kueuev1beta2.FlavorFungibilityPreference) *FlavorFungibilityApplyConfiguration { + b.Preference = &value + return b +} diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml index 35fc0bdd914..0bc089e93ba 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml @@ -163,6 +163,22 @@ spec: flavorFungibility defines whether a workload should try the next flavor before borrowing or preempting in the flavor being evaluated. properties: + preference: + description: |- + preference guides the choosing of the flavor for admission in case all candidate flavors + require either preemption, borrowing, or both. The possible values are: + - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption + when such a choice is possible. More technically it minimizes the borrowing distance + in the cohort tree, and solves tie-breaks by preferring better preemption mode + (reclaim over preemption within ClusterQueue). + - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing + when such a choice is possible. More technically it optimizes the preemption mode + (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing + the borrowing distance in the cohort tree. + enum: + - BorrowingOverPreemption + - PreemptionOverBorrowing + type: string whenCanBorrow: default: MayStopSearch description: |- @@ -195,6 +211,11 @@ spec: - Preempt type: string type: object + x-kubernetes-validations: + - message: preference can only be set when both whenCanBorrow and + whenCanPreempt are TryNextFlavor + rule: '!has(self.preference) || (self.whenCanBorrow == ''TryNextFlavor'' + && self.whenCanPreempt == ''TryNextFlavor'')' namespaceSelector: description: |- namespaceSelector defines which namespaces are allowed to submit workloads to @@ -929,6 +950,22 @@ spec: flavorFungibility defines whether a workload should try the next flavor before borrowing or preempting in the flavor being evaluated. properties: + preference: + description: |- + preference guides the choosing of the flavor for admission in case all candidate flavors + require either preemption, borrowing, or both. The possible values are: + - `BorrowingOverPreemption` (default): prefer to use borrowing rather than preemption + when such a choice is possible. More technically it minimizes the borrowing distance + in the cohort tree, and solves tie-breaks by preferring better preemption mode + (reclaim over preemption within ClusterQueue). + - `PreemptionOverBorrowing`: prefer to use preemption rather than borrowing + when such a choice is possible. More technically it optimizes the preemption mode + (reclaim over preemption within ClusterQueue), and solves tie-breaks by minimizing + the borrowing distance in the cohort tree. + enum: + - BorrowingOverPreemption + - PreemptionOverBorrowing + type: string whenCanBorrow: default: MayStopSearch description: |- @@ -957,6 +994,11 @@ spec: - TryNextFlavor type: string type: object + x-kubernetes-validations: + - message: preference can only be set when both whenCanBorrow and + whenCanPreempt are TryNextFlavor + rule: '!has(self.preference) || (self.whenCanBorrow == ''TryNextFlavor'' + && self.whenCanPreempt == ''TryNextFlavor'')' namespaceSelector: description: |- namespaceSelector defines which namespaces are allowed to submit workloads to diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 52a4794d2f5..7cc2a86971f 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -160,8 +160,8 @@ const ( // owner: @pajakd // kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/582-preempt-based-on-flavor-order // - // In flavor fungibility, the preference whether to preempt or borrow is inferred from flavor fungibility policy - // This feature gate is going to be replaced by an API before graduation or deprecation. + // In flavor fungibility, the preference whether to preempt or borrow is inferred from flavor fungibility policy. + // Deprecated: planned to be removed in v0.16. FlavorFungibilityImplicitPreferenceDefault featuregate.Feature = "FlavorFungibilityImplicitPreferenceDefault" // owner: @alaypatel07 @@ -297,6 +297,7 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned }, FlavorFungibilityImplicitPreferenceDefault: { {Version: version.MustParse("0.13"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("0.15"), Default: false, PreRelease: featuregate.Deprecated}, // remove in 0.16 }, DynamicResourceAllocation: { {Version: version.MustParse("0.14"), Default: false, PreRelease: featuregate.Alpha}, diff --git a/pkg/scheduler/flavorassigner/flavorassigner.go b/pkg/scheduler/flavorassigner/flavorassigner.go index 2306acc850c..328251fe27b 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner.go +++ b/pkg/scheduler/flavorassigner/flavorassigner.go @@ -379,25 +379,39 @@ func isPreferred(a, b granularMode, fungibilityConfig kueue.FlavorFungibility) b return true } - if !features.Enabled(features.FlavorFungibilityImplicitPreferenceDefault) { + borrowingOverPreemption := func() bool { if a.preemptionMode != b.preemptionMode { return a.preemptionMode > b.preemptionMode - } else { - return a.borrowingLevel.betterThan(b.borrowingLevel) } + return a.borrowingLevel.betterThan(b.borrowingLevel) } - - if fungibilityConfig.WhenCanBorrow == kueue.TryNextFlavor { + preemptionOverBorrowing := func() bool { if a.borrowingLevel != b.borrowingLevel { return a.borrowingLevel.betterThan(b.borrowingLevel) } return a.preemptionMode > b.preemptionMode - } else { - if a.preemptionMode != b.preemptionMode { - return a.preemptionMode > b.preemptionMode + } + + if fungibilityConfig.Preference != nil { + switch *fungibilityConfig.Preference { + case kueue.BorrowingOverPreemption: + return preemptionOverBorrowing() + case kueue.PreemptionOverBorrowing: + return borrowingOverPreemption() } - return a.borrowingLevel.betterThan(b.borrowingLevel) } + + // BorrowingOverPreemption preference + if !features.Enabled(features.FlavorFungibilityImplicitPreferenceDefault) { + return borrowingOverPreemption() + } + + // PreemptionOverBorrowing preference + if fungibilityConfig.WhenCanBorrow == kueue.TryNextFlavor { + return preemptionOverBorrowing() + } + // BorrowingOverPreemption preference + return borrowingOverPreemption() } func fromPreemptionPossibility(preemptionPossibility preemptioncommon.PreemptionPossibility) preemptionMode { diff --git a/pkg/scheduler/flavorassigner/flavorassigner_test.go b/pkg/scheduler/flavorassigner/flavorassigner_test.go index dc33b997681..154bfda7e95 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner_test.go +++ b/pkg/scheduler/flavorassigner/flavorassigner_test.go @@ -3404,6 +3404,93 @@ func TestHierarchical(t *testing.T) { } } +func TestIsPreferred(t *testing.T) { + makePref := func(p kueue.FlavorFungibilityPreference) *kueue.FlavorFungibilityPreference { + return &p + } + + cases := map[string]struct { + enableGate bool + a granularMode + b granularMode + config kueue.FlavorFungibility + wantPreferred bool + }{ + "feature gate disabled prioritises preemption": { + a: granularMode{preemptionMode: fit, borrowingLevel: 0}, + b: granularMode{preemptionMode: preempt, borrowingLevel: 0}, + config: kueue.FlavorFungibility{ + WhenCanBorrow: kueue.TryNextFlavor, + WhenCanPreempt: kueue.TryNextFlavor, + }, + wantPreferred: true, + }, + "both policies try default to borrowing preference": { + enableGate: true, + a: granularMode{preemptionMode: preempt, borrowingLevel: 1}, + b: granularMode{preemptionMode: fit, borrowingLevel: 2}, + config: kueue.FlavorFungibility{ + WhenCanBorrow: kueue.TryNextFlavor, + WhenCanPreempt: kueue.TryNextFlavor, + }, + wantPreferred: true, + }, + "mismatched policies default to preemption preference": { + enableGate: true, + a: granularMode{preemptionMode: preempt, borrowingLevel: 0}, + b: granularMode{preemptionMode: fit, borrowingLevel: 1}, + config: kueue.FlavorFungibility{ + WhenCanBorrow: kueue.MayStopSearch, + WhenCanPreempt: kueue.TryNextFlavor, + }, + wantPreferred: false, + }, + "explicit BorrowingOverPreemption prioritises borrowing distance": { + enableGate: false, + a: granularMode{preemptionMode: preempt, borrowingLevel: 1}, + b: granularMode{preemptionMode: fit, borrowingLevel: 2}, + config: kueue.FlavorFungibility{ + WhenCanBorrow: kueue.TryNextFlavor, + WhenCanPreempt: kueue.TryNextFlavor, + Preference: makePref(kueue.BorrowingOverPreemption), + }, + wantPreferred: true, + }, + "explicit PreemptionOverBorrowing prioritises lower preemption": { + enableGate: false, + a: granularMode{preemptionMode: preempt, borrowingLevel: 1}, + b: granularMode{preemptionMode: fit, borrowingLevel: 2}, + config: kueue.FlavorFungibility{ + WhenCanBorrow: kueue.TryNextFlavor, + WhenCanPreempt: kueue.TryNextFlavor, + Preference: makePref(kueue.PreemptionOverBorrowing), + }, + wantPreferred: false, + }, + "explicit PreemptionOverBorrowing breaks borrowing ties with preemption": { + enableGate: false, + a: granularMode{preemptionMode: preempt, borrowingLevel: 1}, + b: granularMode{preemptionMode: fit, borrowingLevel: 1}, + config: kueue.FlavorFungibility{ + WhenCanBorrow: kueue.TryNextFlavor, + WhenCanPreempt: kueue.TryNextFlavor, + Preference: makePref(kueue.PreemptionOverBorrowing), + }, + wantPreferred: false, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + features.SetFeatureGateDuringTest(t, features.FlavorFungibilityImplicitPreferenceDefault, tc.enableGate) + + if got := isPreferred(tc.a, tc.b, tc.config); got != tc.wantPreferred { + t.Fatalf("isPreferred(%+v, %+v, %+v)=%t, want %t", tc.a, tc.b, tc.config, got, tc.wantPreferred) + } + }) + } +} + func TestWorkloadsTopologyRequests_ErrorBranches(t *testing.T) { cases := map[string]struct { cq schdcache.ClusterQueueSnapshot diff --git a/pkg/webhooks/clusterqueue_webhook.go b/pkg/webhooks/clusterqueue_webhook.go index c23c5a0a85a..a8fc692eb3b 100644 --- a/pkg/webhooks/clusterqueue_webhook.go +++ b/pkg/webhooks/clusterqueue_webhook.go @@ -108,6 +108,9 @@ func ValidateClusterQueue(cq *kueue.ClusterQueue) field.ErrorList { if cq.Spec.Preemption != nil { allErrs = append(allErrs, validatePreemption(cq.Spec.Preemption, path.Child("preemption"))...) } + if cq.Spec.FlavorFungibility != nil { + allErrs = append(allErrs, validateFlavorFungibility(cq.Spec.FlavorFungibility, path.Child("flavorFungibility"))...) + } allErrs = append(allErrs, validateFairSharing(cq.Spec.FairSharing, path.Child("fairSharing"))...) allErrs = append(allErrs, validateTotalFlavors(cq.Spec.ResourceGroups, path.Child("resourceGroups"))...) allErrs = append(allErrs, validateTotalCoveredResources(cq.Spec.ResourceGroups, path.Child("resourceGroups"))...) @@ -173,6 +176,20 @@ func validatePreemption(preemption *kueue.ClusterQueuePreemption, path *field.Pa return allErrs } +func validateFlavorFungibility(fungibility *kueue.FlavorFungibility, path *field.Path) field.ErrorList { + var allErrs field.ErrorList + if fungibility.Preference != nil { + if fungibility.WhenCanBorrow != kueue.TryNextFlavor || fungibility.WhenCanPreempt != kueue.TryNextFlavor { + allErrs = append(allErrs, field.Invalid( + path.Child("preference"), + *fungibility.Preference, + fmt.Sprintf("preference %q requires both whenCanBorrow and whenCanPreempt to be TryNextFlavor", *fungibility.Preference), + )) + } + } + return allErrs +} + func validateResourceGroups(resourceGroups []kueue.ResourceGroup, config validationConfig, path *field.Path, isCohort bool) field.ErrorList { var allErrs field.ErrorList seenResources := sets.New[corev1.ResourceName]() diff --git a/pkg/webhooks/clusterqueue_webhook_test.go b/pkg/webhooks/clusterqueue_webhook_test.go index 3ab4a85c83b..c585d9e48d0 100644 --- a/pkg/webhooks/clusterqueue_webhook_test.go +++ b/pkg/webhooks/clusterqueue_webhook_test.go @@ -42,6 +42,8 @@ func TestValidateClusterQueue(t *testing.T) { clusterQueue *kueue.ClusterQueue wantErr field.ErrorList disableLendingLimit bool + wantDetail string + wantBadValue string }{ { name: "built-in resources with qualified names", @@ -317,6 +319,66 @@ func TestValidateClusterQueue(t *testing.T) { }, }, }, + { + name: "flavorFungibility preference set but whenCanPreempt != TryNextFlavor", + clusterQueue: utiltestingapi.MakeClusterQueue("cluster-queue"). + FlavorFungibility(kueue.FlavorFungibility{ + WhenCanBorrow: kueue.TryNextFlavor, + WhenCanPreempt: kueue.MayStopSearch, + Preference: ptr.To(kueue.BorrowingOverPreemption), + }).Obj(), + wantErr: field.ErrorList{ + field.Invalid(specPath.Child("flavorFungibility", "preference"), "", ""), + }, + wantDetail: `preference "BorrowingOverPreemption" requires both whenCanBorrow and whenCanPreempt to be TryNextFlavor`, + wantBadValue: string(kueue.BorrowingOverPreemption), + }, + { + name: "flavorFungibility preference set but whenCanBorrow != TryNextFlavor", + clusterQueue: utiltestingapi.MakeClusterQueue("cluster-queue"). + FlavorFungibility(kueue.FlavorFungibility{ + WhenCanBorrow: kueue.MayStopSearch, + WhenCanPreempt: kueue.MayStopSearch, + Preference: ptr.To(kueue.BorrowingOverPreemption), + }).Obj(), + wantErr: field.ErrorList{ + field.Invalid(specPath.Child("flavorFungibility", "preference"), "", ""), + }, + wantDetail: `preference "BorrowingOverPreemption" requires both whenCanBorrow and whenCanPreempt to be TryNextFlavor`, + wantBadValue: string(kueue.BorrowingOverPreemption), + }, + { + name: "flavorFungibility preference BorrowingOverPreemption with both TryNextFlavor is valid", + clusterQueue: utiltestingapi.MakeClusterQueue("cluster-queue"). + FlavorFungibility(kueue.FlavorFungibility{ + WhenCanBorrow: kueue.TryNextFlavor, + WhenCanPreempt: kueue.TryNextFlavor, + Preference: ptr.To(kueue.BorrowingOverPreemption), + }).Obj(), + }, + { + name: "flavorFungibility preference PreemptionOverBorrowing with both TryNextFlavor is valid", + clusterQueue: utiltestingapi.MakeClusterQueue("cluster-queue"). + FlavorFungibility(kueue.FlavorFungibility{ + WhenCanBorrow: kueue.TryNextFlavor, + WhenCanPreempt: kueue.TryNextFlavor, + Preference: ptr.To(kueue.PreemptionOverBorrowing), + }).Obj(), + }, + { + name: "flavorFungibility preference PreemptionOverBorrowing but whenCanBorrow != TryNextFlavor", + clusterQueue: utiltestingapi.MakeClusterQueue("cluster-queue"). + FlavorFungibility(kueue.FlavorFungibility{ + WhenCanBorrow: kueue.MayStopSearch, + WhenCanPreempt: kueue.TryNextFlavor, + Preference: ptr.To(kueue.PreemptionOverBorrowing), + }).Obj(), + wantErr: field.ErrorList{ + field.Invalid(specPath.Child("flavorFungibility", "preference"), "", ""), + }, + wantDetail: `preference "PreemptionOverBorrowing" requires both whenCanBorrow and whenCanPreempt to be TryNextFlavor`, + wantBadValue: string(kueue.PreemptionOverBorrowing), + }, } for _, tc := range testcases { @@ -328,6 +390,20 @@ func TestValidateClusterQueue(t *testing.T) { if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.IgnoreFields(field.Error{}, "Detail", "BadValue")); diff != "" { t.Errorf("ValidateResources() mismatch (-want +got):\n%s", diff) } + if tc.wantDetail != "" || tc.wantBadValue != "" { + if len(gotErr) == 0 { + t.Fatalf("expected an error but got none") + } + if tc.wantDetail != "" && gotErr[0].Detail != tc.wantDetail { + t.Fatalf("unexpected error detail, want %q got %q", tc.wantDetail, gotErr[0].Detail) + } + if tc.wantBadValue != "" { + gotBad := fmt.Sprint(gotErr[0].BadValue) + if gotBad != tc.wantBadValue { + t.Fatalf("unexpected bad value, want %q got %q", tc.wantBadValue, gotBad) + } + } + } }) } } diff --git a/site/content/en/docs/concepts/cluster_queue.md b/site/content/en/docs/concepts/cluster_queue.md index 85dae3cd429..8d026740365 100644 --- a/site/content/en/docs/concepts/cluster_queue.md +++ b/site/content/en/docs/concepts/cluster_queue.md @@ -517,22 +517,20 @@ ResourceFlavors, Kueue selects a one that fits the workload using borrowing (without preemptions). If there is no such ResourceFlavor, Kueue selects a Flavor that uses preemption and is preferably not borrowing. -By default Kueue avoids preemptions and prefers borrowing when assigning Flavors. -Borrowing is not disruptive to other workloads but a -workload that borrows risks being prempted (since it is using nominal quota -from some other Cluster Queue). If you prefer to preempt rather than borrow when possible, -you can enable the feature gate `FlavorFungibilityImplicitPreferenceDefault`, which -changes the default preference as follows: If `.spec.flavorFungibility.whenCanBorrow` is `TryNextFlavor`, -it assumes that preemption is preferred over borrowing and otherwise it assumes -that borrowing is preferred over preemption. +When both policies are set to `TryNextFlavor`, you can steer how those feasible +assignments are compared by setting `.spec.flavorFungibility.preference`: + +- `BorrowingOverPreemption` (default) keeps the historic behavior: + (`Fit`, `NoBorrow`) → (`Fit`, `Borrow`) → (`Preempt`, `NoBorrow`) → (`Preempt`, `Borrow`). +- `PreemptionOverBorrowing` reverses the tie-breaker to prefer reclaiming quota over borrowing: + (`Fit`, `NoBorrow`) → (`Preempt`, `NoBorrow`) → (`Fit`, `Borrow`) → (`Preempt`, `Borrow`). {{% alert title="Note" color="primary" %}} -`FlavorFungibilityImplicitPreferenceDefault` is currently an alpha feature, -introduced to Kueue in version 0.13 and it is not enabled by default. +The feature gate `FlavorFungibilityImplicitPreferenceDefault` is scheduled for removal in v0.15. -To enable the feature, you have to set the `FlavorFungibilityImplicitPreferenceDefault` -feature gate to `true`. Check the [Installation](/docs/installation/#change-the-feature-gates-configuration) -guide for details on feature gate configuration. +Do not rely on this feature gate; +instead configure flavor selection preference using the ClusterQueue field `spec.flavorFungibility.preference` +(see [FlavorFungibility](/docs/concepts/cluster_queue/#flavorfungibility) for details). {{% /alert %}} diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 13dda163ea8..23b78b075f7 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -296,7 +296,7 @@ spec: | `ElasticJobsViaWorkloadSlices` | `false` | Alpha | 0.13 | | | `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | 0.15 | | `ManagedJobsNamespaceSelectorAlwaysRespected` | `true` | Beta | 0.15 | | -| `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | +| `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | 0.16 | | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | | `SanitizePodSets` | `true` | Beta | 0.13 | | | `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.13 | | diff --git a/site/content/en/docs/reference/kueue.v1beta1.md b/site/content/en/docs/reference/kueue.v1beta1.md index c6e40fb2955..d523eb9aa60 100644 --- a/site/content/en/docs/reference/kueue.v1beta1.md +++ b/site/content/en/docs/reference/kueue.v1beta1.md @@ -1471,6 +1471,24 @@ to fit in current flavor. +preference
+FlavorFungibilityPreference + + +

preference guides the choosing of the flavor for admission in case all candidate flavors +require either preemption, borrowing, or both. The possible values are:

+ + + @@ -1486,6 +1504,18 @@ to fit in current flavor. +## `FlavorFungibilityPreference` {#kueue-x-k8s-io-v1beta1-FlavorFungibilityPreference} + +(Alias of `string`) + +**Appears in:** + +- [FlavorFungibility](#kueue-x-k8s-io-v1beta1-FlavorFungibility) + + + + + ## `FlavorQuotas` {#kueue-x-k8s-io-v1beta1-FlavorQuotas} diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md index 4a061fcd503..e89c6db7bc1 100644 --- a/site/content/en/docs/reference/kueue.v1beta2.md +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -1375,6 +1375,24 @@ to fit in current flavor. +preference
+FlavorFungibilityPreference + + +

preference guides the choosing of the flavor for admission in case all candidate flavors +require either preemption, borrowing, or both. The possible values are:

+ + + @@ -1390,6 +1408,18 @@ to fit in current flavor. +## `FlavorFungibilityPreference` {#kueue-x-k8s-io-v1beta2-FlavorFungibilityPreference} + +(Alias of `string`) + +**Appears in:** + +- [FlavorFungibility](#kueue-x-k8s-io-v1beta2-FlavorFungibility) + + + + + ## `FlavorQuotas` {#kueue-x-k8s-io-v1beta2-FlavorQuotas} diff --git a/site/content/zh-CN/docs/installation/_index.md b/site/content/zh-CN/docs/installation/_index.md index 35635f8e5fc..17d73a3dca1 100644 --- a/site/content/zh-CN/docs/installation/_index.md +++ b/site/content/zh-CN/docs/installation/_index.md @@ -294,7 +294,7 @@ spec: | `TASReplaceNodeOnPodTermination` | `true` | Beta | 0.14 | | | `ElasticJobsViaWorkloadSlices` | `false` | Alpha | 0.13 | | | `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | -| `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | +| `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | 0.16 | | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | | `SanitizePodSets` | `true` | Beta | 0.13 | | | `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.13 | |