From 07e9f65b9a73bf8de1df038fc75d175e030d5e1c Mon Sep 17 00:00:00 2001 From: Singularity23x0 Date: Thu, 23 Oct 2025 09:26:39 +0000 Subject: [PATCH 001/119] Impelemented the alfa version of the deletion integration solution. --- apis/kueue/v1beta1/constants.go | 1 + pkg/controller/core/workload_controller.go | 127 ++++++++++++++++----- pkg/controller/jobframework/interface.go | 2 +- 3 files changed, 101 insertions(+), 29 deletions(-) diff --git a/apis/kueue/v1beta1/constants.go b/apis/kueue/v1beta1/constants.go index 9ee27e1fab3..c401d6768a4 100644 --- a/apis/kueue/v1beta1/constants.go +++ b/apis/kueue/v1beta1/constants.go @@ -18,6 +18,7 @@ package v1beta1 const ( ResourceInUseFinalizerName = "kueue.x-k8s.io/resource-in-use" + SafeDeleteFinalizerName = "kueue.x-k8s.io/delete-safeguard" DefaultPodSetName PodSetReference = "main" // ElasticJobSchedulingGate is the name of the scheduling gate applied to Pods diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index 201443f0a06..382130ae8a0 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -19,6 +19,7 @@ package core import ( "cmp" "context" + "errors" "fmt" "slices" "time" @@ -42,6 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -158,17 +160,53 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, client.IgnoreNotFound(err) } - log := ctrl.LoggerFrom(ctx) + status := workload.Status(&wl) + log := ctrl.LoggerFrom(ctx).WithValues("workload", klog.KObj(&wl), "queue", wl.Spec.QueueName, "status", status) log.V(2).Info("Reconcile Workload") - if len(wl.OwnerReferences) == 0 && !wl.DeletionTimestamp.IsZero() { - // manual deletion triggered by the user - err := workload.RemoveFinalizer(ctx, r.client, &wl) - return ctrl.Result{}, client.IgnoreNotFound(err) + if !wl.DeletionTimestamp.IsZero() { + // Deletion requested (some finalizers must be present). Try to finalize and remove. + hasInUseFinalizer := controllerutil.ContainsFinalizer(&wl, kueue.ResourceInUseFinalizerName) + hasSafeDeleteFinlaizer := controllerutil.ContainsFinalizer(&wl, kueue.SafeDeleteFinalizerName) + switch { + case hasInUseFinalizer && hasSafeDeleteFinlaizer: + { + // Manual delete request by a user. + if len(wl.OwnerReferences) == 0 { + // Delete legal - proceed. + err := r.cleanUp(ctx, &wl, log) + return ctrl.Result{}, err + } else { + // Delete illegal at this time, ignore and continue the reconciliation loop. + log.Info("Manual delete by user when workload still has owners. Delete request ignored.") + } + } + case !hasInUseFinalizer && hasSafeDeleteFinlaizer: + { + // Natural deleteion. + err := r.cleanUp(ctx, &wl, log) + return ctrl.Result{}, err + } + case hasInUseFinalizer && !hasSafeDeleteFinlaizer: + { + err := errors.New("Illegal finalizer configuration for workload") + log.Error(err, "Illegal finalizer configuration for workload") + return ctrl.Result{}, err + } + default: + { + // Unknown finalizer preventing deletion. Continuing the reconciliation loop. + log.Info("Unknown finalizers set for workload.") + } + } } + // TODO: integrate cache and queue updates based on newObject status (make "oldObject status"-agnostic). + finishedCond := apimeta.FindStatusCondition(wl.Status.Conditions, kueue.WorkloadFinished) if finishedCond != nil && finishedCond.Status == metav1.ConditionTrue { + // (new) workload is in state FINISHED + if !features.Enabled(features.ObjectRetentionPolicies) || r.workloadRetention == nil || r.workloadRetention.afterFinished == nil { return ctrl.Result{}, nil } @@ -470,6 +508,37 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, nil } +func (r *WorkloadReconciler) cleanUp(ctx context.Context, wl *kueue.Workload, log logr.Logger) error { + log.Info("Cleaning up for workload deletion") + defer r.notifyWatchers(wl, nil) + var resultError error = nil + + if workload.HasQuotaReservation(wl) { + var cleanupErrors []error + r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, wl, func() { + if err := r.cache.DeleteWorkload(log, wl); err != nil { + append(cleanupErrors, err) + log.Error(err, "Failed to delete workload from cache") + } + }) + if len(cleanupErrors) > 0 { + resultError = errors.Join(cleanupErrors...) + } + } else { + r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, wl, func() { + if err := r.cache.DeleteWorkload(log, wl); err != nil { + log.Info("Failed to delete workload from cache. This may be indended behavior.", "error", err) + } + }) + } + + r.queues.DeleteWorkload(wl) + controllerutil.RemoveFinalizer(wl, kueue.ResourceInUseFinalizerName) + controllerutil.RemoveFinalizer(wl, kueue.SafeDeleteFinalizerName) + + return resultError +} + // isDisabledRequeuedByClusterQueueStopped returns true if the workload is unset requeued by cluster queue stopped. func isDisabledRequeuedByClusterQueueStopped(w *kueue.Workload) bool { return isDisabledRequeuedByReason(w, kueue.WorkloadEvictedByClusterQueueStopped) @@ -755,35 +824,37 @@ func (r *WorkloadReconciler) Create(e event.TypedCreateEvent[*kueue.Workload]) b } func (r *WorkloadReconciler) Delete(e event.TypedDeleteEvent[*kueue.Workload]) bool { - defer r.notifyWatchers(e.Object, nil) + // TODO: delete commented + + // defer r.notifyWatchers(e.Object, nil) status := "unknown" if !e.DeleteStateUnknown { status = workload.Status(e.Object) } log := r.log.WithValues("workload", klog.KObj(e.Object), "queue", e.Object.Spec.QueueName, "status", status) log.V(2).Info("Workload delete event") - ctx := ctrl.LoggerInto(context.Background(), log) - - // When assigning a clusterQueue to a workload, we assume it in the cache. If - // the state is unknown, the workload could have been assumed, and we need - // to clear it from the cache. - if workload.HasQuotaReservation(e.Object) || e.DeleteStateUnknown { - // trigger the move of associated inadmissibleWorkloads if required. - r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, e.Object, func() { - // Delete the workload from cache while holding the queues lock - // to guarantee that requeued workloads are taken into account before - // the next scheduling cycle. - if err := r.cache.DeleteWorkload(log, e.Object); err != nil { - if !e.DeleteStateUnknown { - log.Error(err, "Failed to delete workload from cache") - } - } - }) - } - - // Even if the state is unknown, the last cached state tells us whether the - // workload was in the queues and should be cleared from them. - r.queues.DeleteWorkload(e.Object) + // ctx := ctrl.LoggerInto(context.Background(), log) + + // // When assigning a clusterQueue to a workload, we assume it in the cache. If + // // the state is unknown, the workload could have been assumed, and we need + // // to clear it from the cache. + // if workload.HasQuotaReservation(e.Object) || e.DeleteStateUnknown { + // // trigger the move of associated inadmissibleWorkloads if required. + // r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, e.Object, func() { + // // Delete the workload from cache while holding the queues lock + // // to guarantee that requeued workloads are taken into account before + // // the next scheduling cycle. + // if err := r.cache.DeleteWorkload(log, e.Object); err != nil { + // if !e.DeleteStateUnknown { + // log.Error(err, "Failed to delete workload from cache") + // } + // } + // }) + // } + + // // Even if the state is unknown, the last cached state tells us whether the + // // workload was in the queues and should be cleared from them. + // r.queues.DeleteWorkload(e.Object) return true } diff --git a/pkg/controller/jobframework/interface.go b/pkg/controller/jobframework/interface.go index ef717688680..aa91d3d8df2 100644 --- a/pkg/controller/jobframework/interface.go +++ b/pkg/controller/jobframework/interface.go @@ -222,7 +222,7 @@ func NewWorkload(name string, obj client.Object, podSets []kueue.PodSet, labelKe Name: name, Namespace: obj.GetNamespace(), Labels: maps.FilterKeys(obj.GetLabels(), labelKeysToCopy), - Finalizers: []string{kueue.ResourceInUseFinalizerName}, + Finalizers: []string{kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName}, Annotations: admissioncheck.FilterProvReqAnnotations(obj.GetAnnotations()), }, Spec: kueue.WorkloadSpec{ From 6728e2b97adfa7397ecdf5a7b18496334f3a0d79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Oct 2025 05:14:11 -0700 Subject: [PATCH 002/119] Bump github.com/cert-manager/cert-manager from 1.19.0 to 1.19.1 (#7321) Bumps [github.com/cert-manager/cert-manager](https://github.com/cert-manager/cert-manager) from 1.19.0 to 1.19.1. - [Release notes](https://github.com/cert-manager/cert-manager/releases) - [Changelog](https://github.com/cert-manager/cert-manager/blob/master/RELEASE.md) - [Commits](https://github.com/cert-manager/cert-manager/compare/v1.19.0...v1.19.1) --- updated-dependencies: - dependency-name: github.com/cert-manager/cert-manager dependency-version: 1.19.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- .../cert-manager/cert-manager/pkg/apis/meta/v1/types.go | 4 ++-- vendor/modules.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 41a31750db6..9b13cd84276 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module sigs.k8s.io/kueue go 1.25.0 require ( - github.com/cert-manager/cert-manager v1.19.0 + github.com/cert-manager/cert-manager v1.19.1 github.com/fsnotify/fsnotify v1.9.0 github.com/go-logr/logr v1.4.3 github.com/google/go-cmp v0.7.0 diff --git a/go.sum b/go.sum index a75b9686a63..a3bd6af7602 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/cert-manager/cert-manager v1.19.0 h1:pPKtK06zPriliwq7B12xa7DGKbc4SvnfOjE0BXaAspk= -github.com/cert-manager/cert-manager v1.19.0/go.mod h1:7VBp/ihp5Xgz/ZJuGy1fNIWbDouQKBonvEyWoJLuhmA= +github.com/cert-manager/cert-manager v1.19.1 h1:Txh8L/nLWTDcb7ZnXuXbTe15BxQnLbLirXmbNk0fGgY= +github.com/cert-manager/cert-manager v1.19.1/go.mod h1:8Ps1VXCQRGKT8zNvLQlhDK1gFKWmYKdIPQFmvTS2JeA= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= diff --git a/vendor/github.com/cert-manager/cert-manager/pkg/apis/meta/v1/types.go b/vendor/github.com/cert-manager/cert-manager/pkg/apis/meta/v1/types.go index 6c36d40ec68..2b294e1a922 100644 --- a/vendor/github.com/cert-manager/cert-manager/pkg/apis/meta/v1/types.go +++ b/vendor/github.com/cert-manager/cert-manager/pkg/apis/meta/v1/types.go @@ -53,12 +53,12 @@ type IssuerReference struct { // Name of the issuer being referred to. Name string `json:"name"` // Kind of the issuer being referred to. + // Defaults to 'Issuer'. // +optional - // +default="Issuer" Kind string `json:"kind,omitempty"` // Group of the issuer being referred to. + // Defaults to 'cert-manager.io'. // +optional - // +default="cert-manager.io" Group string `json:"group,omitempty"` } diff --git a/vendor/modules.txt b/vendor/modules.txt index 6b58d833176..3b555377a1f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -26,7 +26,7 @@ github.com/blang/semver/v4 # github.com/cenkalti/backoff/v5 v5.0.3 ## explicit; go 1.23 github.com/cenkalti/backoff/v5 -# github.com/cert-manager/cert-manager v1.19.0 +# github.com/cert-manager/cert-manager v1.19.1 ## explicit; go 1.25.0 github.com/cert-manager/cert-manager/pkg/apis/acme github.com/cert-manager/cert-manager/pkg/apis/acme/v1 From 71b8511f0e461e3687c22b0cb09169e6113ec9dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Paj=C4=85k?= Date: Mon, 20 Oct 2025 15:16:39 +0200 Subject: [PATCH 003/119] Deflake test (#7325) --- .../fairsharing/fair_sharing_test.go | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go b/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go index 2ccb030bad8..4cef95808e6 100644 --- a/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go +++ b/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go @@ -918,12 +918,13 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Validate pending workloads") util.ExpectPendingWorkloadsMetric(cq1, 1, 1) - gomega.Eventually(func(g gomega.Gomega) { - util.FinishEvictionsOfAnyWorkloadsInCq(ctx, k8sClient, cq2) - util.ExpectAdmittedWorkloadsTotalMetric(cq1, "", 1) - util.ExpectClusterQueueWeightedShareMetric(cq1, 0) - util.ExpectClusterQueueWeightedShareMetric(cq2, 0) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) + ginkgo.By("Complete preemption") + util.FinishEvictionOfWorkloadsInCQ(ctx, k8sClient, cq2, 2) + + ginkgo.By("Expected Total Admitted Workloads and Weighted Share") + util.ExpectAdmittedWorkloadsTotalMetric(cq1, "", 1) + util.ExpectClusterQueueWeightedShareMetric(cq1, 0) + util.ExpectClusterQueueWeightedShareMetric(cq2, 0) }) ginkgo.It("sticky workload deleted, next workload can admit", func() { @@ -950,13 +951,13 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Validate pending workloads") util.ExpectPendingWorkloadsMetric(cq1, 1, 1) + ginkgo.By("Complete preemption") + util.FinishEvictionOfWorkloadsInCQ(ctx, k8sClient, cq2, 2) + ginkgo.By("Expected Total Admitted Workloads and Weighted Share") - gomega.Eventually(func(g gomega.Gomega) { - util.FinishEvictionsOfAnyWorkloadsInCq(ctx, k8sClient, cq2) - util.ExpectAdmittedWorkloadsTotalMetric(cq1, "", 1) - util.ExpectClusterQueueWeightedShareMetric(cq1, 0) - util.ExpectClusterQueueWeightedShareMetric(cq2, 0) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) + util.ExpectAdmittedWorkloadsTotalMetric(cq1, "", 1) + util.ExpectClusterQueueWeightedShareMetric(cq1, 0) + util.ExpectClusterQueueWeightedShareMetric(cq2, 0) }) }) From 636383fdb1485ca0bb78fcd03616ff247d665d5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Mon, 20 Oct 2025 18:06:41 +0200 Subject: [PATCH 004/119] [Bugfix] Allow to set ClusterName with ElasticJob (#7278) * Remove dispatcherName from workload webhook * Update tests post webhook dispatcherName removal * Allow to set ClusterName with ElasticJob even though it is not one of the nominatedClusterNames. --- cmd/kueue/main.go | 2 +- pkg/webhooks/webhooks.go | 4 +- pkg/webhooks/workload_webhook.go | 26 ++++----- pkg/webhooks/workload_webhook_test.go | 23 +++++++- .../multikueue/external_job_test.go | 3 +- test/integration/multikueue/suite_test.go | 3 +- .../provisioning/suite_test.go | 2 +- .../controller/core/suite_test.go | 2 +- .../controller/dra/suite_test.go | 2 +- .../controller/jobs/pod/suite_test.go | 2 +- .../singlecluster/importer/suite_test.go | 2 +- .../singlecluster/kueuectl/suite_test.go | 2 +- .../scheduler/fairsharing/suite_test.go | 2 +- .../scheduler/podsready/suite_test.go | 2 +- .../singlecluster/scheduler/suite_test.go | 2 +- .../singlecluster/tas/suite_test.go | 2 +- .../webhook/core/admissioncheck_test.go | 3 +- .../webhook/core/clusterqueue_test.go | 3 +- .../singlecluster/webhook/core/cohort_test.go | 3 +- .../webhook/core/localqueue_test.go | 3 +- .../webhook/core/resourceflavor_test.go | 3 +- .../singlecluster/webhook/core/suite_test.go | 4 +- .../webhook/core/workload_test.go | 55 ++++++++++++------- .../webhook/jobs/raycluster_webhook_test.go | 3 +- 24 files changed, 91 insertions(+), 67 deletions(-) diff --git a/cmd/kueue/main.go b/cmd/kueue/main.go index d6f702cb736..1f8cff3dd37 100644 --- a/cmd/kueue/main.go +++ b/cmd/kueue/main.go @@ -389,7 +389,7 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, cCache *schdcache.C } } - if failedWebhook, err := webhooks.Setup(mgr, ptr.Deref(cfg.MultiKueue.DispatcherName, configapi.MultiKueueDispatcherModeAllAtOnce)); err != nil { + if failedWebhook, err := webhooks.Setup(mgr); err != nil { return fmt.Errorf("unable to create webhook %s: %w", failedWebhook, err) } diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go index c367d94d0f9..cb7c832bbed 100644 --- a/pkg/webhooks/webhooks.go +++ b/pkg/webhooks/webhooks.go @@ -22,8 +22,8 @@ import ( // Setup sets up the webhooks for core controllers. It returns the name of the // webhook that failed to create and an error, if any. -func Setup(mgr ctrl.Manager, dispatcherName string) (string, error) { - if err := setupWebhookForWorkload(mgr, dispatcherName); err != nil { +func Setup(mgr ctrl.Manager) (string, error) { + if err := setupWebhookForWorkload(mgr); err != nil { return "Workload", err } diff --git a/pkg/webhooks/workload_webhook.go b/pkg/webhooks/workload_webhook.go index af6184ca1ff..7bacc236d5d 100644 --- a/pkg/webhooks/workload_webhook.go +++ b/pkg/webhooks/workload_webhook.go @@ -33,22 +33,18 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" utilslices "sigs.k8s.io/kueue/pkg/util/slices" "sigs.k8s.io/kueue/pkg/workload" + "sigs.k8s.io/kueue/pkg/workloadslicing" ) -type WorkloadWebhook struct { - dispatcherName string -} +type WorkloadWebhook struct{} -func setupWebhookForWorkload(mgr ctrl.Manager, dispatcherName string) error { - wh := &WorkloadWebhook{ - dispatcherName: dispatcherName, - } +func setupWebhookForWorkload(mgr ctrl.Manager) error { + wh := &WorkloadWebhook{} return ctrl.NewWebhookManagedBy(mgr). For(&kueue.Workload{}). WithDefaulter(wh). @@ -94,7 +90,7 @@ func (w *WorkloadWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj run oldWL := oldObj.(*kueue.Workload) log := ctrl.LoggerFrom(ctx).WithName("workload-webhook") log.V(5).Info("Validating update") - return nil, ValidateWorkloadUpdate(newWL, oldWL, w.dispatcherName).ToAggregate() + return nil, ValidateWorkloadUpdate(newWL, oldWL).ToAggregate() } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type @@ -273,7 +269,7 @@ func validateReclaimablePods(obj *kueue.Workload, basePath *field.Path) field.Er return ret } -func ValidateWorkloadUpdate(newObj, oldObj *kueue.Workload, dispatcherName string) field.ErrorList { +func ValidateWorkloadUpdate(newObj, oldObj *kueue.Workload) field.ErrorList { var allErrs field.ErrorList specPath := field.NewPath("spec") statusPath := field.NewPath("status") @@ -287,7 +283,7 @@ func ValidateWorkloadUpdate(newObj, oldObj *kueue.Workload, dispatcherName strin } allErrs = append(allErrs, validateAdmissionUpdate(newObj.Status.Admission, oldObj.Status.Admission, field.NewPath("status", "admission"))...) allErrs = append(allErrs, validateImmutablePodSetUpdates(newObj, oldObj, statusPath.Child("admissionChecks"))...) - allErrs = append(allErrs, validateClusterNameUpdate(newObj, oldObj, dispatcherName, statusPath)...) + allErrs = append(allErrs, validateClusterNameUpdate(newObj, oldObj, statusPath)...) return allErrs } @@ -372,9 +368,13 @@ func validateImmutablePodSets(new, old []kueue.PodSet, path *field.Path) field.E return allErrs } -func validateClusterNameUpdate(newObj, oldObj *kueue.Workload, dispatcherName string, statusPath *field.Path) field.ErrorList { +func validateClusterNameUpdate(newObj, oldObj *kueue.Workload, statusPath *field.Path) field.ErrorList { var allErrs field.ErrorList - if oldObj.Status.ClusterName == nil && newObj.Status.ClusterName != nil && dispatcherName != configapi.MultiKueueDispatcherModeAllAtOnce { + if features.Enabled(features.ElasticJobsViaWorkloadSlices) && workloadslicing.ReplacementForKey(newObj) != nil { + // Allow setting clusterName when the workload is a valid elastic job replacement. + return allErrs + } + if oldObj.Status.ClusterName == nil && newObj.Status.ClusterName != nil { found := slices.Contains(oldObj.Status.NominatedClusterNames, *newObj.Status.ClusterName) if !found { allErrs = append(allErrs, field.Invalid(statusPath.Child("clusterName"), newObj.Status.ClusterName, "when setting clusterName it must be one of the nominatedClusterNames")) diff --git a/pkg/webhooks/workload_webhook_test.go b/pkg/webhooks/workload_webhook_test.go index 5dbc0d6be9d..3db54b92df1 100644 --- a/pkg/webhooks/workload_webhook_test.go +++ b/pkg/webhooks/workload_webhook_test.go @@ -27,11 +27,12 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + "sigs.k8s.io/kueue/pkg/workload" + "sigs.k8s.io/kueue/pkg/workloadslicing" ) const ( @@ -532,12 +533,30 @@ func TestValidateWorkloadUpdate(t *testing.T) { kueue.PodSetAssignment{Name: "ps1"}, kueue.PodSetAssignment{Name: "ps2"}).Obj()).Obj(), }, + "ClusterName doesn't have to be one of the nominatedClusterNames with ElasticJobs feature gate": { + enableElasticJobsFeature: true, + before: utiltestingapi.MakeWorkload(testWorkloadName, testWorkloadNamespace). + PodSets( + *utiltestingapi.MakePodSet("ps1", 3).Obj(), + *utiltestingapi.MakePodSet("ps2", 4).Obj()). + Obj(), + after: utiltestingapi.MakeWorkload(testWorkloadName, testWorkloadNamespace). + PodSets( + *utiltestingapi.MakePodSet("ps1", 3).Obj(), + *utiltestingapi.MakePodSet("ps2", 4).Obj()). + ReserveQuota(utiltestingapi.MakeAdmission("cluster-queue").PodSets( + kueue.PodSetAssignment{Name: "ps1"}, + kueue.PodSetAssignment{Name: "ps2"}).Obj()). + ClusterName("worker1"). + Annotation(workloadslicing.WorkloadSliceReplacementFor, string(workload.NewReference(testWorkloadNamespace, testWorkloadName))). + Obj(), + }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { features.SetFeatureGateDuringTest(t, features.ElasticJobsViaWorkloadSlices, tc.enableElasticJobsFeature) features.SetFeatureGateDuringTest(t, features.TopologyAwareScheduling, tc.enableTopologyAwareScheduling) - errList := ValidateWorkloadUpdate(tc.after, tc.before, configapi.MultiKueueDispatcherModeAllAtOnce) + errList := ValidateWorkloadUpdate(tc.after, tc.before) if diff := cmp.Diff(tc.wantErr, errList, cmpopts.IgnoreFields(field.Error{}, "Detail", "BadValue")); diff != "" { t.Errorf("ValidateWorkloadUpdate() mismatch (-want +got):\n%s", diff) } diff --git a/test/integration/multikueue/external_job_test.go b/test/integration/multikueue/external_job_test.go index cbf44e97a8f..ac2cd90d7ff 100644 --- a/test/integration/multikueue/external_job_test.go +++ b/test/integration/multikueue/external_job_test.go @@ -27,7 +27,6 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -89,7 +88,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) - failedWebhook, err := webhooks.Setup(mgr, ptr.Deref(configuration.MultiKueue.DispatcherName, config.MultiKueueDispatcherModeAllAtOnce)) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) // Set up RayJob webhook (but not MultiKueue integration) diff --git a/test/integration/multikueue/suite_test.go b/test/integration/multikueue/suite_test.go index 4ba15ee5496..9a8428d0bb1 100644 --- a/test/integration/multikueue/suite_test.go +++ b/test/integration/multikueue/suite_test.go @@ -30,7 +30,6 @@ import ( versionutil "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -131,7 +130,7 @@ func managerSetup(ctx context.Context, mgr manager.Manager) { failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) - failedWebhook, err := webhooks.Setup(mgr, ptr.Deref(configuration.MultiKueue.DispatcherName, config.MultiKueueDispatcherModeAllAtOnce)) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) err = workloadjob.SetupIndexes(ctx, mgr.GetFieldIndexer()) diff --git a/test/integration/singlecluster/controller/admissionchecks/provisioning/suite_test.go b/test/integration/singlecluster/controller/admissionchecks/provisioning/suite_test.go index 73d7a31fdeb..ed341020b12 100644 --- a/test/integration/singlecluster/controller/admissionchecks/provisioning/suite_test.go +++ b/test/integration/singlecluster/controller/admissionchecks/provisioning/suite_test.go @@ -98,7 +98,7 @@ func managerSetup(options ...managerSetupOption) framework.ManagerSetup { err := indexer.Setup(ctx, mgr.GetFieldIndexer()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) controllersCfg := &config.Configuration{} diff --git a/test/integration/singlecluster/controller/core/suite_test.go b/test/integration/singlecluster/controller/core/suite_test.go index aac94715691..3c5cb9f8c98 100644 --- a/test/integration/singlecluster/controller/core/suite_test.go +++ b/test/integration/singlecluster/controller/core/suite_test.go @@ -72,7 +72,7 @@ func managerAndControllerSetup(controllersCfg *config.Configuration) framework.M err := indexer.Setup(ctx, mgr.GetFieldIndexer()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) if controllersCfg == nil { diff --git a/test/integration/singlecluster/controller/dra/suite_test.go b/test/integration/singlecluster/controller/dra/suite_test.go index bc4c5595fac..d337546dc1b 100644 --- a/test/integration/singlecluster/controller/dra/suite_test.go +++ b/test/integration/singlecluster/controller/dra/suite_test.go @@ -87,7 +87,7 @@ func managerSetup(modifyConfig func(*config.Configuration)) framework.ManagerSet gomega.Expect(err).NotTo(gomega.HaveOccurred()) // Webhooks - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) mappings := []config.DeviceClassMapping{ diff --git a/test/integration/singlecluster/controller/jobs/pod/suite_test.go b/test/integration/singlecluster/controller/jobs/pod/suite_test.go index ace45eb72f5..fff6ceba3e3 100644 --- a/test/integration/singlecluster/controller/jobs/pod/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/pod/suite_test.go @@ -132,7 +132,7 @@ func managerSetup( gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = pod.SetupWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) if setupTASControllers { diff --git a/test/integration/singlecluster/importer/suite_test.go b/test/integration/singlecluster/importer/suite_test.go index aca05ef4e7a..547c5859211 100644 --- a/test/integration/singlecluster/importer/suite_test.go +++ b/test/integration/singlecluster/importer/suite_test.go @@ -77,7 +77,7 @@ func managerAndSchedulerSetup(ctx context.Context, mgr manager.Manager) { failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) err = workloadjob.SetupIndexes(ctx, mgr.GetFieldIndexer()) diff --git a/test/integration/singlecluster/kueuectl/suite_test.go b/test/integration/singlecluster/kueuectl/suite_test.go index 2f318604223..c2c13274d3e 100644 --- a/test/integration/singlecluster/kueuectl/suite_test.go +++ b/test/integration/singlecluster/kueuectl/suite_test.go @@ -79,7 +79,7 @@ func managerSetup(ctx context.Context, mgr manager.Manager) { err := indexer.Setup(ctx, mgr.GetFieldIndexer()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) controllersCfg := &config.Configuration{} diff --git a/test/integration/singlecluster/scheduler/fairsharing/suite_test.go b/test/integration/singlecluster/scheduler/fairsharing/suite_test.go index d4375951b14..e1730bc1d80 100644 --- a/test/integration/singlecluster/scheduler/fairsharing/suite_test.go +++ b/test/integration/singlecluster/scheduler/fairsharing/suite_test.go @@ -87,7 +87,7 @@ func managerAndSchedulerSetup(admissionFairSharing *config.AdmissionFairSharing) failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) err = workloadjob.SetupIndexes(ctx, mgr.GetFieldIndexer()) diff --git a/test/integration/singlecluster/scheduler/podsready/suite_test.go b/test/integration/singlecluster/scheduler/podsready/suite_test.go index 7abfe87051d..c0a868e82a7 100644 --- a/test/integration/singlecluster/scheduler/podsready/suite_test.go +++ b/test/integration/singlecluster/scheduler/podsready/suite_test.go @@ -101,7 +101,7 @@ func managerAndSchedulerSetup(configuration *config.Configuration) framework.Man failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) err = workloadjob.SetupIndexes(ctx, mgr.GetFieldIndexer()) diff --git a/test/integration/singlecluster/scheduler/suite_test.go b/test/integration/singlecluster/scheduler/suite_test.go index 04857d527a6..3ad1ea3d359 100644 --- a/test/integration/singlecluster/scheduler/suite_test.go +++ b/test/integration/singlecluster/scheduler/suite_test.go @@ -90,7 +90,7 @@ func managerAndSchedulerSetup(ctx context.Context, mgr manager.Manager) { failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) err = workloadjob.SetupIndexes(ctx, mgr.GetFieldIndexer()) diff --git a/test/integration/singlecluster/tas/suite_test.go b/test/integration/singlecluster/tas/suite_test.go index 1f4357749d2..be1d3c55ec4 100644 --- a/test/integration/singlecluster/tas/suite_test.go +++ b/test/integration/singlecluster/tas/suite_test.go @@ -75,7 +75,7 @@ func managerSetup(ctx context.Context, mgr manager.Manager) { err := indexer.Setup(ctx, mgr.GetFieldIndexer()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) controllersCfg := &config.Configuration{} diff --git a/test/integration/singlecluster/webhook/core/admissioncheck_test.go b/test/integration/singlecluster/webhook/core/admissioncheck_test.go index 6b3158249b1..239ebb08e11 100644 --- a/test/integration/singlecluster/webhook/core/admissioncheck_test.go +++ b/test/integration/singlecluster/webhook/core/admissioncheck_test.go @@ -28,7 +28,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" @@ -38,7 +37,7 @@ import ( var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { - managerSetup(ctx, mgr, config.MultiKueueDispatcherModeAllAtOnce) + managerSetup(ctx, mgr) }) }) ginkgo.AfterAll(func() { diff --git a/test/integration/singlecluster/webhook/core/clusterqueue_test.go b/test/integration/singlecluster/webhook/core/clusterqueue_test.go index 08e3740c6bd..92102439de5 100644 --- a/test/integration/singlecluster/webhook/core/clusterqueue_test.go +++ b/test/integration/singlecluster/webhook/core/clusterqueue_test.go @@ -31,7 +31,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" @@ -46,7 +45,7 @@ const ( var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { - managerSetup(ctx, mgr, config.MultiKueueDispatcherModeAllAtOnce) + managerSetup(ctx, mgr) }) }) ginkgo.AfterAll(func() { diff --git a/test/integration/singlecluster/webhook/core/cohort_test.go b/test/integration/singlecluster/webhook/core/cohort_test.go index 114eb814140..fe6c7d0fb9f 100644 --- a/test/integration/singlecluster/webhook/core/cohort_test.go +++ b/test/integration/singlecluster/webhook/core/cohort_test.go @@ -29,7 +29,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" @@ -39,7 +38,7 @@ import ( var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { - managerSetup(ctx, mgr, config.MultiKueueDispatcherModeAllAtOnce) + managerSetup(ctx, mgr) }) }) ginkgo.AfterAll(func() { diff --git a/test/integration/singlecluster/webhook/core/localqueue_test.go b/test/integration/singlecluster/webhook/core/localqueue_test.go index 01da0f5ffa8..d9718e5e2b9 100644 --- a/test/integration/singlecluster/webhook/core/localqueue_test.go +++ b/test/integration/singlecluster/webhook/core/localqueue_test.go @@ -24,7 +24,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" @@ -36,7 +35,7 @@ const queueName = "queue-test" var _ = ginkgo.Describe("Queue validating webhook", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { - managerSetup(ctx, mgr, config.MultiKueueDispatcherModeAllAtOnce) + managerSetup(ctx, mgr) }) }) ginkgo.AfterAll(func() { diff --git a/test/integration/singlecluster/webhook/core/resourceflavor_test.go b/test/integration/singlecluster/webhook/core/resourceflavor_test.go index ac9d45d8dba..a0423cd450e 100644 --- a/test/integration/singlecluster/webhook/core/resourceflavor_test.go +++ b/test/integration/singlecluster/webhook/core/resourceflavor_test.go @@ -28,7 +28,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" @@ -43,7 +42,7 @@ const ( var _ = ginkgo.Describe("ResourceFlavor Webhook", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { - managerSetup(ctx, mgr, config.MultiKueueDispatcherModeAllAtOnce) + managerSetup(ctx, mgr) }) }) ginkgo.AfterAll(func() { diff --git a/test/integration/singlecluster/webhook/core/suite_test.go b/test/integration/singlecluster/webhook/core/suite_test.go index 4952223c755..66b66d61bea 100644 --- a/test/integration/singlecluster/webhook/core/suite_test.go +++ b/test/integration/singlecluster/webhook/core/suite_test.go @@ -62,11 +62,11 @@ var _ = ginkgo.AfterSuite(func() { fwk.Teardown() }) -func managerSetup(ctx context.Context, mgr manager.Manager, dispatcherName string) { +func managerSetup(ctx context.Context, mgr manager.Manager) { err := indexer.Setup(ctx, mgr.GetFieldIndexer()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - failedWebhook, err := webhooks.Setup(mgr, dispatcherName) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) cCache := schdcache.New(mgr.GetClient()) diff --git a/test/integration/singlecluster/webhook/core/workload_test.go b/test/integration/singlecluster/webhook/core/workload_test.go index 9befc27abad..dabd56fa0fc 100644 --- a/test/integration/singlecluster/webhook/core/workload_test.go +++ b/test/integration/singlecluster/webhook/core/workload_test.go @@ -32,13 +32,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" "sigs.k8s.io/kueue/pkg/workload" + "sigs.k8s.io/kueue/pkg/workloadslicing" "sigs.k8s.io/kueue/test/util" ) @@ -60,7 +60,7 @@ var _ = ginkgo.AfterEach(func() { var _ = ginkgo.Describe("Workload defaulting webhook", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { - managerSetup(ctx, mgr, config.MultiKueueDispatcherModeAllAtOnce) + managerSetup(ctx, mgr) }) }) ginkgo.AfterAll(func() { @@ -116,7 +116,7 @@ var _ = ginkgo.Describe("Workload defaulting webhook", ginkgo.Ordered, func() { var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { - managerSetup(ctx, mgr, config.MultiKueueDispatcherModeAllAtOnce) + managerSetup(ctx, mgr) }) }) ginkgo.AfterAll(func() { @@ -1062,7 +1062,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher AllAtOnce", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { - managerSetup(ctx, mgr, config.MultiKueueDispatcherModeAllAtOnce) + managerSetup(ctx, mgr) }) }) ginkgo.AfterAll(func() { @@ -1117,7 +1117,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al gomega.Succeed(), gomega.Succeed(), ), - ginkgo.Entry("Valid: ClusterName is not in NominatedClusters", + ginkgo.Entry("Invalid: ClusterName is not in NominatedClusters", func(wl *kueue.Workload) { wl.Status.NominatedClusterNames = []string{"worker1", "worker2"} wl.Status.ClusterName = nil @@ -1127,21 +1127,9 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al wl.Status.ClusterName = ptr.To("worker3") }, gomega.Succeed(), - gomega.Succeed(), - ), - ginkgo.Entry("Invalid: ClusterName is changed", - func(wl *kueue.Workload) { - wl.Status.NominatedClusterNames = nil - wl.Status.ClusterName = ptr.To("worker1") - }, - func(wl *kueue.Workload) { - wl.Status.NominatedClusterNames = nil - wl.Status.ClusterName = ptr.To("worker2") - }, - gomega.Succeed(), - testing.BeInvalidError(), + testing.BeForbiddenError(), ), - ginkgo.Entry("Valid: ClusterName is set when NominatedClusters is empty", + ginkgo.Entry("Invalid: ClusterName is set when NominatedClusters is empty", func(wl *kueue.Workload) { wl.Status.NominatedClusterNames = nil wl.Status.ClusterName = nil @@ -1151,7 +1139,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al wl.Status.NominatedClusterNames = nil }, gomega.Succeed(), - gomega.Succeed(), + testing.BeForbiddenError(), ), ginkgo.Entry("Invalid: ClusterName and NominatedClusters are mutually exclusive", func(wl *kueue.Workload) {}, @@ -1178,7 +1166,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Incremental", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { - managerSetup(ctx, mgr, config.MultiKueueDispatcherModeIncremental) + managerSetup(ctx, mgr) }) }) ginkgo.AfterAll(func() { @@ -1288,5 +1276,30 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher In gomega.Succeed(), ), ) + + ginkgo.It("Should allow to set ClusterName without previous nominatedClusterNames when ElasticJobsViaWorkloadSlices feature gate is enabled", func() { + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.ElasticJobsViaWorkloadSlices, true) + + ginkgo.By("Creating a new Workload") + wl := utiltestingapi.MakeWorkload(workloadName, ns.Name). + PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). + Request(corev1.ResourceCPU, "1"). + Obj()). + Annotation(workloadslicing.WorkloadSliceReplacementFor, string(workload.NewReference(ns.Name, workloadName))). + Obj() + util.MustCreate(ctx, k8sClient, wl) + + ginkgo.By("mimic scheduler by setting the status.clusterName during quota reservation") + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), wl)).To(gomega.Succeed()) + wl.Status.Admission = utiltestingapi.MakeAdmission("default"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj() + wl.Status.ClusterName = ptr.To("worker1") + g.Expect(k8sClient.Status().Update(ctx, wl)).Should(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) }) }) diff --git a/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go b/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go index 05d54b7c9d2..d6f1471dab0 100644 --- a/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go @@ -26,7 +26,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/core/indexer" @@ -101,7 +100,7 @@ var _ = ginkgo.Describe("RayCluster Webhook", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) jobframework.EnableIntegration(workloadrayjob.FrameworkName) - failedWebhook, err := webhooks.Setup(mgr, config.MultiKueueDispatcherModeAllAtOnce) + failedWebhook, err := webhooks.Setup(mgr) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) return nil From b9466d517b7dba9854c6f601f39355ecd4261e10 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Tue, 21 Oct 2025 14:08:40 +0530 Subject: [PATCH 005/119] Bump kueueviz frontend dependencies. (#7335) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cmd/kueueviz/frontend/package-lock.json | 152 ++++++++++++------------ cmd/kueueviz/frontend/package.json | 12 +- 2 files changed, 82 insertions(+), 82 deletions(-) diff --git a/cmd/kueueviz/frontend/package-lock.json b/cmd/kueueviz/frontend/package-lock.json index 7803332557b..487912dd7d2 100644 --- a/cmd/kueueviz/frontend/package-lock.json +++ b/cmd/kueueviz/frontend/package-lock.json @@ -11,19 +11,19 @@ "dependencies": { "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", - "@mui/icons-material": "^7.3.2", + "@mui/icons-material": "^7.3.4", "@mui/material": "^7.3.1", - "ace-builds": "^1.43.3", - "react": "^19.1.1", + "ace-builds": "^1.43.4", + "react": "^19.2.0", "react-ace": "^14.0.1", - "react-dom": "^19.1.1", - "react-router-dom": "^7.9.3", + "react-dom": "^19.2.0", + "react-router-dom": "^7.9.4", "react-toastify": "^11.0.5" }, "devDependencies": { "@vitejs/plugin-react": "^5.0.4", "depcheck": "^1.4.7", - "vite": "^7.1.7" + "vite": "^7.1.11" } }, "node_modules/@babel/code-frame": { @@ -924,9 +924,9 @@ } }, "node_modules/@mui/core-downloads-tracker": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-7.3.2.tgz", - "integrity": "sha512-AOyfHjyDKVPGJJFtxOlept3EYEdLoar/RvssBTWVAvDJGIE676dLi2oT/Kx+FoVXFoA/JdV7DEMq/BVWV3KHRw==", + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-7.3.4.tgz", + "integrity": "sha512-BIktMapG3r4iXwIhYNpvk97ZfYWTreBBQTWjQKbNbzI64+ULHfYavQEX2w99aSWHS58DvXESWIgbD9adKcUOBw==", "license": "MIT", "funding": { "type": "opencollective", @@ -934,12 +934,12 @@ } }, "node_modules/@mui/icons-material": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-7.3.2.tgz", - "integrity": "sha512-TZWazBjWXBjR6iGcNkbKklnwodcwj0SrChCNHc9BhD9rBgET22J1eFhHsEmvSvru9+opDy3umqAimQjokhfJlQ==", + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-7.3.4.tgz", + "integrity": "sha512-9n6Xcq7molXWYb680N2Qx+FRW8oT6j/LXF5PZFH3ph9X/Rct0B/BlLAsFI7iL9ySI6LVLuQIVtrLiPT82R7OZw==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.28.3" + "@babel/runtime": "^7.28.4" }, "engines": { "node": ">=14.0.0" @@ -949,7 +949,7 @@ "url": "https://opencollective.com/mui-org" }, "peerDependencies": { - "@mui/material": "^7.3.2", + "@mui/material": "^7.3.4", "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^17.0.0 || ^18.0.0 || ^19.0.0" }, @@ -960,16 +960,16 @@ } }, "node_modules/@mui/material": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/@mui/material/-/material-7.3.2.tgz", - "integrity": "sha512-qXvbnawQhqUVfH1LMgMaiytP+ZpGoYhnGl7yYq2x57GYzcFL/iPzSZ3L30tlbwEjSVKNYcbiKO8tANR1tadjUg==", + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/@mui/material/-/material-7.3.4.tgz", + "integrity": "sha512-gEQL9pbJZZHT7lYJBKQCS723v1MGys2IFc94COXbUIyCTWa+qC77a7hUax4Yjd5ggEm35dk4AyYABpKKWC4MLw==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.28.3", - "@mui/core-downloads-tracker": "^7.3.2", - "@mui/system": "^7.3.2", - "@mui/types": "^7.4.6", - "@mui/utils": "^7.3.2", + "@babel/runtime": "^7.28.4", + "@mui/core-downloads-tracker": "^7.3.4", + "@mui/system": "^7.3.3", + "@mui/types": "^7.4.7", + "@mui/utils": "^7.3.3", "@popperjs/core": "^2.11.8", "@types/react-transition-group": "^4.4.12", "clsx": "^2.1.1", @@ -988,7 +988,7 @@ "peerDependencies": { "@emotion/react": "^11.5.0", "@emotion/styled": "^11.3.0", - "@mui/material-pigment-css": "^7.3.2", + "@mui/material-pigment-css": "^7.3.3", "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" @@ -1009,13 +1009,13 @@ } }, "node_modules/@mui/private-theming": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/@mui/private-theming/-/private-theming-7.3.2.tgz", - "integrity": "sha512-ha7mFoOyZGJr75xeiO9lugS3joRROjc8tG1u4P50dH0KR7bwhHznVMcYg7MouochUy0OxooJm/OOSpJ7gKcMvg==", + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/@mui/private-theming/-/private-theming-7.3.3.tgz", + "integrity": "sha512-OJM+9nj5JIyPUvsZ5ZjaeC9PfktmK+W5YaVLToLR8L0lB/DGmv1gcKE43ssNLSvpoW71Hct0necfade6+kW3zQ==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.28.3", - "@mui/utils": "^7.3.2", + "@babel/runtime": "^7.28.4", + "@mui/utils": "^7.3.3", "prop-types": "^15.8.1" }, "engines": { @@ -1036,12 +1036,12 @@ } }, "node_modules/@mui/styled-engine": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/@mui/styled-engine/-/styled-engine-7.3.2.tgz", - "integrity": "sha512-PkJzW+mTaek4e0nPYZ6qLnW5RGa0KN+eRTf5FA2nc7cFZTeM+qebmGibaTLrgQBy3UpcpemaqfzToBNkzuxqew==", + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/@mui/styled-engine/-/styled-engine-7.3.3.tgz", + "integrity": "sha512-CmFxvRJIBCEaWdilhXMw/5wFJ1+FT9f3xt+m2pPXhHPeVIbBg9MnMvNSJjdALvnQJMPw8jLhrUtXmN7QAZV2fw==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.28.3", + "@babel/runtime": "^7.28.4", "@emotion/cache": "^11.14.0", "@emotion/serialize": "^1.3.3", "@emotion/sheet": "^1.4.0", @@ -1070,16 +1070,16 @@ } }, "node_modules/@mui/system": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/@mui/system/-/system-7.3.2.tgz", - "integrity": "sha512-9d8JEvZW+H6cVkaZ+FK56R53vkJe3HsTpcjMUtH8v1xK6Y1TjzHdZ7Jck02mGXJsE6MQGWVs3ogRHTQmS9Q/rA==", + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/@mui/system/-/system-7.3.3.tgz", + "integrity": "sha512-Lqq3emZr5IzRLKaHPuMaLBDVaGvxoh6z7HMWd1RPKawBM5uMRaQ4ImsmmgXWtwJdfZux5eugfDhXJUo2mliS8Q==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.28.3", - "@mui/private-theming": "^7.3.2", - "@mui/styled-engine": "^7.3.2", - "@mui/types": "^7.4.6", - "@mui/utils": "^7.3.2", + "@babel/runtime": "^7.28.4", + "@mui/private-theming": "^7.3.3", + "@mui/styled-engine": "^7.3.3", + "@mui/types": "^7.4.7", + "@mui/utils": "^7.3.3", "clsx": "^2.1.1", "csstype": "^3.1.3", "prop-types": "^15.8.1" @@ -1110,12 +1110,12 @@ } }, "node_modules/@mui/types": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/@mui/types/-/types-7.4.6.tgz", - "integrity": "sha512-NVBbIw+4CDMMppNamVxyTccNv0WxtDb7motWDlMeSC8Oy95saj1TIZMGynPpFLePt3yOD8TskzumeqORCgRGWw==", + "version": "7.4.7", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-7.4.7.tgz", + "integrity": "sha512-8vVje9rdEr1rY8oIkYgP+Su5Kwl6ik7O3jQ0wl78JGSmiZhRHV+vkjooGdKD8pbtZbutXFVTWQYshu2b3sG9zw==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.28.3" + "@babel/runtime": "^7.28.4" }, "peerDependencies": { "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" @@ -1127,13 +1127,13 @@ } }, "node_modules/@mui/utils": { - "version": "7.3.2", - "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-7.3.2.tgz", - "integrity": "sha512-4DMWQGenOdLnM3y/SdFQFwKsCLM+mqxzvoWp9+x2XdEzXapkznauHLiXtSohHs/mc0+5/9UACt1GdugCX2te5g==", + "version": "7.3.3", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-7.3.3.tgz", + "integrity": "sha512-kwNAUh7bLZ7mRz9JZ+6qfRnnxbE4Zuc+RzXnhSpRSxjTlSTj7b4JxRLXpG+MVtPVtqks5k/XC8No1Vs3x4Z2gg==", "license": "MIT", "dependencies": { - "@babel/runtime": "^7.28.3", - "@mui/types": "^7.4.6", + "@babel/runtime": "^7.28.4", + "@mui/types": "^7.4.7", "@types/prop-types": "^15.7.15", "clsx": "^2.1.1", "prop-types": "^15.8.1", @@ -1638,9 +1638,9 @@ "license": "MIT" }, "node_modules/ace-builds": { - "version": "1.43.3", - "resolved": "https://registry.npmjs.org/ace-builds/-/ace-builds-1.43.3.tgz", - "integrity": "sha512-MCl9rALmXwIty/4Qboijo/yNysx1r6hBTzG+6n/TiOm5LFhZpEvEIcIITPFiEOEFDfgBOEmxu+a4f54LEFM6Sg==", + "version": "1.43.4", + "resolved": "https://registry.npmjs.org/ace-builds/-/ace-builds-1.43.4.tgz", + "integrity": "sha512-8hAxVfo2ImICd69BWlZwZlxe9rxDGDjuUhh+WeWgGDvfBCE+r3lkynkQvIovDz4jcMi8O7bsEaFygaDT+h9sBA==", "license": "BSD-3-Clause" }, "node_modules/ansi-regex": { @@ -2809,9 +2809,9 @@ "license": "MIT" }, "node_modules/react": { - "version": "19.1.1", - "resolved": "https://registry.npmjs.org/react/-/react-19.1.1.tgz", - "integrity": "sha512-w8nqGImo45dmMIfljjMwOGtbmC/mk4CMYhWIicdSflH91J9TyCyczcPFXJzrZ/ZXcgGRFeP6BU0BEJTw6tZdfQ==", + "version": "19.2.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.0.tgz", + "integrity": "sha512-tmbWg6W31tQLeB5cdIBOicJDJRR2KzXsV7uSK9iNfLWQ5bIZfxuPEHp7M8wiHyHnn0DD1i7w3Zmin0FtkrwoCQ==", "license": "MIT", "engines": { "node": ">=0.10.0" @@ -2835,21 +2835,21 @@ } }, "node_modules/react-dom": { - "version": "19.1.1", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.1.1.tgz", - "integrity": "sha512-Dlq/5LAZgF0Gaz6yiqZCf6VCcZs1ghAJyrsu84Q/GT0gV+mCxbfmKNoGRKBYMJ8IEdGPqu49YWXD02GCknEDkw==", + "version": "19.2.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.0.tgz", + "integrity": "sha512-UlbRu4cAiGaIewkPyiRGJk0imDN2T3JjieT6spoL2UeSf5od4n5LB/mQ4ejmxhCFT1tYe8IvaFulzynWovsEFQ==", "license": "MIT", "dependencies": { - "scheduler": "^0.26.0" + "scheduler": "^0.27.0" }, "peerDependencies": { - "react": "^19.1.1" + "react": "^19.2.0" } }, "node_modules/react-is": { - "version": "19.1.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.1.1.tgz", - "integrity": "sha512-tr41fA15Vn8p4X9ntI+yCyeGSf1TlYaY5vlTZfQmeLBrFo3psOPX6HhTDnFNL9uj3EhP0KAQ80cugCl4b4BERA==", + "version": "19.2.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.2.0.tgz", + "integrity": "sha512-x3Ax3kNSMIIkyVYhWPyO09bu0uttcAIoecO/um/rKGQ4EltYWVYtyiGkS/3xMynrbVQdS69Jhlv8FXUEZehlzA==", "license": "MIT" }, "node_modules/react-refresh": { @@ -2863,9 +2863,9 @@ } }, "node_modules/react-router": { - "version": "7.9.3", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.9.3.tgz", - "integrity": "sha512-4o2iWCFIwhI/eYAIL43+cjORXYn/aRQPgtFRRZb3VzoyQ5Uej0Bmqj7437L97N9NJW4wnicSwLOLS+yCXfAPgg==", + "version": "7.9.4", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.9.4.tgz", + "integrity": "sha512-SD3G8HKviFHg9xj7dNODUKDFgpG4xqD5nhyd0mYoB5iISepuZAvzSr8ywxgxKJ52yRzf/HWtVHc9AWwoTbljvA==", "license": "MIT", "dependencies": { "cookie": "^1.0.1", @@ -2885,12 +2885,12 @@ } }, "node_modules/react-router-dom": { - "version": "7.9.3", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.9.3.tgz", - "integrity": "sha512-1QSbA0TGGFKTAc/aWjpfW/zoEukYfU4dc1dLkT/vvf54JoGMkW+fNA+3oyo2gWVW1GM7BxjJVHz5GnPJv40rvg==", + "version": "7.9.4", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.9.4.tgz", + "integrity": "sha512-f30P6bIkmYvnHHa5Gcu65deIXoA2+r3Eb6PJIAddvsT9aGlchMatJ51GgpU470aSqRRbFX22T70yQNUGuW3DfA==", "license": "MIT", "dependencies": { - "react-router": "7.9.3" + "react-router": "7.9.4" }, "engines": { "node": ">=20.0.0" @@ -3044,9 +3044,9 @@ } }, "node_modules/scheduler": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.26.0.tgz", - "integrity": "sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA==", + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", "license": "MIT" }, "node_modules/semver": { @@ -3245,9 +3245,9 @@ } }, "node_modules/vite": { - "version": "7.1.7", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.1.7.tgz", - "integrity": "sha512-VbA8ScMvAISJNJVbRDTJdCwqQoAareR/wutevKanhR2/1EkoXVZVkkORaYm/tNVCjP/UDTKtcw3bAkwOUdedmA==", + "version": "7.1.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.1.11.tgz", + "integrity": "sha512-uzcxnSDVjAopEUjljkWh8EIrg6tlzrjFUfMcR1EVsRDGwf/ccef0qQPRyOrROwhrTDaApueq+ja+KLPlzR/zdg==", "dev": true, "license": "MIT", "dependencies": { diff --git a/cmd/kueueviz/frontend/package.json b/cmd/kueueviz/frontend/package.json index e03c59fce4b..e0c171eb313 100644 --- a/cmd/kueueviz/frontend/package.json +++ b/cmd/kueueviz/frontend/package.json @@ -15,19 +15,19 @@ "dependencies": { "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", - "@mui/icons-material": "^7.3.2", + "@mui/icons-material": "^7.3.4", "@mui/material": "^7.3.1", - "ace-builds": "^1.43.3", - "react": "^19.1.1", + "ace-builds": "^1.43.4", + "react": "^19.2.0", "react-ace": "^14.0.1", - "react-dom": "^19.1.1", - "react-router-dom": "^7.9.3", + "react-dom": "^19.2.0", + "react-router-dom": "^7.9.4", "react-toastify": "^11.0.5" }, "devDependencies": { "@vitejs/plugin-react": "^5.0.4", "depcheck": "^1.4.7", - "vite": "^7.1.7" + "vite": "^7.1.11" }, "browserslist": { "production": [ From 5d3187fb1537bb2142d05cca60c16b4e09157ccd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 02:26:39 -0700 Subject: [PATCH 006/119] Bump cypress in /test/e2e/kueueviz in the all group (#7239) Bumps the all group in /test/e2e/kueueviz with 1 update: [cypress](https://github.com/cypress-io/cypress). Updates `cypress` from 15.3.0 to 15.4.0 - [Release notes](https://github.com/cypress-io/cypress/releases) - [Changelog](https://github.com/cypress-io/cypress/blob/develop/CHANGELOG.md) - [Commits](https://github.com/cypress-io/cypress/compare/v15.3.0...v15.4.0) --- updated-dependencies: - dependency-name: cypress dependency-version: 15.4.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: all ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test/e2e/kueueviz/package-lock.json | 8 ++++---- test/e2e/kueueviz/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/kueueviz/package-lock.json b/test/e2e/kueueviz/package-lock.json index bf780d474d3..d077f79d74a 100644 --- a/test/e2e/kueueviz/package-lock.json +++ b/test/e2e/kueueviz/package-lock.json @@ -9,7 +9,7 @@ "version": "0.14.1", "devDependencies": { "@testing-library/cypress": "^10.1.0", - "cypress": "^15.3.0", + "cypress": "^15.5.0", "depcheck": "^1.4.7" } }, @@ -1015,9 +1015,9 @@ } }, "node_modules/cypress": { - "version": "15.3.0", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-15.3.0.tgz", - "integrity": "sha512-g9rDhoK9y8wW4Vx3Ppr8dtfvThXxPL3mJsV5e98fG+6EerrhXKmeRT2sL86cvNRtEZouXJfsuVL1lqiMuGNGcg==", + "version": "15.5.0", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-15.5.0.tgz", + "integrity": "sha512-7jXBsh5hTfjxr9QQONC2IbdTj0nxSyU8x4eiarMZBzXzCj3pedKviUx8JnLcE4vL8e0TsOzp70WSLRORjEssRA==", "dev": true, "hasInstallScript": true, "license": "MIT", diff --git a/test/e2e/kueueviz/package.json b/test/e2e/kueueviz/package.json index b5cf5917f8f..8afaa260a8a 100644 --- a/test/e2e/kueueviz/package.json +++ b/test/e2e/kueueviz/package.json @@ -7,7 +7,7 @@ }, "devDependencies": { "@testing-library/cypress": "^10.1.0", - "cypress": "^15.3.0", + "cypress": "^15.5.0", "depcheck": "^1.4.7" } } From 0b08d3d8b048f25e56ed50943aad7aeb67536114 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 02:26:46 -0700 Subject: [PATCH 007/119] Bump node from 24-alpine to 25-alpine in /hack/depcheck (#7323) Bumps node from 24-alpine to 25-alpine. --- updated-dependencies: - dependency-name: node dependency-version: 25-alpine dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- hack/depcheck/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/depcheck/Dockerfile b/hack/depcheck/Dockerfile index a1afe9f6ac4..ed627e3bcd5 100644 --- a/hack/depcheck/Dockerfile +++ b/hack/depcheck/Dockerfile @@ -1,4 +1,4 @@ -FROM node:24-alpine +FROM node:25-alpine # Set working directory WORKDIR /app From 9a97939e257975dc14cf8351ac5d42ae2fdd2b09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Oct 2025 02:26:53 -0700 Subject: [PATCH 008/119] Bump node from 24-slim to 25-slim in /cmd/kueueviz/frontend (#7324) Bumps node from 24-slim to 25-slim. --- updated-dependencies: - dependency-name: node dependency-version: 25-slim dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cmd/kueueviz/frontend/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/kueueviz/frontend/Dockerfile b/cmd/kueueviz/frontend/Dockerfile index 72acbdc4941..6e809e4e90a 100644 --- a/cmd/kueueviz/frontend/Dockerfile +++ b/cmd/kueueviz/frontend/Dockerfile @@ -1,5 +1,5 @@ # Build stage -FROM node:24-slim AS builder +FROM node:25-slim AS builder # Set working directory WORKDIR /app @@ -17,7 +17,7 @@ COPY . . RUN npm run build # Runtime stage -FROM node:24-slim AS runtime +FROM node:25-slim AS runtime WORKDIR /app From d7bedeeac49395b7b49446d55c38c5b06f93adea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Paj=C4=85k?= Date: Wed, 22 Oct 2025 13:18:40 +0200 Subject: [PATCH 009/119] E2e test for Node HotSwap in TAS with slices (#7142) * E2e test for hotswap with slices * Address comments * Fix after merge * Timeout -> LongTimeout * Address comments --- test/e2e/tas/hotswap_test.go | 307 +++++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 test/e2e/tas/hotswap_test.go diff --git a/test/e2e/tas/hotswap_test.go b/test/e2e/tas/hotswap_test.go new file mode 100644 index 00000000000..914e3fae873 --- /dev/null +++ b/test/e2e/tas/hotswap_test.go @@ -0,0 +1,307 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tase2e + +import ( + "context" + "fmt" + + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + "sigs.k8s.io/kueue/pkg/util/testing" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" + testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" + "sigs.k8s.io/kueue/test/util" +) + +var _ = ginkgo.Describe("Hotswap for Topology Aware Scheduling", ginkgo.Ordered, func() { + var ns *corev1.Namespace + ginkgo.BeforeEach(func() { + ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "e2e-tas-hotswap-") + }) + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + util.ExpectAllPodsInNamespaceDeleted(ctx, k8sClient, ns) + }) + // The topology of the e2e cluster looks as follows + // Block: b1 b2 + // / \ / \ + // Rack: r1 r2 r3 r4 + // / \ / \ / \ / \ + // Hostname: worker worker2 worker3 worker4 worker5 worker6 worker7 worker8 + // Each node has 1 GPU (extraResource) + ginkgo.When("Creating a JobSet with slices", func() { + var ( + topology *kueue.Topology + tasFlavor *kueue.ResourceFlavor + localQueue *kueue.LocalQueue + clusterQueue *kueue.ClusterQueue + nodeToRestore *corev1.Node + ) + ginkgo.BeforeEach(func() { + topology = utiltestingapi.MakeDefaultThreeLevelTopology("datacenter") + util.MustCreate(ctx, k8sClient, topology) + + tasFlavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). + NodeLabel(tasNodeGroupLabel, instanceType).TopologyName(topology.Name).Obj() + util.MustCreate(ctx, k8sClient, tasFlavor) + clusterQueue = utiltestingapi.MakeClusterQueue("cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("tas-flavor"). + Resource(extraResource, "8"). + Resource(corev1.ResourceCPU, "8"). + Obj(), + ). + Obj() + util.MustCreate(ctx, k8sClient, clusterQueue) + util.ExpectClusterQueuesToBeActive(ctx, k8sClient, clusterQueue) + + localQueue = utiltestingapi.MakeLocalQueue("main", ns.Name).ClusterQueue("cluster-queue").Obj() + util.MustCreate(ctx, k8sClient, localQueue) + }) + ginkgo.AfterEach(func() { + if nodeToRestore != nil { + ginkgo.By(fmt.Sprintf("Re-creating node %s", nodeToRestore.Name)) + nodeToRestore.ResourceVersion = "" + nodeToRestore.UID = "" + nodeToRestore.ManagedFields = nil + util.MustCreate(ctx, k8sClient, nodeToRestore) + + util.SetNodeCondition(ctx, k8sClient, nodeToRestore, &corev1.NodeCondition{ + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }) + waitForDummyWorkloadToRunOnNode(nodeToRestore, localQueue) + nodeToRestore = nil + } + gomega.Expect(util.DeleteAllJobsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed()) + gomega.Expect(util.DeleteAllJobSetsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed()) + gomega.Expect(util.DeleteWorkloadsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed()) + util.ExpectObjectToBeDeleted(ctx, k8sClient, localQueue, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, clusterQueue, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, tasFlavor, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, topology, true) + util.ExpectAllPodsInNamespaceDeleted(ctx, k8sClient, ns) + }) + // In this test we use a jobset with SliceSize = 3 and SliceRequiredTopology = Block + // Each pod requires 1 "extraResource" so the jobSet will use three nodes from a Block. + // Since each Block has 4 nodes (see the image above), one node will be free. + // When one of the nodes fail, the replacement mechanism should find the available node + // and replace the failed one. + ginkgo.It("Should replace a failed node with a new one within the same domain", func() { + replicas := 1 + parallelism := 3 + numPods := replicas * parallelism + jobName := "ranks-jobset" + replicatedJobName := "replicated-job-1" + sampleJob := testingjobset.MakeJobSet(jobName, ns.Name). + Queue(localQueue.Name). + ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: replicatedJobName, + Image: util.GetAgnHostImage(), + Args: util.BehaviorWaitForDeletion, + Replicas: int32(replicas), + Parallelism: int32(parallelism), + Completions: int32(parallelism), + PodAnnotations: map[string]string{ + kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetSliceSizeAnnotation: "3", + }, + }, + ). + RequestAndLimit(replicatedJobName, extraResource, "1"). + RequestAndLimit(replicatedJobName, corev1.ResourceCPU, "200m"). + Obj() + util.MustCreate(ctx, k8sClient, sampleJob) + + ginkgo.By("JobSet is unsuspended", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(sampleJob), sampleJob)).To(gomega.Succeed()) + g.Expect(sampleJob.Spec.Suspend).Should(gomega.Equal(ptr.To(false))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + pods := &corev1.PodList{} + ginkgo.By("ensure all pods are created, scheduled and running", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.AndSelectors( + fields.OneTermNotEqualSelector("spec.nodeName", ""), + fields.OneTermEqualSelector("status.phase", string(corev1.PodRunning)), + ), + LabelSelector: labels.SelectorFromSet(map[string]string{ + "jobset.sigs.k8s.io/jobset-name": jobName, + }), + } + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name), listOpts)).To(gomega.Succeed(), "listing running pods") + g.Expect(pods.Items).Should(gomega.HaveLen(numPods)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + wlName := pods.Items[0].Annotations[kueue.WorkloadAnnotation] + wlKey := client.ObjectKey{Name: wlName, Namespace: ns.Name} + ginkgo.By("Verify initial topology assignment of the workload", func() { + expectWorkloadTopologyAssignment(ctx, k8sClient, wlKey, numPods, []string{ + "kind-worker", "kind-worker2", "kind-worker3", + }) + }) + chosenPod := pods.Items[0] + node := &corev1.Node{} + + ginkgo.By(fmt.Sprintf("Simulate failure of node hosting pod %s", chosenPod.Name), func() { + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: chosenPod.Spec.NodeName}, node)).To(gomega.Succeed()) + nodeToRestore = node.DeepCopy() + gomega.Expect(k8sClient.Delete(ctx, node)).To(gomega.Succeed()) + }) + ginkgo.By("Check that the topology assignment is updated with the new node in the same block", func() { + expectWorkloadTopologyAssignment(ctx, k8sClient, wlKey, numPods, []string{ + "kind-worker2", "kind-worker3", "kind-worker4", + }) + }) + }) + // In this test we use a jobset with SliceSize = 2 and SliceRequiredTopology = Rack + // Each pod requires 1 "extraResource" so the jobSet will use both nodes from a Rack. + // When one of the nodes fail, the replacement mechanism would need to find the + // replacement within the same rack, which is not possible, thus the workload + // will be evicted. + ginkgo.It("Should evict the workload if replacement is not possible", func() { + replicas := 1 + parallelism := 2 + numPods := replicas * parallelism + jobName := "ranks-jobset" + replicatedJobName := "replicated-job-1" + sampleJob := testingjobset.MakeJobSet(jobName, ns.Name). + Queue(localQueue.Name). + ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: replicatedJobName, + Image: util.GetAgnHostImage(), + Args: util.BehaviorWaitForDeletion, + Replicas: int32(replicas), + Parallelism: int32(parallelism), + Completions: int32(parallelism), + PodAnnotations: map[string]string{ + kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetSliceSizeAnnotation: "2", + }, + }, + ). + RequestAndLimit(replicatedJobName, extraResource, "1"). + RequestAndLimit(replicatedJobName, corev1.ResourceCPU, "200m"). + Obj() + util.MustCreate(ctx, k8sClient, sampleJob) + + ginkgo.By("JobSet is unsuspended", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(sampleJob), sampleJob)).To(gomega.Succeed()) + g.Expect(sampleJob.Spec.Suspend).Should(gomega.Equal(ptr.To(false))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + pods := &corev1.PodList{} + + ginkgo.By("ensure all pods are created, scheduled and running", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.AndSelectors( + fields.OneTermNotEqualSelector("spec.nodeName", ""), + fields.OneTermEqualSelector("status.phase", string(corev1.PodRunning)), + ), + LabelSelector: labels.SelectorFromSet(map[string]string{"jobset.sigs.k8s.io/jobset-name": jobName}), + } + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name), listOpts)).To(gomega.Succeed(), "listing running pods") + g.Expect(pods.Items).Should(gomega.HaveLen(numPods)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + wlName := pods.Items[0].Annotations[kueue.WorkloadAnnotation] + wlKey := client.ObjectKey{Name: wlName, Namespace: ns.Name} + ginkgo.By("Verify initial topology assignment of the workload", func() { + expectWorkloadTopologyAssignment(ctx, k8sClient, wlKey, numPods, []string{ + "kind-worker", "kind-worker2", + }) + }) + chosenPod := pods.Items[0] + node := &corev1.Node{} + ginkgo.By(fmt.Sprintf("Simulate failure of node hosting pod %s", chosenPod.Name), func() { + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: chosenPod.Spec.NodeName}, node)).To(gomega.Succeed()) + nodeToRestore = node.DeepCopy() + gomega.Expect(k8sClient.Delete(ctx, node)).To(gomega.Succeed()) + }) + wl := &kueue.Workload{} + ginkgo.By("Check that the workload is evicted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlKey, wl)).To(gomega.Succeed()) + g.Expect(wl.Status.Admission).To(gomega.BeNil()) + g.Expect(apimeta.IsStatusConditionTrue(wl.Status.Conditions, kueue.WorkloadEvicted)).To(gomega.BeTrue()) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) +}) + +func waitForDummyWorkloadToRunOnNode(node *corev1.Node, lq *kueue.LocalQueue) { + ginkgo.By(fmt.Sprintf("Waiting for a dummy workload to run on the recovered node %s", node.Name), func() { + dummyJob := testingjob.MakeJob(fmt.Sprintf("dummy-job-%s", node.Name), lq.Namespace). + Queue(kueue.LocalQueueName(lq.Name)). + NodeSelector(corev1.LabelHostname, node.Name). + Image(util.GetAgnHostImage(), util.BehaviorExitFast). + Obj() + gomega.Expect(k8sClient.Create(ctx, dummyJob)).To(gomega.Succeed()) + gomega.Eventually(func(g gomega.Gomega) { + var createdDummyJob batchv1.Job + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(dummyJob), &createdDummyJob)).To(gomega.Succeed()) + g.Expect(createdDummyJob.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(batchv1.JobCondition{ + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, cmpopts.IgnoreFields(batchv1.JobCondition{}, "LastTransitionTime", "LastProbeTime", "Reason", "Message")))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) +} + +func expectWorkloadTopologyAssignment(ctx context.Context, k8sClient client.Client, wlKey client.ObjectKey, numPods int, expectedNodes []string) { + gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { + wl := &kueue.Workload{} + g.Expect(k8sClient.Get(ctx, wlKey, wl)).To(gomega.Succeed()) + g.Expect(wl.Status.Admission).NotTo(gomega.BeNil()) + g.Expect(wl.Status.Admission.PodSetAssignments).To(gomega.HaveLen(1)) + topologyAssignment := wl.Status.Admission.PodSetAssignments[0].TopologyAssignment + g.Expect(topologyAssignment).NotTo(gomega.BeNil()) + g.Expect(topologyAssignment.Levels).To(gomega.BeEquivalentTo([]string{corev1.LabelHostname})) + g.Expect(topologyAssignment.Domains).To(gomega.HaveLen(numPods)) + chosenNodes := []string{} + for _, domain := range topologyAssignment.Domains { + g.Expect(domain.Count).To(gomega.Equal(int32(1))) + chosenNodes = append(chosenNodes, domain.Values...) + } + g.Expect(chosenNodes).To(gomega.BeEquivalentTo(expectedNodes)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) +} From 02c34b826093b361c279439acc33193f8211c169 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Thu, 23 Oct 2025 02:02:38 -0400 Subject: [PATCH 010/119] Enable cache in pod integration tests to fix failure with ManagedJob feature gate is turned on (#7346) * MultiKueueManagedByJob is failing when jobs manage a pod with pod integration * add cache to options --- .../jobs/pod/pod_controller_test.go | 37 ++++++++++++++++++- .../controller/jobs/pod/suite_test.go | 2 +- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go index 0c04874ae30..741ad5bf2e0 100644 --- a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go @@ -39,6 +39,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" podcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/pod" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" + "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" @@ -354,7 +355,41 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu }) ginkgo.When("Pod owner is managed by Kueue", func() { - ginkgo.It("Should skip the pod", func() { + ginkgo.It("Should skip the pod with MultiKueueBatchJobWithManagedBy by off", func() { + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.MultiKueueBatchJobWithManagedBy, false) + job := testingjob.MakeJob("parent-job", ns.Name).Queue(kueue.LocalQueueName(lq.Name)).Obj() + util.MustCreate(ctx, k8sClient, job) + + pod := testingpod.MakePod(podName, ns.Name).Queue(lq.Name).Obj() + gomega.Expect(controllerutil.SetControllerReference(job, pod, k8sClient.Scheme())).To(gomega.Succeed()) + util.MustCreate(ctx, k8sClient, pod) + + createdPod := &corev1.Pod{} + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, lookupKey, createdPod)).Should(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + + gomega.Expect(createdPod.Spec.SchedulingGates).NotTo( + gomega.ContainElement(corev1.PodSchedulingGate{Name: podconstants.SchedulingGateName}), + "Pod shouldn't have scheduling gate", + ) + + gomega.Expect(createdPod.Labels).NotTo( + gomega.HaveKeyWithValue(constants.ManagedByKueueLabelKey, constants.ManagedByKueueLabelValue), + "Pod shouldn't have the label", + ) + + gomega.Expect(createdPod.Finalizers).NotTo(gomega.ContainElement(constants.ManagedByKueueLabelKey), + "Pod shouldn't have finalizer") + + wlLookupKey := types.NamespacedName{Name: podcontroller.GetWorkloadNameForPod(createdPod.Name, createdPod.UID), Namespace: ns.Name} + ginkgo.By(fmt.Sprintf("checking that workload '%s' is not created", wlLookupKey)) + createdWorkload := &kueue.Workload{} + + gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(testing.BeNotFoundError()) + }) + ginkgo.It("Should skip the pod with MultiKueueBatchJobWithManagedBy on", func() { + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.MultiKueueBatchJobWithManagedBy, true) job := testingjob.MakeJob("parent-job", ns.Name).Queue(kueue.LocalQueueName(lq.Name)).Obj() util.MustCreate(ctx, k8sClient, job) diff --git a/test/integration/singlecluster/controller/jobs/pod/suite_test.go b/test/integration/singlecluster/controller/jobs/pod/suite_test.go index fff6ceba3e3..9ace5306ede 100644 --- a/test/integration/singlecluster/controller/jobs/pod/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/pod/suite_test.go @@ -121,7 +121,7 @@ func managerSetup( cCache := schdcache.New(mgr.GetClient()) queues := qcache.NewManager(mgr.GetClient(), cCache, queueOptions...) - opts = append(opts, jobframework.WithQueues(queues)) + opts = append(opts, jobframework.WithQueues(queues), jobframework.WithCache(cCache)) mgr.GetScheme().Default(configuration) From b231d05555335ad7a0575bbdaa6676f94c79efbe Mon Sep 17 00:00:00 2001 From: Ravi Gudimetla Date: Wed, 22 Oct 2025 23:54:39 -0700 Subject: [PATCH 011/119] Fix MultiKueue workload re-evaluation bug (#6732) * Fix workload re-evaluation when clusters added to MultiKueueConfig Add handler to watch MultiKueueConfig changes and re-queue affected workloads for re-evaluation. Previously, existing workloads would not see newly added clusters until they were manually re-queued. Changes: - Add mkConfig event handler to watch MultiKueueConfig updates - Add workload indexer by admission check for efficient queries - Update integration test to validate re-evaluation behavior - Add unit tests for handler logic and indexer functionality The handler uses indexed queries to find workloads affected by config changes. Fixes workload re-evaluation bug where existing pending workloads wouldn't dispatch to newly available worker clusters. * Changes requested --- .../admissionchecks/multikueue/indexer.go | 21 +- .../multikueue/indexer_test.go | 74 ++++++ .../admissionchecks/multikueue/workload.go | 60 +++++ .../multikueue/workload_test.go | 165 ++++++++++++- .../integration/multikueue/dispatcher_test.go | 221 ++++++++++++++++-- test/util/util.go | 10 + 6 files changed, 521 insertions(+), 30 deletions(-) diff --git a/pkg/controller/admissionchecks/multikueue/indexer.go b/pkg/controller/admissionchecks/multikueue/indexer.go index 5a5582715e9..7be067f7ed1 100644 --- a/pkg/controller/admissionchecks/multikueue/indexer.go +++ b/pkg/controller/admissionchecks/multikueue/indexer.go @@ -25,12 +25,14 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/util/admissioncheck" + "sigs.k8s.io/kueue/pkg/util/slices" ) const ( - UsingKubeConfigs = "spec.kubeconfigs" - UsingMultiKueueClusters = "spec.multiKueueClusters" - AdmissionCheckUsingConfigKey = "spec.multiKueueConfig" + UsingKubeConfigs = "spec.kubeconfigs" + UsingMultiKueueClusters = "spec.multiKueueClusters" + AdmissionCheckUsingConfigKey = "spec.multiKueueConfig" + WorkloadsWithAdmissionCheckKey = "status.admissionChecks" ) var ( @@ -55,6 +57,16 @@ func indexUsingMultiKueueClusters(obj client.Object) []string { return config.Spec.Clusters } +func indexWorkloadsAdmissionChecks(obj client.Object) []string { + workload, isWorkload := obj.(*kueue.Workload) + if !isWorkload || len(workload.Status.AdmissionChecks) == 0 { + return nil + } + return slices.Map(workload.Status.AdmissionChecks, func(checkState *kueue.AdmissionCheckState) string { + return string(checkState.Name) + }) +} + func SetupIndexer(ctx context.Context, indexer client.FieldIndexer, configNamespace string) error { if err := indexer.IndexField(ctx, &kueue.MultiKueueCluster{}, UsingKubeConfigs, getIndexUsingKubeConfigs(configNamespace)); err != nil { return fmt.Errorf("setting index on clusters using kubeconfig: %w", err) @@ -65,5 +77,8 @@ func SetupIndexer(ctx context.Context, indexer client.FieldIndexer, configNamesp if err := indexer.IndexField(ctx, &kueue.AdmissionCheck{}, AdmissionCheckUsingConfigKey, admissioncheck.IndexerByConfigFunction(kueue.MultiKueueControllerName, configGVK)); err != nil { return fmt.Errorf("setting index on admission checks config: %w", err) } + if err := indexer.IndexField(ctx, &kueue.Workload{}, WorkloadsWithAdmissionCheckKey, indexWorkloadsAdmissionChecks); err != nil { + return fmt.Errorf("setting index on workloads admission checks: %w", err) + } return nil } diff --git a/pkg/controller/admissionchecks/multikueue/indexer_test.go b/pkg/controller/admissionchecks/multikueue/indexer_test.go index a90f997b8bf..3c2f70a9c1f 100644 --- a/pkg/controller/admissionchecks/multikueue/indexer_test.go +++ b/pkg/controller/admissionchecks/multikueue/indexer_test.go @@ -172,3 +172,77 @@ func TestListMultiKueueConfigsUsingMultiKueueClusters(t *testing.T) { }) } } + +func TestListWorkloadsWithAdmissionCheck(t *testing.T) { + cases := map[string]struct { + workloads []*kueue.Workload + filter client.ListOption + wantListError error + wantList []string + }{ + "no workloads": { + filter: client.MatchingFields{WorkloadsWithAdmissionCheckKey: "ac1"}, + }, + "single workload, single match": { + workloads: []*kueue.Workload{ + utiltestingapi.MakeWorkload("wl1", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{ + Name: "ac1", + State: kueue.CheckStatePending, + }).Obj(), + }, + filter: client.MatchingFields{WorkloadsWithAdmissionCheckKey: "ac1"}, + wantList: []string{"wl1"}, + }, + "single workload, no match": { + workloads: []*kueue.Workload{ + utiltestingapi.MakeWorkload("wl2", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{ + Name: "ac2", + State: kueue.CheckStatePending, + }).Obj(), + }, + filter: client.MatchingFields{WorkloadsWithAdmissionCheckKey: "ac1"}, + }, + "multiple workloads, single match": { + workloads: []*kueue.Workload{ + utiltestingapi.MakeWorkload("wl1", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{ + Name: "ac1", + State: kueue.CheckStatePending, + }).Obj(), + utiltestingapi.MakeWorkload("wl2", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{ + Name: "ac2", + State: kueue.CheckStatePending, + }).Obj(), + }, + filter: client.MatchingFields{WorkloadsWithAdmissionCheckKey: "ac1"}, + wantList: []string{"wl1"}, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ctx, _ := utiltesting.ContextWithLog(t) + builder := getClientBuilder(ctx) + k8sClient := builder.Build() + for _, wl := range tc.workloads { + if err := k8sClient.Create(ctx, wl); err != nil { + t.Fatalf("Unable to create %q workload: %v", client.ObjectKeyFromObject(wl), err) + } + } + + lst := &kueue.WorkloadList{} + + gotListErr := k8sClient.List(ctx, lst, tc.filter) + if diff := cmp.Diff(tc.wantListError, gotListErr); diff != "" { + t.Errorf("unexpected error (-want/+got):\n%s", diff) + } + + gotList := slices.Map(lst.Items, func(wl *kueue.Workload) string { return wl.Name }) + if diff := cmp.Diff(tc.wantList, gotList, cmpopts.EquateEmpty(), cmpopts.SortSlices(func(a, b string) bool { return a < b })); diff != "" { + t.Errorf("unexpected (-want/+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/controller/admissionchecks/multikueue/workload.go b/pkg/controller/admissionchecks/multikueue/workload.go index db7a23e0f99..6ad58de7158 100644 --- a/pkg/controller/admissionchecks/multikueue/workload.go +++ b/pkg/controller/admissionchecks/multikueue/workload.go @@ -559,6 +559,65 @@ func newWlReconciler(c client.Client, helper *admissioncheck.MultiKueueStoreHelp return r } +type configHandler struct { + client client.Client + eventsBatchPeriod time.Duration +} + +func (c *configHandler) Create(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + // no-op as we don't need to react to new configs +} + +func (c *configHandler) Update(ctx context.Context, e event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + oldConfig, isOldConfig := e.ObjectOld.(*kueue.MultiKueueConfig) + newConfig, isNewConfig := e.ObjectNew.(*kueue.MultiKueueConfig) + if !isOldConfig || !isNewConfig { + return + } + if equality.Semantic.DeepEqual(oldConfig.Spec.Clusters, newConfig.Spec.Clusters) { + return + } + if err := c.queueWorkloadsForConfig(ctx, oldConfig.Name, q); err != nil { + ctrl.LoggerFrom(ctx).V(2).Error(err, "Failed to queue workloads on config update", "multiKueueConfig", klog.KObj(oldConfig)) + } +} + +func (c *configHandler) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + config, isConfig := e.Object.(*kueue.MultiKueueConfig) + if !isConfig { + return + } + if err := c.queueWorkloadsForConfig(ctx, config.Name, q); err != nil { + ctrl.LoggerFrom(ctx).V(2).Error(err, "Failed to queue workloads on config deletion", "multiKueueConfig", klog.KObj(config)) + } +} + +func (c *configHandler) Generic(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + // no-op as we don't need to react to generic +} + +func (c *configHandler) queueWorkloadsForConfig(ctx context.Context, configName string, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + admissionChecks := &kueue.AdmissionCheckList{} + var errs []error + + if err := c.client.List(ctx, admissionChecks, client.MatchingFields{AdmissionCheckUsingConfigKey: configName}); err != nil { + errs = append(errs, err) + return errors.Join(errs...) + } + + for _, admissionCheck := range admissionChecks.Items { + workloads := &kueue.WorkloadList{} + if err := c.client.List(ctx, workloads, client.MatchingFields{WorkloadsWithAdmissionCheckKey: admissionCheck.Name}); err != nil { + errs = append(errs, err) + continue + } + for _, workload := range workloads.Items { + q.AddAfter(reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&workload)}, c.eventsBatchPeriod) + } + } + return errors.Join(errs...) +} + func (w *wlReconciler) setupWithManager(mgr ctrl.Manager) error { syncHndl := handler.Funcs{ GenericFunc: func(_ context.Context, e event.GenericEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { @@ -573,6 +632,7 @@ func (w *wlReconciler) setupWithManager(mgr ctrl.Manager) error { Named("multikueue_workload"). For(&kueue.Workload{}). WatchesRawSource(source.Channel(w.clusters.wlUpdateCh, syncHndl)). + Watches(&kueue.MultiKueueConfig{}, &configHandler{client: w.client, eventsBatchPeriod: w.eventsBatchPeriod}). WithEventFilter(w). Complete(w) } diff --git a/pkg/controller/admissionchecks/multikueue/workload_test.go b/pkg/controller/admissionchecks/multikueue/workload_test.go index 0613a8f820a..6453997bfc1 100644 --- a/pkg/controller/admissionchecks/multikueue/workload_test.go +++ b/pkg/controller/admissionchecks/multikueue/workload_test.go @@ -55,9 +55,7 @@ import ( _ "sigs.k8s.io/kueue/pkg/controller/jobs" ) -var ( - errFake = errors.New("fake error") -) +var errFake = errors.New("fake error") func TestWlReconcile(t *testing.T) { now := time.Now() @@ -1747,3 +1745,164 @@ func TestNominateAndSynchronizeWorkers_MoreCases(t *testing.T) { }) } } + +// mockQueue implements workqueue.TypedRateLimitingInterface for testing +type mockQueue struct { + addedItems []reconcile.Request +} + +func (m *mockQueue) Add(item reconcile.Request) { + m.addedItems = append(m.addedItems, item) +} + +func (m *mockQueue) Len() int { return 0 } +func (m *mockQueue) Get() (reconcile.Request, bool) { return reconcile.Request{}, false } +func (m *mockQueue) Done(reconcile.Request) {} +func (m *mockQueue) Forget(reconcile.Request) {} +func (m *mockQueue) NumRequeues(reconcile.Request) int { return 0 } +func (m *mockQueue) AddRateLimited(reconcile.Request) {} +func (m *mockQueue) AddAfter(item reconcile.Request, duration time.Duration) { + m.addedItems = append(m.addedItems, item) +} +func (m *mockQueue) ShutDown() {} +func (m *mockQueue) ShutDownWithDrain() {} +func (m *mockQueue) ShuttingDown() bool { return false } + +func TestConfigHandlerUpdate(t *testing.T) { + cases := map[string]struct { + admissionChecks []kueue.AdmissionCheck + workloads []kueue.Workload + oldConfig *kueue.MultiKueueConfig + newConfig *kueue.MultiKueueConfig + expectedQueuedWLs []string + }{ + "clusters unchanged - no workloads queued": { + admissionChecks: []kueue.AdmissionCheck{ + *utiltestingapi.MakeAdmissionCheck("ac1"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "config1"). + Obj(), + }, + workloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("wl1", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{Name: "ac1", State: kueue.CheckStatePending}). + Obj(), + }, + oldConfig: utiltestingapi.MakeMultiKueueConfig("config1").Clusters("cluster1", "cluster2").Obj(), + newConfig: utiltestingapi.MakeMultiKueueConfig("config1").Clusters("cluster1", "cluster2").Obj(), + expectedQueuedWLs: nil, // No workloads should be queued + }, + "clusters changed - workloads queued": { + admissionChecks: []kueue.AdmissionCheck{ + *utiltestingapi.MakeAdmissionCheck("ac1"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "config1"). + Obj(), + }, + workloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("wl1", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{Name: "ac1", State: kueue.CheckStatePending}). + Obj(), + *utiltestingapi.MakeWorkload("wl2", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{Name: "ac1", State: kueue.CheckStateReady}). + Obj(), + }, + oldConfig: utiltestingapi.MakeMultiKueueConfig("config1").Clusters("cluster1").Obj(), + newConfig: utiltestingapi.MakeMultiKueueConfig("config1").Clusters("cluster1", "cluster2").Obj(), + expectedQueuedWLs: []string{"wl1", "wl2"}, // Both workloads should be queued + }, + "multiple configs - only affected workloads queued": { + admissionChecks: []kueue.AdmissionCheck{ + *utiltestingapi.MakeAdmissionCheck("ac1"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "config1"). + Obj(), + *utiltestingapi.MakeAdmissionCheck("ac2"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "other-config"). + Obj(), + }, + workloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("wl1", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{Name: "ac1", State: kueue.CheckStatePending}). + Obj(), + *utiltestingapi.MakeWorkload("wl2", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{Name: "ac2", State: kueue.CheckStatePending}). + Obj(), + }, + oldConfig: utiltestingapi.MakeMultiKueueConfig("config1").Clusters("cluster1").Obj(), + newConfig: utiltestingapi.MakeMultiKueueConfig("config1").Clusters("cluster1", "cluster2").Obj(), + expectedQueuedWLs: []string{"wl1"}, // Only wl1 uses config1, wl2 uses other-config + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ctx, _ := utiltesting.ContextWithLog(t) + clientBuilder := getClientBuilder(ctx) + + for i := range tc.admissionChecks { + clientBuilder = clientBuilder.WithObjects(&tc.admissionChecks[i]) + } + for i := range tc.workloads { + clientBuilder = clientBuilder.WithObjects(&tc.workloads[i]) + } + + fakeClient := clientBuilder.Build() + handler := &configHandler{client: fakeClient, eventsBatchPeriod: time.Second} + mockQ := &mockQueue{} + + updateEvent := event.UpdateEvent{ + ObjectOld: tc.oldConfig, + ObjectNew: tc.newConfig, + } + + handler.Update(ctx, updateEvent, mockQ) + + var actualQueuedWLs []string + for _, req := range mockQ.addedItems { + actualQueuedWLs = append(actualQueuedWLs, req.Name) + } + sort.Strings(actualQueuedWLs) + sort.Strings(tc.expectedQueuedWLs) + + if diff := cmp.Diff(tc.expectedQueuedWLs, actualQueuedWLs); diff != "" { + t.Errorf("unexpected queued workloads (-want/+got):\n%s", diff) + } + }) + } +} + +func TestConfigHandlerDelete(t *testing.T) { + ctx, _ := utiltesting.ContextWithLog(t) + + admissionCheck := utiltestingapi.MakeAdmissionCheck("ac1"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "config1"). + Obj() + + workload := utiltestingapi.MakeWorkload("wl1", TestNamespace). + AdmissionCheck(kueue.AdmissionCheckState{Name: "ac1", State: kueue.CheckStatePending}). + Obj() + + clientBuilder := getClientBuilder(ctx) + clientBuilder = clientBuilder.WithObjects(admissionCheck, workload) + fakeClient := clientBuilder.Build() + + handler := &configHandler{client: fakeClient, eventsBatchPeriod: time.Second} + mockQ := &mockQueue{} + + config := utiltestingapi.MakeMultiKueueConfig("config1").Clusters("cluster1").Obj() + deleteEvent := event.DeleteEvent{ + Object: config, + } + + handler.Delete(ctx, deleteEvent, mockQ) + + if len(mockQ.addedItems) != 1 { + t.Errorf("expected 1 workload to be queued, got %d", len(mockQ.addedItems)) + } + if mockQ.addedItems[0].Name != "wl1" { + t.Errorf("expected workload wl1 to be queued, got %s", mockQ.addedItems[0].Name) + } +} diff --git a/test/integration/multikueue/dispatcher_test.go b/test/integration/multikueue/dispatcher_test.go index 4691376ab56..2d47498643f 100644 --- a/test/integration/multikueue/dispatcher_test.go +++ b/test/integration/multikueue/dispatcher_test.go @@ -26,6 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -123,14 +124,7 @@ var _ = ginkgo.Describe("MultiKueueDispatcherIncremental", ginkgo.Ordered, ginkg Obj() gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, multiKueueAC)).Should(gomega.Succeed()) - ginkgo.By("wait for check active", func() { - updatedAc := kueue.AdmissionCheck{} - acKey := client.ObjectKeyFromObject(multiKueueAC) - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatedAc)).To(gomega.Succeed()) - g.Expect(updatedAc.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.AdmissionCheckActive)) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) - }) + util.ExpectAdmissionChecksToBeActive(managerTestCluster.ctx, managerTestCluster.client, multiKueueAC) managerCq = utiltestingapi.MakeClusterQueue("q1"). AdmissionChecks(kueue.AdmissionCheckReference(multiKueueAC.Name)). @@ -319,14 +313,7 @@ var _ = ginkgo.Describe("MultiKueueDispatcherExternal", ginkgo.Ordered, ginkgo.C Obj() gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, multiKueueAC)).Should(gomega.Succeed()) - ginkgo.By("wait for check active", func() { - updatedAc := kueue.AdmissionCheck{} - acKey := client.ObjectKeyFromObject(multiKueueAC) - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatedAc)).To(gomega.Succeed()) - g.Expect(updatedAc.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.AdmissionCheckActive)) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) - }) + util.ExpectAdmissionChecksToBeActive(managerTestCluster.ctx, managerTestCluster.client, multiKueueAC) managerCq = utiltestingapi.MakeClusterQueue("q1"). AdmissionChecks(kueue.AdmissionCheckReference(multiKueueAC.Name)). @@ -566,14 +553,7 @@ var _ = ginkgo.Describe("MultiKueueDispatcherAllAtOnce", ginkgo.Ordered, ginkgo. Obj() gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, multiKueueAC)).Should(gomega.Succeed()) - ginkgo.By("wait for check active", func() { - updatedAc := kueue.AdmissionCheck{} - acKey := client.ObjectKeyFromObject(multiKueueAC) - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatedAc)).To(gomega.Succeed()) - g.Expect(updatedAc.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.AdmissionCheckActive)) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) - }) + util.ExpectAdmissionChecksToBeActive(managerTestCluster.ctx, managerTestCluster.client, multiKueueAC) managerCq = utiltestingapi.MakeClusterQueue("q1"). AdmissionChecks(kueue.AdmissionCheckReference(multiKueueAC.Name)). @@ -722,3 +702,196 @@ var _ = ginkgo.Describe("MultiKueueDispatcherAllAtOnce", ginkgo.Ordered, ginkgo. }) }) }) + +var _ = ginkgo.Describe("MultiKueueConfig Re-evaluation", ginkgo.Ordered, func() { + var ( + managerNs *corev1.Namespace + worker1Ns *corev1.Namespace + worker2Ns *corev1.Namespace + + managerMultiKueueSecret1 *corev1.Secret + managerMultiKueueSecret2 *corev1.Secret + workerCluster1 *kueue.MultiKueueCluster + workerCluster2 *kueue.MultiKueueCluster + managerMultiKueueConfig *kueue.MultiKueueConfig + multiKueueAC *kueue.AdmissionCheck + managerCq *kueue.ClusterQueue + managerLq *kueue.LocalQueue + + worker1Cq *kueue.ClusterQueue + worker2Cq *kueue.ClusterQueue + ) + + ginkgo.BeforeAll(func() { + managerTestCluster.fwk.StartManager(managerTestCluster.ctx, managerTestCluster.cfg, func(ctx context.Context, mgr manager.Manager) { + managerAndMultiKueueSetup(ctx, mgr, 2*time.Second, defaultEnabledIntegrations, config.MultiKueueDispatcherModeAllAtOnce) + }) + }) + + ginkgo.AfterAll(func() { + managerTestCluster.fwk.StopManager(managerTestCluster.ctx) + }) + + ginkgo.BeforeEach(func() { + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.MultiKueueBatchJobWithManagedBy, true) + + managerNs = util.CreateNamespaceFromPrefixWithLog(managerTestCluster.ctx, managerTestCluster.client, "multikueue-re-eval-") + worker1Ns = util.CreateNamespaceWithLog(worker1TestCluster.ctx, worker1TestCluster.client, managerNs.Name) + worker2Ns = util.CreateNamespaceWithLog(worker2TestCluster.ctx, worker2TestCluster.client, managerNs.Name) + + w1Kubeconfig, err := worker1TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + w2Kubeconfig, err := worker2TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + managerMultiKueueSecret1 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multikueue1-re-eval", + Namespace: managersConfigNamespace.Name, + }, + Data: map[string][]byte{ + kueue.MultiKueueConfigSecretKey: w1Kubeconfig, + }, + } + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultiKueueSecret1)).To(gomega.Succeed()) + + managerMultiKueueSecret2 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multikueue2-re-eval", + Namespace: managersConfigNamespace.Name, + }, + Data: map[string][]byte{ + kueue.MultiKueueConfigSecretKey: w2Kubeconfig, + }, + } + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultiKueueSecret2)).To(gomega.Succeed()) + + workerCluster1 = utiltestingapi.MakeMultiKueueCluster("worker1-re-eval").KubeConfig(kueue.SecretLocationType, managerMultiKueueSecret1.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, workerCluster1)).To(gomega.Succeed()) + + workerCluster2 = utiltestingapi.MakeMultiKueueCluster("worker2-re-eval").KubeConfig(kueue.SecretLocationType, managerMultiKueueSecret2.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, workerCluster2)).To(gomega.Succeed()) + + managerMultiKueueConfig = utiltestingapi.MakeMultiKueueConfig("isolated-config").Clusters(workerCluster1.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultiKueueConfig)).To(gomega.Succeed()) + + multiKueueAC = utiltestingapi.MakeAdmissionCheck("isolated-ac"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", managerMultiKueueConfig.Name). + Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, multiKueueAC)).To(gomega.Succeed()) + + util.ExpectAdmissionChecksToBeActive(managerTestCluster.ctx, managerTestCluster.client, multiKueueAC) + + managerCq = utiltestingapi.MakeClusterQueue("isolated-cq"). + AdmissionChecks(kueue.AdmissionCheckReference(multiKueueAC.Name)). + Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerCq)).To(gomega.Succeed()) + util.ExpectClusterQueuesToBeActive(managerTestCluster.ctx, managerTestCluster.client, managerCq) + + managerLq = utiltestingapi.MakeLocalQueue("isolated-lq", managerNs.Name).ClusterQueue(managerCq.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerLq)).To(gomega.Succeed()) + util.ExpectLocalQueuesToBeActive(managerTestCluster.ctx, managerTestCluster.client, managerLq) + + worker1Cq = utiltestingapi.MakeClusterQueue("q1-re-eval").Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Cq)).Should(gomega.Succeed()) + + worker2Cq = utiltestingapi.MakeClusterQueue("q1-re-eval").Obj() + gomega.Expect(worker2TestCluster.client.Create(worker2TestCluster.ctx, worker2Cq)).Should(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(managerTestCluster.ctx, managerTestCluster.client, managerNs)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(worker1TestCluster.ctx, worker1TestCluster.client, worker1Ns)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(worker2TestCluster.ctx, worker2TestCluster.client, worker2Ns)).To(gomega.Succeed()) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerCq, true) + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1Cq, true) + util.ExpectObjectToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, worker2Cq, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, multiKueueAC, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerMultiKueueConfig, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, workerCluster1, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, workerCluster2, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerMultiKueueSecret1, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerMultiKueueSecret2, true) + }) + + ginkgo.Context("when MultiKueueConfig changes", func() { + var ( + workloadLookupKey types.NamespacedName + admission *kueue.Admission + ) + + ginkgo.BeforeEach(func() { + ginkgo.By("Create job and workload") + job := testingjob.MakeJob("cpu-job-reeval", managerNs.Name). + ManagedBy(kueue.MultiKueueControllerName). + Queue(kueue.LocalQueueName(managerLq.Name)). + Request(corev1.ResourceCPU, "100m"). + Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, job)).To(gomega.Succeed()) + workloadLookupKey = types.NamespacedName{ + Name: workloadjob.GetWorkloadNameForJob(job.Name, job.UID), + Namespace: managerNs.Name, + } + + // Set quota reservation to trigger MultiKueue processing + admission = utiltestingapi.MakeAdmission(managerCq.Name).Obj() + util.SetQuotaReservation(managerTestCluster.ctx, managerTestCluster.client, workloadLookupKey, admission) + }) + + ginkgo.It("should re-evaluate existing workload when worker2 is added to config", func() { + ginkgo.By("Verify workload initially only sees worker1") + gomega.Eventually(func(g gomega.Gomega) { + managerWorkload := &kueue.Workload{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, workloadLookupKey, managerWorkload)).To(gomega.Succeed()) + g.Expect(managerWorkload.Status.NominatedClusterNames).To(gomega.ConsistOf(workerCluster1.Name)) + + // Workload should be created on worker1 + remoteWorkload := &kueue.Workload{} + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, workloadLookupKey, remoteWorkload)).To(gomega.Succeed()) + + // But not on worker2 (not in config yet) + g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, workloadLookupKey, remoteWorkload)).To(utiltesting.BeNotFoundError()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + + // Now add worker2 to the MultiKueueConfig + gomega.Eventually(func() error { + if err := managerTestCluster.client.Get(managerTestCluster.ctx, client.ObjectKeyFromObject(managerMultiKueueConfig), managerMultiKueueConfig); err != nil { + return err + } + managerMultiKueueConfig.Spec.Clusters = []string{workerCluster1.Name, workerCluster2.Name} + return managerTestCluster.client.Update(managerTestCluster.ctx, managerMultiKueueConfig) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + + ginkgo.By("Wait for admission check to remain active with both clusters") + util.ExpectAdmissionChecksToBeActive(managerTestCluster.ctx, managerTestCluster.client, multiKueueAC) + + ginkgo.By("Wait for worker2 cluster to become active") + gomega.Eventually(func(g gomega.Gomega) { + cluster2 := &kueue.MultiKueueCluster{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, client.ObjectKeyFromObject(workerCluster2), cluster2)).To(gomega.Succeed()) + g.Expect(cluster2.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.MultiKueueClusterActive)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + + ginkgo.By("Verify existing workload gets re-evaluated and sees both workers") + gomega.Eventually(func(g gomega.Gomega) { + managerWorkload := &kueue.Workload{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, workloadLookupKey, managerWorkload)).To(gomega.Succeed()) + actualClusters := sets.New(managerWorkload.Status.NominatedClusterNames...) + g.Expect(actualClusters.Has(workerCluster2.Name)).To(gomega.BeTrue(), + "workload should see worker2 after it's added to MultiKueueConfig") + g.Expect(actualClusters.Has(workerCluster1.Name)).To(gomega.BeTrue(), + "workload should still see worker1") + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + + ginkgo.By("Verify workloads get created on both worker clusters") + gomega.Eventually(func(g gomega.Gomega) { + remoteWorkload1 := &kueue.Workload{} + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, workloadLookupKey, remoteWorkload1)).To(gomega.Succeed()) + remoteWorkload2 := &kueue.Workload{} + g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, workloadLookupKey, remoteWorkload2)).To(gomega.Succeed()) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + }) +}) diff --git a/test/util/util.go b/test/util/util.go index 83700669552..25e7495c825 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -1042,6 +1042,16 @@ func ExpectLocalQueuesToBeActive(ctx context.Context, c client.Client, lqs ...*k }, Timeout, Interval).Should(gomega.Succeed()) } +func ExpectAdmissionChecksToBeActive(ctx context.Context, c client.Client, acs ...*kueue.AdmissionCheck) { + gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { + readAc := &kueue.AdmissionCheck{} + for _, ac := range acs { + g.Expect(c.Get(ctx, client.ObjectKeyFromObject(ac), readAc)).To(gomega.Succeed()) + g.Expect(readAc.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.AdmissionCheckActive)) + } + }, Timeout, Interval).Should(gomega.Succeed()) +} + func ExpectJobUnsuspendedWithNodeSelectors(ctx context.Context, c client.Client, key types.NamespacedName, nodeSelector map[string]string) { job := &batchv1.Job{} gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { From e05dd5e67e2d7e55190407557e5a2428aafe6914 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 23 Oct 2025 14:19:04 +0530 Subject: [PATCH 012/119] Remove unnecessary error check. (#7352) --- test/e2e/multikueue/suite_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/multikueue/suite_test.go b/test/e2e/multikueue/suite_test.go index 0b47f400b1f..202cb954f86 100644 --- a/test/e2e/multikueue/suite_test.go +++ b/test/e2e/multikueue/suite_test.go @@ -266,7 +266,6 @@ var _ = ginkgo.BeforeSuite(func() { gomega.Expect(worker2ClusterName).NotTo(gomega.BeEmpty(), "WORKER2_KIND_CLUSTER_NAME should not be empty") var err error - gomega.Expect(err).NotTo(gomega.HaveOccurred()) k8sManagerClient, managerCfg, err = util.CreateClientUsingCluster("kind-" + managerClusterName) gomega.Expect(err).NotTo(gomega.HaveOccurred()) k8sWorker1Client, worker1Cfg, err = util.CreateClientUsingCluster("kind-" + worker1ClusterName) From 122318e3958061d371f20786dff16a04b3797abf Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 23 Oct 2025 14:53:04 +0530 Subject: [PATCH 013/119] Use default cluster names. (#7353) --- test/e2e/multikueue/suite_test.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/test/e2e/multikueue/suite_test.go b/test/e2e/multikueue/suite_test.go index 202cb954f86..07c50f0b6b9 100644 --- a/test/e2e/multikueue/suite_test.go +++ b/test/e2e/multikueue/suite_test.go @@ -17,6 +17,7 @@ limitations under the License. package mke2e import ( + "cmp" "context" "fmt" "os" @@ -256,14 +257,9 @@ func TestAPIs(t *testing.T) { var _ = ginkgo.BeforeSuite(func() { util.SetupLogger() - managerClusterName = os.Getenv("MANAGER_KIND_CLUSTER_NAME") - gomega.Expect(managerClusterName).NotTo(gomega.BeEmpty(), "MANAGER_KIND_CLUSTER_NAME should not be empty") - - worker1ClusterName = os.Getenv("WORKER1_KIND_CLUSTER_NAME") - gomega.Expect(worker1ClusterName).NotTo(gomega.BeEmpty(), "WORKER1_KIND_CLUSTER_NAME should not be empty") - - worker2ClusterName = os.Getenv("WORKER2_KIND_CLUSTER_NAME") - gomega.Expect(worker2ClusterName).NotTo(gomega.BeEmpty(), "WORKER2_KIND_CLUSTER_NAME should not be empty") + managerClusterName = cmp.Or(os.Getenv("MANAGER_KIND_CLUSTER_NAME"), "kind-manager") + worker1ClusterName = cmp.Or(os.Getenv("WORKER1_KIND_CLUSTER_NAME"), "kind-worker1") + worker2ClusterName = cmp.Or(os.Getenv("WORKER2_KIND_CLUSTER_NAME"), "kind-worker2") var err error k8sManagerClient, managerCfg, err = util.CreateClientUsingCluster("kind-" + managerClusterName) From 93e6b5012971e3e7bd287422c365373d83385494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Thu, 23 Oct 2025 12:53:02 +0200 Subject: [PATCH 014/119] Enable conversion webhooks for v1beta2: LocalQueue, ClusterQueue, Workload (#7318) * Enable conversion webhooks for v1beta2 * Refactor the test to prepare for workloads * Add conversion for workload --- .gitignore | 1 + Makefile | 6 + Makefile-test.mk | 4 +- apis/kueue/v1beta1/clusterqueue_conversion.go | 47 + apis/kueue/v1beta1/doc.go | 1 + apis/kueue/v1beta1/groupversion_info.go | 14 + apis/kueue/v1beta1/localqueue_conversion.go | 57 + apis/kueue/v1beta1/workload_conversion.go | 47 + apis/kueue/v1beta1/zz_generated.conversion.go | 2772 +++++++++++++++++ apis/kueue/v1beta1/zz_generated.deepcopy.go | 2 +- apis/kueue/v1beta2/clusterqueue_types.go | 8 +- apis/kueue/v1beta2/localqueue_types.go | 6 +- apis/kueue/v1beta2/workload_types.go | 8 +- apis/kueue/v1beta2/zz_generated.deepcopy.go | 8 +- .../crd/kueue.x-k8s.io_clusterqueues.yaml | 6 +- .../crd/kueue.x-k8s.io_localqueues.yaml | 84 +- .../crd/kueue.x-k8s.io_workloads.yaml | 6 +- charts/kueue/templates/rbac/role.yaml | 9 + .../kueue/v1beta2/clusterqueuespec.go | 10 +- .../kueue/v1beta2/localqueuestatus.go | 12 +- .../kueue/v1beta2/workloadstatus.go | 30 +- .../bases/kueue.x-k8s.io_clusterqueues.yaml | 6 +- .../crd/bases/kueue.x-k8s.io_localqueues.yaml | 86 +- .../crd/bases/kueue.x-k8s.io_workloads.yaml | 13 +- config/components/crd/kustomization.yaml | 6 +- config/components/rbac/role.yaml | 9 + pkg/util/cert/cert.go | 10 + pkg/util/testing/v1beta2/wrappers.go | 4 +- pkg/webhooks/webhooks.go | 6 + test/integration/framework/framework.go | 16 +- test/integration/multikueue/suite_test.go | 8 + .../conversion/conversions_v1beta2_test.go | 335 ++ .../singlecluster/conversion/suite_test.go | 104 + 33 files changed, 3595 insertions(+), 146 deletions(-) create mode 100644 apis/kueue/v1beta1/clusterqueue_conversion.go create mode 100644 apis/kueue/v1beta1/localqueue_conversion.go create mode 100644 apis/kueue/v1beta1/workload_conversion.go create mode 100644 apis/kueue/v1beta1/zz_generated.conversion.go create mode 100644 test/integration/singlecluster/conversion/conversions_v1beta2_test.go create mode 100644 test/integration/singlecluster/conversion/suite_test.go diff --git a/.gitignore b/.gitignore index da528897b65..beadfce66ec 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.dylib bin artifacts +config/components/crd/_output testbin/* .DS_Store go.work* diff --git a/Makefile b/Makefile index ad0470cd5a1..ddf7859365f 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,12 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust rbac:roleName=manager-role output:rbac:artifacts:config=config/components/rbac\ webhook output:webhook:artifacts:config=config/components/webhook\ paths="./pkg/controller/...;./pkg/webhooks/...;./pkg/util/cert/...;./pkg/visibility/..." + $(MAKE) compile-crd-manifests + +.PHONY: compile-crd-manifests +compile-crd-manifests: kustomize + @mkdir -p config/components/crd/_output + $(KUSTOMIZE) build config/components/crd > config/components/crd/_output/crds-with-webhooks.yaml .PHONY: update-helm update-helm: manifests yq yaml-processor diff --git a/Makefile-test.mk b/Makefile-test.mk index eec13e1dc0c..b058cba231d 100644 --- a/Makefile-test.mk +++ b/Makefile-test.mk @@ -68,7 +68,7 @@ test: gotestsum ## Run tests. TEST_LOG_LEVEL=$(TEST_LOG_LEVEL) $(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.xml -- $(GOFLAGS) $(GO_TEST_FLAGS) $(shell $(GO_CMD) list $(GO_TEST_TARGET)/... | grep -v '/test/') -coverpkg=$(GO_TEST_TARGET)/... -coverprofile $(ARTIFACTS)/cover.out .PHONY: test-integration -test-integration: gomod-download envtest ginkgo dep-crds kueuectl ginkgo-top ## Run integration tests for all singlecluster suites. +test-integration: compile-crd-manifests gomod-download envtest ginkgo dep-crds kueuectl ginkgo-top ## Run integration tests for all singlecluster suites. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \ PROJECT_DIR=$(PROJECT_DIR)/ \ KUEUE_BIN=$(BIN_DIR) \ @@ -86,7 +86,7 @@ test-integration-extended: INTEGRATION_FILTERS= --label-filter="slow || redundan test-integration-extended: test-integration ## Run extended integration tests for singlecluster suites. .PHONY: test-multikueue-integration -test-multikueue-integration: gomod-download envtest ginkgo dep-crds ginkgo-top ## Run integration tests for MultiKueue suite. +test-multikueue-integration: compile-crd-manifests gomod-download envtest ginkgo dep-crds ginkgo-top ## Run integration tests for MultiKueue suite. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \ PROJECT_DIR=$(PROJECT_DIR)/ \ KUEUE_BIN=$(BIN_DIR) \ diff --git a/apis/kueue/v1beta1/clusterqueue_conversion.go b/apis/kueue/v1beta1/clusterqueue_conversion.go new file mode 100644 index 00000000000..0b30c586416 --- /dev/null +++ b/apis/kueue/v1beta1/clusterqueue_conversion.go @@ -0,0 +1,47 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + conversionapi "k8s.io/apimachinery/pkg/conversion" + "sigs.k8s.io/controller-runtime/pkg/conversion" + + "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +//lint:file-ignore ST1003 "generated Convert_* calls below use underscores" +//revive:disable:var-naming + +func (src *ClusterQueue) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1beta2.ClusterQueue) + return Convert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(src, dst, nil) +} + +func (dst *ClusterQueue) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1beta2.ClusterQueue) + return Convert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(src, dst, nil) +} + +func Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in *ClusterQueueSpec, out *v1beta2.ClusterQueueSpec, s conversionapi.Scope) error { + out.CohortName = v1beta2.CohortReference(in.Cohort) + return autoConvert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in, out, s) +} + +func Convert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec(in *v1beta2.ClusterQueueSpec, out *ClusterQueueSpec, s conversionapi.Scope) error { + out.Cohort = CohortReference(in.CohortName) + return autoConvert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec(in, out, s) +} diff --git a/apis/kueue/v1beta1/doc.go b/apis/kueue/v1beta1/doc.go index 90512688121..17bdf9cc80b 100644 --- a/apis/kueue/v1beta1/doc.go +++ b/apis/kueue/v1beta1/doc.go @@ -15,6 +15,7 @@ limitations under the License. */ // +kubebuilder:object:generate=true +// +k8s:conversion-gen=sigs.k8s.io/kueue/apis/kueue/v1beta2 // +groupName=kueue.x-k8s.io package v1beta1 diff --git a/apis/kueue/v1beta1/groupversion_info.go b/apis/kueue/v1beta1/groupversion_info.go index 1d1d3ed6dc2..9882f6b5fb5 100644 --- a/apis/kueue/v1beta1/groupversion_info.go +++ b/apis/kueue/v1beta1/groupversion_info.go @@ -20,8 +20,11 @@ limitations under the License. package v1beta1 import ( + apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/scheme" + + "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) var ( @@ -37,9 +40,20 @@ var ( // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = SchemeBuilder.AddToScheme + + // required by zz_generated.conversion.go + localSchemeBuilder = &SchemeBuilder.SchemeBuilder ) // Resource is required by pkg/client/listers/... func Resource(resource string) schema.GroupResource { return GroupVersion.WithResource(resource).GroupResource() } + +var convScheme = apiruntime.NewScheme() + +func init() { + // register generated conversions into our local scheme + _ = AddToScheme(convScheme) + _ = v1beta2.AddToScheme(convScheme) +} diff --git a/apis/kueue/v1beta1/localqueue_conversion.go b/apis/kueue/v1beta1/localqueue_conversion.go new file mode 100644 index 00000000000..c50d2809d3a --- /dev/null +++ b/apis/kueue/v1beta1/localqueue_conversion.go @@ -0,0 +1,57 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + conversionapi "k8s.io/apimachinery/pkg/conversion" + "sigs.k8s.io/controller-runtime/pkg/conversion" + + "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +//lint:file-ignore ST1003 "generated Convert_* calls below use underscores" +//revive:disable:var-naming + +func (src *LocalQueue) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1beta2.LocalQueue) + return Convert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(src, dst, nil) +} + +func (dst *LocalQueue) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1beta2.LocalQueue) + return Convert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(src, dst, nil) +} + +func Convert_v1beta2_LocalQueueStatus_To_v1beta1_LocalQueueStatus(in *v1beta2.LocalQueueStatus, out *LocalQueueStatus, s conversionapi.Scope) error { + out.FlavorUsage = make([]LocalQueueFlavorUsage, len(in.FlavorsUsage)) + for i := range in.FlavorsUsage { + if err := autoConvert_v1beta2_LocalQueueFlavorUsage_To_v1beta1_LocalQueueFlavorUsage(&in.FlavorsUsage[i], &out.FlavorUsage[i], s); err != nil { + return err + } + } + return autoConvert_v1beta2_LocalQueueStatus_To_v1beta1_LocalQueueStatus(in, out, s) +} + +func Convert_v1beta1_LocalQueueStatus_To_v1beta2_LocalQueueStatus(in *LocalQueueStatus, out *v1beta2.LocalQueueStatus, s conversionapi.Scope) error { + out.FlavorsUsage = make([]v1beta2.LocalQueueFlavorUsage, len(in.FlavorUsage)) + for i := range in.FlavorUsage { + if err := autoConvert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage(&in.FlavorUsage[i], &out.FlavorsUsage[i], s); err != nil { + return err + } + } + return autoConvert_v1beta1_LocalQueueStatus_To_v1beta2_LocalQueueStatus(in, out, s) +} diff --git a/apis/kueue/v1beta1/workload_conversion.go b/apis/kueue/v1beta1/workload_conversion.go new file mode 100644 index 00000000000..55d407ec5bb --- /dev/null +++ b/apis/kueue/v1beta1/workload_conversion.go @@ -0,0 +1,47 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + conversionapi "k8s.io/apimachinery/pkg/conversion" + "sigs.k8s.io/controller-runtime/pkg/conversion" + + "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +//lint:file-ignore ST1003 "generated Convert_* calls below use underscores" +//revive:disable:var-naming + +func (src *Workload) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1beta2.Workload) + return Convert_v1beta1_Workload_To_v1beta2_Workload(src, dst, nil) +} + +func (dst *Workload) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1beta2.Workload) + return Convert_v1beta2_Workload_To_v1beta1_Workload(src, dst, nil) +} + +func Convert_v1beta2_WorkloadStatus_To_v1beta1_WorkloadStatus(in *v1beta2.WorkloadStatus, out *WorkloadStatus, s conversionapi.Scope) error { + out.AccumulatedPastExexcutionTimeSeconds = in.AccumulatedPastExecutionTimeSeconds + return autoConvert_v1beta2_WorkloadStatus_To_v1beta1_WorkloadStatus(in, out, s) +} + +func Convert_v1beta1_WorkloadStatus_To_v1beta2_WorkloadStatus(in *WorkloadStatus, out *v1beta2.WorkloadStatus, s conversionapi.Scope) error { + out.AccumulatedPastExecutionTimeSeconds = in.AccumulatedPastExexcutionTimeSeconds + return autoConvert_v1beta1_WorkloadStatus_To_v1beta2_WorkloadStatus(in, out, s) +} diff --git a/apis/kueue/v1beta1/zz_generated.conversion.go b/apis/kueue/v1beta1/zz_generated.conversion.go new file mode 100644 index 00000000000..7fb80707c5c --- /dev/null +++ b/apis/kueue/v1beta1/zz_generated.conversion.go @@ -0,0 +1,2772 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by conversion-gen. DO NOT EDIT. + +package v1beta1 + +import ( + unsafe "unsafe" + + corev1 "k8s.io/api/core/v1" + resource "k8s.io/apimachinery/pkg/api/resource" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + conversion "k8s.io/apimachinery/pkg/conversion" + runtime "k8s.io/apimachinery/pkg/runtime" + v1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +func init() { + localSchemeBuilder.Register(RegisterConversions) +} + +// RegisterConversions adds conversion functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*Admission)(nil), (*v1beta2.Admission)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Admission_To_v1beta2_Admission(a.(*Admission), b.(*v1beta2.Admission), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.Admission)(nil), (*Admission)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_Admission_To_v1beta1_Admission(a.(*v1beta2.Admission), b.(*Admission), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionCheck)(nil), (*v1beta2.AdmissionCheck)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionCheck_To_v1beta2_AdmissionCheck(a.(*AdmissionCheck), b.(*v1beta2.AdmissionCheck), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionCheck)(nil), (*AdmissionCheck)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionCheck_To_v1beta1_AdmissionCheck(a.(*v1beta2.AdmissionCheck), b.(*AdmissionCheck), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionCheckList)(nil), (*v1beta2.AdmissionCheckList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionCheckList_To_v1beta2_AdmissionCheckList(a.(*AdmissionCheckList), b.(*v1beta2.AdmissionCheckList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionCheckList)(nil), (*AdmissionCheckList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionCheckList_To_v1beta1_AdmissionCheckList(a.(*v1beta2.AdmissionCheckList), b.(*AdmissionCheckList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionCheckParametersReference)(nil), (*v1beta2.AdmissionCheckParametersReference)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionCheckParametersReference_To_v1beta2_AdmissionCheckParametersReference(a.(*AdmissionCheckParametersReference), b.(*v1beta2.AdmissionCheckParametersReference), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionCheckParametersReference)(nil), (*AdmissionCheckParametersReference)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionCheckParametersReference_To_v1beta1_AdmissionCheckParametersReference(a.(*v1beta2.AdmissionCheckParametersReference), b.(*AdmissionCheckParametersReference), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionCheckSpec)(nil), (*v1beta2.AdmissionCheckSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(a.(*AdmissionCheckSpec), b.(*v1beta2.AdmissionCheckSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionCheckSpec)(nil), (*AdmissionCheckSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionCheckSpec_To_v1beta1_AdmissionCheckSpec(a.(*v1beta2.AdmissionCheckSpec), b.(*AdmissionCheckSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionCheckState)(nil), (*v1beta2.AdmissionCheckState)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionCheckState_To_v1beta2_AdmissionCheckState(a.(*AdmissionCheckState), b.(*v1beta2.AdmissionCheckState), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionCheckState)(nil), (*AdmissionCheckState)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionCheckState_To_v1beta1_AdmissionCheckState(a.(*v1beta2.AdmissionCheckState), b.(*AdmissionCheckState), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionCheckStatus)(nil), (*v1beta2.AdmissionCheckStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionCheckStatus_To_v1beta2_AdmissionCheckStatus(a.(*AdmissionCheckStatus), b.(*v1beta2.AdmissionCheckStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionCheckStatus)(nil), (*AdmissionCheckStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionCheckStatus_To_v1beta1_AdmissionCheckStatus(a.(*v1beta2.AdmissionCheckStatus), b.(*AdmissionCheckStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionCheckStrategyRule)(nil), (*v1beta2.AdmissionCheckStrategyRule)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionCheckStrategyRule_To_v1beta2_AdmissionCheckStrategyRule(a.(*AdmissionCheckStrategyRule), b.(*v1beta2.AdmissionCheckStrategyRule), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionCheckStrategyRule)(nil), (*AdmissionCheckStrategyRule)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionCheckStrategyRule_To_v1beta1_AdmissionCheckStrategyRule(a.(*v1beta2.AdmissionCheckStrategyRule), b.(*AdmissionCheckStrategyRule), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionChecksStrategy)(nil), (*v1beta2.AdmissionChecksStrategy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionChecksStrategy_To_v1beta2_AdmissionChecksStrategy(a.(*AdmissionChecksStrategy), b.(*v1beta2.AdmissionChecksStrategy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionChecksStrategy)(nil), (*AdmissionChecksStrategy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionChecksStrategy_To_v1beta1_AdmissionChecksStrategy(a.(*v1beta2.AdmissionChecksStrategy), b.(*AdmissionChecksStrategy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionFairSharingStatus)(nil), (*v1beta2.AdmissionFairSharingStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionFairSharingStatus_To_v1beta2_AdmissionFairSharingStatus(a.(*AdmissionFairSharingStatus), b.(*v1beta2.AdmissionFairSharingStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionFairSharingStatus)(nil), (*AdmissionFairSharingStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionFairSharingStatus_To_v1beta1_AdmissionFairSharingStatus(a.(*v1beta2.AdmissionFairSharingStatus), b.(*AdmissionFairSharingStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*AdmissionScope)(nil), (*v1beta2.AdmissionScope)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionScope_To_v1beta2_AdmissionScope(a.(*AdmissionScope), b.(*v1beta2.AdmissionScope), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionScope)(nil), (*AdmissionScope)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionScope_To_v1beta1_AdmissionScope(a.(*v1beta2.AdmissionScope), b.(*AdmissionScope), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*BorrowWithinCohort)(nil), (*v1beta2.BorrowWithinCohort)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_BorrowWithinCohort_To_v1beta2_BorrowWithinCohort(a.(*BorrowWithinCohort), b.(*v1beta2.BorrowWithinCohort), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.BorrowWithinCohort)(nil), (*BorrowWithinCohort)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_BorrowWithinCohort_To_v1beta1_BorrowWithinCohort(a.(*v1beta2.BorrowWithinCohort), b.(*BorrowWithinCohort), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ClusterQueue)(nil), (*v1beta2.ClusterQueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(a.(*ClusterQueue), b.(*v1beta2.ClusterQueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueue)(nil), (*ClusterQueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(a.(*v1beta2.ClusterQueue), b.(*ClusterQueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ClusterQueueList)(nil), (*v1beta2.ClusterQueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList(a.(*ClusterQueueList), b.(*v1beta2.ClusterQueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueueList)(nil), (*ClusterQueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(a.(*v1beta2.ClusterQueueList), b.(*ClusterQueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ClusterQueuePendingWorkload)(nil), (*v1beta2.ClusterQueuePendingWorkload)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload(a.(*ClusterQueuePendingWorkload), b.(*v1beta2.ClusterQueuePendingWorkload), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueuePendingWorkload)(nil), (*ClusterQueuePendingWorkload)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload(a.(*v1beta2.ClusterQueuePendingWorkload), b.(*ClusterQueuePendingWorkload), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ClusterQueuePendingWorkloadsStatus)(nil), (*v1beta2.ClusterQueuePendingWorkloadsStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus(a.(*ClusterQueuePendingWorkloadsStatus), b.(*v1beta2.ClusterQueuePendingWorkloadsStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueuePendingWorkloadsStatus)(nil), (*ClusterQueuePendingWorkloadsStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus(a.(*v1beta2.ClusterQueuePendingWorkloadsStatus), b.(*ClusterQueuePendingWorkloadsStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ClusterQueuePreemption)(nil), (*v1beta2.ClusterQueuePreemption)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueuePreemption_To_v1beta2_ClusterQueuePreemption(a.(*ClusterQueuePreemption), b.(*v1beta2.ClusterQueuePreemption), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueuePreemption)(nil), (*ClusterQueuePreemption)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueuePreemption_To_v1beta1_ClusterQueuePreemption(a.(*v1beta2.ClusterQueuePreemption), b.(*ClusterQueuePreemption), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ClusterQueueStatus)(nil), (*v1beta2.ClusterQueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(a.(*ClusterQueueStatus), b.(*v1beta2.ClusterQueueStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueueStatus)(nil), (*ClusterQueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueueStatus_To_v1beta1_ClusterQueueStatus(a.(*v1beta2.ClusterQueueStatus), b.(*ClusterQueueStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Cohort)(nil), (*v1beta2.Cohort)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Cohort_To_v1beta2_Cohort(a.(*Cohort), b.(*v1beta2.Cohort), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.Cohort)(nil), (*Cohort)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_Cohort_To_v1beta1_Cohort(a.(*v1beta2.Cohort), b.(*Cohort), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*CohortList)(nil), (*v1beta2.CohortList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_CohortList_To_v1beta2_CohortList(a.(*CohortList), b.(*v1beta2.CohortList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.CohortList)(nil), (*CohortList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_CohortList_To_v1beta1_CohortList(a.(*v1beta2.CohortList), b.(*CohortList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*CohortSpec)(nil), (*v1beta2.CohortSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_CohortSpec_To_v1beta2_CohortSpec(a.(*CohortSpec), b.(*v1beta2.CohortSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.CohortSpec)(nil), (*CohortSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_CohortSpec_To_v1beta1_CohortSpec(a.(*v1beta2.CohortSpec), b.(*CohortSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*CohortStatus)(nil), (*v1beta2.CohortStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_CohortStatus_To_v1beta2_CohortStatus(a.(*CohortStatus), b.(*v1beta2.CohortStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.CohortStatus)(nil), (*CohortStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_CohortStatus_To_v1beta1_CohortStatus(a.(*v1beta2.CohortStatus), b.(*CohortStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*FairSharing)(nil), (*v1beta2.FairSharing)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_FairSharing_To_v1beta2_FairSharing(a.(*FairSharing), b.(*v1beta2.FairSharing), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.FairSharing)(nil), (*FairSharing)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_FairSharing_To_v1beta1_FairSharing(a.(*v1beta2.FairSharing), b.(*FairSharing), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*FairSharingStatus)(nil), (*v1beta2.FairSharingStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_FairSharingStatus_To_v1beta2_FairSharingStatus(a.(*FairSharingStatus), b.(*v1beta2.FairSharingStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.FairSharingStatus)(nil), (*FairSharingStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_FairSharingStatus_To_v1beta1_FairSharingStatus(a.(*v1beta2.FairSharingStatus), b.(*FairSharingStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*FlavorFungibility)(nil), (*v1beta2.FlavorFungibility)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_FlavorFungibility_To_v1beta2_FlavorFungibility(a.(*FlavorFungibility), b.(*v1beta2.FlavorFungibility), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.FlavorFungibility)(nil), (*FlavorFungibility)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_FlavorFungibility_To_v1beta1_FlavorFungibility(a.(*v1beta2.FlavorFungibility), b.(*FlavorFungibility), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*FlavorQuotas)(nil), (*v1beta2.FlavorQuotas)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_FlavorQuotas_To_v1beta2_FlavorQuotas(a.(*FlavorQuotas), b.(*v1beta2.FlavorQuotas), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.FlavorQuotas)(nil), (*FlavorQuotas)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_FlavorQuotas_To_v1beta1_FlavorQuotas(a.(*v1beta2.FlavorQuotas), b.(*FlavorQuotas), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*FlavorUsage)(nil), (*v1beta2.FlavorUsage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_FlavorUsage_To_v1beta2_FlavorUsage(a.(*FlavorUsage), b.(*v1beta2.FlavorUsage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.FlavorUsage)(nil), (*FlavorUsage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_FlavorUsage_To_v1beta1_FlavorUsage(a.(*v1beta2.FlavorUsage), b.(*FlavorUsage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*KubeConfig)(nil), (*v1beta2.KubeConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_KubeConfig_To_v1beta2_KubeConfig(a.(*KubeConfig), b.(*v1beta2.KubeConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.KubeConfig)(nil), (*KubeConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_KubeConfig_To_v1beta1_KubeConfig(a.(*v1beta2.KubeConfig), b.(*KubeConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LocalQueue)(nil), (*v1beta2.LocalQueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(a.(*LocalQueue), b.(*v1beta2.LocalQueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.LocalQueue)(nil), (*LocalQueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(a.(*v1beta2.LocalQueue), b.(*LocalQueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LocalQueueFlavorStatus)(nil), (*v1beta2.LocalQueueFlavorStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus(a.(*LocalQueueFlavorStatus), b.(*v1beta2.LocalQueueFlavorStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.LocalQueueFlavorStatus)(nil), (*LocalQueueFlavorStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus(a.(*v1beta2.LocalQueueFlavorStatus), b.(*LocalQueueFlavorStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LocalQueueFlavorUsage)(nil), (*v1beta2.LocalQueueFlavorUsage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage(a.(*LocalQueueFlavorUsage), b.(*v1beta2.LocalQueueFlavorUsage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.LocalQueueFlavorUsage)(nil), (*LocalQueueFlavorUsage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_LocalQueueFlavorUsage_To_v1beta1_LocalQueueFlavorUsage(a.(*v1beta2.LocalQueueFlavorUsage), b.(*LocalQueueFlavorUsage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LocalQueueList)(nil), (*v1beta2.LocalQueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList(a.(*LocalQueueList), b.(*v1beta2.LocalQueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.LocalQueueList)(nil), (*LocalQueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList(a.(*v1beta2.LocalQueueList), b.(*LocalQueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LocalQueueResourceUsage)(nil), (*v1beta2.LocalQueueResourceUsage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LocalQueueResourceUsage_To_v1beta2_LocalQueueResourceUsage(a.(*LocalQueueResourceUsage), b.(*v1beta2.LocalQueueResourceUsage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.LocalQueueResourceUsage)(nil), (*LocalQueueResourceUsage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_LocalQueueResourceUsage_To_v1beta1_LocalQueueResourceUsage(a.(*v1beta2.LocalQueueResourceUsage), b.(*LocalQueueResourceUsage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LocalQueueSpec)(nil), (*v1beta2.LocalQueueSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LocalQueueSpec_To_v1beta2_LocalQueueSpec(a.(*LocalQueueSpec), b.(*v1beta2.LocalQueueSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.LocalQueueSpec)(nil), (*LocalQueueSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_LocalQueueSpec_To_v1beta1_LocalQueueSpec(a.(*v1beta2.LocalQueueSpec), b.(*LocalQueueSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*MultiKueueCluster)(nil), (*v1beta2.MultiKueueCluster)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MultiKueueCluster_To_v1beta2_MultiKueueCluster(a.(*MultiKueueCluster), b.(*v1beta2.MultiKueueCluster), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.MultiKueueCluster)(nil), (*MultiKueueCluster)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_MultiKueueCluster_To_v1beta1_MultiKueueCluster(a.(*v1beta2.MultiKueueCluster), b.(*MultiKueueCluster), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*MultiKueueClusterList)(nil), (*v1beta2.MultiKueueClusterList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MultiKueueClusterList_To_v1beta2_MultiKueueClusterList(a.(*MultiKueueClusterList), b.(*v1beta2.MultiKueueClusterList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.MultiKueueClusterList)(nil), (*MultiKueueClusterList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_MultiKueueClusterList_To_v1beta1_MultiKueueClusterList(a.(*v1beta2.MultiKueueClusterList), b.(*MultiKueueClusterList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*MultiKueueClusterSpec)(nil), (*v1beta2.MultiKueueClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MultiKueueClusterSpec_To_v1beta2_MultiKueueClusterSpec(a.(*MultiKueueClusterSpec), b.(*v1beta2.MultiKueueClusterSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.MultiKueueClusterSpec)(nil), (*MultiKueueClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_MultiKueueClusterSpec_To_v1beta1_MultiKueueClusterSpec(a.(*v1beta2.MultiKueueClusterSpec), b.(*MultiKueueClusterSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*MultiKueueClusterStatus)(nil), (*v1beta2.MultiKueueClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MultiKueueClusterStatus_To_v1beta2_MultiKueueClusterStatus(a.(*MultiKueueClusterStatus), b.(*v1beta2.MultiKueueClusterStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.MultiKueueClusterStatus)(nil), (*MultiKueueClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_MultiKueueClusterStatus_To_v1beta1_MultiKueueClusterStatus(a.(*v1beta2.MultiKueueClusterStatus), b.(*MultiKueueClusterStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*MultiKueueConfig)(nil), (*v1beta2.MultiKueueConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MultiKueueConfig_To_v1beta2_MultiKueueConfig(a.(*MultiKueueConfig), b.(*v1beta2.MultiKueueConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.MultiKueueConfig)(nil), (*MultiKueueConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_MultiKueueConfig_To_v1beta1_MultiKueueConfig(a.(*v1beta2.MultiKueueConfig), b.(*MultiKueueConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*MultiKueueConfigList)(nil), (*v1beta2.MultiKueueConfigList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MultiKueueConfigList_To_v1beta2_MultiKueueConfigList(a.(*MultiKueueConfigList), b.(*v1beta2.MultiKueueConfigList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.MultiKueueConfigList)(nil), (*MultiKueueConfigList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_MultiKueueConfigList_To_v1beta1_MultiKueueConfigList(a.(*v1beta2.MultiKueueConfigList), b.(*MultiKueueConfigList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*MultiKueueConfigSpec)(nil), (*v1beta2.MultiKueueConfigSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MultiKueueConfigSpec_To_v1beta2_MultiKueueConfigSpec(a.(*MultiKueueConfigSpec), b.(*v1beta2.MultiKueueConfigSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.MultiKueueConfigSpec)(nil), (*MultiKueueConfigSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_MultiKueueConfigSpec_To_v1beta1_MultiKueueConfigSpec(a.(*v1beta2.MultiKueueConfigSpec), b.(*MultiKueueConfigSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodSet)(nil), (*v1beta2.PodSet)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodSet_To_v1beta2_PodSet(a.(*PodSet), b.(*v1beta2.PodSet), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.PodSet)(nil), (*PodSet)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_PodSet_To_v1beta1_PodSet(a.(*v1beta2.PodSet), b.(*PodSet), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodSetAssignment)(nil), (*v1beta2.PodSetAssignment)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodSetAssignment_To_v1beta2_PodSetAssignment(a.(*PodSetAssignment), b.(*v1beta2.PodSetAssignment), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.PodSetAssignment)(nil), (*PodSetAssignment)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_PodSetAssignment_To_v1beta1_PodSetAssignment(a.(*v1beta2.PodSetAssignment), b.(*PodSetAssignment), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodSetRequest)(nil), (*v1beta2.PodSetRequest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodSetRequest_To_v1beta2_PodSetRequest(a.(*PodSetRequest), b.(*v1beta2.PodSetRequest), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.PodSetRequest)(nil), (*PodSetRequest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_PodSetRequest_To_v1beta1_PodSetRequest(a.(*v1beta2.PodSetRequest), b.(*PodSetRequest), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodSetTopologyRequest)(nil), (*v1beta2.PodSetTopologyRequest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodSetTopologyRequest_To_v1beta2_PodSetTopologyRequest(a.(*PodSetTopologyRequest), b.(*v1beta2.PodSetTopologyRequest), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.PodSetTopologyRequest)(nil), (*PodSetTopologyRequest)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_PodSetTopologyRequest_To_v1beta1_PodSetTopologyRequest(a.(*v1beta2.PodSetTopologyRequest), b.(*PodSetTopologyRequest), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodSetUpdate)(nil), (*v1beta2.PodSetUpdate)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodSetUpdate_To_v1beta2_PodSetUpdate(a.(*PodSetUpdate), b.(*v1beta2.PodSetUpdate), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.PodSetUpdate)(nil), (*PodSetUpdate)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_PodSetUpdate_To_v1beta1_PodSetUpdate(a.(*v1beta2.PodSetUpdate), b.(*PodSetUpdate), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ProvisioningRequestConfig)(nil), (*v1beta2.ProvisioningRequestConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ProvisioningRequestConfig_To_v1beta2_ProvisioningRequestConfig(a.(*ProvisioningRequestConfig), b.(*v1beta2.ProvisioningRequestConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ProvisioningRequestConfig)(nil), (*ProvisioningRequestConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ProvisioningRequestConfig_To_v1beta1_ProvisioningRequestConfig(a.(*v1beta2.ProvisioningRequestConfig), b.(*ProvisioningRequestConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ProvisioningRequestConfigList)(nil), (*v1beta2.ProvisioningRequestConfigList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ProvisioningRequestConfigList_To_v1beta2_ProvisioningRequestConfigList(a.(*ProvisioningRequestConfigList), b.(*v1beta2.ProvisioningRequestConfigList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ProvisioningRequestConfigList)(nil), (*ProvisioningRequestConfigList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ProvisioningRequestConfigList_To_v1beta1_ProvisioningRequestConfigList(a.(*v1beta2.ProvisioningRequestConfigList), b.(*ProvisioningRequestConfigList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ProvisioningRequestConfigSpec)(nil), (*v1beta2.ProvisioningRequestConfigSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ProvisioningRequestConfigSpec_To_v1beta2_ProvisioningRequestConfigSpec(a.(*ProvisioningRequestConfigSpec), b.(*v1beta2.ProvisioningRequestConfigSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ProvisioningRequestConfigSpec)(nil), (*ProvisioningRequestConfigSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ProvisioningRequestConfigSpec_To_v1beta1_ProvisioningRequestConfigSpec(a.(*v1beta2.ProvisioningRequestConfigSpec), b.(*ProvisioningRequestConfigSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ProvisioningRequestPodSetUpdates)(nil), (*v1beta2.ProvisioningRequestPodSetUpdates)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ProvisioningRequestPodSetUpdates_To_v1beta2_ProvisioningRequestPodSetUpdates(a.(*ProvisioningRequestPodSetUpdates), b.(*v1beta2.ProvisioningRequestPodSetUpdates), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ProvisioningRequestPodSetUpdates)(nil), (*ProvisioningRequestPodSetUpdates)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ProvisioningRequestPodSetUpdates_To_v1beta1_ProvisioningRequestPodSetUpdates(a.(*v1beta2.ProvisioningRequestPodSetUpdates), b.(*ProvisioningRequestPodSetUpdates), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ProvisioningRequestPodSetUpdatesNodeSelector)(nil), (*v1beta2.ProvisioningRequestPodSetUpdatesNodeSelector)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector(a.(*ProvisioningRequestPodSetUpdatesNodeSelector), b.(*v1beta2.ProvisioningRequestPodSetUpdatesNodeSelector), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ProvisioningRequestPodSetUpdatesNodeSelector)(nil), (*ProvisioningRequestPodSetUpdatesNodeSelector)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector(a.(*v1beta2.ProvisioningRequestPodSetUpdatesNodeSelector), b.(*ProvisioningRequestPodSetUpdatesNodeSelector), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ProvisioningRequestRetryStrategy)(nil), (*v1beta2.ProvisioningRequestRetryStrategy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ProvisioningRequestRetryStrategy_To_v1beta2_ProvisioningRequestRetryStrategy(a.(*ProvisioningRequestRetryStrategy), b.(*v1beta2.ProvisioningRequestRetryStrategy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ProvisioningRequestRetryStrategy)(nil), (*ProvisioningRequestRetryStrategy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ProvisioningRequestRetryStrategy_To_v1beta1_ProvisioningRequestRetryStrategy(a.(*v1beta2.ProvisioningRequestRetryStrategy), b.(*ProvisioningRequestRetryStrategy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ReclaimablePod)(nil), (*v1beta2.ReclaimablePod)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ReclaimablePod_To_v1beta2_ReclaimablePod(a.(*ReclaimablePod), b.(*v1beta2.ReclaimablePod), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ReclaimablePod)(nil), (*ReclaimablePod)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ReclaimablePod_To_v1beta1_ReclaimablePod(a.(*v1beta2.ReclaimablePod), b.(*ReclaimablePod), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*RequeueState)(nil), (*v1beta2.RequeueState)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_RequeueState_To_v1beta2_RequeueState(a.(*RequeueState), b.(*v1beta2.RequeueState), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.RequeueState)(nil), (*RequeueState)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_RequeueState_To_v1beta1_RequeueState(a.(*v1beta2.RequeueState), b.(*RequeueState), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ResourceFlavor)(nil), (*v1beta2.ResourceFlavor)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ResourceFlavor_To_v1beta2_ResourceFlavor(a.(*ResourceFlavor), b.(*v1beta2.ResourceFlavor), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ResourceFlavor)(nil), (*ResourceFlavor)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ResourceFlavor_To_v1beta1_ResourceFlavor(a.(*v1beta2.ResourceFlavor), b.(*ResourceFlavor), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ResourceFlavorList)(nil), (*v1beta2.ResourceFlavorList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ResourceFlavorList_To_v1beta2_ResourceFlavorList(a.(*ResourceFlavorList), b.(*v1beta2.ResourceFlavorList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ResourceFlavorList)(nil), (*ResourceFlavorList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ResourceFlavorList_To_v1beta1_ResourceFlavorList(a.(*v1beta2.ResourceFlavorList), b.(*ResourceFlavorList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ResourceFlavorSpec)(nil), (*v1beta2.ResourceFlavorSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ResourceFlavorSpec_To_v1beta2_ResourceFlavorSpec(a.(*ResourceFlavorSpec), b.(*v1beta2.ResourceFlavorSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ResourceFlavorSpec)(nil), (*ResourceFlavorSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ResourceFlavorSpec_To_v1beta1_ResourceFlavorSpec(a.(*v1beta2.ResourceFlavorSpec), b.(*ResourceFlavorSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ResourceGroup)(nil), (*v1beta2.ResourceGroup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ResourceGroup_To_v1beta2_ResourceGroup(a.(*ResourceGroup), b.(*v1beta2.ResourceGroup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ResourceGroup)(nil), (*ResourceGroup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ResourceGroup_To_v1beta1_ResourceGroup(a.(*v1beta2.ResourceGroup), b.(*ResourceGroup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ResourceQuota)(nil), (*v1beta2.ResourceQuota)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ResourceQuota_To_v1beta2_ResourceQuota(a.(*ResourceQuota), b.(*v1beta2.ResourceQuota), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ResourceQuota)(nil), (*ResourceQuota)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ResourceQuota_To_v1beta1_ResourceQuota(a.(*v1beta2.ResourceQuota), b.(*ResourceQuota), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ResourceUsage)(nil), (*v1beta2.ResourceUsage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ResourceUsage_To_v1beta2_ResourceUsage(a.(*ResourceUsage), b.(*v1beta2.ResourceUsage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ResourceUsage)(nil), (*ResourceUsage)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ResourceUsage_To_v1beta1_ResourceUsage(a.(*v1beta2.ResourceUsage), b.(*ResourceUsage), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*SchedulingStats)(nil), (*v1beta2.SchedulingStats)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_SchedulingStats_To_v1beta2_SchedulingStats(a.(*SchedulingStats), b.(*v1beta2.SchedulingStats), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.SchedulingStats)(nil), (*SchedulingStats)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_SchedulingStats_To_v1beta1_SchedulingStats(a.(*v1beta2.SchedulingStats), b.(*SchedulingStats), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Topology)(nil), (*v1beta2.Topology)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Topology_To_v1beta2_Topology(a.(*Topology), b.(*v1beta2.Topology), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.Topology)(nil), (*Topology)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_Topology_To_v1beta1_Topology(a.(*v1beta2.Topology), b.(*Topology), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*TopologyAssignment)(nil), (*v1beta2.TopologyAssignment)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_TopologyAssignment_To_v1beta2_TopologyAssignment(a.(*TopologyAssignment), b.(*v1beta2.TopologyAssignment), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.TopologyAssignment)(nil), (*TopologyAssignment)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_TopologyAssignment_To_v1beta1_TopologyAssignment(a.(*v1beta2.TopologyAssignment), b.(*TopologyAssignment), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*TopologyDomainAssignment)(nil), (*v1beta2.TopologyDomainAssignment)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_TopologyDomainAssignment_To_v1beta2_TopologyDomainAssignment(a.(*TopologyDomainAssignment), b.(*v1beta2.TopologyDomainAssignment), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.TopologyDomainAssignment)(nil), (*TopologyDomainAssignment)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_TopologyDomainAssignment_To_v1beta1_TopologyDomainAssignment(a.(*v1beta2.TopologyDomainAssignment), b.(*TopologyDomainAssignment), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*TopologyInfo)(nil), (*v1beta2.TopologyInfo)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_TopologyInfo_To_v1beta2_TopologyInfo(a.(*TopologyInfo), b.(*v1beta2.TopologyInfo), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.TopologyInfo)(nil), (*TopologyInfo)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_TopologyInfo_To_v1beta1_TopologyInfo(a.(*v1beta2.TopologyInfo), b.(*TopologyInfo), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*TopologyLevel)(nil), (*v1beta2.TopologyLevel)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_TopologyLevel_To_v1beta2_TopologyLevel(a.(*TopologyLevel), b.(*v1beta2.TopologyLevel), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.TopologyLevel)(nil), (*TopologyLevel)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_TopologyLevel_To_v1beta1_TopologyLevel(a.(*v1beta2.TopologyLevel), b.(*TopologyLevel), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*TopologyList)(nil), (*v1beta2.TopologyList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_TopologyList_To_v1beta2_TopologyList(a.(*TopologyList), b.(*v1beta2.TopologyList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.TopologyList)(nil), (*TopologyList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_TopologyList_To_v1beta1_TopologyList(a.(*v1beta2.TopologyList), b.(*TopologyList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*TopologySpec)(nil), (*v1beta2.TopologySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_TopologySpec_To_v1beta2_TopologySpec(a.(*TopologySpec), b.(*v1beta2.TopologySpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.TopologySpec)(nil), (*TopologySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_TopologySpec_To_v1beta1_TopologySpec(a.(*v1beta2.TopologySpec), b.(*TopologySpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*UnhealthyNode)(nil), (*v1beta2.UnhealthyNode)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_UnhealthyNode_To_v1beta2_UnhealthyNode(a.(*UnhealthyNode), b.(*v1beta2.UnhealthyNode), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.UnhealthyNode)(nil), (*UnhealthyNode)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_UnhealthyNode_To_v1beta1_UnhealthyNode(a.(*v1beta2.UnhealthyNode), b.(*UnhealthyNode), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Workload)(nil), (*v1beta2.Workload)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Workload_To_v1beta2_Workload(a.(*Workload), b.(*v1beta2.Workload), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.Workload)(nil), (*Workload)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_Workload_To_v1beta1_Workload(a.(*v1beta2.Workload), b.(*Workload), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*WorkloadList)(nil), (*v1beta2.WorkloadList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_WorkloadList_To_v1beta2_WorkloadList(a.(*WorkloadList), b.(*v1beta2.WorkloadList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.WorkloadList)(nil), (*WorkloadList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_WorkloadList_To_v1beta1_WorkloadList(a.(*v1beta2.WorkloadList), b.(*WorkloadList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*WorkloadPriorityClass)(nil), (*v1beta2.WorkloadPriorityClass)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_WorkloadPriorityClass_To_v1beta2_WorkloadPriorityClass(a.(*WorkloadPriorityClass), b.(*v1beta2.WorkloadPriorityClass), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.WorkloadPriorityClass)(nil), (*WorkloadPriorityClass)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_WorkloadPriorityClass_To_v1beta1_WorkloadPriorityClass(a.(*v1beta2.WorkloadPriorityClass), b.(*WorkloadPriorityClass), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*WorkloadPriorityClassList)(nil), (*v1beta2.WorkloadPriorityClassList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_WorkloadPriorityClassList_To_v1beta2_WorkloadPriorityClassList(a.(*WorkloadPriorityClassList), b.(*v1beta2.WorkloadPriorityClassList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.WorkloadPriorityClassList)(nil), (*WorkloadPriorityClassList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_WorkloadPriorityClassList_To_v1beta1_WorkloadPriorityClassList(a.(*v1beta2.WorkloadPriorityClassList), b.(*WorkloadPriorityClassList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*WorkloadSchedulingStatsEviction)(nil), (*v1beta2.WorkloadSchedulingStatsEviction)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_WorkloadSchedulingStatsEviction_To_v1beta2_WorkloadSchedulingStatsEviction(a.(*WorkloadSchedulingStatsEviction), b.(*v1beta2.WorkloadSchedulingStatsEviction), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.WorkloadSchedulingStatsEviction)(nil), (*WorkloadSchedulingStatsEviction)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_WorkloadSchedulingStatsEviction_To_v1beta1_WorkloadSchedulingStatsEviction(a.(*v1beta2.WorkloadSchedulingStatsEviction), b.(*WorkloadSchedulingStatsEviction), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*WorkloadSpec)(nil), (*v1beta2.WorkloadSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(a.(*WorkloadSpec), b.(*v1beta2.WorkloadSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.WorkloadSpec)(nil), (*WorkloadSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(a.(*v1beta2.WorkloadSpec), b.(*WorkloadSpec), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*ClusterQueueSpec)(nil), (*v1beta2.ClusterQueueSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(a.(*ClusterQueueSpec), b.(*v1beta2.ClusterQueueSpec), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*LocalQueueStatus)(nil), (*v1beta2.LocalQueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LocalQueueStatus_To_v1beta2_LocalQueueStatus(a.(*LocalQueueStatus), b.(*v1beta2.LocalQueueStatus), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*WorkloadStatus)(nil), (*v1beta2.WorkloadStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_WorkloadStatus_To_v1beta2_WorkloadStatus(a.(*WorkloadStatus), b.(*v1beta2.WorkloadStatus), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta2.ClusterQueueSpec)(nil), (*ClusterQueueSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec(a.(*v1beta2.ClusterQueueSpec), b.(*ClusterQueueSpec), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta2.LocalQueueStatus)(nil), (*LocalQueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_LocalQueueStatus_To_v1beta1_LocalQueueStatus(a.(*v1beta2.LocalQueueStatus), b.(*LocalQueueStatus), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta2.WorkloadStatus)(nil), (*WorkloadStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_WorkloadStatus_To_v1beta1_WorkloadStatus(a.(*v1beta2.WorkloadStatus), b.(*WorkloadStatus), scope) + }); err != nil { + return err + } + return nil +} + +func autoConvert_v1beta1_Admission_To_v1beta2_Admission(in *Admission, out *v1beta2.Admission, s conversion.Scope) error { + out.ClusterQueue = v1beta2.ClusterQueueReference(in.ClusterQueue) + out.PodSetAssignments = *(*[]v1beta2.PodSetAssignment)(unsafe.Pointer(&in.PodSetAssignments)) + return nil +} + +// Convert_v1beta1_Admission_To_v1beta2_Admission is an autogenerated conversion function. +func Convert_v1beta1_Admission_To_v1beta2_Admission(in *Admission, out *v1beta2.Admission, s conversion.Scope) error { + return autoConvert_v1beta1_Admission_To_v1beta2_Admission(in, out, s) +} + +func autoConvert_v1beta2_Admission_To_v1beta1_Admission(in *v1beta2.Admission, out *Admission, s conversion.Scope) error { + out.ClusterQueue = ClusterQueueReference(in.ClusterQueue) + out.PodSetAssignments = *(*[]PodSetAssignment)(unsafe.Pointer(&in.PodSetAssignments)) + return nil +} + +// Convert_v1beta2_Admission_To_v1beta1_Admission is an autogenerated conversion function. +func Convert_v1beta2_Admission_To_v1beta1_Admission(in *v1beta2.Admission, out *Admission, s conversion.Scope) error { + return autoConvert_v1beta2_Admission_To_v1beta1_Admission(in, out, s) +} + +func autoConvert_v1beta1_AdmissionCheck_To_v1beta2_AdmissionCheck(in *AdmissionCheck, out *v1beta2.AdmissionCheck, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta1_AdmissionCheckStatus_To_v1beta2_AdmissionCheckStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_AdmissionCheck_To_v1beta2_AdmissionCheck is an autogenerated conversion function. +func Convert_v1beta1_AdmissionCheck_To_v1beta2_AdmissionCheck(in *AdmissionCheck, out *v1beta2.AdmissionCheck, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionCheck_To_v1beta2_AdmissionCheck(in, out, s) +} + +func autoConvert_v1beta2_AdmissionCheck_To_v1beta1_AdmissionCheck(in *v1beta2.AdmissionCheck, out *AdmissionCheck, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_AdmissionCheckSpec_To_v1beta1_AdmissionCheckSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta2_AdmissionCheckStatus_To_v1beta1_AdmissionCheckStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_AdmissionCheck_To_v1beta1_AdmissionCheck is an autogenerated conversion function. +func Convert_v1beta2_AdmissionCheck_To_v1beta1_AdmissionCheck(in *v1beta2.AdmissionCheck, out *AdmissionCheck, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionCheck_To_v1beta1_AdmissionCheck(in, out, s) +} + +func autoConvert_v1beta1_AdmissionCheckList_To_v1beta2_AdmissionCheckList(in *AdmissionCheckList, out *v1beta2.AdmissionCheckList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.AdmissionCheck)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_AdmissionCheckList_To_v1beta2_AdmissionCheckList is an autogenerated conversion function. +func Convert_v1beta1_AdmissionCheckList_To_v1beta2_AdmissionCheckList(in *AdmissionCheckList, out *v1beta2.AdmissionCheckList, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionCheckList_To_v1beta2_AdmissionCheckList(in, out, s) +} + +func autoConvert_v1beta2_AdmissionCheckList_To_v1beta1_AdmissionCheckList(in *v1beta2.AdmissionCheckList, out *AdmissionCheckList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]AdmissionCheck)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_AdmissionCheckList_To_v1beta1_AdmissionCheckList is an autogenerated conversion function. +func Convert_v1beta2_AdmissionCheckList_To_v1beta1_AdmissionCheckList(in *v1beta2.AdmissionCheckList, out *AdmissionCheckList, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionCheckList_To_v1beta1_AdmissionCheckList(in, out, s) +} + +func autoConvert_v1beta1_AdmissionCheckParametersReference_To_v1beta2_AdmissionCheckParametersReference(in *AdmissionCheckParametersReference, out *v1beta2.AdmissionCheckParametersReference, s conversion.Scope) error { + out.APIGroup = in.APIGroup + out.Kind = in.Kind + out.Name = in.Name + return nil +} + +// Convert_v1beta1_AdmissionCheckParametersReference_To_v1beta2_AdmissionCheckParametersReference is an autogenerated conversion function. +func Convert_v1beta1_AdmissionCheckParametersReference_To_v1beta2_AdmissionCheckParametersReference(in *AdmissionCheckParametersReference, out *v1beta2.AdmissionCheckParametersReference, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionCheckParametersReference_To_v1beta2_AdmissionCheckParametersReference(in, out, s) +} + +func autoConvert_v1beta2_AdmissionCheckParametersReference_To_v1beta1_AdmissionCheckParametersReference(in *v1beta2.AdmissionCheckParametersReference, out *AdmissionCheckParametersReference, s conversion.Scope) error { + out.APIGroup = in.APIGroup + out.Kind = in.Kind + out.Name = in.Name + return nil +} + +// Convert_v1beta2_AdmissionCheckParametersReference_To_v1beta1_AdmissionCheckParametersReference is an autogenerated conversion function. +func Convert_v1beta2_AdmissionCheckParametersReference_To_v1beta1_AdmissionCheckParametersReference(in *v1beta2.AdmissionCheckParametersReference, out *AdmissionCheckParametersReference, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionCheckParametersReference_To_v1beta1_AdmissionCheckParametersReference(in, out, s) +} + +func autoConvert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(in *AdmissionCheckSpec, out *v1beta2.AdmissionCheckSpec, s conversion.Scope) error { + out.ControllerName = in.ControllerName + out.RetryDelayMinutes = (*int64)(unsafe.Pointer(in.RetryDelayMinutes)) + out.Parameters = (*v1beta2.AdmissionCheckParametersReference)(unsafe.Pointer(in.Parameters)) + return nil +} + +// Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec is an autogenerated conversion function. +func Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(in *AdmissionCheckSpec, out *v1beta2.AdmissionCheckSpec, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(in, out, s) +} + +func autoConvert_v1beta2_AdmissionCheckSpec_To_v1beta1_AdmissionCheckSpec(in *v1beta2.AdmissionCheckSpec, out *AdmissionCheckSpec, s conversion.Scope) error { + out.ControllerName = in.ControllerName + out.RetryDelayMinutes = (*int64)(unsafe.Pointer(in.RetryDelayMinutes)) + out.Parameters = (*AdmissionCheckParametersReference)(unsafe.Pointer(in.Parameters)) + return nil +} + +// Convert_v1beta2_AdmissionCheckSpec_To_v1beta1_AdmissionCheckSpec is an autogenerated conversion function. +func Convert_v1beta2_AdmissionCheckSpec_To_v1beta1_AdmissionCheckSpec(in *v1beta2.AdmissionCheckSpec, out *AdmissionCheckSpec, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionCheckSpec_To_v1beta1_AdmissionCheckSpec(in, out, s) +} + +func autoConvert_v1beta1_AdmissionCheckState_To_v1beta2_AdmissionCheckState(in *AdmissionCheckState, out *v1beta2.AdmissionCheckState, s conversion.Scope) error { + out.Name = v1beta2.AdmissionCheckReference(in.Name) + out.State = v1beta2.CheckState(in.State) + out.LastTransitionTime = in.LastTransitionTime + out.Message = in.Message + out.PodSetUpdates = *(*[]v1beta2.PodSetUpdate)(unsafe.Pointer(&in.PodSetUpdates)) + return nil +} + +// Convert_v1beta1_AdmissionCheckState_To_v1beta2_AdmissionCheckState is an autogenerated conversion function. +func Convert_v1beta1_AdmissionCheckState_To_v1beta2_AdmissionCheckState(in *AdmissionCheckState, out *v1beta2.AdmissionCheckState, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionCheckState_To_v1beta2_AdmissionCheckState(in, out, s) +} + +func autoConvert_v1beta2_AdmissionCheckState_To_v1beta1_AdmissionCheckState(in *v1beta2.AdmissionCheckState, out *AdmissionCheckState, s conversion.Scope) error { + out.Name = AdmissionCheckReference(in.Name) + out.State = CheckState(in.State) + out.LastTransitionTime = in.LastTransitionTime + out.Message = in.Message + out.PodSetUpdates = *(*[]PodSetUpdate)(unsafe.Pointer(&in.PodSetUpdates)) + return nil +} + +// Convert_v1beta2_AdmissionCheckState_To_v1beta1_AdmissionCheckState is an autogenerated conversion function. +func Convert_v1beta2_AdmissionCheckState_To_v1beta1_AdmissionCheckState(in *v1beta2.AdmissionCheckState, out *AdmissionCheckState, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionCheckState_To_v1beta1_AdmissionCheckState(in, out, s) +} + +func autoConvert_v1beta1_AdmissionCheckStatus_To_v1beta2_AdmissionCheckStatus(in *AdmissionCheckStatus, out *v1beta2.AdmissionCheckStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + return nil +} + +// Convert_v1beta1_AdmissionCheckStatus_To_v1beta2_AdmissionCheckStatus is an autogenerated conversion function. +func Convert_v1beta1_AdmissionCheckStatus_To_v1beta2_AdmissionCheckStatus(in *AdmissionCheckStatus, out *v1beta2.AdmissionCheckStatus, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionCheckStatus_To_v1beta2_AdmissionCheckStatus(in, out, s) +} + +func autoConvert_v1beta2_AdmissionCheckStatus_To_v1beta1_AdmissionCheckStatus(in *v1beta2.AdmissionCheckStatus, out *AdmissionCheckStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + return nil +} + +// Convert_v1beta2_AdmissionCheckStatus_To_v1beta1_AdmissionCheckStatus is an autogenerated conversion function. +func Convert_v1beta2_AdmissionCheckStatus_To_v1beta1_AdmissionCheckStatus(in *v1beta2.AdmissionCheckStatus, out *AdmissionCheckStatus, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionCheckStatus_To_v1beta1_AdmissionCheckStatus(in, out, s) +} + +func autoConvert_v1beta1_AdmissionCheckStrategyRule_To_v1beta2_AdmissionCheckStrategyRule(in *AdmissionCheckStrategyRule, out *v1beta2.AdmissionCheckStrategyRule, s conversion.Scope) error { + out.Name = v1beta2.AdmissionCheckReference(in.Name) + out.OnFlavors = *(*[]v1beta2.ResourceFlavorReference)(unsafe.Pointer(&in.OnFlavors)) + return nil +} + +// Convert_v1beta1_AdmissionCheckStrategyRule_To_v1beta2_AdmissionCheckStrategyRule is an autogenerated conversion function. +func Convert_v1beta1_AdmissionCheckStrategyRule_To_v1beta2_AdmissionCheckStrategyRule(in *AdmissionCheckStrategyRule, out *v1beta2.AdmissionCheckStrategyRule, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionCheckStrategyRule_To_v1beta2_AdmissionCheckStrategyRule(in, out, s) +} + +func autoConvert_v1beta2_AdmissionCheckStrategyRule_To_v1beta1_AdmissionCheckStrategyRule(in *v1beta2.AdmissionCheckStrategyRule, out *AdmissionCheckStrategyRule, s conversion.Scope) error { + out.Name = AdmissionCheckReference(in.Name) + out.OnFlavors = *(*[]ResourceFlavorReference)(unsafe.Pointer(&in.OnFlavors)) + return nil +} + +// Convert_v1beta2_AdmissionCheckStrategyRule_To_v1beta1_AdmissionCheckStrategyRule is an autogenerated conversion function. +func Convert_v1beta2_AdmissionCheckStrategyRule_To_v1beta1_AdmissionCheckStrategyRule(in *v1beta2.AdmissionCheckStrategyRule, out *AdmissionCheckStrategyRule, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionCheckStrategyRule_To_v1beta1_AdmissionCheckStrategyRule(in, out, s) +} + +func autoConvert_v1beta1_AdmissionChecksStrategy_To_v1beta2_AdmissionChecksStrategy(in *AdmissionChecksStrategy, out *v1beta2.AdmissionChecksStrategy, s conversion.Scope) error { + out.AdmissionChecks = *(*[]v1beta2.AdmissionCheckStrategyRule)(unsafe.Pointer(&in.AdmissionChecks)) + return nil +} + +// Convert_v1beta1_AdmissionChecksStrategy_To_v1beta2_AdmissionChecksStrategy is an autogenerated conversion function. +func Convert_v1beta1_AdmissionChecksStrategy_To_v1beta2_AdmissionChecksStrategy(in *AdmissionChecksStrategy, out *v1beta2.AdmissionChecksStrategy, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionChecksStrategy_To_v1beta2_AdmissionChecksStrategy(in, out, s) +} + +func autoConvert_v1beta2_AdmissionChecksStrategy_To_v1beta1_AdmissionChecksStrategy(in *v1beta2.AdmissionChecksStrategy, out *AdmissionChecksStrategy, s conversion.Scope) error { + out.AdmissionChecks = *(*[]AdmissionCheckStrategyRule)(unsafe.Pointer(&in.AdmissionChecks)) + return nil +} + +// Convert_v1beta2_AdmissionChecksStrategy_To_v1beta1_AdmissionChecksStrategy is an autogenerated conversion function. +func Convert_v1beta2_AdmissionChecksStrategy_To_v1beta1_AdmissionChecksStrategy(in *v1beta2.AdmissionChecksStrategy, out *AdmissionChecksStrategy, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionChecksStrategy_To_v1beta1_AdmissionChecksStrategy(in, out, s) +} + +func autoConvert_v1beta1_AdmissionFairSharingStatus_To_v1beta2_AdmissionFairSharingStatus(in *AdmissionFairSharingStatus, out *v1beta2.AdmissionFairSharingStatus, s conversion.Scope) error { + out.ConsumedResources = *(*corev1.ResourceList)(unsafe.Pointer(&in.ConsumedResources)) + out.LastUpdate = in.LastUpdate + return nil +} + +// Convert_v1beta1_AdmissionFairSharingStatus_To_v1beta2_AdmissionFairSharingStatus is an autogenerated conversion function. +func Convert_v1beta1_AdmissionFairSharingStatus_To_v1beta2_AdmissionFairSharingStatus(in *AdmissionFairSharingStatus, out *v1beta2.AdmissionFairSharingStatus, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionFairSharingStatus_To_v1beta2_AdmissionFairSharingStatus(in, out, s) +} + +func autoConvert_v1beta2_AdmissionFairSharingStatus_To_v1beta1_AdmissionFairSharingStatus(in *v1beta2.AdmissionFairSharingStatus, out *AdmissionFairSharingStatus, s conversion.Scope) error { + out.ConsumedResources = *(*corev1.ResourceList)(unsafe.Pointer(&in.ConsumedResources)) + out.LastUpdate = in.LastUpdate + return nil +} + +// Convert_v1beta2_AdmissionFairSharingStatus_To_v1beta1_AdmissionFairSharingStatus is an autogenerated conversion function. +func Convert_v1beta2_AdmissionFairSharingStatus_To_v1beta1_AdmissionFairSharingStatus(in *v1beta2.AdmissionFairSharingStatus, out *AdmissionFairSharingStatus, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionFairSharingStatus_To_v1beta1_AdmissionFairSharingStatus(in, out, s) +} + +func autoConvert_v1beta1_AdmissionScope_To_v1beta2_AdmissionScope(in *AdmissionScope, out *v1beta2.AdmissionScope, s conversion.Scope) error { + out.AdmissionMode = v1beta2.AdmissionMode(in.AdmissionMode) + return nil +} + +// Convert_v1beta1_AdmissionScope_To_v1beta2_AdmissionScope is an autogenerated conversion function. +func Convert_v1beta1_AdmissionScope_To_v1beta2_AdmissionScope(in *AdmissionScope, out *v1beta2.AdmissionScope, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionScope_To_v1beta2_AdmissionScope(in, out, s) +} + +func autoConvert_v1beta2_AdmissionScope_To_v1beta1_AdmissionScope(in *v1beta2.AdmissionScope, out *AdmissionScope, s conversion.Scope) error { + out.AdmissionMode = AdmissionMode(in.AdmissionMode) + return nil +} + +// Convert_v1beta2_AdmissionScope_To_v1beta1_AdmissionScope is an autogenerated conversion function. +func Convert_v1beta2_AdmissionScope_To_v1beta1_AdmissionScope(in *v1beta2.AdmissionScope, out *AdmissionScope, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionScope_To_v1beta1_AdmissionScope(in, out, s) +} + +func autoConvert_v1beta1_BorrowWithinCohort_To_v1beta2_BorrowWithinCohort(in *BorrowWithinCohort, out *v1beta2.BorrowWithinCohort, s conversion.Scope) error { + out.Policy = v1beta2.BorrowWithinCohortPolicy(in.Policy) + out.MaxPriorityThreshold = (*int32)(unsafe.Pointer(in.MaxPriorityThreshold)) + return nil +} + +// Convert_v1beta1_BorrowWithinCohort_To_v1beta2_BorrowWithinCohort is an autogenerated conversion function. +func Convert_v1beta1_BorrowWithinCohort_To_v1beta2_BorrowWithinCohort(in *BorrowWithinCohort, out *v1beta2.BorrowWithinCohort, s conversion.Scope) error { + return autoConvert_v1beta1_BorrowWithinCohort_To_v1beta2_BorrowWithinCohort(in, out, s) +} + +func autoConvert_v1beta2_BorrowWithinCohort_To_v1beta1_BorrowWithinCohort(in *v1beta2.BorrowWithinCohort, out *BorrowWithinCohort, s conversion.Scope) error { + out.Policy = BorrowWithinCohortPolicy(in.Policy) + out.MaxPriorityThreshold = (*int32)(unsafe.Pointer(in.MaxPriorityThreshold)) + return nil +} + +// Convert_v1beta2_BorrowWithinCohort_To_v1beta1_BorrowWithinCohort is an autogenerated conversion function. +func Convert_v1beta2_BorrowWithinCohort_To_v1beta1_BorrowWithinCohort(in *v1beta2.BorrowWithinCohort, out *BorrowWithinCohort, s conversion.Scope) error { + return autoConvert_v1beta2_BorrowWithinCohort_To_v1beta1_BorrowWithinCohort(in, out, s) +} + +func autoConvert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(in *ClusterQueue, out *v1beta2.ClusterQueue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue is an autogenerated conversion function. +func Convert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(in *ClusterQueue, out *v1beta2.ClusterQueue, s conversion.Scope) error { + return autoConvert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(in, out, s) +} + +func autoConvert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(in *v1beta2.ClusterQueue, out *ClusterQueue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta2_ClusterQueueStatus_To_v1beta1_ClusterQueueStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue is an autogenerated conversion function. +func Convert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(in *v1beta2.ClusterQueue, out *ClusterQueue, s conversion.Scope) error { + return autoConvert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(in, out, s) +} + +func autoConvert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList(in *ClusterQueueList, out *v1beta2.ClusterQueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1beta2.ClusterQueue, len(*in)) + for i := range *in { + if err := Convert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } + return nil +} + +// Convert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList is an autogenerated conversion function. +func Convert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList(in *ClusterQueueList, out *v1beta2.ClusterQueueList, s conversion.Scope) error { + return autoConvert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList(in, out, s) +} + +func autoConvert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(in *v1beta2.ClusterQueueList, out *ClusterQueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterQueue, len(*in)) + for i := range *in { + if err := Convert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } + return nil +} + +// Convert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList is an autogenerated conversion function. +func Convert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(in *v1beta2.ClusterQueueList, out *ClusterQueueList, s conversion.Scope) error { + return autoConvert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(in, out, s) +} + +func autoConvert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload(in *ClusterQueuePendingWorkload, out *v1beta2.ClusterQueuePendingWorkload, s conversion.Scope) error { + out.Name = in.Name + out.Namespace = in.Namespace + return nil +} + +// Convert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload is an autogenerated conversion function. +func Convert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload(in *ClusterQueuePendingWorkload, out *v1beta2.ClusterQueuePendingWorkload, s conversion.Scope) error { + return autoConvert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload(in, out, s) +} + +func autoConvert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload(in *v1beta2.ClusterQueuePendingWorkload, out *ClusterQueuePendingWorkload, s conversion.Scope) error { + out.Name = in.Name + out.Namespace = in.Namespace + return nil +} + +// Convert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload is an autogenerated conversion function. +func Convert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload(in *v1beta2.ClusterQueuePendingWorkload, out *ClusterQueuePendingWorkload, s conversion.Scope) error { + return autoConvert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload(in, out, s) +} + +func autoConvert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus(in *ClusterQueuePendingWorkloadsStatus, out *v1beta2.ClusterQueuePendingWorkloadsStatus, s conversion.Scope) error { + out.Head = *(*[]v1beta2.ClusterQueuePendingWorkload)(unsafe.Pointer(&in.Head)) + out.LastChangeTime = in.LastChangeTime + return nil +} + +// Convert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus is an autogenerated conversion function. +func Convert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus(in *ClusterQueuePendingWorkloadsStatus, out *v1beta2.ClusterQueuePendingWorkloadsStatus, s conversion.Scope) error { + return autoConvert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus(in, out, s) +} + +func autoConvert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus(in *v1beta2.ClusterQueuePendingWorkloadsStatus, out *ClusterQueuePendingWorkloadsStatus, s conversion.Scope) error { + out.Head = *(*[]ClusterQueuePendingWorkload)(unsafe.Pointer(&in.Head)) + out.LastChangeTime = in.LastChangeTime + return nil +} + +// Convert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus is an autogenerated conversion function. +func Convert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus(in *v1beta2.ClusterQueuePendingWorkloadsStatus, out *ClusterQueuePendingWorkloadsStatus, s conversion.Scope) error { + return autoConvert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus(in, out, s) +} + +func autoConvert_v1beta1_ClusterQueuePreemption_To_v1beta2_ClusterQueuePreemption(in *ClusterQueuePreemption, out *v1beta2.ClusterQueuePreemption, s conversion.Scope) error { + out.ReclaimWithinCohort = v1beta2.PreemptionPolicy(in.ReclaimWithinCohort) + out.BorrowWithinCohort = (*v1beta2.BorrowWithinCohort)(unsafe.Pointer(in.BorrowWithinCohort)) + out.WithinClusterQueue = v1beta2.PreemptionPolicy(in.WithinClusterQueue) + return nil +} + +// Convert_v1beta1_ClusterQueuePreemption_To_v1beta2_ClusterQueuePreemption is an autogenerated conversion function. +func Convert_v1beta1_ClusterQueuePreemption_To_v1beta2_ClusterQueuePreemption(in *ClusterQueuePreemption, out *v1beta2.ClusterQueuePreemption, s conversion.Scope) error { + return autoConvert_v1beta1_ClusterQueuePreemption_To_v1beta2_ClusterQueuePreemption(in, out, s) +} + +func autoConvert_v1beta2_ClusterQueuePreemption_To_v1beta1_ClusterQueuePreemption(in *v1beta2.ClusterQueuePreemption, out *ClusterQueuePreemption, s conversion.Scope) error { + out.ReclaimWithinCohort = PreemptionPolicy(in.ReclaimWithinCohort) + out.BorrowWithinCohort = (*BorrowWithinCohort)(unsafe.Pointer(in.BorrowWithinCohort)) + out.WithinClusterQueue = PreemptionPolicy(in.WithinClusterQueue) + return nil +} + +// Convert_v1beta2_ClusterQueuePreemption_To_v1beta1_ClusterQueuePreemption is an autogenerated conversion function. +func Convert_v1beta2_ClusterQueuePreemption_To_v1beta1_ClusterQueuePreemption(in *v1beta2.ClusterQueuePreemption, out *ClusterQueuePreemption, s conversion.Scope) error { + return autoConvert_v1beta2_ClusterQueuePreemption_To_v1beta1_ClusterQueuePreemption(in, out, s) +} + +func autoConvert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in *ClusterQueueSpec, out *v1beta2.ClusterQueueSpec, s conversion.Scope) error { + out.ResourceGroups = *(*[]v1beta2.ResourceGroup)(unsafe.Pointer(&in.ResourceGroups)) + // WARNING: in.Cohort requires manual conversion: does not exist in peer-type + out.QueueingStrategy = v1beta2.QueueingStrategy(in.QueueingStrategy) + out.NamespaceSelector = (*v1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) + out.FlavorFungibility = (*v1beta2.FlavorFungibility)(unsafe.Pointer(in.FlavorFungibility)) + out.Preemption = (*v1beta2.ClusterQueuePreemption)(unsafe.Pointer(in.Preemption)) + out.AdmissionChecks = *(*[]v1beta2.AdmissionCheckReference)(unsafe.Pointer(&in.AdmissionChecks)) + out.AdmissionChecksStrategy = (*v1beta2.AdmissionChecksStrategy)(unsafe.Pointer(in.AdmissionChecksStrategy)) + out.StopPolicy = (*v1beta2.StopPolicy)(unsafe.Pointer(in.StopPolicy)) + out.FairSharing = (*v1beta2.FairSharing)(unsafe.Pointer(in.FairSharing)) + out.AdmissionScope = (*v1beta2.AdmissionScope)(unsafe.Pointer(in.AdmissionScope)) + return nil +} + +func autoConvert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec(in *v1beta2.ClusterQueueSpec, out *ClusterQueueSpec, s conversion.Scope) error { + out.ResourceGroups = *(*[]ResourceGroup)(unsafe.Pointer(&in.ResourceGroups)) + // WARNING: in.CohortName requires manual conversion: does not exist in peer-type + out.QueueingStrategy = QueueingStrategy(in.QueueingStrategy) + out.NamespaceSelector = (*v1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) + out.FlavorFungibility = (*FlavorFungibility)(unsafe.Pointer(in.FlavorFungibility)) + out.Preemption = (*ClusterQueuePreemption)(unsafe.Pointer(in.Preemption)) + out.AdmissionChecks = *(*[]AdmissionCheckReference)(unsafe.Pointer(&in.AdmissionChecks)) + out.AdmissionChecksStrategy = (*AdmissionChecksStrategy)(unsafe.Pointer(in.AdmissionChecksStrategy)) + out.StopPolicy = (*StopPolicy)(unsafe.Pointer(in.StopPolicy)) + out.FairSharing = (*FairSharing)(unsafe.Pointer(in.FairSharing)) + out.AdmissionScope = (*AdmissionScope)(unsafe.Pointer(in.AdmissionScope)) + return nil +} + +func autoConvert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(in *ClusterQueueStatus, out *v1beta2.ClusterQueueStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + out.FlavorsReservation = *(*[]v1beta2.FlavorUsage)(unsafe.Pointer(&in.FlavorsReservation)) + out.FlavorsUsage = *(*[]v1beta2.FlavorUsage)(unsafe.Pointer(&in.FlavorsUsage)) + out.PendingWorkloads = in.PendingWorkloads + out.ReservingWorkloads = in.ReservingWorkloads + out.AdmittedWorkloads = in.AdmittedWorkloads + out.PendingWorkloadsStatus = (*v1beta2.ClusterQueuePendingWorkloadsStatus)(unsafe.Pointer(in.PendingWorkloadsStatus)) + out.FairSharing = (*v1beta2.FairSharingStatus)(unsafe.Pointer(in.FairSharing)) + return nil +} + +// Convert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus is an autogenerated conversion function. +func Convert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(in *ClusterQueueStatus, out *v1beta2.ClusterQueueStatus, s conversion.Scope) error { + return autoConvert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(in, out, s) +} + +func autoConvert_v1beta2_ClusterQueueStatus_To_v1beta1_ClusterQueueStatus(in *v1beta2.ClusterQueueStatus, out *ClusterQueueStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + out.FlavorsReservation = *(*[]FlavorUsage)(unsafe.Pointer(&in.FlavorsReservation)) + out.FlavorsUsage = *(*[]FlavorUsage)(unsafe.Pointer(&in.FlavorsUsage)) + out.PendingWorkloads = in.PendingWorkloads + out.ReservingWorkloads = in.ReservingWorkloads + out.AdmittedWorkloads = in.AdmittedWorkloads + out.PendingWorkloadsStatus = (*ClusterQueuePendingWorkloadsStatus)(unsafe.Pointer(in.PendingWorkloadsStatus)) + out.FairSharing = (*FairSharingStatus)(unsafe.Pointer(in.FairSharing)) + return nil +} + +// Convert_v1beta2_ClusterQueueStatus_To_v1beta1_ClusterQueueStatus is an autogenerated conversion function. +func Convert_v1beta2_ClusterQueueStatus_To_v1beta1_ClusterQueueStatus(in *v1beta2.ClusterQueueStatus, out *ClusterQueueStatus, s conversion.Scope) error { + return autoConvert_v1beta2_ClusterQueueStatus_To_v1beta1_ClusterQueueStatus(in, out, s) +} + +func autoConvert_v1beta1_Cohort_To_v1beta2_Cohort(in *Cohort, out *v1beta2.Cohort, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_CohortSpec_To_v1beta2_CohortSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta1_CohortStatus_To_v1beta2_CohortStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_Cohort_To_v1beta2_Cohort is an autogenerated conversion function. +func Convert_v1beta1_Cohort_To_v1beta2_Cohort(in *Cohort, out *v1beta2.Cohort, s conversion.Scope) error { + return autoConvert_v1beta1_Cohort_To_v1beta2_Cohort(in, out, s) +} + +func autoConvert_v1beta2_Cohort_To_v1beta1_Cohort(in *v1beta2.Cohort, out *Cohort, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_CohortSpec_To_v1beta1_CohortSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta2_CohortStatus_To_v1beta1_CohortStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_Cohort_To_v1beta1_Cohort is an autogenerated conversion function. +func Convert_v1beta2_Cohort_To_v1beta1_Cohort(in *v1beta2.Cohort, out *Cohort, s conversion.Scope) error { + return autoConvert_v1beta2_Cohort_To_v1beta1_Cohort(in, out, s) +} + +func autoConvert_v1beta1_CohortList_To_v1beta2_CohortList(in *CohortList, out *v1beta2.CohortList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.Cohort)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_CohortList_To_v1beta2_CohortList is an autogenerated conversion function. +func Convert_v1beta1_CohortList_To_v1beta2_CohortList(in *CohortList, out *v1beta2.CohortList, s conversion.Scope) error { + return autoConvert_v1beta1_CohortList_To_v1beta2_CohortList(in, out, s) +} + +func autoConvert_v1beta2_CohortList_To_v1beta1_CohortList(in *v1beta2.CohortList, out *CohortList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]Cohort)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_CohortList_To_v1beta1_CohortList is an autogenerated conversion function. +func Convert_v1beta2_CohortList_To_v1beta1_CohortList(in *v1beta2.CohortList, out *CohortList, s conversion.Scope) error { + return autoConvert_v1beta2_CohortList_To_v1beta1_CohortList(in, out, s) +} + +func autoConvert_v1beta1_CohortSpec_To_v1beta2_CohortSpec(in *CohortSpec, out *v1beta2.CohortSpec, s conversion.Scope) error { + out.ParentName = v1beta2.CohortReference(in.ParentName) + out.ResourceGroups = *(*[]v1beta2.ResourceGroup)(unsafe.Pointer(&in.ResourceGroups)) + out.FairSharing = (*v1beta2.FairSharing)(unsafe.Pointer(in.FairSharing)) + return nil +} + +// Convert_v1beta1_CohortSpec_To_v1beta2_CohortSpec is an autogenerated conversion function. +func Convert_v1beta1_CohortSpec_To_v1beta2_CohortSpec(in *CohortSpec, out *v1beta2.CohortSpec, s conversion.Scope) error { + return autoConvert_v1beta1_CohortSpec_To_v1beta2_CohortSpec(in, out, s) +} + +func autoConvert_v1beta2_CohortSpec_To_v1beta1_CohortSpec(in *v1beta2.CohortSpec, out *CohortSpec, s conversion.Scope) error { + out.ParentName = CohortReference(in.ParentName) + out.ResourceGroups = *(*[]ResourceGroup)(unsafe.Pointer(&in.ResourceGroups)) + out.FairSharing = (*FairSharing)(unsafe.Pointer(in.FairSharing)) + return nil +} + +// Convert_v1beta2_CohortSpec_To_v1beta1_CohortSpec is an autogenerated conversion function. +func Convert_v1beta2_CohortSpec_To_v1beta1_CohortSpec(in *v1beta2.CohortSpec, out *CohortSpec, s conversion.Scope) error { + return autoConvert_v1beta2_CohortSpec_To_v1beta1_CohortSpec(in, out, s) +} + +func autoConvert_v1beta1_CohortStatus_To_v1beta2_CohortStatus(in *CohortStatus, out *v1beta2.CohortStatus, s conversion.Scope) error { + out.FairSharing = (*v1beta2.FairSharingStatus)(unsafe.Pointer(in.FairSharing)) + return nil +} + +// Convert_v1beta1_CohortStatus_To_v1beta2_CohortStatus is an autogenerated conversion function. +func Convert_v1beta1_CohortStatus_To_v1beta2_CohortStatus(in *CohortStatus, out *v1beta2.CohortStatus, s conversion.Scope) error { + return autoConvert_v1beta1_CohortStatus_To_v1beta2_CohortStatus(in, out, s) +} + +func autoConvert_v1beta2_CohortStatus_To_v1beta1_CohortStatus(in *v1beta2.CohortStatus, out *CohortStatus, s conversion.Scope) error { + out.FairSharing = (*FairSharingStatus)(unsafe.Pointer(in.FairSharing)) + return nil +} + +// Convert_v1beta2_CohortStatus_To_v1beta1_CohortStatus is an autogenerated conversion function. +func Convert_v1beta2_CohortStatus_To_v1beta1_CohortStatus(in *v1beta2.CohortStatus, out *CohortStatus, s conversion.Scope) error { + return autoConvert_v1beta2_CohortStatus_To_v1beta1_CohortStatus(in, out, s) +} + +func autoConvert_v1beta1_FairSharing_To_v1beta2_FairSharing(in *FairSharing, out *v1beta2.FairSharing, s conversion.Scope) error { + out.Weight = (*resource.Quantity)(unsafe.Pointer(in.Weight)) + return nil +} + +// Convert_v1beta1_FairSharing_To_v1beta2_FairSharing is an autogenerated conversion function. +func Convert_v1beta1_FairSharing_To_v1beta2_FairSharing(in *FairSharing, out *v1beta2.FairSharing, s conversion.Scope) error { + return autoConvert_v1beta1_FairSharing_To_v1beta2_FairSharing(in, out, s) +} + +func autoConvert_v1beta2_FairSharing_To_v1beta1_FairSharing(in *v1beta2.FairSharing, out *FairSharing, s conversion.Scope) error { + out.Weight = (*resource.Quantity)(unsafe.Pointer(in.Weight)) + return nil +} + +// Convert_v1beta2_FairSharing_To_v1beta1_FairSharing is an autogenerated conversion function. +func Convert_v1beta2_FairSharing_To_v1beta1_FairSharing(in *v1beta2.FairSharing, out *FairSharing, s conversion.Scope) error { + return autoConvert_v1beta2_FairSharing_To_v1beta1_FairSharing(in, out, s) +} + +func autoConvert_v1beta1_FairSharingStatus_To_v1beta2_FairSharingStatus(in *FairSharingStatus, out *v1beta2.FairSharingStatus, s conversion.Scope) error { + out.WeightedShare = in.WeightedShare + out.AdmissionFairSharingStatus = (*v1beta2.AdmissionFairSharingStatus)(unsafe.Pointer(in.AdmissionFairSharingStatus)) + return nil +} + +// Convert_v1beta1_FairSharingStatus_To_v1beta2_FairSharingStatus is an autogenerated conversion function. +func Convert_v1beta1_FairSharingStatus_To_v1beta2_FairSharingStatus(in *FairSharingStatus, out *v1beta2.FairSharingStatus, s conversion.Scope) error { + return autoConvert_v1beta1_FairSharingStatus_To_v1beta2_FairSharingStatus(in, out, s) +} + +func autoConvert_v1beta2_FairSharingStatus_To_v1beta1_FairSharingStatus(in *v1beta2.FairSharingStatus, out *FairSharingStatus, s conversion.Scope) error { + out.WeightedShare = in.WeightedShare + out.AdmissionFairSharingStatus = (*AdmissionFairSharingStatus)(unsafe.Pointer(in.AdmissionFairSharingStatus)) + return nil +} + +// Convert_v1beta2_FairSharingStatus_To_v1beta1_FairSharingStatus is an autogenerated conversion function. +func Convert_v1beta2_FairSharingStatus_To_v1beta1_FairSharingStatus(in *v1beta2.FairSharingStatus, out *FairSharingStatus, s conversion.Scope) error { + return autoConvert_v1beta2_FairSharingStatus_To_v1beta1_FairSharingStatus(in, out, s) +} + +func autoConvert_v1beta1_FlavorFungibility_To_v1beta2_FlavorFungibility(in *FlavorFungibility, out *v1beta2.FlavorFungibility, s conversion.Scope) error { + out.WhenCanBorrow = v1beta2.FlavorFungibilityPolicy(in.WhenCanBorrow) + out.WhenCanPreempt = v1beta2.FlavorFungibilityPolicy(in.WhenCanPreempt) + return nil +} + +// Convert_v1beta1_FlavorFungibility_To_v1beta2_FlavorFungibility is an autogenerated conversion function. +func Convert_v1beta1_FlavorFungibility_To_v1beta2_FlavorFungibility(in *FlavorFungibility, out *v1beta2.FlavorFungibility, s conversion.Scope) error { + return autoConvert_v1beta1_FlavorFungibility_To_v1beta2_FlavorFungibility(in, out, s) +} + +func autoConvert_v1beta2_FlavorFungibility_To_v1beta1_FlavorFungibility(in *v1beta2.FlavorFungibility, out *FlavorFungibility, s conversion.Scope) error { + out.WhenCanBorrow = FlavorFungibilityPolicy(in.WhenCanBorrow) + out.WhenCanPreempt = FlavorFungibilityPolicy(in.WhenCanPreempt) + return nil +} + +// Convert_v1beta2_FlavorFungibility_To_v1beta1_FlavorFungibility is an autogenerated conversion function. +func Convert_v1beta2_FlavorFungibility_To_v1beta1_FlavorFungibility(in *v1beta2.FlavorFungibility, out *FlavorFungibility, s conversion.Scope) error { + return autoConvert_v1beta2_FlavorFungibility_To_v1beta1_FlavorFungibility(in, out, s) +} + +func autoConvert_v1beta1_FlavorQuotas_To_v1beta2_FlavorQuotas(in *FlavorQuotas, out *v1beta2.FlavorQuotas, s conversion.Scope) error { + out.Name = v1beta2.ResourceFlavorReference(in.Name) + out.Resources = *(*[]v1beta2.ResourceQuota)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1beta1_FlavorQuotas_To_v1beta2_FlavorQuotas is an autogenerated conversion function. +func Convert_v1beta1_FlavorQuotas_To_v1beta2_FlavorQuotas(in *FlavorQuotas, out *v1beta2.FlavorQuotas, s conversion.Scope) error { + return autoConvert_v1beta1_FlavorQuotas_To_v1beta2_FlavorQuotas(in, out, s) +} + +func autoConvert_v1beta2_FlavorQuotas_To_v1beta1_FlavorQuotas(in *v1beta2.FlavorQuotas, out *FlavorQuotas, s conversion.Scope) error { + out.Name = ResourceFlavorReference(in.Name) + out.Resources = *(*[]ResourceQuota)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1beta2_FlavorQuotas_To_v1beta1_FlavorQuotas is an autogenerated conversion function. +func Convert_v1beta2_FlavorQuotas_To_v1beta1_FlavorQuotas(in *v1beta2.FlavorQuotas, out *FlavorQuotas, s conversion.Scope) error { + return autoConvert_v1beta2_FlavorQuotas_To_v1beta1_FlavorQuotas(in, out, s) +} + +func autoConvert_v1beta1_FlavorUsage_To_v1beta2_FlavorUsage(in *FlavorUsage, out *v1beta2.FlavorUsage, s conversion.Scope) error { + out.Name = v1beta2.ResourceFlavorReference(in.Name) + out.Resources = *(*[]v1beta2.ResourceUsage)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1beta1_FlavorUsage_To_v1beta2_FlavorUsage is an autogenerated conversion function. +func Convert_v1beta1_FlavorUsage_To_v1beta2_FlavorUsage(in *FlavorUsage, out *v1beta2.FlavorUsage, s conversion.Scope) error { + return autoConvert_v1beta1_FlavorUsage_To_v1beta2_FlavorUsage(in, out, s) +} + +func autoConvert_v1beta2_FlavorUsage_To_v1beta1_FlavorUsage(in *v1beta2.FlavorUsage, out *FlavorUsage, s conversion.Scope) error { + out.Name = ResourceFlavorReference(in.Name) + out.Resources = *(*[]ResourceUsage)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1beta2_FlavorUsage_To_v1beta1_FlavorUsage is an autogenerated conversion function. +func Convert_v1beta2_FlavorUsage_To_v1beta1_FlavorUsage(in *v1beta2.FlavorUsage, out *FlavorUsage, s conversion.Scope) error { + return autoConvert_v1beta2_FlavorUsage_To_v1beta1_FlavorUsage(in, out, s) +} + +func autoConvert_v1beta1_KubeConfig_To_v1beta2_KubeConfig(in *KubeConfig, out *v1beta2.KubeConfig, s conversion.Scope) error { + out.Location = in.Location + out.LocationType = v1beta2.LocationType(in.LocationType) + return nil +} + +// Convert_v1beta1_KubeConfig_To_v1beta2_KubeConfig is an autogenerated conversion function. +func Convert_v1beta1_KubeConfig_To_v1beta2_KubeConfig(in *KubeConfig, out *v1beta2.KubeConfig, s conversion.Scope) error { + return autoConvert_v1beta1_KubeConfig_To_v1beta2_KubeConfig(in, out, s) +} + +func autoConvert_v1beta2_KubeConfig_To_v1beta1_KubeConfig(in *v1beta2.KubeConfig, out *KubeConfig, s conversion.Scope) error { + out.Location = in.Location + out.LocationType = LocationType(in.LocationType) + return nil +} + +// Convert_v1beta2_KubeConfig_To_v1beta1_KubeConfig is an autogenerated conversion function. +func Convert_v1beta2_KubeConfig_To_v1beta1_KubeConfig(in *v1beta2.KubeConfig, out *KubeConfig, s conversion.Scope) error { + return autoConvert_v1beta2_KubeConfig_To_v1beta1_KubeConfig(in, out, s) +} + +func autoConvert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(in *LocalQueue, out *v1beta2.LocalQueue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_LocalQueueSpec_To_v1beta2_LocalQueueSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta1_LocalQueueStatus_To_v1beta2_LocalQueueStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_LocalQueue_To_v1beta2_LocalQueue is an autogenerated conversion function. +func Convert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(in *LocalQueue, out *v1beta2.LocalQueue, s conversion.Scope) error { + return autoConvert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(in, out, s) +} + +func autoConvert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(in *v1beta2.LocalQueue, out *LocalQueue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_LocalQueueSpec_To_v1beta1_LocalQueueSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta2_LocalQueueStatus_To_v1beta1_LocalQueueStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_LocalQueue_To_v1beta1_LocalQueue is an autogenerated conversion function. +func Convert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(in *v1beta2.LocalQueue, out *LocalQueue, s conversion.Scope) error { + return autoConvert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(in, out, s) +} + +func autoConvert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus(in *LocalQueueFlavorStatus, out *v1beta2.LocalQueueFlavorStatus, s conversion.Scope) error { + out.Name = v1beta2.ResourceFlavorReference(in.Name) + out.Resources = *(*[]corev1.ResourceName)(unsafe.Pointer(&in.Resources)) + out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels)) + out.NodeTaints = *(*[]corev1.Taint)(unsafe.Pointer(&in.NodeTaints)) + out.Topology = (*v1beta2.TopologyInfo)(unsafe.Pointer(in.Topology)) + return nil +} + +// Convert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus is an autogenerated conversion function. +func Convert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus(in *LocalQueueFlavorStatus, out *v1beta2.LocalQueueFlavorStatus, s conversion.Scope) error { + return autoConvert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus(in, out, s) +} + +func autoConvert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus(in *v1beta2.LocalQueueFlavorStatus, out *LocalQueueFlavorStatus, s conversion.Scope) error { + out.Name = ResourceFlavorReference(in.Name) + out.Resources = *(*[]corev1.ResourceName)(unsafe.Pointer(&in.Resources)) + out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels)) + out.NodeTaints = *(*[]corev1.Taint)(unsafe.Pointer(&in.NodeTaints)) + out.Topology = (*TopologyInfo)(unsafe.Pointer(in.Topology)) + return nil +} + +// Convert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus is an autogenerated conversion function. +func Convert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus(in *v1beta2.LocalQueueFlavorStatus, out *LocalQueueFlavorStatus, s conversion.Scope) error { + return autoConvert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus(in, out, s) +} + +func autoConvert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage(in *LocalQueueFlavorUsage, out *v1beta2.LocalQueueFlavorUsage, s conversion.Scope) error { + out.Name = v1beta2.ResourceFlavorReference(in.Name) + out.Resources = *(*[]v1beta2.LocalQueueResourceUsage)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage is an autogenerated conversion function. +func Convert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage(in *LocalQueueFlavorUsage, out *v1beta2.LocalQueueFlavorUsage, s conversion.Scope) error { + return autoConvert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage(in, out, s) +} + +func autoConvert_v1beta2_LocalQueueFlavorUsage_To_v1beta1_LocalQueueFlavorUsage(in *v1beta2.LocalQueueFlavorUsage, out *LocalQueueFlavorUsage, s conversion.Scope) error { + out.Name = ResourceFlavorReference(in.Name) + out.Resources = *(*[]LocalQueueResourceUsage)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1beta2_LocalQueueFlavorUsage_To_v1beta1_LocalQueueFlavorUsage is an autogenerated conversion function. +func Convert_v1beta2_LocalQueueFlavorUsage_To_v1beta1_LocalQueueFlavorUsage(in *v1beta2.LocalQueueFlavorUsage, out *LocalQueueFlavorUsage, s conversion.Scope) error { + return autoConvert_v1beta2_LocalQueueFlavorUsage_To_v1beta1_LocalQueueFlavorUsage(in, out, s) +} + +func autoConvert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList(in *LocalQueueList, out *v1beta2.LocalQueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1beta2.LocalQueue, len(*in)) + for i := range *in { + if err := Convert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } + return nil +} + +// Convert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList is an autogenerated conversion function. +func Convert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList(in *LocalQueueList, out *v1beta2.LocalQueueList, s conversion.Scope) error { + return autoConvert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList(in, out, s) +} + +func autoConvert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList(in *v1beta2.LocalQueueList, out *LocalQueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LocalQueue, len(*in)) + for i := range *in { + if err := Convert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } + return nil +} + +// Convert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList is an autogenerated conversion function. +func Convert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList(in *v1beta2.LocalQueueList, out *LocalQueueList, s conversion.Scope) error { + return autoConvert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList(in, out, s) +} + +func autoConvert_v1beta1_LocalQueueResourceUsage_To_v1beta2_LocalQueueResourceUsage(in *LocalQueueResourceUsage, out *v1beta2.LocalQueueResourceUsage, s conversion.Scope) error { + out.Name = corev1.ResourceName(in.Name) + out.Total = in.Total + return nil +} + +// Convert_v1beta1_LocalQueueResourceUsage_To_v1beta2_LocalQueueResourceUsage is an autogenerated conversion function. +func Convert_v1beta1_LocalQueueResourceUsage_To_v1beta2_LocalQueueResourceUsage(in *LocalQueueResourceUsage, out *v1beta2.LocalQueueResourceUsage, s conversion.Scope) error { + return autoConvert_v1beta1_LocalQueueResourceUsage_To_v1beta2_LocalQueueResourceUsage(in, out, s) +} + +func autoConvert_v1beta2_LocalQueueResourceUsage_To_v1beta1_LocalQueueResourceUsage(in *v1beta2.LocalQueueResourceUsage, out *LocalQueueResourceUsage, s conversion.Scope) error { + out.Name = corev1.ResourceName(in.Name) + out.Total = in.Total + return nil +} + +// Convert_v1beta2_LocalQueueResourceUsage_To_v1beta1_LocalQueueResourceUsage is an autogenerated conversion function. +func Convert_v1beta2_LocalQueueResourceUsage_To_v1beta1_LocalQueueResourceUsage(in *v1beta2.LocalQueueResourceUsage, out *LocalQueueResourceUsage, s conversion.Scope) error { + return autoConvert_v1beta2_LocalQueueResourceUsage_To_v1beta1_LocalQueueResourceUsage(in, out, s) +} + +func autoConvert_v1beta1_LocalQueueSpec_To_v1beta2_LocalQueueSpec(in *LocalQueueSpec, out *v1beta2.LocalQueueSpec, s conversion.Scope) error { + out.ClusterQueue = v1beta2.ClusterQueueReference(in.ClusterQueue) + out.StopPolicy = (*v1beta2.StopPolicy)(unsafe.Pointer(in.StopPolicy)) + out.FairSharing = (*v1beta2.FairSharing)(unsafe.Pointer(in.FairSharing)) + return nil +} + +// Convert_v1beta1_LocalQueueSpec_To_v1beta2_LocalQueueSpec is an autogenerated conversion function. +func Convert_v1beta1_LocalQueueSpec_To_v1beta2_LocalQueueSpec(in *LocalQueueSpec, out *v1beta2.LocalQueueSpec, s conversion.Scope) error { + return autoConvert_v1beta1_LocalQueueSpec_To_v1beta2_LocalQueueSpec(in, out, s) +} + +func autoConvert_v1beta2_LocalQueueSpec_To_v1beta1_LocalQueueSpec(in *v1beta2.LocalQueueSpec, out *LocalQueueSpec, s conversion.Scope) error { + out.ClusterQueue = ClusterQueueReference(in.ClusterQueue) + out.StopPolicy = (*StopPolicy)(unsafe.Pointer(in.StopPolicy)) + out.FairSharing = (*FairSharing)(unsafe.Pointer(in.FairSharing)) + return nil +} + +// Convert_v1beta2_LocalQueueSpec_To_v1beta1_LocalQueueSpec is an autogenerated conversion function. +func Convert_v1beta2_LocalQueueSpec_To_v1beta1_LocalQueueSpec(in *v1beta2.LocalQueueSpec, out *LocalQueueSpec, s conversion.Scope) error { + return autoConvert_v1beta2_LocalQueueSpec_To_v1beta1_LocalQueueSpec(in, out, s) +} + +func autoConvert_v1beta1_LocalQueueStatus_To_v1beta2_LocalQueueStatus(in *LocalQueueStatus, out *v1beta2.LocalQueueStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + out.PendingWorkloads = in.PendingWorkloads + out.ReservingWorkloads = in.ReservingWorkloads + out.AdmittedWorkloads = in.AdmittedWorkloads + out.FlavorsReservation = *(*[]v1beta2.LocalQueueFlavorUsage)(unsafe.Pointer(&in.FlavorsReservation)) + // WARNING: in.FlavorUsage requires manual conversion: does not exist in peer-type + out.Flavors = *(*[]v1beta2.LocalQueueFlavorStatus)(unsafe.Pointer(&in.Flavors)) + out.FairSharing = (*v1beta2.FairSharingStatus)(unsafe.Pointer(in.FairSharing)) + return nil +} + +func autoConvert_v1beta2_LocalQueueStatus_To_v1beta1_LocalQueueStatus(in *v1beta2.LocalQueueStatus, out *LocalQueueStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + out.PendingWorkloads = in.PendingWorkloads + out.ReservingWorkloads = in.ReservingWorkloads + out.AdmittedWorkloads = in.AdmittedWorkloads + out.FlavorsReservation = *(*[]LocalQueueFlavorUsage)(unsafe.Pointer(&in.FlavorsReservation)) + // WARNING: in.FlavorsUsage requires manual conversion: does not exist in peer-type + out.Flavors = *(*[]LocalQueueFlavorStatus)(unsafe.Pointer(&in.Flavors)) + out.FairSharing = (*FairSharingStatus)(unsafe.Pointer(in.FairSharing)) + return nil +} + +func autoConvert_v1beta1_MultiKueueCluster_To_v1beta2_MultiKueueCluster(in *MultiKueueCluster, out *v1beta2.MultiKueueCluster, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_MultiKueueClusterSpec_To_v1beta2_MultiKueueClusterSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta1_MultiKueueClusterStatus_To_v1beta2_MultiKueueClusterStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_MultiKueueCluster_To_v1beta2_MultiKueueCluster is an autogenerated conversion function. +func Convert_v1beta1_MultiKueueCluster_To_v1beta2_MultiKueueCluster(in *MultiKueueCluster, out *v1beta2.MultiKueueCluster, s conversion.Scope) error { + return autoConvert_v1beta1_MultiKueueCluster_To_v1beta2_MultiKueueCluster(in, out, s) +} + +func autoConvert_v1beta2_MultiKueueCluster_To_v1beta1_MultiKueueCluster(in *v1beta2.MultiKueueCluster, out *MultiKueueCluster, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_MultiKueueClusterSpec_To_v1beta1_MultiKueueClusterSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta2_MultiKueueClusterStatus_To_v1beta1_MultiKueueClusterStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_MultiKueueCluster_To_v1beta1_MultiKueueCluster is an autogenerated conversion function. +func Convert_v1beta2_MultiKueueCluster_To_v1beta1_MultiKueueCluster(in *v1beta2.MultiKueueCluster, out *MultiKueueCluster, s conversion.Scope) error { + return autoConvert_v1beta2_MultiKueueCluster_To_v1beta1_MultiKueueCluster(in, out, s) +} + +func autoConvert_v1beta1_MultiKueueClusterList_To_v1beta2_MultiKueueClusterList(in *MultiKueueClusterList, out *v1beta2.MultiKueueClusterList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.MultiKueueCluster)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_MultiKueueClusterList_To_v1beta2_MultiKueueClusterList is an autogenerated conversion function. +func Convert_v1beta1_MultiKueueClusterList_To_v1beta2_MultiKueueClusterList(in *MultiKueueClusterList, out *v1beta2.MultiKueueClusterList, s conversion.Scope) error { + return autoConvert_v1beta1_MultiKueueClusterList_To_v1beta2_MultiKueueClusterList(in, out, s) +} + +func autoConvert_v1beta2_MultiKueueClusterList_To_v1beta1_MultiKueueClusterList(in *v1beta2.MultiKueueClusterList, out *MultiKueueClusterList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]MultiKueueCluster)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_MultiKueueClusterList_To_v1beta1_MultiKueueClusterList is an autogenerated conversion function. +func Convert_v1beta2_MultiKueueClusterList_To_v1beta1_MultiKueueClusterList(in *v1beta2.MultiKueueClusterList, out *MultiKueueClusterList, s conversion.Scope) error { + return autoConvert_v1beta2_MultiKueueClusterList_To_v1beta1_MultiKueueClusterList(in, out, s) +} + +func autoConvert_v1beta1_MultiKueueClusterSpec_To_v1beta2_MultiKueueClusterSpec(in *MultiKueueClusterSpec, out *v1beta2.MultiKueueClusterSpec, s conversion.Scope) error { + if err := Convert_v1beta1_KubeConfig_To_v1beta2_KubeConfig(&in.KubeConfig, &out.KubeConfig, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_MultiKueueClusterSpec_To_v1beta2_MultiKueueClusterSpec is an autogenerated conversion function. +func Convert_v1beta1_MultiKueueClusterSpec_To_v1beta2_MultiKueueClusterSpec(in *MultiKueueClusterSpec, out *v1beta2.MultiKueueClusterSpec, s conversion.Scope) error { + return autoConvert_v1beta1_MultiKueueClusterSpec_To_v1beta2_MultiKueueClusterSpec(in, out, s) +} + +func autoConvert_v1beta2_MultiKueueClusterSpec_To_v1beta1_MultiKueueClusterSpec(in *v1beta2.MultiKueueClusterSpec, out *MultiKueueClusterSpec, s conversion.Scope) error { + if err := Convert_v1beta2_KubeConfig_To_v1beta1_KubeConfig(&in.KubeConfig, &out.KubeConfig, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_MultiKueueClusterSpec_To_v1beta1_MultiKueueClusterSpec is an autogenerated conversion function. +func Convert_v1beta2_MultiKueueClusterSpec_To_v1beta1_MultiKueueClusterSpec(in *v1beta2.MultiKueueClusterSpec, out *MultiKueueClusterSpec, s conversion.Scope) error { + return autoConvert_v1beta2_MultiKueueClusterSpec_To_v1beta1_MultiKueueClusterSpec(in, out, s) +} + +func autoConvert_v1beta1_MultiKueueClusterStatus_To_v1beta2_MultiKueueClusterStatus(in *MultiKueueClusterStatus, out *v1beta2.MultiKueueClusterStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + return nil +} + +// Convert_v1beta1_MultiKueueClusterStatus_To_v1beta2_MultiKueueClusterStatus is an autogenerated conversion function. +func Convert_v1beta1_MultiKueueClusterStatus_To_v1beta2_MultiKueueClusterStatus(in *MultiKueueClusterStatus, out *v1beta2.MultiKueueClusterStatus, s conversion.Scope) error { + return autoConvert_v1beta1_MultiKueueClusterStatus_To_v1beta2_MultiKueueClusterStatus(in, out, s) +} + +func autoConvert_v1beta2_MultiKueueClusterStatus_To_v1beta1_MultiKueueClusterStatus(in *v1beta2.MultiKueueClusterStatus, out *MultiKueueClusterStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + return nil +} + +// Convert_v1beta2_MultiKueueClusterStatus_To_v1beta1_MultiKueueClusterStatus is an autogenerated conversion function. +func Convert_v1beta2_MultiKueueClusterStatus_To_v1beta1_MultiKueueClusterStatus(in *v1beta2.MultiKueueClusterStatus, out *MultiKueueClusterStatus, s conversion.Scope) error { + return autoConvert_v1beta2_MultiKueueClusterStatus_To_v1beta1_MultiKueueClusterStatus(in, out, s) +} + +func autoConvert_v1beta1_MultiKueueConfig_To_v1beta2_MultiKueueConfig(in *MultiKueueConfig, out *v1beta2.MultiKueueConfig, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_MultiKueueConfigSpec_To_v1beta2_MultiKueueConfigSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_MultiKueueConfig_To_v1beta2_MultiKueueConfig is an autogenerated conversion function. +func Convert_v1beta1_MultiKueueConfig_To_v1beta2_MultiKueueConfig(in *MultiKueueConfig, out *v1beta2.MultiKueueConfig, s conversion.Scope) error { + return autoConvert_v1beta1_MultiKueueConfig_To_v1beta2_MultiKueueConfig(in, out, s) +} + +func autoConvert_v1beta2_MultiKueueConfig_To_v1beta1_MultiKueueConfig(in *v1beta2.MultiKueueConfig, out *MultiKueueConfig, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_MultiKueueConfigSpec_To_v1beta1_MultiKueueConfigSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_MultiKueueConfig_To_v1beta1_MultiKueueConfig is an autogenerated conversion function. +func Convert_v1beta2_MultiKueueConfig_To_v1beta1_MultiKueueConfig(in *v1beta2.MultiKueueConfig, out *MultiKueueConfig, s conversion.Scope) error { + return autoConvert_v1beta2_MultiKueueConfig_To_v1beta1_MultiKueueConfig(in, out, s) +} + +func autoConvert_v1beta1_MultiKueueConfigList_To_v1beta2_MultiKueueConfigList(in *MultiKueueConfigList, out *v1beta2.MultiKueueConfigList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.MultiKueueConfig)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_MultiKueueConfigList_To_v1beta2_MultiKueueConfigList is an autogenerated conversion function. +func Convert_v1beta1_MultiKueueConfigList_To_v1beta2_MultiKueueConfigList(in *MultiKueueConfigList, out *v1beta2.MultiKueueConfigList, s conversion.Scope) error { + return autoConvert_v1beta1_MultiKueueConfigList_To_v1beta2_MultiKueueConfigList(in, out, s) +} + +func autoConvert_v1beta2_MultiKueueConfigList_To_v1beta1_MultiKueueConfigList(in *v1beta2.MultiKueueConfigList, out *MultiKueueConfigList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]MultiKueueConfig)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_MultiKueueConfigList_To_v1beta1_MultiKueueConfigList is an autogenerated conversion function. +func Convert_v1beta2_MultiKueueConfigList_To_v1beta1_MultiKueueConfigList(in *v1beta2.MultiKueueConfigList, out *MultiKueueConfigList, s conversion.Scope) error { + return autoConvert_v1beta2_MultiKueueConfigList_To_v1beta1_MultiKueueConfigList(in, out, s) +} + +func autoConvert_v1beta1_MultiKueueConfigSpec_To_v1beta2_MultiKueueConfigSpec(in *MultiKueueConfigSpec, out *v1beta2.MultiKueueConfigSpec, s conversion.Scope) error { + out.Clusters = *(*[]string)(unsafe.Pointer(&in.Clusters)) + return nil +} + +// Convert_v1beta1_MultiKueueConfigSpec_To_v1beta2_MultiKueueConfigSpec is an autogenerated conversion function. +func Convert_v1beta1_MultiKueueConfigSpec_To_v1beta2_MultiKueueConfigSpec(in *MultiKueueConfigSpec, out *v1beta2.MultiKueueConfigSpec, s conversion.Scope) error { + return autoConvert_v1beta1_MultiKueueConfigSpec_To_v1beta2_MultiKueueConfigSpec(in, out, s) +} + +func autoConvert_v1beta2_MultiKueueConfigSpec_To_v1beta1_MultiKueueConfigSpec(in *v1beta2.MultiKueueConfigSpec, out *MultiKueueConfigSpec, s conversion.Scope) error { + out.Clusters = *(*[]string)(unsafe.Pointer(&in.Clusters)) + return nil +} + +// Convert_v1beta2_MultiKueueConfigSpec_To_v1beta1_MultiKueueConfigSpec is an autogenerated conversion function. +func Convert_v1beta2_MultiKueueConfigSpec_To_v1beta1_MultiKueueConfigSpec(in *v1beta2.MultiKueueConfigSpec, out *MultiKueueConfigSpec, s conversion.Scope) error { + return autoConvert_v1beta2_MultiKueueConfigSpec_To_v1beta1_MultiKueueConfigSpec(in, out, s) +} + +func autoConvert_v1beta1_PodSet_To_v1beta2_PodSet(in *PodSet, out *v1beta2.PodSet, s conversion.Scope) error { + out.Name = v1beta2.PodSetReference(in.Name) + out.Template = in.Template + out.Count = in.Count + out.MinCount = (*int32)(unsafe.Pointer(in.MinCount)) + out.TopologyRequest = (*v1beta2.PodSetTopologyRequest)(unsafe.Pointer(in.TopologyRequest)) + return nil +} + +// Convert_v1beta1_PodSet_To_v1beta2_PodSet is an autogenerated conversion function. +func Convert_v1beta1_PodSet_To_v1beta2_PodSet(in *PodSet, out *v1beta2.PodSet, s conversion.Scope) error { + return autoConvert_v1beta1_PodSet_To_v1beta2_PodSet(in, out, s) +} + +func autoConvert_v1beta2_PodSet_To_v1beta1_PodSet(in *v1beta2.PodSet, out *PodSet, s conversion.Scope) error { + out.Name = PodSetReference(in.Name) + out.Template = in.Template + out.Count = in.Count + out.MinCount = (*int32)(unsafe.Pointer(in.MinCount)) + out.TopologyRequest = (*PodSetTopologyRequest)(unsafe.Pointer(in.TopologyRequest)) + return nil +} + +// Convert_v1beta2_PodSet_To_v1beta1_PodSet is an autogenerated conversion function. +func Convert_v1beta2_PodSet_To_v1beta1_PodSet(in *v1beta2.PodSet, out *PodSet, s conversion.Scope) error { + return autoConvert_v1beta2_PodSet_To_v1beta1_PodSet(in, out, s) +} + +func autoConvert_v1beta1_PodSetAssignment_To_v1beta2_PodSetAssignment(in *PodSetAssignment, out *v1beta2.PodSetAssignment, s conversion.Scope) error { + out.Name = v1beta2.PodSetReference(in.Name) + out.Flavors = *(*map[corev1.ResourceName]v1beta2.ResourceFlavorReference)(unsafe.Pointer(&in.Flavors)) + out.ResourceUsage = *(*corev1.ResourceList)(unsafe.Pointer(&in.ResourceUsage)) + out.Count = (*int32)(unsafe.Pointer(in.Count)) + out.TopologyAssignment = (*v1beta2.TopologyAssignment)(unsafe.Pointer(in.TopologyAssignment)) + out.DelayedTopologyRequest = (*v1beta2.DelayedTopologyRequestState)(unsafe.Pointer(in.DelayedTopologyRequest)) + return nil +} + +// Convert_v1beta1_PodSetAssignment_To_v1beta2_PodSetAssignment is an autogenerated conversion function. +func Convert_v1beta1_PodSetAssignment_To_v1beta2_PodSetAssignment(in *PodSetAssignment, out *v1beta2.PodSetAssignment, s conversion.Scope) error { + return autoConvert_v1beta1_PodSetAssignment_To_v1beta2_PodSetAssignment(in, out, s) +} + +func autoConvert_v1beta2_PodSetAssignment_To_v1beta1_PodSetAssignment(in *v1beta2.PodSetAssignment, out *PodSetAssignment, s conversion.Scope) error { + out.Name = PodSetReference(in.Name) + out.Flavors = *(*map[corev1.ResourceName]ResourceFlavorReference)(unsafe.Pointer(&in.Flavors)) + out.ResourceUsage = *(*corev1.ResourceList)(unsafe.Pointer(&in.ResourceUsage)) + out.Count = (*int32)(unsafe.Pointer(in.Count)) + out.TopologyAssignment = (*TopologyAssignment)(unsafe.Pointer(in.TopologyAssignment)) + out.DelayedTopologyRequest = (*DelayedTopologyRequestState)(unsafe.Pointer(in.DelayedTopologyRequest)) + return nil +} + +// Convert_v1beta2_PodSetAssignment_To_v1beta1_PodSetAssignment is an autogenerated conversion function. +func Convert_v1beta2_PodSetAssignment_To_v1beta1_PodSetAssignment(in *v1beta2.PodSetAssignment, out *PodSetAssignment, s conversion.Scope) error { + return autoConvert_v1beta2_PodSetAssignment_To_v1beta1_PodSetAssignment(in, out, s) +} + +func autoConvert_v1beta1_PodSetRequest_To_v1beta2_PodSetRequest(in *PodSetRequest, out *v1beta2.PodSetRequest, s conversion.Scope) error { + out.Name = v1beta2.PodSetReference(in.Name) + out.Resources = *(*corev1.ResourceList)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1beta1_PodSetRequest_To_v1beta2_PodSetRequest is an autogenerated conversion function. +func Convert_v1beta1_PodSetRequest_To_v1beta2_PodSetRequest(in *PodSetRequest, out *v1beta2.PodSetRequest, s conversion.Scope) error { + return autoConvert_v1beta1_PodSetRequest_To_v1beta2_PodSetRequest(in, out, s) +} + +func autoConvert_v1beta2_PodSetRequest_To_v1beta1_PodSetRequest(in *v1beta2.PodSetRequest, out *PodSetRequest, s conversion.Scope) error { + out.Name = PodSetReference(in.Name) + out.Resources = *(*corev1.ResourceList)(unsafe.Pointer(&in.Resources)) + return nil +} + +// Convert_v1beta2_PodSetRequest_To_v1beta1_PodSetRequest is an autogenerated conversion function. +func Convert_v1beta2_PodSetRequest_To_v1beta1_PodSetRequest(in *v1beta2.PodSetRequest, out *PodSetRequest, s conversion.Scope) error { + return autoConvert_v1beta2_PodSetRequest_To_v1beta1_PodSetRequest(in, out, s) +} + +func autoConvert_v1beta1_PodSetTopologyRequest_To_v1beta2_PodSetTopologyRequest(in *PodSetTopologyRequest, out *v1beta2.PodSetTopologyRequest, s conversion.Scope) error { + out.Required = (*string)(unsafe.Pointer(in.Required)) + out.Preferred = (*string)(unsafe.Pointer(in.Preferred)) + out.Unconstrained = (*bool)(unsafe.Pointer(in.Unconstrained)) + out.PodIndexLabel = (*string)(unsafe.Pointer(in.PodIndexLabel)) + out.SubGroupIndexLabel = (*string)(unsafe.Pointer(in.SubGroupIndexLabel)) + out.SubGroupCount = (*int32)(unsafe.Pointer(in.SubGroupCount)) + out.PodSetGroupName = (*string)(unsafe.Pointer(in.PodSetGroupName)) + out.PodSetSliceRequiredTopology = (*string)(unsafe.Pointer(in.PodSetSliceRequiredTopology)) + out.PodSetSliceSize = (*int32)(unsafe.Pointer(in.PodSetSliceSize)) + return nil +} + +// Convert_v1beta1_PodSetTopologyRequest_To_v1beta2_PodSetTopologyRequest is an autogenerated conversion function. +func Convert_v1beta1_PodSetTopologyRequest_To_v1beta2_PodSetTopologyRequest(in *PodSetTopologyRequest, out *v1beta2.PodSetTopologyRequest, s conversion.Scope) error { + return autoConvert_v1beta1_PodSetTopologyRequest_To_v1beta2_PodSetTopologyRequest(in, out, s) +} + +func autoConvert_v1beta2_PodSetTopologyRequest_To_v1beta1_PodSetTopologyRequest(in *v1beta2.PodSetTopologyRequest, out *PodSetTopologyRequest, s conversion.Scope) error { + out.Required = (*string)(unsafe.Pointer(in.Required)) + out.Preferred = (*string)(unsafe.Pointer(in.Preferred)) + out.Unconstrained = (*bool)(unsafe.Pointer(in.Unconstrained)) + out.PodIndexLabel = (*string)(unsafe.Pointer(in.PodIndexLabel)) + out.SubGroupIndexLabel = (*string)(unsafe.Pointer(in.SubGroupIndexLabel)) + out.SubGroupCount = (*int32)(unsafe.Pointer(in.SubGroupCount)) + out.PodSetGroupName = (*string)(unsafe.Pointer(in.PodSetGroupName)) + out.PodSetSliceRequiredTopology = (*string)(unsafe.Pointer(in.PodSetSliceRequiredTopology)) + out.PodSetSliceSize = (*int32)(unsafe.Pointer(in.PodSetSliceSize)) + return nil +} + +// Convert_v1beta2_PodSetTopologyRequest_To_v1beta1_PodSetTopologyRequest is an autogenerated conversion function. +func Convert_v1beta2_PodSetTopologyRequest_To_v1beta1_PodSetTopologyRequest(in *v1beta2.PodSetTopologyRequest, out *PodSetTopologyRequest, s conversion.Scope) error { + return autoConvert_v1beta2_PodSetTopologyRequest_To_v1beta1_PodSetTopologyRequest(in, out, s) +} + +func autoConvert_v1beta1_PodSetUpdate_To_v1beta2_PodSetUpdate(in *PodSetUpdate, out *v1beta2.PodSetUpdate, s conversion.Scope) error { + out.Name = v1beta2.PodSetReference(in.Name) + out.Labels = *(*map[string]string)(unsafe.Pointer(&in.Labels)) + out.Annotations = *(*map[string]string)(unsafe.Pointer(&in.Annotations)) + out.NodeSelector = *(*map[string]string)(unsafe.Pointer(&in.NodeSelector)) + out.Tolerations = *(*[]corev1.Toleration)(unsafe.Pointer(&in.Tolerations)) + return nil +} + +// Convert_v1beta1_PodSetUpdate_To_v1beta2_PodSetUpdate is an autogenerated conversion function. +func Convert_v1beta1_PodSetUpdate_To_v1beta2_PodSetUpdate(in *PodSetUpdate, out *v1beta2.PodSetUpdate, s conversion.Scope) error { + return autoConvert_v1beta1_PodSetUpdate_To_v1beta2_PodSetUpdate(in, out, s) +} + +func autoConvert_v1beta2_PodSetUpdate_To_v1beta1_PodSetUpdate(in *v1beta2.PodSetUpdate, out *PodSetUpdate, s conversion.Scope) error { + out.Name = PodSetReference(in.Name) + out.Labels = *(*map[string]string)(unsafe.Pointer(&in.Labels)) + out.Annotations = *(*map[string]string)(unsafe.Pointer(&in.Annotations)) + out.NodeSelector = *(*map[string]string)(unsafe.Pointer(&in.NodeSelector)) + out.Tolerations = *(*[]corev1.Toleration)(unsafe.Pointer(&in.Tolerations)) + return nil +} + +// Convert_v1beta2_PodSetUpdate_To_v1beta1_PodSetUpdate is an autogenerated conversion function. +func Convert_v1beta2_PodSetUpdate_To_v1beta1_PodSetUpdate(in *v1beta2.PodSetUpdate, out *PodSetUpdate, s conversion.Scope) error { + return autoConvert_v1beta2_PodSetUpdate_To_v1beta1_PodSetUpdate(in, out, s) +} + +func autoConvert_v1beta1_ProvisioningRequestConfig_To_v1beta2_ProvisioningRequestConfig(in *ProvisioningRequestConfig, out *v1beta2.ProvisioningRequestConfig, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_ProvisioningRequestConfigSpec_To_v1beta2_ProvisioningRequestConfigSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_ProvisioningRequestConfig_To_v1beta2_ProvisioningRequestConfig is an autogenerated conversion function. +func Convert_v1beta1_ProvisioningRequestConfig_To_v1beta2_ProvisioningRequestConfig(in *ProvisioningRequestConfig, out *v1beta2.ProvisioningRequestConfig, s conversion.Scope) error { + return autoConvert_v1beta1_ProvisioningRequestConfig_To_v1beta2_ProvisioningRequestConfig(in, out, s) +} + +func autoConvert_v1beta2_ProvisioningRequestConfig_To_v1beta1_ProvisioningRequestConfig(in *v1beta2.ProvisioningRequestConfig, out *ProvisioningRequestConfig, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_ProvisioningRequestConfigSpec_To_v1beta1_ProvisioningRequestConfigSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_ProvisioningRequestConfig_To_v1beta1_ProvisioningRequestConfig is an autogenerated conversion function. +func Convert_v1beta2_ProvisioningRequestConfig_To_v1beta1_ProvisioningRequestConfig(in *v1beta2.ProvisioningRequestConfig, out *ProvisioningRequestConfig, s conversion.Scope) error { + return autoConvert_v1beta2_ProvisioningRequestConfig_To_v1beta1_ProvisioningRequestConfig(in, out, s) +} + +func autoConvert_v1beta1_ProvisioningRequestConfigList_To_v1beta2_ProvisioningRequestConfigList(in *ProvisioningRequestConfigList, out *v1beta2.ProvisioningRequestConfigList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.ProvisioningRequestConfig)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_ProvisioningRequestConfigList_To_v1beta2_ProvisioningRequestConfigList is an autogenerated conversion function. +func Convert_v1beta1_ProvisioningRequestConfigList_To_v1beta2_ProvisioningRequestConfigList(in *ProvisioningRequestConfigList, out *v1beta2.ProvisioningRequestConfigList, s conversion.Scope) error { + return autoConvert_v1beta1_ProvisioningRequestConfigList_To_v1beta2_ProvisioningRequestConfigList(in, out, s) +} + +func autoConvert_v1beta2_ProvisioningRequestConfigList_To_v1beta1_ProvisioningRequestConfigList(in *v1beta2.ProvisioningRequestConfigList, out *ProvisioningRequestConfigList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]ProvisioningRequestConfig)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_ProvisioningRequestConfigList_To_v1beta1_ProvisioningRequestConfigList is an autogenerated conversion function. +func Convert_v1beta2_ProvisioningRequestConfigList_To_v1beta1_ProvisioningRequestConfigList(in *v1beta2.ProvisioningRequestConfigList, out *ProvisioningRequestConfigList, s conversion.Scope) error { + return autoConvert_v1beta2_ProvisioningRequestConfigList_To_v1beta1_ProvisioningRequestConfigList(in, out, s) +} + +func autoConvert_v1beta1_ProvisioningRequestConfigSpec_To_v1beta2_ProvisioningRequestConfigSpec(in *ProvisioningRequestConfigSpec, out *v1beta2.ProvisioningRequestConfigSpec, s conversion.Scope) error { + out.ProvisioningClassName = in.ProvisioningClassName + out.Parameters = *(*map[string]v1beta2.Parameter)(unsafe.Pointer(&in.Parameters)) + out.ManagedResources = *(*[]corev1.ResourceName)(unsafe.Pointer(&in.ManagedResources)) + out.RetryStrategy = (*v1beta2.ProvisioningRequestRetryStrategy)(unsafe.Pointer(in.RetryStrategy)) + out.PodSetUpdates = (*v1beta2.ProvisioningRequestPodSetUpdates)(unsafe.Pointer(in.PodSetUpdates)) + out.PodSetMergePolicy = (*v1beta2.ProvisioningRequestConfigPodSetMergePolicy)(unsafe.Pointer(in.PodSetMergePolicy)) + return nil +} + +// Convert_v1beta1_ProvisioningRequestConfigSpec_To_v1beta2_ProvisioningRequestConfigSpec is an autogenerated conversion function. +func Convert_v1beta1_ProvisioningRequestConfigSpec_To_v1beta2_ProvisioningRequestConfigSpec(in *ProvisioningRequestConfigSpec, out *v1beta2.ProvisioningRequestConfigSpec, s conversion.Scope) error { + return autoConvert_v1beta1_ProvisioningRequestConfigSpec_To_v1beta2_ProvisioningRequestConfigSpec(in, out, s) +} + +func autoConvert_v1beta2_ProvisioningRequestConfigSpec_To_v1beta1_ProvisioningRequestConfigSpec(in *v1beta2.ProvisioningRequestConfigSpec, out *ProvisioningRequestConfigSpec, s conversion.Scope) error { + out.ProvisioningClassName = in.ProvisioningClassName + out.Parameters = *(*map[string]Parameter)(unsafe.Pointer(&in.Parameters)) + out.ManagedResources = *(*[]corev1.ResourceName)(unsafe.Pointer(&in.ManagedResources)) + out.RetryStrategy = (*ProvisioningRequestRetryStrategy)(unsafe.Pointer(in.RetryStrategy)) + out.PodSetUpdates = (*ProvisioningRequestPodSetUpdates)(unsafe.Pointer(in.PodSetUpdates)) + out.PodSetMergePolicy = (*ProvisioningRequestConfigPodSetMergePolicy)(unsafe.Pointer(in.PodSetMergePolicy)) + return nil +} + +// Convert_v1beta2_ProvisioningRequestConfigSpec_To_v1beta1_ProvisioningRequestConfigSpec is an autogenerated conversion function. +func Convert_v1beta2_ProvisioningRequestConfigSpec_To_v1beta1_ProvisioningRequestConfigSpec(in *v1beta2.ProvisioningRequestConfigSpec, out *ProvisioningRequestConfigSpec, s conversion.Scope) error { + return autoConvert_v1beta2_ProvisioningRequestConfigSpec_To_v1beta1_ProvisioningRequestConfigSpec(in, out, s) +} + +func autoConvert_v1beta1_ProvisioningRequestPodSetUpdates_To_v1beta2_ProvisioningRequestPodSetUpdates(in *ProvisioningRequestPodSetUpdates, out *v1beta2.ProvisioningRequestPodSetUpdates, s conversion.Scope) error { + out.NodeSelector = *(*[]v1beta2.ProvisioningRequestPodSetUpdatesNodeSelector)(unsafe.Pointer(&in.NodeSelector)) + return nil +} + +// Convert_v1beta1_ProvisioningRequestPodSetUpdates_To_v1beta2_ProvisioningRequestPodSetUpdates is an autogenerated conversion function. +func Convert_v1beta1_ProvisioningRequestPodSetUpdates_To_v1beta2_ProvisioningRequestPodSetUpdates(in *ProvisioningRequestPodSetUpdates, out *v1beta2.ProvisioningRequestPodSetUpdates, s conversion.Scope) error { + return autoConvert_v1beta1_ProvisioningRequestPodSetUpdates_To_v1beta2_ProvisioningRequestPodSetUpdates(in, out, s) +} + +func autoConvert_v1beta2_ProvisioningRequestPodSetUpdates_To_v1beta1_ProvisioningRequestPodSetUpdates(in *v1beta2.ProvisioningRequestPodSetUpdates, out *ProvisioningRequestPodSetUpdates, s conversion.Scope) error { + out.NodeSelector = *(*[]ProvisioningRequestPodSetUpdatesNodeSelector)(unsafe.Pointer(&in.NodeSelector)) + return nil +} + +// Convert_v1beta2_ProvisioningRequestPodSetUpdates_To_v1beta1_ProvisioningRequestPodSetUpdates is an autogenerated conversion function. +func Convert_v1beta2_ProvisioningRequestPodSetUpdates_To_v1beta1_ProvisioningRequestPodSetUpdates(in *v1beta2.ProvisioningRequestPodSetUpdates, out *ProvisioningRequestPodSetUpdates, s conversion.Scope) error { + return autoConvert_v1beta2_ProvisioningRequestPodSetUpdates_To_v1beta1_ProvisioningRequestPodSetUpdates(in, out, s) +} + +func autoConvert_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector(in *ProvisioningRequestPodSetUpdatesNodeSelector, out *v1beta2.ProvisioningRequestPodSetUpdatesNodeSelector, s conversion.Scope) error { + out.Key = in.Key + out.ValueFromProvisioningClassDetail = in.ValueFromProvisioningClassDetail + return nil +} + +// Convert_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector is an autogenerated conversion function. +func Convert_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector(in *ProvisioningRequestPodSetUpdatesNodeSelector, out *v1beta2.ProvisioningRequestPodSetUpdatesNodeSelector, s conversion.Scope) error { + return autoConvert_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector(in, out, s) +} + +func autoConvert_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector(in *v1beta2.ProvisioningRequestPodSetUpdatesNodeSelector, out *ProvisioningRequestPodSetUpdatesNodeSelector, s conversion.Scope) error { + out.Key = in.Key + out.ValueFromProvisioningClassDetail = in.ValueFromProvisioningClassDetail + return nil +} + +// Convert_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector is an autogenerated conversion function. +func Convert_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector(in *v1beta2.ProvisioningRequestPodSetUpdatesNodeSelector, out *ProvisioningRequestPodSetUpdatesNodeSelector, s conversion.Scope) error { + return autoConvert_v1beta2_ProvisioningRequestPodSetUpdatesNodeSelector_To_v1beta1_ProvisioningRequestPodSetUpdatesNodeSelector(in, out, s) +} + +func autoConvert_v1beta1_ProvisioningRequestRetryStrategy_To_v1beta2_ProvisioningRequestRetryStrategy(in *ProvisioningRequestRetryStrategy, out *v1beta2.ProvisioningRequestRetryStrategy, s conversion.Scope) error { + out.BackoffLimitCount = (*int32)(unsafe.Pointer(in.BackoffLimitCount)) + out.BackoffBaseSeconds = (*int32)(unsafe.Pointer(in.BackoffBaseSeconds)) + out.BackoffMaxSeconds = (*int32)(unsafe.Pointer(in.BackoffMaxSeconds)) + return nil +} + +// Convert_v1beta1_ProvisioningRequestRetryStrategy_To_v1beta2_ProvisioningRequestRetryStrategy is an autogenerated conversion function. +func Convert_v1beta1_ProvisioningRequestRetryStrategy_To_v1beta2_ProvisioningRequestRetryStrategy(in *ProvisioningRequestRetryStrategy, out *v1beta2.ProvisioningRequestRetryStrategy, s conversion.Scope) error { + return autoConvert_v1beta1_ProvisioningRequestRetryStrategy_To_v1beta2_ProvisioningRequestRetryStrategy(in, out, s) +} + +func autoConvert_v1beta2_ProvisioningRequestRetryStrategy_To_v1beta1_ProvisioningRequestRetryStrategy(in *v1beta2.ProvisioningRequestRetryStrategy, out *ProvisioningRequestRetryStrategy, s conversion.Scope) error { + out.BackoffLimitCount = (*int32)(unsafe.Pointer(in.BackoffLimitCount)) + out.BackoffBaseSeconds = (*int32)(unsafe.Pointer(in.BackoffBaseSeconds)) + out.BackoffMaxSeconds = (*int32)(unsafe.Pointer(in.BackoffMaxSeconds)) + return nil +} + +// Convert_v1beta2_ProvisioningRequestRetryStrategy_To_v1beta1_ProvisioningRequestRetryStrategy is an autogenerated conversion function. +func Convert_v1beta2_ProvisioningRequestRetryStrategy_To_v1beta1_ProvisioningRequestRetryStrategy(in *v1beta2.ProvisioningRequestRetryStrategy, out *ProvisioningRequestRetryStrategy, s conversion.Scope) error { + return autoConvert_v1beta2_ProvisioningRequestRetryStrategy_To_v1beta1_ProvisioningRequestRetryStrategy(in, out, s) +} + +func autoConvert_v1beta1_ReclaimablePod_To_v1beta2_ReclaimablePod(in *ReclaimablePod, out *v1beta2.ReclaimablePod, s conversion.Scope) error { + out.Name = v1beta2.PodSetReference(in.Name) + out.Count = in.Count + return nil +} + +// Convert_v1beta1_ReclaimablePod_To_v1beta2_ReclaimablePod is an autogenerated conversion function. +func Convert_v1beta1_ReclaimablePod_To_v1beta2_ReclaimablePod(in *ReclaimablePod, out *v1beta2.ReclaimablePod, s conversion.Scope) error { + return autoConvert_v1beta1_ReclaimablePod_To_v1beta2_ReclaimablePod(in, out, s) +} + +func autoConvert_v1beta2_ReclaimablePod_To_v1beta1_ReclaimablePod(in *v1beta2.ReclaimablePod, out *ReclaimablePod, s conversion.Scope) error { + out.Name = PodSetReference(in.Name) + out.Count = in.Count + return nil +} + +// Convert_v1beta2_ReclaimablePod_To_v1beta1_ReclaimablePod is an autogenerated conversion function. +func Convert_v1beta2_ReclaimablePod_To_v1beta1_ReclaimablePod(in *v1beta2.ReclaimablePod, out *ReclaimablePod, s conversion.Scope) error { + return autoConvert_v1beta2_ReclaimablePod_To_v1beta1_ReclaimablePod(in, out, s) +} + +func autoConvert_v1beta1_RequeueState_To_v1beta2_RequeueState(in *RequeueState, out *v1beta2.RequeueState, s conversion.Scope) error { + out.Count = (*int32)(unsafe.Pointer(in.Count)) + out.RequeueAt = (*v1.Time)(unsafe.Pointer(in.RequeueAt)) + return nil +} + +// Convert_v1beta1_RequeueState_To_v1beta2_RequeueState is an autogenerated conversion function. +func Convert_v1beta1_RequeueState_To_v1beta2_RequeueState(in *RequeueState, out *v1beta2.RequeueState, s conversion.Scope) error { + return autoConvert_v1beta1_RequeueState_To_v1beta2_RequeueState(in, out, s) +} + +func autoConvert_v1beta2_RequeueState_To_v1beta1_RequeueState(in *v1beta2.RequeueState, out *RequeueState, s conversion.Scope) error { + out.Count = (*int32)(unsafe.Pointer(in.Count)) + out.RequeueAt = (*v1.Time)(unsafe.Pointer(in.RequeueAt)) + return nil +} + +// Convert_v1beta2_RequeueState_To_v1beta1_RequeueState is an autogenerated conversion function. +func Convert_v1beta2_RequeueState_To_v1beta1_RequeueState(in *v1beta2.RequeueState, out *RequeueState, s conversion.Scope) error { + return autoConvert_v1beta2_RequeueState_To_v1beta1_RequeueState(in, out, s) +} + +func autoConvert_v1beta1_ResourceFlavor_To_v1beta2_ResourceFlavor(in *ResourceFlavor, out *v1beta2.ResourceFlavor, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_ResourceFlavorSpec_To_v1beta2_ResourceFlavorSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_ResourceFlavor_To_v1beta2_ResourceFlavor is an autogenerated conversion function. +func Convert_v1beta1_ResourceFlavor_To_v1beta2_ResourceFlavor(in *ResourceFlavor, out *v1beta2.ResourceFlavor, s conversion.Scope) error { + return autoConvert_v1beta1_ResourceFlavor_To_v1beta2_ResourceFlavor(in, out, s) +} + +func autoConvert_v1beta2_ResourceFlavor_To_v1beta1_ResourceFlavor(in *v1beta2.ResourceFlavor, out *ResourceFlavor, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_ResourceFlavorSpec_To_v1beta1_ResourceFlavorSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_ResourceFlavor_To_v1beta1_ResourceFlavor is an autogenerated conversion function. +func Convert_v1beta2_ResourceFlavor_To_v1beta1_ResourceFlavor(in *v1beta2.ResourceFlavor, out *ResourceFlavor, s conversion.Scope) error { + return autoConvert_v1beta2_ResourceFlavor_To_v1beta1_ResourceFlavor(in, out, s) +} + +func autoConvert_v1beta1_ResourceFlavorList_To_v1beta2_ResourceFlavorList(in *ResourceFlavorList, out *v1beta2.ResourceFlavorList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.ResourceFlavor)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_ResourceFlavorList_To_v1beta2_ResourceFlavorList is an autogenerated conversion function. +func Convert_v1beta1_ResourceFlavorList_To_v1beta2_ResourceFlavorList(in *ResourceFlavorList, out *v1beta2.ResourceFlavorList, s conversion.Scope) error { + return autoConvert_v1beta1_ResourceFlavorList_To_v1beta2_ResourceFlavorList(in, out, s) +} + +func autoConvert_v1beta2_ResourceFlavorList_To_v1beta1_ResourceFlavorList(in *v1beta2.ResourceFlavorList, out *ResourceFlavorList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]ResourceFlavor)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_ResourceFlavorList_To_v1beta1_ResourceFlavorList is an autogenerated conversion function. +func Convert_v1beta2_ResourceFlavorList_To_v1beta1_ResourceFlavorList(in *v1beta2.ResourceFlavorList, out *ResourceFlavorList, s conversion.Scope) error { + return autoConvert_v1beta2_ResourceFlavorList_To_v1beta1_ResourceFlavorList(in, out, s) +} + +func autoConvert_v1beta1_ResourceFlavorSpec_To_v1beta2_ResourceFlavorSpec(in *ResourceFlavorSpec, out *v1beta2.ResourceFlavorSpec, s conversion.Scope) error { + out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels)) + out.NodeTaints = *(*[]corev1.Taint)(unsafe.Pointer(&in.NodeTaints)) + out.Tolerations = *(*[]corev1.Toleration)(unsafe.Pointer(&in.Tolerations)) + out.TopologyName = (*v1beta2.TopologyReference)(unsafe.Pointer(in.TopologyName)) + return nil +} + +// Convert_v1beta1_ResourceFlavorSpec_To_v1beta2_ResourceFlavorSpec is an autogenerated conversion function. +func Convert_v1beta1_ResourceFlavorSpec_To_v1beta2_ResourceFlavorSpec(in *ResourceFlavorSpec, out *v1beta2.ResourceFlavorSpec, s conversion.Scope) error { + return autoConvert_v1beta1_ResourceFlavorSpec_To_v1beta2_ResourceFlavorSpec(in, out, s) +} + +func autoConvert_v1beta2_ResourceFlavorSpec_To_v1beta1_ResourceFlavorSpec(in *v1beta2.ResourceFlavorSpec, out *ResourceFlavorSpec, s conversion.Scope) error { + out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels)) + out.NodeTaints = *(*[]corev1.Taint)(unsafe.Pointer(&in.NodeTaints)) + out.Tolerations = *(*[]corev1.Toleration)(unsafe.Pointer(&in.Tolerations)) + out.TopologyName = (*TopologyReference)(unsafe.Pointer(in.TopologyName)) + return nil +} + +// Convert_v1beta2_ResourceFlavorSpec_To_v1beta1_ResourceFlavorSpec is an autogenerated conversion function. +func Convert_v1beta2_ResourceFlavorSpec_To_v1beta1_ResourceFlavorSpec(in *v1beta2.ResourceFlavorSpec, out *ResourceFlavorSpec, s conversion.Scope) error { + return autoConvert_v1beta2_ResourceFlavorSpec_To_v1beta1_ResourceFlavorSpec(in, out, s) +} + +func autoConvert_v1beta1_ResourceGroup_To_v1beta2_ResourceGroup(in *ResourceGroup, out *v1beta2.ResourceGroup, s conversion.Scope) error { + out.CoveredResources = *(*[]corev1.ResourceName)(unsafe.Pointer(&in.CoveredResources)) + out.Flavors = *(*[]v1beta2.FlavorQuotas)(unsafe.Pointer(&in.Flavors)) + return nil +} + +// Convert_v1beta1_ResourceGroup_To_v1beta2_ResourceGroup is an autogenerated conversion function. +func Convert_v1beta1_ResourceGroup_To_v1beta2_ResourceGroup(in *ResourceGroup, out *v1beta2.ResourceGroup, s conversion.Scope) error { + return autoConvert_v1beta1_ResourceGroup_To_v1beta2_ResourceGroup(in, out, s) +} + +func autoConvert_v1beta2_ResourceGroup_To_v1beta1_ResourceGroup(in *v1beta2.ResourceGroup, out *ResourceGroup, s conversion.Scope) error { + out.CoveredResources = *(*[]corev1.ResourceName)(unsafe.Pointer(&in.CoveredResources)) + out.Flavors = *(*[]FlavorQuotas)(unsafe.Pointer(&in.Flavors)) + return nil +} + +// Convert_v1beta2_ResourceGroup_To_v1beta1_ResourceGroup is an autogenerated conversion function. +func Convert_v1beta2_ResourceGroup_To_v1beta1_ResourceGroup(in *v1beta2.ResourceGroup, out *ResourceGroup, s conversion.Scope) error { + return autoConvert_v1beta2_ResourceGroup_To_v1beta1_ResourceGroup(in, out, s) +} + +func autoConvert_v1beta1_ResourceQuota_To_v1beta2_ResourceQuota(in *ResourceQuota, out *v1beta2.ResourceQuota, s conversion.Scope) error { + out.Name = corev1.ResourceName(in.Name) + out.NominalQuota = in.NominalQuota + out.BorrowingLimit = (*resource.Quantity)(unsafe.Pointer(in.BorrowingLimit)) + out.LendingLimit = (*resource.Quantity)(unsafe.Pointer(in.LendingLimit)) + return nil +} + +// Convert_v1beta1_ResourceQuota_To_v1beta2_ResourceQuota is an autogenerated conversion function. +func Convert_v1beta1_ResourceQuota_To_v1beta2_ResourceQuota(in *ResourceQuota, out *v1beta2.ResourceQuota, s conversion.Scope) error { + return autoConvert_v1beta1_ResourceQuota_To_v1beta2_ResourceQuota(in, out, s) +} + +func autoConvert_v1beta2_ResourceQuota_To_v1beta1_ResourceQuota(in *v1beta2.ResourceQuota, out *ResourceQuota, s conversion.Scope) error { + out.Name = corev1.ResourceName(in.Name) + out.NominalQuota = in.NominalQuota + out.BorrowingLimit = (*resource.Quantity)(unsafe.Pointer(in.BorrowingLimit)) + out.LendingLimit = (*resource.Quantity)(unsafe.Pointer(in.LendingLimit)) + return nil +} + +// Convert_v1beta2_ResourceQuota_To_v1beta1_ResourceQuota is an autogenerated conversion function. +func Convert_v1beta2_ResourceQuota_To_v1beta1_ResourceQuota(in *v1beta2.ResourceQuota, out *ResourceQuota, s conversion.Scope) error { + return autoConvert_v1beta2_ResourceQuota_To_v1beta1_ResourceQuota(in, out, s) +} + +func autoConvert_v1beta1_ResourceUsage_To_v1beta2_ResourceUsage(in *ResourceUsage, out *v1beta2.ResourceUsage, s conversion.Scope) error { + out.Name = corev1.ResourceName(in.Name) + out.Total = in.Total + out.Borrowed = in.Borrowed + return nil +} + +// Convert_v1beta1_ResourceUsage_To_v1beta2_ResourceUsage is an autogenerated conversion function. +func Convert_v1beta1_ResourceUsage_To_v1beta2_ResourceUsage(in *ResourceUsage, out *v1beta2.ResourceUsage, s conversion.Scope) error { + return autoConvert_v1beta1_ResourceUsage_To_v1beta2_ResourceUsage(in, out, s) +} + +func autoConvert_v1beta2_ResourceUsage_To_v1beta1_ResourceUsage(in *v1beta2.ResourceUsage, out *ResourceUsage, s conversion.Scope) error { + out.Name = corev1.ResourceName(in.Name) + out.Total = in.Total + out.Borrowed = in.Borrowed + return nil +} + +// Convert_v1beta2_ResourceUsage_To_v1beta1_ResourceUsage is an autogenerated conversion function. +func Convert_v1beta2_ResourceUsage_To_v1beta1_ResourceUsage(in *v1beta2.ResourceUsage, out *ResourceUsage, s conversion.Scope) error { + return autoConvert_v1beta2_ResourceUsage_To_v1beta1_ResourceUsage(in, out, s) +} + +func autoConvert_v1beta1_SchedulingStats_To_v1beta2_SchedulingStats(in *SchedulingStats, out *v1beta2.SchedulingStats, s conversion.Scope) error { + out.Evictions = *(*[]v1beta2.WorkloadSchedulingStatsEviction)(unsafe.Pointer(&in.Evictions)) + return nil +} + +// Convert_v1beta1_SchedulingStats_To_v1beta2_SchedulingStats is an autogenerated conversion function. +func Convert_v1beta1_SchedulingStats_To_v1beta2_SchedulingStats(in *SchedulingStats, out *v1beta2.SchedulingStats, s conversion.Scope) error { + return autoConvert_v1beta1_SchedulingStats_To_v1beta2_SchedulingStats(in, out, s) +} + +func autoConvert_v1beta2_SchedulingStats_To_v1beta1_SchedulingStats(in *v1beta2.SchedulingStats, out *SchedulingStats, s conversion.Scope) error { + out.Evictions = *(*[]WorkloadSchedulingStatsEviction)(unsafe.Pointer(&in.Evictions)) + return nil +} + +// Convert_v1beta2_SchedulingStats_To_v1beta1_SchedulingStats is an autogenerated conversion function. +func Convert_v1beta2_SchedulingStats_To_v1beta1_SchedulingStats(in *v1beta2.SchedulingStats, out *SchedulingStats, s conversion.Scope) error { + return autoConvert_v1beta2_SchedulingStats_To_v1beta1_SchedulingStats(in, out, s) +} + +func autoConvert_v1beta1_Topology_To_v1beta2_Topology(in *Topology, out *v1beta2.Topology, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_TopologySpec_To_v1beta2_TopologySpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_Topology_To_v1beta2_Topology is an autogenerated conversion function. +func Convert_v1beta1_Topology_To_v1beta2_Topology(in *Topology, out *v1beta2.Topology, s conversion.Scope) error { + return autoConvert_v1beta1_Topology_To_v1beta2_Topology(in, out, s) +} + +func autoConvert_v1beta2_Topology_To_v1beta1_Topology(in *v1beta2.Topology, out *Topology, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_TopologySpec_To_v1beta1_TopologySpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_Topology_To_v1beta1_Topology is an autogenerated conversion function. +func Convert_v1beta2_Topology_To_v1beta1_Topology(in *v1beta2.Topology, out *Topology, s conversion.Scope) error { + return autoConvert_v1beta2_Topology_To_v1beta1_Topology(in, out, s) +} + +func autoConvert_v1beta1_TopologyAssignment_To_v1beta2_TopologyAssignment(in *TopologyAssignment, out *v1beta2.TopologyAssignment, s conversion.Scope) error { + out.Levels = *(*[]string)(unsafe.Pointer(&in.Levels)) + out.Domains = *(*[]v1beta2.TopologyDomainAssignment)(unsafe.Pointer(&in.Domains)) + return nil +} + +// Convert_v1beta1_TopologyAssignment_To_v1beta2_TopologyAssignment is an autogenerated conversion function. +func Convert_v1beta1_TopologyAssignment_To_v1beta2_TopologyAssignment(in *TopologyAssignment, out *v1beta2.TopologyAssignment, s conversion.Scope) error { + return autoConvert_v1beta1_TopologyAssignment_To_v1beta2_TopologyAssignment(in, out, s) +} + +func autoConvert_v1beta2_TopologyAssignment_To_v1beta1_TopologyAssignment(in *v1beta2.TopologyAssignment, out *TopologyAssignment, s conversion.Scope) error { + out.Levels = *(*[]string)(unsafe.Pointer(&in.Levels)) + out.Domains = *(*[]TopologyDomainAssignment)(unsafe.Pointer(&in.Domains)) + return nil +} + +// Convert_v1beta2_TopologyAssignment_To_v1beta1_TopologyAssignment is an autogenerated conversion function. +func Convert_v1beta2_TopologyAssignment_To_v1beta1_TopologyAssignment(in *v1beta2.TopologyAssignment, out *TopologyAssignment, s conversion.Scope) error { + return autoConvert_v1beta2_TopologyAssignment_To_v1beta1_TopologyAssignment(in, out, s) +} + +func autoConvert_v1beta1_TopologyDomainAssignment_To_v1beta2_TopologyDomainAssignment(in *TopologyDomainAssignment, out *v1beta2.TopologyDomainAssignment, s conversion.Scope) error { + out.Values = *(*[]string)(unsafe.Pointer(&in.Values)) + out.Count = in.Count + return nil +} + +// Convert_v1beta1_TopologyDomainAssignment_To_v1beta2_TopologyDomainAssignment is an autogenerated conversion function. +func Convert_v1beta1_TopologyDomainAssignment_To_v1beta2_TopologyDomainAssignment(in *TopologyDomainAssignment, out *v1beta2.TopologyDomainAssignment, s conversion.Scope) error { + return autoConvert_v1beta1_TopologyDomainAssignment_To_v1beta2_TopologyDomainAssignment(in, out, s) +} + +func autoConvert_v1beta2_TopologyDomainAssignment_To_v1beta1_TopologyDomainAssignment(in *v1beta2.TopologyDomainAssignment, out *TopologyDomainAssignment, s conversion.Scope) error { + out.Values = *(*[]string)(unsafe.Pointer(&in.Values)) + out.Count = in.Count + return nil +} + +// Convert_v1beta2_TopologyDomainAssignment_To_v1beta1_TopologyDomainAssignment is an autogenerated conversion function. +func Convert_v1beta2_TopologyDomainAssignment_To_v1beta1_TopologyDomainAssignment(in *v1beta2.TopologyDomainAssignment, out *TopologyDomainAssignment, s conversion.Scope) error { + return autoConvert_v1beta2_TopologyDomainAssignment_To_v1beta1_TopologyDomainAssignment(in, out, s) +} + +func autoConvert_v1beta1_TopologyInfo_To_v1beta2_TopologyInfo(in *TopologyInfo, out *v1beta2.TopologyInfo, s conversion.Scope) error { + out.Name = v1beta2.TopologyReference(in.Name) + out.Levels = *(*[]string)(unsafe.Pointer(&in.Levels)) + return nil +} + +// Convert_v1beta1_TopologyInfo_To_v1beta2_TopologyInfo is an autogenerated conversion function. +func Convert_v1beta1_TopologyInfo_To_v1beta2_TopologyInfo(in *TopologyInfo, out *v1beta2.TopologyInfo, s conversion.Scope) error { + return autoConvert_v1beta1_TopologyInfo_To_v1beta2_TopologyInfo(in, out, s) +} + +func autoConvert_v1beta2_TopologyInfo_To_v1beta1_TopologyInfo(in *v1beta2.TopologyInfo, out *TopologyInfo, s conversion.Scope) error { + out.Name = TopologyReference(in.Name) + out.Levels = *(*[]string)(unsafe.Pointer(&in.Levels)) + return nil +} + +// Convert_v1beta2_TopologyInfo_To_v1beta1_TopologyInfo is an autogenerated conversion function. +func Convert_v1beta2_TopologyInfo_To_v1beta1_TopologyInfo(in *v1beta2.TopologyInfo, out *TopologyInfo, s conversion.Scope) error { + return autoConvert_v1beta2_TopologyInfo_To_v1beta1_TopologyInfo(in, out, s) +} + +func autoConvert_v1beta1_TopologyLevel_To_v1beta2_TopologyLevel(in *TopologyLevel, out *v1beta2.TopologyLevel, s conversion.Scope) error { + out.NodeLabel = in.NodeLabel + return nil +} + +// Convert_v1beta1_TopologyLevel_To_v1beta2_TopologyLevel is an autogenerated conversion function. +func Convert_v1beta1_TopologyLevel_To_v1beta2_TopologyLevel(in *TopologyLevel, out *v1beta2.TopologyLevel, s conversion.Scope) error { + return autoConvert_v1beta1_TopologyLevel_To_v1beta2_TopologyLevel(in, out, s) +} + +func autoConvert_v1beta2_TopologyLevel_To_v1beta1_TopologyLevel(in *v1beta2.TopologyLevel, out *TopologyLevel, s conversion.Scope) error { + out.NodeLabel = in.NodeLabel + return nil +} + +// Convert_v1beta2_TopologyLevel_To_v1beta1_TopologyLevel is an autogenerated conversion function. +func Convert_v1beta2_TopologyLevel_To_v1beta1_TopologyLevel(in *v1beta2.TopologyLevel, out *TopologyLevel, s conversion.Scope) error { + return autoConvert_v1beta2_TopologyLevel_To_v1beta1_TopologyLevel(in, out, s) +} + +func autoConvert_v1beta1_TopologyList_To_v1beta2_TopologyList(in *TopologyList, out *v1beta2.TopologyList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.Topology)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_TopologyList_To_v1beta2_TopologyList is an autogenerated conversion function. +func Convert_v1beta1_TopologyList_To_v1beta2_TopologyList(in *TopologyList, out *v1beta2.TopologyList, s conversion.Scope) error { + return autoConvert_v1beta1_TopologyList_To_v1beta2_TopologyList(in, out, s) +} + +func autoConvert_v1beta2_TopologyList_To_v1beta1_TopologyList(in *v1beta2.TopologyList, out *TopologyList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]Topology)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_TopologyList_To_v1beta1_TopologyList is an autogenerated conversion function. +func Convert_v1beta2_TopologyList_To_v1beta1_TopologyList(in *v1beta2.TopologyList, out *TopologyList, s conversion.Scope) error { + return autoConvert_v1beta2_TopologyList_To_v1beta1_TopologyList(in, out, s) +} + +func autoConvert_v1beta1_TopologySpec_To_v1beta2_TopologySpec(in *TopologySpec, out *v1beta2.TopologySpec, s conversion.Scope) error { + out.Levels = *(*[]v1beta2.TopologyLevel)(unsafe.Pointer(&in.Levels)) + return nil +} + +// Convert_v1beta1_TopologySpec_To_v1beta2_TopologySpec is an autogenerated conversion function. +func Convert_v1beta1_TopologySpec_To_v1beta2_TopologySpec(in *TopologySpec, out *v1beta2.TopologySpec, s conversion.Scope) error { + return autoConvert_v1beta1_TopologySpec_To_v1beta2_TopologySpec(in, out, s) +} + +func autoConvert_v1beta2_TopologySpec_To_v1beta1_TopologySpec(in *v1beta2.TopologySpec, out *TopologySpec, s conversion.Scope) error { + out.Levels = *(*[]TopologyLevel)(unsafe.Pointer(&in.Levels)) + return nil +} + +// Convert_v1beta2_TopologySpec_To_v1beta1_TopologySpec is an autogenerated conversion function. +func Convert_v1beta2_TopologySpec_To_v1beta1_TopologySpec(in *v1beta2.TopologySpec, out *TopologySpec, s conversion.Scope) error { + return autoConvert_v1beta2_TopologySpec_To_v1beta1_TopologySpec(in, out, s) +} + +func autoConvert_v1beta1_UnhealthyNode_To_v1beta2_UnhealthyNode(in *UnhealthyNode, out *v1beta2.UnhealthyNode, s conversion.Scope) error { + out.Name = in.Name + return nil +} + +// Convert_v1beta1_UnhealthyNode_To_v1beta2_UnhealthyNode is an autogenerated conversion function. +func Convert_v1beta1_UnhealthyNode_To_v1beta2_UnhealthyNode(in *UnhealthyNode, out *v1beta2.UnhealthyNode, s conversion.Scope) error { + return autoConvert_v1beta1_UnhealthyNode_To_v1beta2_UnhealthyNode(in, out, s) +} + +func autoConvert_v1beta2_UnhealthyNode_To_v1beta1_UnhealthyNode(in *v1beta2.UnhealthyNode, out *UnhealthyNode, s conversion.Scope) error { + out.Name = in.Name + return nil +} + +// Convert_v1beta2_UnhealthyNode_To_v1beta1_UnhealthyNode is an autogenerated conversion function. +func Convert_v1beta2_UnhealthyNode_To_v1beta1_UnhealthyNode(in *v1beta2.UnhealthyNode, out *UnhealthyNode, s conversion.Scope) error { + return autoConvert_v1beta2_UnhealthyNode_To_v1beta1_UnhealthyNode(in, out, s) +} + +func autoConvert_v1beta1_Workload_To_v1beta2_Workload(in *Workload, out *v1beta2.Workload, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta1_WorkloadStatus_To_v1beta2_WorkloadStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_Workload_To_v1beta2_Workload is an autogenerated conversion function. +func Convert_v1beta1_Workload_To_v1beta2_Workload(in *Workload, out *v1beta2.Workload, s conversion.Scope) error { + return autoConvert_v1beta1_Workload_To_v1beta2_Workload(in, out, s) +} + +func autoConvert_v1beta2_Workload_To_v1beta1_Workload(in *v1beta2.Workload, out *Workload, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta2_WorkloadStatus_To_v1beta1_WorkloadStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_Workload_To_v1beta1_Workload is an autogenerated conversion function. +func Convert_v1beta2_Workload_To_v1beta1_Workload(in *v1beta2.Workload, out *Workload, s conversion.Scope) error { + return autoConvert_v1beta2_Workload_To_v1beta1_Workload(in, out, s) +} + +func autoConvert_v1beta1_WorkloadList_To_v1beta2_WorkloadList(in *WorkloadList, out *v1beta2.WorkloadList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1beta2.Workload, len(*in)) + for i := range *in { + if err := Convert_v1beta1_Workload_To_v1beta2_Workload(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } + return nil +} + +// Convert_v1beta1_WorkloadList_To_v1beta2_WorkloadList is an autogenerated conversion function. +func Convert_v1beta1_WorkloadList_To_v1beta2_WorkloadList(in *WorkloadList, out *v1beta2.WorkloadList, s conversion.Scope) error { + return autoConvert_v1beta1_WorkloadList_To_v1beta2_WorkloadList(in, out, s) +} + +func autoConvert_v1beta2_WorkloadList_To_v1beta1_WorkloadList(in *v1beta2.WorkloadList, out *WorkloadList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Workload, len(*in)) + for i := range *in { + if err := Convert_v1beta2_Workload_To_v1beta1_Workload(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } + return nil +} + +// Convert_v1beta2_WorkloadList_To_v1beta1_WorkloadList is an autogenerated conversion function. +func Convert_v1beta2_WorkloadList_To_v1beta1_WorkloadList(in *v1beta2.WorkloadList, out *WorkloadList, s conversion.Scope) error { + return autoConvert_v1beta2_WorkloadList_To_v1beta1_WorkloadList(in, out, s) +} + +func autoConvert_v1beta1_WorkloadPriorityClass_To_v1beta2_WorkloadPriorityClass(in *WorkloadPriorityClass, out *v1beta2.WorkloadPriorityClass, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + out.Value = in.Value + out.Description = in.Description + return nil +} + +// Convert_v1beta1_WorkloadPriorityClass_To_v1beta2_WorkloadPriorityClass is an autogenerated conversion function. +func Convert_v1beta1_WorkloadPriorityClass_To_v1beta2_WorkloadPriorityClass(in *WorkloadPriorityClass, out *v1beta2.WorkloadPriorityClass, s conversion.Scope) error { + return autoConvert_v1beta1_WorkloadPriorityClass_To_v1beta2_WorkloadPriorityClass(in, out, s) +} + +func autoConvert_v1beta2_WorkloadPriorityClass_To_v1beta1_WorkloadPriorityClass(in *v1beta2.WorkloadPriorityClass, out *WorkloadPriorityClass, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + out.Value = in.Value + out.Description = in.Description + return nil +} + +// Convert_v1beta2_WorkloadPriorityClass_To_v1beta1_WorkloadPriorityClass is an autogenerated conversion function. +func Convert_v1beta2_WorkloadPriorityClass_To_v1beta1_WorkloadPriorityClass(in *v1beta2.WorkloadPriorityClass, out *WorkloadPriorityClass, s conversion.Scope) error { + return autoConvert_v1beta2_WorkloadPriorityClass_To_v1beta1_WorkloadPriorityClass(in, out, s) +} + +func autoConvert_v1beta1_WorkloadPriorityClassList_To_v1beta2_WorkloadPriorityClassList(in *WorkloadPriorityClassList, out *v1beta2.WorkloadPriorityClassList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.WorkloadPriorityClass)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_WorkloadPriorityClassList_To_v1beta2_WorkloadPriorityClassList is an autogenerated conversion function. +func Convert_v1beta1_WorkloadPriorityClassList_To_v1beta2_WorkloadPriorityClassList(in *WorkloadPriorityClassList, out *v1beta2.WorkloadPriorityClassList, s conversion.Scope) error { + return autoConvert_v1beta1_WorkloadPriorityClassList_To_v1beta2_WorkloadPriorityClassList(in, out, s) +} + +func autoConvert_v1beta2_WorkloadPriorityClassList_To_v1beta1_WorkloadPriorityClassList(in *v1beta2.WorkloadPriorityClassList, out *WorkloadPriorityClassList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]WorkloadPriorityClass)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_WorkloadPriorityClassList_To_v1beta1_WorkloadPriorityClassList is an autogenerated conversion function. +func Convert_v1beta2_WorkloadPriorityClassList_To_v1beta1_WorkloadPriorityClassList(in *v1beta2.WorkloadPriorityClassList, out *WorkloadPriorityClassList, s conversion.Scope) error { + return autoConvert_v1beta2_WorkloadPriorityClassList_To_v1beta1_WorkloadPriorityClassList(in, out, s) +} + +func autoConvert_v1beta1_WorkloadSchedulingStatsEviction_To_v1beta2_WorkloadSchedulingStatsEviction(in *WorkloadSchedulingStatsEviction, out *v1beta2.WorkloadSchedulingStatsEviction, s conversion.Scope) error { + out.Reason = in.Reason + out.UnderlyingCause = v1beta2.EvictionUnderlyingCause(in.UnderlyingCause) + out.Count = in.Count + return nil +} + +// Convert_v1beta1_WorkloadSchedulingStatsEviction_To_v1beta2_WorkloadSchedulingStatsEviction is an autogenerated conversion function. +func Convert_v1beta1_WorkloadSchedulingStatsEviction_To_v1beta2_WorkloadSchedulingStatsEviction(in *WorkloadSchedulingStatsEviction, out *v1beta2.WorkloadSchedulingStatsEviction, s conversion.Scope) error { + return autoConvert_v1beta1_WorkloadSchedulingStatsEviction_To_v1beta2_WorkloadSchedulingStatsEviction(in, out, s) +} + +func autoConvert_v1beta2_WorkloadSchedulingStatsEviction_To_v1beta1_WorkloadSchedulingStatsEviction(in *v1beta2.WorkloadSchedulingStatsEviction, out *WorkloadSchedulingStatsEviction, s conversion.Scope) error { + out.Reason = in.Reason + out.UnderlyingCause = EvictionUnderlyingCause(in.UnderlyingCause) + out.Count = in.Count + return nil +} + +// Convert_v1beta2_WorkloadSchedulingStatsEviction_To_v1beta1_WorkloadSchedulingStatsEviction is an autogenerated conversion function. +func Convert_v1beta2_WorkloadSchedulingStatsEviction_To_v1beta1_WorkloadSchedulingStatsEviction(in *v1beta2.WorkloadSchedulingStatsEviction, out *WorkloadSchedulingStatsEviction, s conversion.Scope) error { + return autoConvert_v1beta2_WorkloadSchedulingStatsEviction_To_v1beta1_WorkloadSchedulingStatsEviction(in, out, s) +} + +func autoConvert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(in *WorkloadSpec, out *v1beta2.WorkloadSpec, s conversion.Scope) error { + out.PodSets = *(*[]v1beta2.PodSet)(unsafe.Pointer(&in.PodSets)) + out.QueueName = v1beta2.LocalQueueName(in.QueueName) + out.PriorityClassName = in.PriorityClassName + out.Priority = (*int32)(unsafe.Pointer(in.Priority)) + out.PriorityClassSource = in.PriorityClassSource + out.Active = (*bool)(unsafe.Pointer(in.Active)) + out.MaximumExecutionTimeSeconds = (*int32)(unsafe.Pointer(in.MaximumExecutionTimeSeconds)) + return nil +} + +// Convert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec is an autogenerated conversion function. +func Convert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(in *WorkloadSpec, out *v1beta2.WorkloadSpec, s conversion.Scope) error { + return autoConvert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(in, out, s) +} + +func autoConvert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(in *v1beta2.WorkloadSpec, out *WorkloadSpec, s conversion.Scope) error { + out.PodSets = *(*[]PodSet)(unsafe.Pointer(&in.PodSets)) + out.QueueName = LocalQueueName(in.QueueName) + out.PriorityClassName = in.PriorityClassName + out.Priority = (*int32)(unsafe.Pointer(in.Priority)) + out.PriorityClassSource = in.PriorityClassSource + out.Active = (*bool)(unsafe.Pointer(in.Active)) + out.MaximumExecutionTimeSeconds = (*int32)(unsafe.Pointer(in.MaximumExecutionTimeSeconds)) + return nil +} + +// Convert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec is an autogenerated conversion function. +func Convert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(in *v1beta2.WorkloadSpec, out *WorkloadSpec, s conversion.Scope) error { + return autoConvert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(in, out, s) +} + +func autoConvert_v1beta1_WorkloadStatus_To_v1beta2_WorkloadStatus(in *WorkloadStatus, out *v1beta2.WorkloadStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + out.Admission = (*v1beta2.Admission)(unsafe.Pointer(in.Admission)) + out.RequeueState = (*v1beta2.RequeueState)(unsafe.Pointer(in.RequeueState)) + out.ReclaimablePods = *(*[]v1beta2.ReclaimablePod)(unsafe.Pointer(&in.ReclaimablePods)) + out.AdmissionChecks = *(*[]v1beta2.AdmissionCheckState)(unsafe.Pointer(&in.AdmissionChecks)) + out.ResourceRequests = *(*[]v1beta2.PodSetRequest)(unsafe.Pointer(&in.ResourceRequests)) + // WARNING: in.AccumulatedPastExexcutionTimeSeconds requires manual conversion: does not exist in peer-type + out.SchedulingStats = (*v1beta2.SchedulingStats)(unsafe.Pointer(in.SchedulingStats)) + out.NominatedClusterNames = *(*[]string)(unsafe.Pointer(&in.NominatedClusterNames)) + out.ClusterName = (*string)(unsafe.Pointer(in.ClusterName)) + out.UnhealthyNodes = *(*[]v1beta2.UnhealthyNode)(unsafe.Pointer(&in.UnhealthyNodes)) + return nil +} + +func autoConvert_v1beta2_WorkloadStatus_To_v1beta1_WorkloadStatus(in *v1beta2.WorkloadStatus, out *WorkloadStatus, s conversion.Scope) error { + out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) + out.Admission = (*Admission)(unsafe.Pointer(in.Admission)) + out.RequeueState = (*RequeueState)(unsafe.Pointer(in.RequeueState)) + out.ReclaimablePods = *(*[]ReclaimablePod)(unsafe.Pointer(&in.ReclaimablePods)) + out.AdmissionChecks = *(*[]AdmissionCheckState)(unsafe.Pointer(&in.AdmissionChecks)) + out.ResourceRequests = *(*[]PodSetRequest)(unsafe.Pointer(&in.ResourceRequests)) + // WARNING: in.AccumulatedPastExecutionTimeSeconds requires manual conversion: does not exist in peer-type + out.SchedulingStats = (*SchedulingStats)(unsafe.Pointer(in.SchedulingStats)) + out.NominatedClusterNames = *(*[]string)(unsafe.Pointer(&in.NominatedClusterNames)) + out.ClusterName = (*string)(unsafe.Pointer(in.ClusterName)) + out.UnhealthyNodes = *(*[]UnhealthyNode)(unsafe.Pointer(&in.UnhealthyNodes)) + return nil +} diff --git a/apis/kueue/v1beta1/zz_generated.deepcopy.go b/apis/kueue/v1beta1/zz_generated.deepcopy.go index 951a494459a..7277dd8b717 100644 --- a/apis/kueue/v1beta1/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1beta1 import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index 868a6188df1..9f749757bf6 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -53,7 +53,7 @@ type ClusterQueueReference string type CohortReference string // ClusterQueueSpec defines the desired state of ClusterQueue -// +kubebuilder:validation:XValidation:rule="!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true", message="borrowingLimit must be nil when cohort is empty" +// +kubebuilder:validation:XValidation:rule="!has(self.cohortName) && has(self.resourceGroups) ? self.resourceGroups.all(rg, rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true", message="borrowingLimit must be nil when cohort is empty" type ClusterQueueSpec struct { // resourceGroups describes groups of resources. // Each resource group defines the list of resources and a list of flavors @@ -64,7 +64,7 @@ type ClusterQueueSpec struct { // +kubebuilder:validation:MaxItems=16 ResourceGroups []ResourceGroup `json:"resourceGroups,omitempty"` - // cohort that this ClusterQueue belongs to. CQs that belong to the + // cohortName that this ClusterQueue belongs to. CQs that belong to the // same cohort can borrow unused resources from each other. // // A CQ can be a member of a single borrowing cohort. A workload submitted @@ -76,7 +76,7 @@ type ClusterQueueSpec struct { // // A cohort is a name that links CQs together, but it doesn't reference any // object. - Cohort CohortReference `json:"cohort,omitempty"` + CohortName CohortReference `json:"cohortName,omitempty"` // queueingStrategy indicates the queueing strategy of the workloads // across the queues in this ClusterQueue. @@ -548,3 +548,5 @@ type ClusterQueueList struct { func init() { SchemeBuilder.Register(&ClusterQueue{}, &ClusterQueueList{}) } + +func (*ClusterQueue) Hub() {} diff --git a/apis/kueue/v1beta2/localqueue_types.go b/apis/kueue/v1beta2/localqueue_types.go index a774c2746a1..9ad483b1c40 100644 --- a/apis/kueue/v1beta2/localqueue_types.go +++ b/apis/kueue/v1beta2/localqueue_types.go @@ -141,13 +141,13 @@ type LocalQueueStatus struct { // +optional FlavorsReservation []LocalQueueFlavorUsage `json:"flavorsReservation"` - // flavorUsage are the used quotas, by flavor currently in use by the + // flavorsUsage are the used quotas, by flavor currently in use by the // workloads assigned to this LocalQueue. // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=16 // +optional - FlavorUsage []LocalQueueFlavorUsage `json:"flavorUsage"` + FlavorsUsage []LocalQueueFlavorUsage `json:"flavorsUsage"` // flavors lists all currently available ResourceFlavors in specified ClusterQueue. // +listType=map @@ -218,3 +218,5 @@ type LocalQueueList struct { func init() { SchemeBuilder.Register(&LocalQueue{}, &LocalQueueList{}) } + +func (*LocalQueue) Hub() {} diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index 119a9cab336..dd1b45599d4 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -405,11 +405,11 @@ type WorkloadStatus struct { // +kubebuilder:validation:MaxItems=8 ResourceRequests []PodSetRequest `json:"resourceRequests,omitempty"` - // accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent + // accumulatedPastExecutionTimeSeconds holds the total time, in seconds, the workload spent // in Admitted state, in the previous `Admit` - `Evict` cycles. // // +optional - AccumulatedPastExexcutionTimeSeconds *int32 `json:"accumulatedPastExexcutionTimeSeconds,omitempty"` + AccumulatedPastExecutionTimeSeconds *int32 `json:"accumulatedPastExecutionTimeSeconds,omitempty"` // schedulingStats tracks scheduling statistics // @@ -764,7 +764,7 @@ const ( // Workload is the Schema for the workloads API // +kubebuilder:validation:XValidation:rule="has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true", message="podSetAssignments must have the same number of podSets as the spec" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? (oldSelf.spec.priorityClassSource == self.spec.priorityClassSource) : true", message="field is immutable" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true", message="field is immutable" // +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true", message="field is immutable" // +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true", message="field is immutable" // +kubebuilder:validation:XValidation:rule="((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true", message="maximumExecutionTimeSeconds is immutable while admitted" @@ -791,3 +791,5 @@ type WorkloadList struct { func init() { SchemeBuilder.Register(&Workload{}, &WorkloadList{}) } + +func (*Workload) Hub() {} diff --git a/apis/kueue/v1beta2/zz_generated.deepcopy.go b/apis/kueue/v1beta2/zz_generated.deepcopy.go index a0f863921e1..281deaf91f6 100644 --- a/apis/kueue/v1beta2/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta2/zz_generated.deepcopy.go @@ -913,8 +913,8 @@ func (in *LocalQueueStatus) DeepCopyInto(out *LocalQueueStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.FlavorUsage != nil { - in, out := &in.FlavorUsage, &out.FlavorUsage + if in.FlavorsUsage != nil { + in, out := &in.FlavorsUsage, &out.FlavorsUsage *out = make([]LocalQueueFlavorUsage, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) @@ -2089,8 +2089,8 @@ func (in *WorkloadStatus) DeepCopyInto(out *WorkloadStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.AccumulatedPastExexcutionTimeSeconds != nil { - in, out := &in.AccumulatedPastExexcutionTimeSeconds, &out.AccumulatedPastExexcutionTimeSeconds + if in.AccumulatedPastExecutionTimeSeconds != nil { + in, out := &in.AccumulatedPastExecutionTimeSeconds, &out.AccumulatedPastExecutionTimeSeconds *out = new(int32) **out = **in } diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml index 9b631f119a8..749a5aa0439 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml @@ -873,9 +873,9 @@ spec: required: - admissionMode type: object - cohort: + cohortName: description: |- - cohort that this ClusterQueue belongs to. CQs that belong to the + cohortName that this ClusterQueue belongs to. CQs that belong to the same cohort can borrow unused resources from each other. A CQ can be a member of a single borrowing cohort. A workload submitted @@ -1251,7 +1251,7 @@ spec: type: object x-kubernetes-validations: - message: borrowingLimit must be nil when cohort is empty - rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' + rule: '!has(self.cohortName) && has(self.resourceGroups) ? self.resourceGroups.all(rg, rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' status: description: status is the status of the ClusterQueue. properties: diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml index 7e019e20684..fb717a951b3 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml @@ -622,48 +622,6 @@ spec: required: - weightedShare type: object - flavorUsage: - description: |- - flavorUsage are the used quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map flavors: description: flavors lists all currently available ResourceFlavors in specified ClusterQueue. items: @@ -796,6 +754,48 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + flavorsUsage: + description: |- + flavorsUsage are the used quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map pendingWorkloads: description: pendingWorkloads is the number of Workloads in the LocalQueue not yet admitted to a ClusterQueue format: int32 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index d37a9c35cf9..154c4d5666e 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -17145,9 +17145,9 @@ spec: status: description: status is the status of the Workload. properties: - accumulatedPastExexcutionTimeSeconds: + accumulatedPastExecutionTimeSeconds: description: |- - accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent + accumulatedPastExecutionTimeSeconds holds the total time, in seconds, the workload spent in Admitted state, in the previous `Admit` - `Evict` cycles. format: int32 type: integer @@ -17669,7 +17669,7 @@ spec: - message: podSetAssignments must have the same number of podSets as the spec rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true' - message: field is immutable - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource == self.spec.priorityClassSource) : true' + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true' - message: field is immutable rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' - message: field is immutable diff --git a/charts/kueue/templates/rbac/role.yaml b/charts/kueue/templates/rbac/role.yaml index 0c01e191726..43931821aa8 100644 --- a/charts/kueue/templates/rbac/role.yaml +++ b/charts/kueue/templates/rbac/role.yaml @@ -72,6 +72,15 @@ rules: - list - update - watch + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - update + - watch - apiGroups: - apps resources: diff --git a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuespec.go b/client-go/applyconfiguration/kueue/v1beta2/clusterqueuespec.go index 48547a5c666..b76cd0f2426 100644 --- a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuespec.go +++ b/client-go/applyconfiguration/kueue/v1beta2/clusterqueuespec.go @@ -26,7 +26,7 @@ import ( // with apply. type ClusterQueueSpecApplyConfiguration struct { ResourceGroups []ResourceGroupApplyConfiguration `json:"resourceGroups,omitempty"` - Cohort *kueuev1beta2.CohortReference `json:"cohort,omitempty"` + CohortName *kueuev1beta2.CohortReference `json:"cohortName,omitempty"` QueueingStrategy *kueuev1beta2.QueueingStrategy `json:"queueingStrategy,omitempty"` NamespaceSelector *v1.LabelSelectorApplyConfiguration `json:"namespaceSelector,omitempty"` FlavorFungibility *FlavorFungibilityApplyConfiguration `json:"flavorFungibility,omitempty"` @@ -57,11 +57,11 @@ func (b *ClusterQueueSpecApplyConfiguration) WithResourceGroups(values ...*Resou return b } -// WithCohort sets the Cohort field in the declarative configuration to the given value +// WithCohortName sets the CohortName field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Cohort field is set to the value of the last call. -func (b *ClusterQueueSpecApplyConfiguration) WithCohort(value kueuev1beta2.CohortReference) *ClusterQueueSpecApplyConfiguration { - b.Cohort = &value +// If called multiple times, the CohortName field is set to the value of the last call. +func (b *ClusterQueueSpecApplyConfiguration) WithCohortName(value kueuev1beta2.CohortReference) *ClusterQueueSpecApplyConfiguration { + b.CohortName = &value return b } diff --git a/client-go/applyconfiguration/kueue/v1beta2/localqueuestatus.go b/client-go/applyconfiguration/kueue/v1beta2/localqueuestatus.go index 575d2905cfe..c16e4ee961a 100644 --- a/client-go/applyconfiguration/kueue/v1beta2/localqueuestatus.go +++ b/client-go/applyconfiguration/kueue/v1beta2/localqueuestatus.go @@ -29,7 +29,7 @@ type LocalQueueStatusApplyConfiguration struct { ReservingWorkloads *int32 `json:"reservingWorkloads,omitempty"` AdmittedWorkloads *int32 `json:"admittedWorkloads,omitempty"` FlavorsReservation []LocalQueueFlavorUsageApplyConfiguration `json:"flavorsReservation,omitempty"` - FlavorUsage []LocalQueueFlavorUsageApplyConfiguration `json:"flavorUsage,omitempty"` + FlavorsUsage []LocalQueueFlavorUsageApplyConfiguration `json:"flavorsUsage,omitempty"` Flavors []LocalQueueFlavorStatusApplyConfiguration `json:"flavors,omitempty"` FairSharing *FairSharingStatusApplyConfiguration `json:"fairSharing,omitempty"` } @@ -90,15 +90,15 @@ func (b *LocalQueueStatusApplyConfiguration) WithFlavorsReservation(values ...*L return b } -// WithFlavorUsage adds the given value to the FlavorUsage field in the declarative configuration +// WithFlavorsUsage adds the given value to the FlavorsUsage field in the declarative configuration // and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the FlavorUsage field. -func (b *LocalQueueStatusApplyConfiguration) WithFlavorUsage(values ...*LocalQueueFlavorUsageApplyConfiguration) *LocalQueueStatusApplyConfiguration { +// If called multiple times, values provided by each call will be appended to the FlavorsUsage field. +func (b *LocalQueueStatusApplyConfiguration) WithFlavorsUsage(values ...*LocalQueueFlavorUsageApplyConfiguration) *LocalQueueStatusApplyConfiguration { for i := range values { if values[i] == nil { - panic("nil value passed to WithFlavorUsage") + panic("nil value passed to WithFlavorsUsage") } - b.FlavorUsage = append(b.FlavorUsage, *values[i]) + b.FlavorsUsage = append(b.FlavorsUsage, *values[i]) } return b } diff --git a/client-go/applyconfiguration/kueue/v1beta2/workloadstatus.go b/client-go/applyconfiguration/kueue/v1beta2/workloadstatus.go index 355389c4111..9d1179a72ad 100644 --- a/client-go/applyconfiguration/kueue/v1beta2/workloadstatus.go +++ b/client-go/applyconfiguration/kueue/v1beta2/workloadstatus.go @@ -24,17 +24,17 @@ import ( // WorkloadStatusApplyConfiguration represents a declarative configuration of the WorkloadStatus type for use // with apply. type WorkloadStatusApplyConfiguration struct { - Conditions []v1.ConditionApplyConfiguration `json:"conditions,omitempty"` - Admission *AdmissionApplyConfiguration `json:"admission,omitempty"` - RequeueState *RequeueStateApplyConfiguration `json:"requeueState,omitempty"` - ReclaimablePods []ReclaimablePodApplyConfiguration `json:"reclaimablePods,omitempty"` - AdmissionChecks []AdmissionCheckStateApplyConfiguration `json:"admissionChecks,omitempty"` - ResourceRequests []PodSetRequestApplyConfiguration `json:"resourceRequests,omitempty"` - AccumulatedPastExexcutionTimeSeconds *int32 `json:"accumulatedPastExexcutionTimeSeconds,omitempty"` - SchedulingStats *SchedulingStatsApplyConfiguration `json:"schedulingStats,omitempty"` - NominatedClusterNames []string `json:"nominatedClusterNames,omitempty"` - ClusterName *string `json:"clusterName,omitempty"` - UnhealthyNodes []UnhealthyNodeApplyConfiguration `json:"unhealthyNodes,omitempty"` + Conditions []v1.ConditionApplyConfiguration `json:"conditions,omitempty"` + Admission *AdmissionApplyConfiguration `json:"admission,omitempty"` + RequeueState *RequeueStateApplyConfiguration `json:"requeueState,omitempty"` + ReclaimablePods []ReclaimablePodApplyConfiguration `json:"reclaimablePods,omitempty"` + AdmissionChecks []AdmissionCheckStateApplyConfiguration `json:"admissionChecks,omitempty"` + ResourceRequests []PodSetRequestApplyConfiguration `json:"resourceRequests,omitempty"` + AccumulatedPastExecutionTimeSeconds *int32 `json:"accumulatedPastExecutionTimeSeconds,omitempty"` + SchedulingStats *SchedulingStatsApplyConfiguration `json:"schedulingStats,omitempty"` + NominatedClusterNames []string `json:"nominatedClusterNames,omitempty"` + ClusterName *string `json:"clusterName,omitempty"` + UnhealthyNodes []UnhealthyNodeApplyConfiguration `json:"unhealthyNodes,omitempty"` } // WorkloadStatusApplyConfiguration constructs a declarative configuration of the WorkloadStatus type for use with @@ -111,11 +111,11 @@ func (b *WorkloadStatusApplyConfiguration) WithResourceRequests(values ...*PodSe return b } -// WithAccumulatedPastExexcutionTimeSeconds sets the AccumulatedPastExexcutionTimeSeconds field in the declarative configuration to the given value +// WithAccumulatedPastExecutionTimeSeconds sets the AccumulatedPastExecutionTimeSeconds field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the AccumulatedPastExexcutionTimeSeconds field is set to the value of the last call. -func (b *WorkloadStatusApplyConfiguration) WithAccumulatedPastExexcutionTimeSeconds(value int32) *WorkloadStatusApplyConfiguration { - b.AccumulatedPastExexcutionTimeSeconds = &value +// If called multiple times, the AccumulatedPastExecutionTimeSeconds field is set to the value of the last call. +func (b *WorkloadStatusApplyConfiguration) WithAccumulatedPastExecutionTimeSeconds(value int32) *WorkloadStatusApplyConfiguration { + b.AccumulatedPastExecutionTimeSeconds = &value return b } diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml index bb64f68cc89..e42809e8f55 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml @@ -877,9 +877,9 @@ spec: required: - admissionMode type: object - cohort: + cohortName: description: |- - cohort that this ClusterQueue belongs to. CQs that belong to the + cohortName that this ClusterQueue belongs to. CQs that belong to the same cohort can borrow unused resources from each other. A CQ can be a member of a single borrowing cohort. A workload submitted @@ -1260,7 +1260,7 @@ spec: type: object x-kubernetes-validations: - message: borrowingLimit must be nil when cohort is empty - rule: '!has(self.cohort) && has(self.resourceGroups) ? self.resourceGroups.all(rg, + rule: '!has(self.cohortName) && has(self.resourceGroups) ? self.resourceGroups.all(rg, rg.flavors.all(f, f.resources.all(r, !has(r.borrowingLimit)))) : true' status: description: status is the status of the ClusterQueue. diff --git a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml index 606ed2fe447..e1f2d8b8a23 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml @@ -624,49 +624,6 @@ spec: required: - weightedShare type: object - flavorUsage: - description: |- - flavorUsage are the used quotas, by flavor currently in use by the - workloads assigned to this LocalQueue. - items: - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - resources: - description: resources lists the quota usage for the resources - in this flavor. - items: - properties: - name: - description: name of the resource. - type: string - total: - anyOf: - - type: integer - - type: string - description: total is the total quantity of used quota. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - required: - - name - - resources - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map flavors: description: flavors lists all currently available ResourceFlavors in specified ClusterQueue. @@ -805,6 +762,49 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + flavorsUsage: + description: |- + flavorsUsage are the used quotas, by flavor currently in use by the + workloads assigned to this LocalQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + resources: + description: resources lists the quota usage for the resources + in this flavor. + items: + properties: + name: + description: name of the resource. + type: string + total: + anyOf: + - type: integer + - type: string + description: total is the total quantity of used quota. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - name + - resources + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map pendingWorkloads: description: pendingWorkloads is the number of Workloads in the LocalQueue not yet admitted to a ClusterQueue diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index 35c96cfb779..03e53c0bf7c 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -18139,9 +18139,9 @@ spec: status: description: status is the status of the Workload. properties: - accumulatedPastExexcutionTimeSeconds: + accumulatedPastExecutionTimeSeconds: description: |- - accumulatedPastExexcutionTimeSeconds holds the total time, in seconds, the workload spent + accumulatedPastExecutionTimeSeconds holds the total time, in seconds, the workload spent in Admitted state, in the previous `Admit` - `Evict` cycles. format: int32 type: integer @@ -18694,8 +18694,13 @@ spec: : true' - message: field is immutable rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource - == self.spec.priorityClassSource) : true' + c.type == ''QuotaReserved'' && c.status == ''True'')) ? ((!has(self.spec.priorityClassSource) + && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) + && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource + == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) + && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() + == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) + && self.spec.priorityClassSource.size() == 0)) : true' - message: field is immutable rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) diff --git a/config/components/crd/kustomization.yaml b/config/components/crd/kustomization.yaml index c59e3a9a26c..c5779dd7088 100644 --- a/config/components/crd/kustomization.yaml +++ b/config/components/crd/kustomization.yaml @@ -18,9 +18,9 @@ resources: patches: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD -#- path: patches/webhook_in_localqueues.yaml -#- path: patches/webhook_in_clusterqueues.yaml -#- path: patches/webhook_in_workloads.yaml +- path: patches/webhook_in_localqueues.yaml +- path: patches/webhook_in_clusterqueues.yaml +- path: patches/webhook_in_workloads.yaml #- path: patches/webhook_in_resourceflavors.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch diff --git a/config/components/rbac/role.yaml b/config/components/rbac/role.yaml index cbe4c80a09d..f6d95530689 100644 --- a/config/components/rbac/role.yaml +++ b/config/components/rbac/role.yaml @@ -69,6 +69,15 @@ rules: - list - update - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - update + - watch - apiGroups: - apps resources: diff --git a/pkg/util/cert/cert.go b/pkg/util/cert/cert.go index 308c72a20b3..7fe7e236ed2 100644 --- a/pkg/util/cert/cert.go +++ b/pkg/util/cert/cert.go @@ -36,6 +36,7 @@ const ( // +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=get;list;watch;update // +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch;update // ManageCerts creates all certs for webhooks. This function is called from main.go. func ManageCerts(mgr ctrl.Manager, cfg config.Configuration, setupFinished chan struct{}) error { @@ -63,6 +64,15 @@ func ManageCerts(mgr ctrl.Manager, cfg config.Configuration, setupFinished chan }, { Type: cert.Mutating, Name: mutatingWebhookName, + }, { + Type: cert.CRDConversion, + Name: "localqueues.kueue.x-k8s.io", + }, { + Type: cert.CRDConversion, + Name: "clusterqueues.kueue.x-k8s.io", + }, { + Type: cert.CRDConversion, + Name: "workloads.kueue.x-k8s.io", }}, // When kueue is running in the leader election mode, // we expect webhook server will run in primary and secondary instance diff --git a/pkg/util/testing/v1beta2/wrappers.go b/pkg/util/testing/v1beta2/wrappers.go index 1b28d7d562d..47a3472a400 100644 --- a/pkg/util/testing/v1beta2/wrappers.go +++ b/pkg/util/testing/v1beta2/wrappers.go @@ -391,7 +391,7 @@ func (w *WorkloadWrapper) MaximumExecutionTimeSeconds(v int32) *WorkloadWrapper } func (w *WorkloadWrapper) PastAdmittedTime(v int32) *WorkloadWrapper { - w.Status.AccumulatedPastExexcutionTimeSeconds = &v + w.Status.AccumulatedPastExecutionTimeSeconds = &v return w } @@ -862,7 +862,7 @@ func (c *ClusterQueueWrapper) Obj() *kueue.ClusterQueue { // Cohort sets the borrowing cohort. func (c *ClusterQueueWrapper) Cohort(cohort kueue.CohortReference) *ClusterQueueWrapper { - c.Spec.Cohort = cohort + c.Spec.CohortName = cohort return c } diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go index cb7c832bbed..7c661634515 100644 --- a/pkg/webhooks/webhooks.go +++ b/pkg/webhooks/webhooks.go @@ -18,6 +18,8 @@ package webhooks import ( ctrl "sigs.k8s.io/controller-runtime" + + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" ) // Setup sets up the webhooks for core controllers. It returns the name of the @@ -39,5 +41,9 @@ func Setup(mgr ctrl.Manager) (string, error) { return "Cohort", err } + if err := ctrl.NewWebhookManagedBy(mgr).For(&kueue.LocalQueue{}).Complete(); err != nil { + return "LocalQueue", err + } + return "", nil } diff --git a/test/integration/framework/framework.go b/test/integration/framework/framework.go index 4b9eaaeab52..3af92c5bb75 100644 --- a/test/integration/framework/framework.go +++ b/test/integration/framework/framework.go @@ -51,6 +51,8 @@ import ( config "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" "sigs.k8s.io/kueue/test/util" ) @@ -74,11 +76,19 @@ func (f *Framework) Init() *rest.Config { var cfg *rest.Config ginkgo.By("bootstrapping test environment", func() { - baseCrdPath := filepath.Join(util.GetProjectBaseDir(), "config", "components", "crd", "bases") + baseCrdPath := filepath.Join(util.GetProjectBaseDir(), "config", "components", "crd", "_output") f.testEnv = &envtest.Environment{ CRDDirectoryPaths: append(f.DepCRDPaths, baseCrdPath), ErrorIfCRDPathMissing: true, } + var err error + f.testEnv.Scheme = scheme.Scheme + err = kueue.AddToScheme(f.testEnv.Scheme) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + + err = kueuev1beta2.AddToScheme(f.testEnv.Scheme) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + if len(f.WebhookPath) > 0 { f.testEnv.WebhookInstallOptions.Paths = []string{f.WebhookPath} } @@ -97,7 +107,6 @@ func (f *Framework) Init() *rest.Config { f.testEnv.ControlPlane.GetAPIServer().Err = ginkgo.GinkgoWriter } - var err error cfg, err = f.testEnv.Start() gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) gomega.ExpectWithOffset(1, cfg).NotTo(gomega.BeNil()) @@ -114,6 +123,9 @@ func (f *Framework) SetupClient(cfg *rest.Config) (context.Context, client.Clien err = kueue.AddToScheme(f.scheme) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + err = kueuev1beta2.AddToScheme(f.scheme) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + err = awv1beta2.AddToScheme(f.scheme) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) diff --git a/test/integration/multikueue/suite_test.go b/test/integration/multikueue/suite_test.go index 9a8428d0bb1..9559d9157f5 100644 --- a/test/integration/multikueue/suite_test.go +++ b/test/integration/multikueue/suite_test.go @@ -82,6 +82,11 @@ var ( worker1TestCluster cluster worker2TestCluster cluster managersConfigNamespace *corev1.Namespace + + // makes sure there is only one fwk.Init and setupClient at the same time + // since these functions are not thread safe due to adding to the common + // schema. + mu sync.Mutex ) func TestMultiKueue(t *testing.T) { @@ -106,8 +111,10 @@ func createCluster(setupFnc framework.ManagerSetup, apiFeatureGates ...string) c }, APIServerFeatureGates: apiFeatureGates, } + mu.Lock() c.cfg = c.fwk.Init() c.ctx, c.client = c.fwk.SetupClient(c.cfg) + mu.Unlock() // skip the manager setup if setup func is not provided if setupFnc != nil { @@ -359,6 +366,7 @@ func managerAndMultiKueueSetup( var _ = ginkgo.BeforeSuite(func() { var managerFeatureGates []string ginkgo.By("creating the clusters", func() { + mu = sync.Mutex{} wg := sync.WaitGroup{} wg.Add(3) go func() { diff --git a/test/integration/singlecluster/conversion/conversions_v1beta2_test.go b/test/integration/singlecluster/conversion/conversions_v1beta2_test.go new file mode 100644 index 00000000000..17e2d1dda49 --- /dev/null +++ b/test/integration/singlecluster/conversion/conversions_v1beta2_test.go @@ -0,0 +1,335 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/pkg/controller/constants" + workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" + "sigs.k8s.io/kueue/test/integration/framework" + "sigs.k8s.io/kueue/test/util" +) + +var _ = ginkgo.Describe("v1beta2 conversions", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { + const ( + flavorModelC = "model-c" + flavorModelD = "model-d" + + resourceGPU corev1.ResourceName = "example.com/gpu" + ) + var ( + ns *corev1.Namespace + localQueue *kueue.LocalQueue + clusterQueues []*kueue.ClusterQueue + jobs []*batchv1.Job + resourceFlavors []kueue.ResourceFlavor + workloadPriorityClasses []kueue.WorkloadPriorityClass + ) + + ginkgo.BeforeEach(func() { + ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "core-queue-") + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + for _, cq := range clusterQueues { + util.ExpectObjectToBeDeleted(ctx, k8sClient, cq, true) + } + for _, rf := range resourceFlavors { + util.ExpectObjectToBeDeleted(ctx, k8sClient, &rf, true) + } + for _, wpc := range workloadPriorityClasses { + util.ExpectObjectToBeDeleted(ctx, k8sClient, &wpc, true) + } + }) + + ginkgo.It("Should update status when workloads are created", framework.SlowSpec, func() { + cq1 := utiltestingapi.MakeClusterQueue("cluster-queue.queue-controller"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(flavorModelC).Resource(resourceGPU, "2", "2").Obj(), + *utiltestingapi.MakeFlavorQuotas(flavorModelD).Resource(resourceGPU, "2", "2").Obj(), + ). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Cohort("cohort"). + Obj() + cq2 := utiltestingapi.MakeClusterQueue("shared-pool"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(flavorModelC).Resource(resourceGPU, "2", "2").Obj(), + *utiltestingapi.MakeFlavorQuotas(flavorModelD).Resource(resourceGPU, "2", "2").Obj(), + ). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Cohort("cohort"). + Obj() + clusterQueues = []*kueue.ClusterQueue{cq1, cq2} + + localQueue = utiltestingapi.MakeLocalQueue("queue", ns.Name).ClusterQueue(cq1.Name).Obj() + util.MustCreate(ctx, k8sClient, localQueue) + + ginkgo.By("Creating resourceFlavors") + resourceFlavors = []kueue.ResourceFlavor{ + *utiltestingapi.MakeResourceFlavor(flavorModelC).NodeLabel(resourceGPU.String(), flavorModelC).Obj(), + *utiltestingapi.MakeResourceFlavor(flavorModelD).NodeLabel(resourceGPU.String(), flavorModelD).Obj(), + } + for _, rf := range resourceFlavors { + util.MustCreate(ctx, k8sClient, &rf) + } + + ginkgo.By("Creating clusterQueues") + for _, cq := range clusterQueues { + util.MustCreate(ctx, k8sClient, cq) + } + + ginkgo.By("Verify the cohort setting on the clusterQueue") + gomega.Eventually(func(g gomega.Gomega) { + var updatedCq kueue.ClusterQueue + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueues[0]), &updatedCq)).To(gomega.Succeed()) + g.Expect(updatedCq.Spec.CohortName).Should(gomega.Equal(kueue.CohortReference("cohort"))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + + ginkgo.By("await for the LocalQueue to be ready") + emptyUsage := []kueue.LocalQueueFlavorUsage{ + { + Name: flavorModelC, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("0"), + }, + }, + }, + { + Name: flavorModelD, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("0"), + }, + }, + }, + } + gomega.Eventually(func(g gomega.Gomega) { + var updatedQueue kueue.LocalQueue + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(localQueue), &updatedQueue)).To(gomega.Succeed()) + g.Expect(updatedQueue.Status).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionTrue, + Reason: "Ready", + Message: "Can submit new workloads to localQueue", + }, + }, + FlavorsReservation: emptyUsage, + FlavorsUsage: emptyUsage, + }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors"))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + + ginkgo.By("Creating workloads wave1") + job1 := testingjob.MakeJob("job1", ns.Name). + Label(constants.QueueLabel, localQueue.Name). + RequestAndLimit(resourceGPU, "4").Obj() + util.MustCreate(ctx, k8sClient, job1) + jobs = append(jobs, job1) + + ginkgo.By("Verify the LocalQueue flavor usage is updated correctly") + partUsage := []kueue.LocalQueueFlavorUsage{ + { + Name: flavorModelC, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("4"), + }, + }, + }, + { + Name: flavorModelD, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("0"), + }, + }, + }, + } + gomega.Eventually(func(g gomega.Gomega) { + var updatedQueue kueue.LocalQueue + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(localQueue), &updatedQueue)).To(gomega.Succeed()) + g.Expect(updatedQueue.Status).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + ReservingWorkloads: 1, + AdmittedWorkloads: 1, + PendingWorkloads: 0, + FlavorsReservation: partUsage, + FlavorsUsage: partUsage, + }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors", "Conditions"))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + + ginkgo.By("Creating workloads wave2") + job2 := testingjob.MakeJob("job2", ns.Name). + Label(constants.QueueLabel, localQueue.Name). + RequestAndLimit(resourceGPU, "4").Obj() + util.MustCreate(ctx, k8sClient, job2) + jobs = append(jobs, job2) + + ginkgo.By("Verify the LocalQueue flavor usage is updated correctly") + fullUsage := []kueue.LocalQueueFlavorUsage{ + { + Name: flavorModelC, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("4"), + }, + }, + }, + { + Name: flavorModelD, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("4"), + }, + }, + }, + } + gomega.Eventually(func(g gomega.Gomega) { + var updatedQueue kueue.LocalQueue + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(localQueue), &updatedQueue)).To(gomega.Succeed()) + g.Expect(updatedQueue.Status).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + ReservingWorkloads: 2, + AdmittedWorkloads: 2, + PendingWorkloads: 0, + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionTrue, + Reason: "Ready", + Message: "Can submit new workloads to localQueue", + }, + }, + FlavorsReservation: fullUsage, + FlavorsUsage: fullUsage, + }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors"))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.It("Should correctly accout for Workload accumulated execution time", framework.SlowSpec, func() { + highWorkloadPriorityClass := utiltestingapi.MakeWorkloadPriorityClass("high-workload").PriorityValue(100).Obj() + util.MustCreate(ctx, k8sClient, highWorkloadPriorityClass) + workloadPriorityClasses = append(workloadPriorityClasses, *highWorkloadPriorityClass) + + ginkgo.By("Creating resourceFlavors") + resourceFlavors = []kueue.ResourceFlavor{ + *utiltestingapi.MakeResourceFlavor(flavorModelC).NodeLabel(resourceGPU.String(), flavorModelC).Obj(), + } + for _, rf := range resourceFlavors { + util.MustCreate(ctx, k8sClient, &rf) + } + + ginkgo.By("Creating clusterQueues") + cq1 := utiltestingapi.MakeClusterQueue("cluster-queue.queue-controller"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(flavorModelC).Resource(resourceGPU, "2").Obj(), + ). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Obj() + clusterQueues = []*kueue.ClusterQueue{cq1} + + for _, cq := range clusterQueues { + util.MustCreate(ctx, k8sClient, cq) + } + + ginkgo.By("creating LocalQueue") + localQueue = utiltestingapi.MakeLocalQueue("queue", ns.Name).ClusterQueue(cq1.Name).Obj() + util.MustCreate(ctx, k8sClient, localQueue) + + ginkgo.By("await for the LocalQueue to be ready") + gomega.Eventually(func(g gomega.Gomega) { + var updatedQueue kueue.LocalQueue + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(localQueue), &updatedQueue)).To(gomega.Succeed()) + g.Expect(updatedQueue.Status).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionTrue, + Reason: "Ready", + Message: "Can submit new workloads to localQueue", + }, + }, + }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors", "FlavorsReservation", "FlavorsUsage"))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + + ginkgo.By("Creating workload1") + job1 := testingjob.MakeJob("job1", ns.Name). + Label(constants.QueueLabel, localQueue.Name). + Label(constants.MaxExecTimeSecondsLabel, "30"). + RequestAndLimit(resourceGPU, "2").Obj() + util.MustCreate(ctx, k8sClient, job1) + jobs = append(jobs, job1) + + ginkgo.By("Verify the LocalQueue has the workload admitted") + gomega.Eventually(func(g gomega.Gomega) { + var updatedQueue kueue.LocalQueue + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(localQueue), &updatedQueue)).To(gomega.Succeed()) + g.Expect(updatedQueue.Status).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + ReservingWorkloads: 1, + AdmittedWorkloads: 1, + PendingWorkloads: 0, + }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors", "FlavorsReservation", "FlavorsUsage", "Conditions"))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + + ginkgo.By("Creating workload2") + job2 := testingjob.MakeJob("job2", ns.Name). + Label(constants.QueueLabel, localQueue.Name). + WorkloadPriorityClass("high-workload"). + RequestAndLimit(resourceGPU, "2").Obj() + util.MustCreate(ctx, k8sClient, job2) + jobs = append(jobs, job2) + + wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job1.Name, job1.UID), Namespace: ns.Name} + + ginkgo.By("Verify workload2 is correct") + gomega.Eventually(func(g gomega.Gomega) { + var updatedWl kueue.Workload + g.Expect(k8sClient.Get(ctx, wlLookupKey, &updatedWl)).To(gomega.Succeed()) + g.Expect(apimeta.IsStatusConditionTrue(updatedWl.Status.Conditions, kueue.WorkloadAdmitted)).Should(gomega.BeFalse()) + g.Expect(apimeta.IsStatusConditionTrue(updatedWl.Status.Conditions, kueue.WorkloadQuotaReserved)).Should(gomega.BeFalse()) + g.Expect(apimeta.IsStatusConditionTrue(updatedWl.Status.Conditions, kueue.WorkloadEvicted)).Should(gomega.BeTrue()) + g.Expect(updatedWl.Spec.MaximumExecutionTimeSeconds).ShouldNot(gomega.BeNil()) + g.Expect(updatedWl.Status.AccumulatedPastExecutionTimeSeconds).ShouldNot(gomega.BeNil()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) +}) diff --git a/test/integration/singlecluster/conversion/suite_test.go b/test/integration/singlecluster/conversion/suite_test.go new file mode 100644 index 00000000000..6d902a5895d --- /dev/null +++ b/test/integration/singlecluster/conversion/suite_test.go @@ -0,0 +1,104 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "context" + "testing" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + config "sigs.k8s.io/kueue/apis/config/v1beta1" + qcache "sigs.k8s.io/kueue/pkg/cache/queue" + schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" + "sigs.k8s.io/kueue/pkg/constants" + "sigs.k8s.io/kueue/pkg/controller/core" + "sigs.k8s.io/kueue/pkg/controller/core/indexer" + "sigs.k8s.io/kueue/pkg/controller/jobframework" + workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" + "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" + "sigs.k8s.io/kueue/test/integration/framework" + "sigs.k8s.io/kueue/test/util" +) + +var ( + cfg *rest.Config + k8sClient client.Client + ctx context.Context + fwk *framework.Framework +) + +func TestConversion(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + + ginkgo.RunSpecs(t, + "Conversion Suite", + ) +} + +var _ = ginkgo.BeforeSuite(func() { + fwk = &framework.Framework{ + WebhookPath: util.WebhookPath, + } + cfg = fwk.Init() + ctx, k8sClient = fwk.SetupClient(cfg) + fwk.StartManager(ctx, cfg, managerAndSchedulerSetup) +}) + +var _ = ginkgo.AfterSuite(func() { + fwk.Teardown() +}) + +func managerAndSchedulerSetup(ctx context.Context, mgr manager.Manager) { + err := indexer.Setup(ctx, mgr.GetFieldIndexer()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + cCache := schdcache.New(mgr.GetClient()) + queues := qcache.NewManager(mgr.GetClient(), cCache) + + configuration := &config.Configuration{} + mgr.GetScheme().Default(configuration) + + failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) + + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) + + jobReconciler, err := workloadjob.NewReconciler( + ctx, + mgr.GetClient(), + mgr.GetFieldIndexer(), + mgr.GetEventRecorderFor(constants.JobControllerName)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = workloadjob.SetupIndexes(ctx, mgr.GetFieldIndexer()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = jobReconciler.SetupWithManager(mgr) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = workloadjob.SetupWebhook(mgr, jobframework.WithCache(cCache), jobframework.WithQueues(queues)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + jobframework.EnableIntegration(workloadjob.FrameworkName) + + sched := scheduler.New(queues, cCache, mgr.GetClient(), mgr.GetEventRecorderFor(constants.AdmissionName)) + err = sched.Start(ctx) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} From 5dc4f0aa4aff9a9e957bc92ec184b3fb26ab3513 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 23 Oct 2025 17:07:34 +0530 Subject: [PATCH 015/119] Extend immutable error messages. (#7354) --- apis/kueue/v1beta1/workload_types.go | 8 ++++---- apis/kueue/v1beta2/workload_types.go | 8 ++++---- .../templates/crd/kueue.x-k8s.io_workloads.yaml | 16 ++++++++-------- .../crd/bases/kueue.x-k8s.io_workloads.yaml | 16 ++++++++-------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/apis/kueue/v1beta1/workload_types.go b/apis/kueue/v1beta1/workload_types.go index 8781aa2e8ce..7d297f96f15 100644 --- a/apis/kueue/v1beta1/workload_types.go +++ b/apis/kueue/v1beta1/workload_types.go @@ -765,10 +765,10 @@ const ( // Workload is the Schema for the workloads API // +kubebuilder:validation:XValidation:rule="has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true", message="podSetAssignments must have the same number of podSets as the spec" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? (oldSelf.spec.priorityClassSource == self.spec.priorityClassSource) : true", message="field is immutable" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true", message="field is immutable" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true", message="field is immutable" -// +kubebuilder:validation:XValidation:rule="((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true", message="maximumExecutionTimeSeconds is immutable while admitted" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? (oldSelf.spec.priorityClassSource == self.spec.priorityClassSource) : true", message="priorityClassSource is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true", message="priorityClassName is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true", message="queueName is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true", message="maximumExecutionTimeSeconds is immutable while workload quota reserved" type Workload struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the Workload. diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index dd1b45599d4..5010fa0e051 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -764,10 +764,10 @@ const ( // Workload is the Schema for the workloads API // +kubebuilder:validation:XValidation:rule="has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true", message="podSetAssignments must have the same number of podSets as the spec" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true", message="field is immutable" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true", message="field is immutable" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true", message="field is immutable" -// +kubebuilder:validation:XValidation:rule="((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true", message="maximumExecutionTimeSeconds is immutable while admitted" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true", message="priorityClassSource is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true",message="priorityClassName is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true", message="queueName is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true", message="maximumExecutionTimeSeconds is immutable while workload quota reserved" type Workload struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the Workload. diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index 154c4d5666e..da71f3d5f1b 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -8844,13 +8844,13 @@ spec: x-kubernetes-validations: - message: podSetAssignments must have the same number of podSets as the spec rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true' - - message: field is immutable + - message: priorityClassSource is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource == self.spec.priorityClassSource) : true' - - message: field is immutable + - message: priorityClassName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' - - message: field is immutable + - message: queueName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true' - - message: maximumExecutionTimeSeconds is immutable while admitted + - message: maximumExecutionTimeSeconds is immutable while workload quota reserved rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true served: true storage: true @@ -17668,13 +17668,13 @@ spec: x-kubernetes-validations: - message: podSetAssignments must have the same number of podSets as the spec rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true' - - message: field is immutable + - message: priorityClassSource is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true' - - message: field is immutable + - message: priorityClassName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' - - message: field is immutable + - message: queueName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true' - - message: maximumExecutionTimeSeconds is immutable while admitted + - message: maximumExecutionTimeSeconds is immutable while workload quota reserved rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true served: true storage: false diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index 03e53c0bf7c..e837266742a 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -9341,23 +9341,23 @@ spec: c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true' - - message: field is immutable + - message: priorityClassSource is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource == self.spec.priorityClassSource) : true' - - message: field is immutable + - message: priorityClassName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' - - message: field is immutable + - message: queueName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true' - - message: maximumExecutionTimeSeconds is immutable while admitted + - message: maximumExecutionTimeSeconds is immutable while workload quota reserved rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == @@ -18692,7 +18692,7 @@ spec: c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true' - - message: field is immutable + - message: priorityClassSource is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) @@ -18701,19 +18701,19 @@ spec: && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true' - - message: field is immutable + - message: priorityClassName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' - - message: field is immutable + - message: queueName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true' - - message: maximumExecutionTimeSeconds is immutable while admitted + - message: maximumExecutionTimeSeconds is immutable while workload quota reserved rule: ((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == From 4f9e8e4909f916401cd806a3a001270017f0e9f2 Mon Sep 17 00:00:00 2001 From: Yuki Iwai Date: Thu, 23 Oct 2025 21:17:34 +0900 Subject: [PATCH 016/119] chore: Use utiltesting context in DRA UTs (#7356) Signed-off-by: Yuki Iwai --- pkg/dra/claims_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/dra/claims_test.go b/pkg/dra/claims_test.go index 8674e45742c..970784b21c5 100644 --- a/pkg/dra/claims_test.go +++ b/pkg/dra/claims_test.go @@ -17,7 +17,6 @@ limitations under the License. package dra_test import ( - "context" "reflect" "testing" @@ -257,7 +256,8 @@ func Test_GetResourceRequests(t *testing.T) { tc.modifyWL(wlCopy) } - got, err := dra.GetResourceRequestsForResourceClaimTemplates(context.Background(), baseClient, wlCopy) + ctx, _ := utiltesting.ContextWithLog(t) + got, err := dra.GetResourceRequestsForResourceClaimTemplates(ctx, baseClient, wlCopy) if (err != nil) != tc.wantErr { t.Fatalf("unexpected error status: gotErr=%v wantErr=%v, err=%v", err != nil, tc.wantErr, err) } From 26d469b06a92388c4ce3ec3cd87ce0d27ccf6b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Thu, 23 Oct 2025 14:17:40 +0200 Subject: [PATCH 017/119] Update main after 0.13.7 (#7360) --- CHANGELOG/CHANGELOG-0.13.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/CHANGELOG/CHANGELOG-0.13.md b/CHANGELOG/CHANGELOG-0.13.md index 189e1294a5a..0a374526785 100644 --- a/CHANGELOG/CHANGELOG-0.13.md +++ b/CHANGELOG/CHANGELOG-0.13.md @@ -1,3 +1,33 @@ +## v0.13.7 + +Changes since `v0.13.6`: + +## Changes by Kind + +### Feature + +- JobFramework: Introduce an optional interface for custom Jobs, called JobWithCustomWorkloadActivation, which can be used to deactivate or active a custom CRD workload. (#7199, @tg123) + +### Bug or Regression + +- Fix existing workloads not being re-evaluated when new clusters are added to MultiKueueConfig. Previously, only newly created workloads would see updated cluster lists. (#7351, @mimowo) +- Fix handling of RayJobs which specify the spec.clusterSelector and the "queue-name" label for Kueue. These jobs should be ignored by kueue as they are being submitted to a RayCluster which is where the resources are being used and was likely already admitted by kueue. No need to double admit. + Fix on a panic on kueue managed jobs if spec.rayClusterSpec wasn't specified. (#7257, @laurafitzgerald) +- Fixed a bug that Kueue would keep sending empty updates to a Workload, along with sending the "UpdatedWorkload" event, even if the Workload didn't change. This would happen for Workloads using any other mechanism for setting + the priority than the WorkloadPriorityClass, eg. for Workloads for PodGroups. (#7306, @mbobrovskyi) +- MultiKueue x ElasticJobs: fix webhook validation bug which prevented scale up operation when any other + than the default "AllAtOnce" MultiKueue dispatcher was used. (#7333, @mszadkow) +- Visibility API: Fix a bug that the Config clientConnection is not respected in the visibility server. (#7224, @tenzen-y) + +### Other (Cleanup or Flake) + +- Improve the messages presented to the user in scheduling events, by clarifying the reason for "insufficient quota" + in case of workloads with multiple PodSets. + + Example: + - before: "insufficient quota for resource-type in flavor example-flavor, request > maximum capacity (24 > 16)" + - after: "insufficient quota for resource-type in flavor example-flavor, previously considered podsets requests (16) + current podset request (8) > maximum capacity (16)" (#7294, @iomarsayed) + ## v0.13.6 Changes since `v0.13.5`: From 071dd5118541dda7cc932445115b6dd2a0ceb0c6 Mon Sep 17 00:00:00 2001 From: Yuki Iwai Date: Thu, 23 Oct 2025 21:47:36 +0900 Subject: [PATCH 018/119] Update main with the latest v0.14.2 (#7359) Signed-off-by: Yuki Iwai --- CHANGELOG/CHANGELOG-0.14.md | 38 +++++++++++++++++++++++++ Makefile | 2 +- README.md | 2 +- SECURITY-INSIGHTS.yaml | 12 ++++---- charts/kueue/Chart.yaml | 4 +-- charts/kueue/README.md | 8 +++--- charts/kueue/README.md.gotmpl | 6 ++-- cmd/kueueviz/INSTALL.md | 6 ++-- cmd/kueueviz/frontend/package-lock.json | 4 +-- cmd/kueueviz/frontend/package.json | 2 +- site/hugo.toml | 4 +-- test/e2e/kueueviz/package-lock.json | 4 +-- test/e2e/kueueviz/package.json | 2 +- 13 files changed, 66 insertions(+), 28 deletions(-) diff --git a/CHANGELOG/CHANGELOG-0.14.md b/CHANGELOG/CHANGELOG-0.14.md index 6468482399a..9bbbe272db0 100644 --- a/CHANGELOG/CHANGELOG-0.14.md +++ b/CHANGELOG/CHANGELOG-0.14.md @@ -1,3 +1,41 @@ +## v0.14.2 + +Changes since `v0.14.1`: + +## Changes by Kind + +### Feature + +- JobFramework: Introduce an optional interface for custom Jobs, called JobWithCustomWorkloadActivation, which can be used to deactivate or active a custom CRD workload. (#7286, @tg123) + +### Bug or Regression + +- Fix existing workloads not being re-evaluated when new clusters are added to MultiKueueConfig. Previously, only newly created workloads would see updated cluster lists. (#7349, @mimowo) +- Fix handling of RayJobs which specify the spec.clusterSelector and the "queue-name" label for Kueue. These jobs should be ignored by kueue as they are being submitted to a RayCluster which is where the resources are being used and was likely already admitted by kueue. No need to double admit. + Fix on a panic on kueue managed jobs if spec.rayClusterSpec wasn't specified. (#7258, @laurafitzgerald) +- Fixed a bug that Kueue would keep sending empty updates to a Workload, along with sending the "UpdatedWorkload" event, even if the Workload didn't change. This would happen for Workloads using any other mechanism for setting + the priority than the WorkloadPriorityClass, eg. for Workloads for PodGroups. (#7305, @mbobrovskyi) +- MultiKueue x ElasticJobs: fix webhook validation bug which prevented scale up operation when any other + than the default "AllAtOnce" MultiKueue dispatcher was used. (#7332, @mszadkow) +- TAS: Introduce missing validation against using incompatible `PodSet` grouping configuration in `JobSet, `MPIJob`, `LeaderWorkerSet`, `RayJob` and `RayCluster`. + + Now, only groups of two `PodSet`s can be defined and one of the grouped `PodSet`s has to have only a single `Pod`. + The `PodSet`s within a group must specify the same topology request via one of the `kueue.x-k8s.io/podset-required-topology` and `kueue.x-k8s.io/podset-preferred-topology` annotations. (#7263, @kshalot) +- Visibility API: Fix a bug that the Config clientConnection is not respected in the visibility server. (#7225, @tenzen-y) +- WorkloadRequestUseMergePatch: use "strict" mode for admission patches during scheduling which + sends the ResourceVersion of the workload being admitted for comparing by kube-apiserver. + This fixes the race-condition issue that Workload conditions added concurrently by other controllers + could be removed during scheduling. (#7279, @mszadkow) + +### Other (Cleanup or Flake) + +- Improve the messages presented to the user in scheduling events, by clarifying the reason for "insufficient quota" + in case of workloads with multiple PodSets. + + Example: + - before: "insufficient quota for resource-type in flavor example-flavor, request > maximum capacity (24 > 16)" + - after: "insufficient quota for resource-type in flavor example-flavor, previously considered podsets requests (16) + current podset request (8) > maximum capacity (16)" (#7293, @iomarsayed) + ## v0.14.1 Changes since `v0.14.0`: diff --git a/Makefile b/Makefile index ddf7859365f..7c9b2db58b7 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ LD_FLAGS += -X '$(version_pkg).BuildDate=$(shell date -u +%Y-%m-%dT%H:%M:%SZ)' # Update these variables when preparing a new release or a release branch. # Then run `make prepare-release-branch` -RELEASE_VERSION=v0.14.1 +RELEASE_VERSION=v0.14.2 RELEASE_BRANCH=main # Application version for Helm and npm (strips leading 'v' from RELEASE_VERSION) APP_VERSION := $(shell echo $(RELEASE_VERSION) | cut -c2-) diff --git a/README.md b/README.md index 92546e7382b..38603c01b53 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Read the [overview](https://kueue.sigs.k8s.io/docs/overview/) and watch the Kueu To install the latest release of Kueue in your cluster, run the following command: ```shell -kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.1/manifests.yaml +kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.2/manifests.yaml ``` The controller runs in the `kueue-system` namespace. diff --git a/SECURITY-INSIGHTS.yaml b/SECURITY-INSIGHTS.yaml index 339de146d32..2384c6af60a 100644 --- a/SECURITY-INSIGHTS.yaml +++ b/SECURITY-INSIGHTS.yaml @@ -1,11 +1,11 @@ header: schema-version: 1.0.0 expiration-date: '2024-09-28T01:00:00.000Z' - last-updated: '2025-10-08' - last-reviewed: '2025-10-08' - commit-hash: "bcdc1e482ba64f56fee107a1e5a27646825ab9e6" + last-updated: '2025-10-23' + last-reviewed: '2025-10-23' + commit-hash: "afb96efdd444eb95e357e6d6b77ce9cd2817fc91" project-url: 'https://github.com/kubernetes-sigs/kueue' - project-release: "0.14.1" + project-release: "0.14.2" changelog: 'https://github.com/kubernetes-sigs/kueue/tree/main/CHANGELOG' license: 'https://github.com/kubernetes-sigs/kueue/blob/main/LICENSE' project-lifecycle: @@ -28,7 +28,7 @@ documentation: - 'https://kueue.sigs.k8s.io/docs/' distribution-points: - >- - https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.1/manifests.yaml + https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.2/manifests.yaml security-artifacts: threat-model: threat-model-created: false @@ -62,5 +62,5 @@ dependencies: dependencies-lists: - 'https://github.com/kubernetes-sigs/kueue/blob/main/go.mod' sbom: - - sbom-file: https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.1/kueue-v0.14.1.spdx.json + - sbom-file: https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.2/kueue-v0.14.2.spdx.json sbom-format: SPDX diff --git a/charts/kueue/Chart.yaml b/charts/kueue/Chart.yaml index 4960d9df204..702b56000a7 100644 --- a/charts/kueue/Chart.yaml +++ b/charts/kueue/Chart.yaml @@ -16,9 +16,9 @@ type: application # NOTE: Do not modify manually. In Kueue, the version and appVersion are # overridden to GIT_TAG when building the artifacts, including the helm charts, # via Makefile. -version: 0.14.1 +version: 0.14.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v0.14.1" +appVersion: "v0.14.2" diff --git a/charts/kueue/README.md b/charts/kueue/README.md index 32707803146..16353fc2cfe 100644 --- a/charts/kueue/README.md +++ b/charts/kueue/README.md @@ -1,6 +1,6 @@ # kueue -![Version: 0.14.1](https://img.shields.io/badge/Version-0.14.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.14.1](https://img.shields.io/badge/AppVersion-v0.14.1-informational?style=flat-square) +![Version: 0.14.2](https://img.shields.io/badge/Version-0.14.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.14.2](https://img.shields.io/badge/AppVersion-v0.14.2-informational?style=flat-square) Kueue is a set of APIs and controllers for job queueing. It is a job-level manager that decides when a job should be admitted to start (as in pods can be created) and when it should stop (as in active pods should be deleted). @@ -28,7 +28,7 @@ $ helm install kueue kueue/ --create-namespace --namespace kueue-system Or use the charts pushed to `oci://registry.k8s.io/kueue/charts/kueue`: ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.1" --create-namespace --namespace=kueue-system +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" --create-namespace --namespace=kueue-system ``` For more advanced parametrization of Kueue, we recommend using a local overrides file, passed via the `--values` flag. For example: @@ -50,7 +50,7 @@ controllerManager: ``` ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.1" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ --create-namespace --namespace=kueue-system \ --values overrides.yaml ``` @@ -58,7 +58,7 @@ helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.1" \ You can also use the `--set` flag. For example, to enable a feature gate (e.g., `TopologyAwareScheduling`): ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.1" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ --create-namespace --namespace=kueue-system \ --set "controllerManager.featureGates[0].name=TopologyAwareScheduling" \ --set "controllerManager.featureGates[0].enabled=true" diff --git a/charts/kueue/README.md.gotmpl b/charts/kueue/README.md.gotmpl index 6cf856979f8..6a93180b739 100644 --- a/charts/kueue/README.md.gotmpl +++ b/charts/kueue/README.md.gotmpl @@ -30,7 +30,7 @@ $ helm install kueue kueue/ --create-namespace --namespace kueue-system Or use the charts pushed to `oci://registry.k8s.io/kueue/charts/kueue`: ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.1" --create-namespace --namespace=kueue-system +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" --create-namespace --namespace=kueue-system ``` For more advanced parametrization of Kueue, we recommend using a local overrides file, passed via the `--values` flag. For example: @@ -52,7 +52,7 @@ controllerManager: ``` ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.1" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ --create-namespace --namespace=kueue-system \ --values overrides.yaml ``` @@ -60,7 +60,7 @@ helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.1" \ You can also use the `--set` flag. For example, to enable a feature gate (e.g., `TopologyAwareScheduling`): ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.1" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ --create-namespace --namespace=kueue-system \ --set "controllerManager.featureGates[0].name=TopologyAwareScheduling" \ --set "controllerManager.featureGates[0].enabled=true" diff --git a/cmd/kueueviz/INSTALL.md b/cmd/kueueviz/INSTALL.md index 82b3449204c..b8c2fd9bc4f 100644 --- a/cmd/kueueviz/INSTALL.md +++ b/cmd/kueueviz/INSTALL.md @@ -3,7 +3,7 @@ KueueViz can be installed using `kubectl` with the following command: ``` -kubectl create -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.1/kueueviz.yaml +kubectl create -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.2/kueueviz.yaml ``` If you are using `kind` and that you don't have an `ingress` controller, you can use `port-forward` to configure and run `KueueViz`: @@ -23,7 +23,7 @@ by ensuring that `enableKueueViz` is set to `true`: ``` helm upgrade --install kueue oci://registry.k8s.io/kueue/charts/kueue \ - --version="0.14.1" + --version="0.14.2" --namespace kueue-system \ --set enableKueueViz=true \ --create-namespace @@ -44,7 +44,7 @@ kind create cluster kind get kubeconfig > kubeconfig export KUBECONFIG=$PWD/kubeconfig helm install kueue oci://us-central1-docker.pkg.dev/k8s-staging-images/charts/kueue \ - --version="0.14.1" --create-namespace --namespace=kueue-system + --version="0.14.2" --create-namespace --namespace=kueue-system ``` ## Build diff --git a/cmd/kueueviz/frontend/package-lock.json b/cmd/kueueviz/frontend/package-lock.json index 487912dd7d2..adc29e4805f 100644 --- a/cmd/kueueviz/frontend/package-lock.json +++ b/cmd/kueueviz/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "kueueviz-frontend", - "version": "0.14.1", + "version": "0.14.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "kueueviz-frontend", - "version": "0.14.1", + "version": "0.14.2", "license": "Apache 2.0", "dependencies": { "@emotion/react": "^11.14.0", diff --git a/cmd/kueueviz/frontend/package.json b/cmd/kueueviz/frontend/package.json index e0c171eb313..e60f3e40fc2 100644 --- a/cmd/kueueviz/frontend/package.json +++ b/cmd/kueueviz/frontend/package.json @@ -1,6 +1,6 @@ { "name": "kueueviz-frontend", - "version": "0.14.1", + "version": "0.14.2", "private": true, "description": "Frontend dashboard for visualizing Kueue status", "main": "src/index.jsx", diff --git a/site/hugo.toml b/site/hugo.toml index 341b6b39ba8..b79c676f2a7 100644 --- a/site/hugo.toml +++ b/site/hugo.toml @@ -99,10 +99,10 @@ ignoreFiles = [] # The major.minor version tag for the version of the docs represented in this # branch of the repository. Used in the "version-banner" partial to display a # version number for this doc set. - version = "v0.14.1" + version = "v0.14.2" # Version of Kueue without the leading "v", as used for Helm charts. - chart_version = "0.14.1" + chart_version = "0.14.2" # Flag used in the "version-banner" partial to decide whether to display a # banner on every page indicating that this is an archived version of the docs. diff --git a/test/e2e/kueueviz/package-lock.json b/test/e2e/kueueviz/package-lock.json index d077f79d74a..6962a8fb3fd 100644 --- a/test/e2e/kueueviz/package-lock.json +++ b/test/e2e/kueueviz/package-lock.json @@ -1,12 +1,12 @@ { "name": "kueueviz-frontend-tests", - "version": "0.14.1", + "version": "0.14.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "kueueviz-frontend-tests", - "version": "0.14.1", + "version": "0.14.2", "devDependencies": { "@testing-library/cypress": "^10.1.0", "cypress": "^15.5.0", diff --git a/test/e2e/kueueviz/package.json b/test/e2e/kueueviz/package.json index 8afaa260a8a..4ec1e1d15da 100644 --- a/test/e2e/kueueviz/package.json +++ b/test/e2e/kueueviz/package.json @@ -1,6 +1,6 @@ { "name": "kueueviz-frontend-tests", - "version": "0.14.1", + "version": "0.14.2", "scripts": { "cypress:run": "cypress run --headless", "check-unused": "depcheck" From db6df11cc29a5a3638c72c855097b0581ec084c4 Mon Sep 17 00:00:00 2001 From: Omar Sayed Date: Fri, 24 Oct 2025 09:21:35 +0200 Subject: [PATCH 019/119] Deprecate LocalQueueFlavorStatus for v1beta1 and v1beta2 (#7337) * Deprecate LocalQueueFlavorStatus for v1beta2 * Generate manifests and helm and improve deprecation comments * Fix Linter * Fix Linter * Fix a typo * Fix typo after main branch merge * remove comments from generated files --- .golangci.yaml | 6 ++++++ apis/kueue/v1beta1/localqueue_types.go | 2 ++ apis/kueue/v1beta2/localqueue_types.go | 2 ++ .../templates/crd/kueue.x-k8s.io_localqueues.yaml | 10 ++++++++-- .../crd/bases/kueue.x-k8s.io_localqueues.yaml | 14 ++++++++++---- pkg/cache/scheduler/cache.go | 3 ++- site/content/en/docs/reference/kueue.v1beta1.md | 5 ++++- site/content/zh-CN/docs/reference/kueue.v1beta1.md | 2 ++ 8 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index e43fff90754..16c9f0296a7 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -90,6 +90,12 @@ linters: - linters: - staticcheck text: 'SA1019: j.Status.RayClusterStatus.State is deprecated: the State field is replaced by the Conditions field.' + - linters: + - staticcheck + text: 'SA1019: kueue.LocalQueueFlavorStatus is deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.' + - linters: + - staticcheck + text: 'SA1019: stats.Flavors is deprecated: Flavors is deprecated and marked for removal in v1beta2.' - linters: - fatcontext path: ^test/* diff --git a/apis/kueue/v1beta1/localqueue_types.go b/apis/kueue/v1beta1/localqueue_types.go index 564d244910c..c46604d9cab 100644 --- a/apis/kueue/v1beta1/localqueue_types.go +++ b/apis/kueue/v1beta1/localqueue_types.go @@ -56,6 +56,7 @@ type LocalQueueSpec struct { FairSharing *FairSharing `json:"fairSharing,omitempty"` } +// Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2. type LocalQueueFlavorStatus struct { // name of the flavor. // +required @@ -154,6 +155,7 @@ type LocalQueueStatus struct { // +listMapKey=name // +kubebuilder:validation:MaxItems=16 // +optional + // Deprecated: Flavors is deprecated and marked for removal in v1beta2. Flavors []LocalQueueFlavorStatus `json:"flavors,omitempty"` // fairSharing contains the information about the current status of fair sharing. diff --git a/apis/kueue/v1beta2/localqueue_types.go b/apis/kueue/v1beta2/localqueue_types.go index 9ad483b1c40..a19f2d900ad 100644 --- a/apis/kueue/v1beta2/localqueue_types.go +++ b/apis/kueue/v1beta2/localqueue_types.go @@ -56,6 +56,7 @@ type LocalQueueSpec struct { FairSharing *FairSharing `json:"fairSharing,omitempty"` } +// Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2. type LocalQueueFlavorStatus struct { // name of the flavor. // +required @@ -154,6 +155,7 @@ type LocalQueueStatus struct { // +listMapKey=name // +kubebuilder:validation:MaxItems=16 // +optional + // Deprecated: Flavors is deprecated and marked for removal in v1beta2. Flavors []LocalQueueFlavorStatus `json:"flavors,omitempty"` // fairSharing contains the information about the current status of fair sharing. diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml index fb717a951b3..b6cf04c6f3c 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml @@ -276,8 +276,11 @@ spec: - name x-kubernetes-list-type: map flavors: - description: flavors lists all currently available ResourceFlavors in specified ClusterQueue. + description: |- + flavors lists all currently available ResourceFlavors in specified ClusterQueue. + Deprecated: Flavors is deprecated and marked for removal in v1beta2. items: + description: 'Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.' properties: name: description: name of the flavor. @@ -623,8 +626,11 @@ spec: - weightedShare type: object flavors: - description: flavors lists all currently available ResourceFlavors in specified ClusterQueue. + description: |- + flavors lists all currently available ResourceFlavors in specified ClusterQueue. + Deprecated: Flavors is deprecated and marked for removal in v1beta2. items: + description: 'Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.' properties: name: description: name of the flavor. diff --git a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml index e1f2d8b8a23..e8c95ba7e79 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml @@ -266,9 +266,12 @@ spec: - name x-kubernetes-list-type: map flavors: - description: flavors lists all currently available ResourceFlavors - in specified ClusterQueue. + description: |- + flavors lists all currently available ResourceFlavors in specified ClusterQueue. + Deprecated: Flavors is deprecated and marked for removal in v1beta2. items: + description: 'Deprecated: LocalQueueFlavorStatus is deprecated and + marked for removal in v1beta2.' properties: name: description: name of the flavor. @@ -625,9 +628,12 @@ spec: - weightedShare type: object flavors: - description: flavors lists all currently available ResourceFlavors - in specified ClusterQueue. + description: |- + flavors lists all currently available ResourceFlavors in specified ClusterQueue. + Deprecated: Flavors is deprecated and marked for removal in v1beta2. items: + description: 'Deprecated: LocalQueueFlavorStatus is deprecated and + marked for removal in v1beta2.' properties: name: description: name of the flavor. diff --git a/pkg/cache/scheduler/cache.go b/pkg/cache/scheduler/cache.go index ac2a06cab3c..233689f5069 100644 --- a/pkg/cache/scheduler/cache.go +++ b/pkg/cache/scheduler/cache.go @@ -805,7 +805,8 @@ type LocalQueueUsageStats struct { ReservingWorkloads int AdmittedResources []kueue.LocalQueueFlavorUsage AdmittedWorkloads int - Flavors []kueue.LocalQueueFlavorStatus + // Deprecated: Flavors is deprecated and marked for removal in v1beta2. + Flavors []kueue.LocalQueueFlavorStatus } func (c *Cache) LocalQueueUsage(qObj *kueue.LocalQueue) (*LocalQueueUsageStats, error) { diff --git a/site/content/en/docs/reference/kueue.v1beta1.md b/site/content/en/docs/reference/kueue.v1beta1.md index 0fff02f80e8..c6e40fb2955 100644 --- a/site/content/en/docs/reference/kueue.v1beta1.md +++ b/site/content/en/docs/reference/kueue.v1beta1.md @@ -1592,6 +1592,8 @@ which the kueue controller manager is running. The config should be stored in th - [LocalQueueStatus](#kueue-x-k8s-io-v1beta1-LocalQueueStatus) +

Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.

+ @@ -1833,7 +1835,8 @@ workloads assigned to this LocalQueue.

[]LocalQueueFlavorStatus
FieldDescription
-

flavors lists all currently available ResourceFlavors in specified ClusterQueue.

+

flavors lists all currently available ResourceFlavors in specified ClusterQueue. +Deprecated: Flavors is deprecated and marked for removal in v1beta2.

fairSharing
diff --git a/site/content/zh-CN/docs/reference/kueue.v1beta1.md b/site/content/zh-CN/docs/reference/kueue.v1beta1.md index 9082b376b96..852b67978d3 100644 --- a/site/content/zh-CN/docs/reference/kueue.v1beta1.md +++ b/site/content/zh-CN/docs/reference/kueue.v1beta1.md @@ -1519,6 +1519,7 @@ which the kueue controller manager is running. The config should be stored in th **Appears in:** - [LocalQueueStatus](#kueue-x-k8s-io-v1beta1-LocalQueueStatus) +

Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.

@@ -1763,6 +1764,7 @@ workloads assigned to this LocalQueue.

flavors lists all currently available ResourceFlavors in specified ClusterQueue.

+

Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.

fairSharing
From 5909cb323ae32502d6e9ad0deca5968b4a875fe5 Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Fri, 24 Oct 2025 10:51:36 +0200 Subject: [PATCH 020/119] Add TAS support to the Kubeflow Trainer integration (#7249) --- Makefile-test.mk | 1 + go.mod | 3 +- go.sum | 6 +- .../jobs/trainjob/trainjob_controller.go | 48 +- .../jobs/trainjob/trainjob_controller_test.go | 108 +- .../trainjob_multikueue_adapter_test.go | 108 +- .../jobs/trainjob/trainjob_webhook.go | 54 +- .../jobs/trainjob/trainjob_webhook_test.go | 144 +- pkg/util/testingjobs/trainjob/wrappers.go | 38 +- test/e2e/singlecluster/tas_test.go | 123 ++ test/e2e/singlecluster/trainjob_test.go | 2 +- test/e2e/tas/trainjob_test.go | 314 +++ test/integration/multikueue/jobs_test.go | 4 +- .../controller/jobs/trainjob/suite_test.go | 12 +- .../jobs/trainjob/trainjob_controller_test.go | 202 +- .../webhook/jobs/trainjob_webhook_test.go | 45 +- .../trainer/v1alpha1/trainingruntime_types.go | 96 +- .../apis/trainer/v1alpha1/trainjob_types.go | 228 ++- .../trainer/v1alpha1/zz_generated.deepcopy.go | 138 +- .../trainer/v1alpha1/zz_generated.defaults.go | 203 +- .../trainer/v1alpha1/zz_generated.openapi.go | 1712 +++++++++++++++-- .../kubeflow/trainer/v2/pkg/apply/apply.go | 6 - .../trainer/v2/pkg/constants/constants.go | 50 +- .../runtime/core/clustertrainingruntime.go | 21 +- .../v2/pkg/runtime/core/trainingruntime.go | 47 +- .../pkg/runtime/framework/core/framework.go | 40 +- .../v2/pkg/runtime/framework/interface.go | 5 +- .../plugins/coscheduling/coscheduling.go | 26 +- .../framework/plugins/coscheduling/indexer.go | 56 - .../framework/plugins/jobset/builder.go | 40 +- .../framework/plugins/jobset/jobset.go | 85 +- .../pkg/runtime/framework/plugins/mpi/mpi.go | 11 +- .../pkg/runtime/framework/plugins/registry.go | 2 + .../runtime/framework/plugins/torch/torch.go | 142 +- .../framework/plugins/torch/torchtune.go | 164 ++ .../framework/plugins/volcano/volcano.go | 359 ++++ .../trainer/v2/pkg/runtime/indexer/indexer.go | 37 + .../trainer/v2/pkg/runtime/interface.go | 3 +- .../trainer/v2/pkg/runtime/runtime.go | 53 +- .../util/trainingruntime/trainingruntime.go | 12 + vendor/modules.txt | 9 +- vendor/volcano.sh/apis/LICENSE | 201 ++ .../apis/pkg/apis/scheduling/doc.go | 17 + .../apis/pkg/apis/scheduling/register.go | 42 + .../apis/pkg/apis/scheduling/types.go | 405 ++++ .../pkg/apis/scheduling/v1beta1/conversion.go | 34 + .../apis/pkg/apis/scheduling/v1beta1/doc.go | 24 + .../pkg/apis/scheduling/v1beta1/labels.go | 73 + .../pkg/apis/scheduling/v1beta1/register.go | 58 + .../apis/pkg/apis/scheduling/v1beta1/types.go | 428 +++++ .../v1beta1/zz_generated.conversion.go | 646 +++++++ .../v1beta1/zz_generated.deepcopy.go | 473 +++++ .../apis/scheduling/zz_generated.deepcopy.go | 473 +++++ .../scheduling/v1beta1/affinity.go | 47 + .../scheduling/v1beta1/cluster.go | 60 + .../scheduling/v1beta1/guarantee.go | 42 + .../scheduling/v1beta1/networktopologyspec.go | 51 + .../scheduling/v1beta1/nodegroupaffinity.go | 51 + .../v1beta1/nodegroupantiaffinity.go | 51 + .../scheduling/v1beta1/podgroup.go | 224 +++ .../scheduling/v1beta1/podgroupcondition.go | 89 + .../scheduling/v1beta1/podgroupspec.go | 93 + .../scheduling/v1beta1/podgroupstatus.go | 83 + .../scheduling/v1beta1/queue.go | 223 +++ .../scheduling/v1beta1/queuespec.go | 128 ++ .../scheduling/v1beta1/queuestatus.go | 106 + .../scheduling/v1beta1/reservation.go | 53 + 67 files changed, 8131 insertions(+), 801 deletions(-) create mode 100644 test/e2e/tas/trainjob_test.go delete mode 100644 vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/coscheduling/indexer.go create mode 100644 vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch/torchtune.go create mode 100644 vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/volcano/volcano.go create mode 100644 vendor/github.com/kubeflow/trainer/v2/pkg/util/trainingruntime/trainingruntime.go create mode 100644 vendor/volcano.sh/apis/LICENSE create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/doc.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/register.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/types.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/conversion.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/doc.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/register.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.deepcopy.go create mode 100644 vendor/volcano.sh/apis/pkg/apis/scheduling/zz_generated.deepcopy.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/affinity.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/cluster.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/guarantee.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/networktopologyspec.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/nodegroupaffinity.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/nodegroupantiaffinity.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroup.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupcondition.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupspec.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupstatus.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queue.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queuespec.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queuestatus.go create mode 100644 vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/reservation.go diff --git a/Makefile-test.mk b/Makefile-test.mk index b058cba231d..d2ec6358631 100644 --- a/Makefile-test.mk +++ b/Makefile-test.mk @@ -170,6 +170,7 @@ run-test-tas-e2e-%: APPWRAPPER_VERSION=$(APPWRAPPER_VERSION) \ LEADERWORKERSET_VERSION=$(LEADERWORKERSET_VERSION) \ KUBERAY_VERSION=$(KUBERAY_VERSION) RAY_VERSION=$(RAY_VERSION) RAYMINI_VERSION=$(RAYMINI_VERSION) USE_RAY_FOR_TESTS=$(USE_RAY_FOR_TESTS) \ + KUBEFLOW_TRAINER_VERSION=$(KUBEFLOW_TRAINER_VERSION) \ KIND_CLUSTER_FILE="tas-kind-cluster.yaml" E2E_TARGET_FOLDER="tas" \ TEST_LOG_LEVEL=$(TEST_LOG_LEVEL) \ E2E_RUN_ONLY_ENV=$(E2E_RUN_ONLY_ENV) \ diff --git a/go.mod b/go.mod index 9b13cd84276..a02321e904e 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/google/go-cmp v0.7.0 github.com/json-iterator/go v1.1.12 github.com/kubeflow/mpi-operator v0.6.0 - github.com/kubeflow/trainer/v2 v2.0.1 + github.com/kubeflow/trainer/v2 v2.1.0-rc.0 github.com/kubeflow/training-operator v1.9.3 github.com/onsi/ginkgo/v2 v2.26.0 github.com/onsi/gomega v1.38.2 @@ -155,4 +155,5 @@ require ( sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/scheduler-plugins v0.31.8 // indirect + volcano.sh/apis v1.12.2 // indirect ) diff --git a/go.sum b/go.sum index a3bd6af7602..c0830049844 100644 --- a/go.sum +++ b/go.sum @@ -136,8 +136,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubeflow/mpi-operator v0.6.0 h1:RzzG03oyQIiHe/SH2k19N+Hp4QkxEsG2l9G1T7US+ys= github.com/kubeflow/mpi-operator v0.6.0/go.mod h1:lu3cj3ESq3SdS29nbHC26TuXiTYYQYTHJ6fcy5Xv20c= -github.com/kubeflow/trainer/v2 v2.0.1 h1:4bMZdFIpQBHjcUene6P+MqbY5v7ccyLUIKkefOhUFG8= -github.com/kubeflow/trainer/v2 v2.0.1/go.mod h1:AH97YWUun+pLQlQHyRIlYcXw+YUuuQhnVWVYTbzGvgo= +github.com/kubeflow/trainer/v2 v2.1.0-rc.0 h1:kr4Gjb7osBIwZm5qE4ajwxe0P4n2lWwD3grhfXc66ds= +github.com/kubeflow/trainer/v2 v2.1.0-rc.0/go.mod h1:Y9hmSGG3A9tFKUjlcWf7Kg/SwibYwrZTK5ZqAxMxyno= github.com/kubeflow/training-operator v1.9.3 h1:aaSHqskOtCY8Dn8sVd+l9p5XAczW6O0nYMPjVLAw/lc= github.com/kubeflow/training-operator v1.9.3/go.mod h1:6zI0hgeCOheiW5Z12IcVkKBuSWm714fz8mUvYu4Fid4= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -436,3 +436,5 @@ sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/ sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= +volcano.sh/apis v1.12.2 h1:KvNyM/kMizFVlALiH/uFHPwYFHRtxuVnBL0upbFbDss= +volcano.sh/apis v1.12.2/go.mod h1:0XNNnIOevJSYNiXRmwhXUrYCcCcWcBeTY0nxrlkk03A= diff --git a/pkg/controller/jobs/trainjob/trainjob_controller.go b/pkg/controller/jobs/trainjob/trainjob_controller.go index 544cd621d4a..5386fac2f48 100644 --- a/pkg/controller/jobs/trainjob/trainjob_controller.go +++ b/pkg/controller/jobs/trainjob/trainjob_controller.go @@ -138,7 +138,8 @@ func (t *TrainJob) IsSuspended() bool { func (t *TrainJob) IsActive() bool { for i := range t.Status.JobsStatus { - if t.Status.JobsStatus[i].Active > 0 { + active := ptr.Deref(t.Status.JobsStatus[i].Active, 0) + if active > 0 { return true } } @@ -229,13 +230,17 @@ func getRuntimeSpec(ctx context.Context, trainJob *kftrainer.TrainJob) (*kftrain } } -func (t *TrainJob) PodSets(ctx context.Context) ([]kueue.PodSet, error) { +func podSets(ctx context.Context, t *TrainJob) ([]kueue.PodSet, error) { jobset, err := getChildJobSet(ctx, t) if err != nil { return nil, err } - podsets, err := (*workloadjobset.JobSet)(jobset).PodSets(ctx) + return (*workloadjobset.JobSet)(jobset).PodSets(ctx) +} + +func (t *TrainJob) PodSets(ctx context.Context) ([]kueue.PodSet, error) { + podsets, err := podSets(ctx, t) if err != nil { return nil, err } @@ -261,26 +266,28 @@ func (t *TrainJob) RunWithPodSetsInfo(ctx context.Context, podSetsInfo []podset. return podset.BadPodSetsInfoLenError(len(jobset.Spec.ReplicatedJobs), len(podSetsInfo)) } - if t.Spec.PodSpecOverrides == nil { - t.Spec.PodSpecOverrides = []kftrainer.PodSpecOverride{} + if t.Spec.PodTemplateOverrides == nil { + t.Spec.PodTemplateOverrides = []kftrainer.PodTemplateOverride{} } if t.Annotations == nil { t.Annotations = map[string]string{} } - t.Annotations[firstOverrideIdx] = strconv.Itoa(len(t.Spec.PodSpecOverrides)) + t.Annotations[firstOverrideIdx] = strconv.Itoa(len(t.Spec.PodTemplateOverrides)) for _, info := range podSetsInfo { // The trainjob controller merges each podSpecOverride sequentially, so any existing user provided override will be processed first - t.Spec.PodSpecOverrides = append(t.Spec.PodSpecOverrides, kftrainer.PodSpecOverride{ - TargetJobs: []kftrainer.PodSpecOverrideTargetJob{ + t.Spec.PodTemplateOverrides = append(t.Spec.PodTemplateOverrides, kftrainer.PodTemplateOverride{ + TargetJobs: []kftrainer.PodTemplateOverrideTargetJob{ {Name: string(info.Name)}, }, - // TODO: Set the labels/annotations when supported. See https://github.com/kubeflow/trainer/pull/2785 - // - // NOTE: Due to the issue above, in TAS mode, missing PodSet-specific labels and annotations - // prevent removal of the scheduling gate, leaving the Pod in a Pending state. - NodeSelector: info.NodeSelector, - Tolerations: info.Tolerations, - SchedulingGates: info.SchedulingGates, + Metadata: &metav1.ObjectMeta{ + Annotations: info.Annotations, + Labels: info.Labels, + }, + Spec: &kftrainer.PodTemplateSpecOverride{ + NodeSelector: info.NodeSelector, + Tolerations: info.Tolerations, + SchedulingGates: info.SchedulingGates, + }, }) } // Update the podSpecOverrides while the job is suspended, since is a requirement from the trainjob admission webhook @@ -327,7 +334,7 @@ func (t *TrainJob) RestorePodSetsInfo(_ []podset.PodSetInfo) bool { if err != nil { return false } - t.Spec.PodSpecOverrides = t.Spec.PodSpecOverrides[:idxInt] + t.Spec.PodTemplateOverrides = t.Spec.PodTemplateOverrides[:idxInt] return true } @@ -353,7 +360,7 @@ func (t *TrainJob) PodsReady(ctx context.Context) bool { } var readyReplicas int32 for _, jobStatus := range t.Status.JobsStatus { - readyReplicas += jobStatus.Ready + jobStatus.Succeeded + readyReplicas += ptr.Deref(jobStatus.Ready, 0) + ptr.Deref(jobStatus.Succeeded, 0) } return replicas == readyReplicas } @@ -372,11 +379,12 @@ func (t *TrainJob) ReclaimablePods(ctx context.Context) ([]kueue.ReclaimablePod, for i := range jobset.Spec.ReplicatedJobs { spec := &jobset.Spec.ReplicatedJobs[i] - if status, found := statuses[spec.Name]; found && status.Succeeded > 0 { - if status.Succeeded > 0 && status.Succeeded <= spec.Replicas { + if status, found := statuses[spec.Name]; found { + succeeded := ptr.Deref(status.Succeeded, 0) + if succeeded > 0 && succeeded <= spec.Replicas { ret = append(ret, kueue.ReclaimablePod{ Name: kueue.NewPodSetReference(spec.Name), - Count: status.Succeeded * workloadjobset.PodsCountPerReplica(spec), + Count: succeeded * workloadjobset.PodsCountPerReplica(spec), }) } } diff --git a/pkg/controller/jobs/trainjob/trainjob_controller_test.go b/pkg/controller/jobs/trainjob/trainjob_controller_test.go index 68cdc7785e1..a5a4757711c 100644 --- a/pkg/controller/jobs/trainjob/trainjob_controller_test.go +++ b/pkg/controller/jobs/trainjob/trainjob_controller_test.go @@ -71,9 +71,9 @@ func TestRunWithPodsetsInfo(t *testing.T) { // Create and refererence a fake ClusterTrainingRuntime testTrainJob := testingtrainjob.MakeTrainJob("trainjob", "ns").RuntimeRef(kftrainerapi.RuntimeRef{ - APIGroup: ptr.To("trainer.kubeflow.org"), + APIGroup: ptr.To(kftrainerapi.GroupVersion.Group), Name: "test", - Kind: ptr.To("ClusterTrainingRuntime"), + Kind: ptr.To(kftrainerapi.ClusterTrainingRuntimeKind), }) testJobset := testingjobset.MakeJobSet("", "").ReplicatedJobs( testingjobset.ReplicatedJobRequirements{ @@ -92,6 +92,8 @@ func TestRunWithPodsetsInfo(t *testing.T) { podsetsInfo: []podset.PodSetInfo{ { Name: "node", + Annotations: map[string]string{"test-annotation": "test"}, + Labels: map[string]string{"test-label": "label"}, NodeSelector: map[string]string{"disktype": "ssd"}, Tolerations: []corev1.Toleration{*toleration1.DeepCopy()}, SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-1"}}, @@ -99,14 +101,20 @@ func TestRunWithPodsetsInfo(t *testing.T) { }, wantTrainJob: testTrainJob.Clone(). Annotation(firstOverrideIdx, "0"). - PodSpecOverrides([]kftrainerapi.PodSpecOverride{ + PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "node"}, }, - NodeSelector: map[string]string{"disktype": "ssd"}, - Tolerations: []corev1.Toleration{*toleration1.DeepCopy()}, - SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-1"}}, + Metadata: &metav1.ObjectMeta{ + Annotations: map[string]string{"test-annotation": "test"}, + Labels: map[string]string{"test-label": "label"}, + }, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"disktype": "ssd"}, + Tolerations: []corev1.Toleration{*toleration1.DeepCopy()}, + SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-1"}}, + }, }, }). Suspend(false). @@ -115,19 +123,23 @@ func TestRunWithPodsetsInfo(t *testing.T) { }, "should respect user provided PodSpecOverrides when adding PodSet info config to the trainjob": { trainJob: testTrainJob.Clone(). - PodSpecOverrides([]kftrainerapi.PodSpecOverride{ + PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "node"}, }, - NodeSelector: map[string]string{"disktype": "sdd"}, - Tolerations: []corev1.Toleration{*toleration1.DeepCopy()}, - SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-4"}}, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"disktype": "sdd"}, + Tolerations: []corev1.Toleration{*toleration1.DeepCopy()}, + SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-4"}}, + }, }, }).Obj(), podsetsInfo: []podset.PodSetInfo{ { Name: "node", + Annotations: map[string]string{"test-annotation": "test"}, + Labels: map[string]string{"test-label": "label"}, NodeSelector: map[string]string{"gpu": "nvidia"}, Tolerations: []corev1.Toleration{*toleration2.DeepCopy()}, SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-2"}}, @@ -135,22 +147,30 @@ func TestRunWithPodsetsInfo(t *testing.T) { }, wantTrainJob: testTrainJob.Clone(). Annotation(firstOverrideIdx, "1"). - PodSpecOverrides([]kftrainerapi.PodSpecOverride{ + PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "node"}, }, - NodeSelector: map[string]string{"disktype": "sdd"}, - Tolerations: []corev1.Toleration{*toleration1.DeepCopy()}, - SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-4"}}, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"disktype": "sdd"}, + Tolerations: []corev1.Toleration{*toleration1.DeepCopy()}, + SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-4"}}, + }, }, { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "node"}, }, - NodeSelector: map[string]string{"gpu": "nvidia"}, - Tolerations: []corev1.Toleration{*toleration2.DeepCopy()}, - SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-2"}}, + Metadata: &metav1.ObjectMeta{ + Annotations: map[string]string{"test-annotation": "test"}, + Labels: map[string]string{"test-label": "label"}, + }, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"gpu": "nvidia"}, + Tolerations: []corev1.Toleration{*toleration2.DeepCopy()}, + SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-2"}}, + }, }, }). Suspend(false). @@ -238,50 +258,62 @@ func TestRestorePodSetsInfo(t *testing.T) { Obj(), wantReturn: false, }, - "should remove all the podSpecOverrides starting from the index specified in the annotation": { + "should remove all the podTemplateOverrides starting from the index specified in the annotation": { trainJob: testTrainJob.Clone(). Annotation(firstOverrideIdx, "2"). - PodSpecOverrides([]kftrainerapi.PodSpecOverride{ + PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "user-provided-1"}, }, - NodeSelector: map[string]string{"disktype": "sdd"}, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"disktype": "sdd"}, + }, }, { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "user-provided-2"}, }, - NodeSelector: map[string]string{"disktype": "sdd"}, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"disktype": "sdd"}, + }, }, { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "kueue-provided-1"}, }, - NodeSelector: map[string]string{"disktype": "sdd"}, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"disktype": "sdd"}, + }, }, { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "kueue-provided-2"}, }, - NodeSelector: map[string]string{"disktype": "sdd"}, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"disktype": "sdd"}, + }, }, }). Obj(), wantTrainJob: testTrainJob.Clone(). Annotation(firstOverrideIdx, "2"). - PodSpecOverrides([]kftrainerapi.PodSpecOverride{ + PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "user-provided-1"}, }, - NodeSelector: map[string]string{"disktype": "sdd"}, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"disktype": "sdd"}, + }, }, { - TargetJobs: []kftrainerapi.PodSpecOverrideTargetJob{ + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "user-provided-2"}, }, - NodeSelector: map[string]string{"disktype": "sdd"}, + Spec: &kftrainerapi.PodTemplateSpecOverride{ + NodeSelector: map[string]string{"disktype": "sdd"}, + }, }, }). Obj(), @@ -306,9 +338,9 @@ func TestReconciler(t *testing.T) { testNamespace := utiltesting.MakeNamespaceWrapper("ns").Label(corev1.LabelMetadataName, "ns").Obj() // Create and refererence a fake ClusterTrainingRuntime testTrainJob := testingtrainjob.MakeTrainJob("trainjob", "ns").RuntimeRef(kftrainerapi.RuntimeRef{ - APIGroup: ptr.To("trainer.kubeflow.org"), + APIGroup: ptr.To(kftrainerapi.GroupVersion.Group), Name: "test", - Kind: ptr.To("ClusterTrainingRuntime"), + Kind: ptr.To(kftrainerapi.ClusterTrainingRuntimeKind), }) testJobset := testingjobset.MakeJobSet("", "").ReplicatedJobs( testingjobset.ReplicatedJobRequirements{ diff --git a/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter_test.go b/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter_test.go index ad1c8ec0f59..c73cfa37199 100644 --- a/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter_test.go +++ b/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter_test.go @@ -87,16 +87,14 @@ func TestMultiKueueAdapter(t *testing.T) { Label(constants.PrebuiltWorkloadLabel, "wl1"). Label(kueue.MultiKueueOriginLabel, "origin1"). JobsStatus( - kftrainerapi.JobStatus{ - Name: "replicated-job-1", - Ready: 1, - Succeeded: 1, - }, - kftrainerapi.JobStatus{ - Name: "replicated-job-2", - Ready: 3, - Succeeded: 0, - }, + testingtrainjob.MakeJobStatusWrapper("replicated-job-1"). + Ready(1). + Succeeded(1). + Obj(), + testingtrainjob.MakeJobStatusWrapper("replicated-job-2"). + Ready(3). + Succeeded(0). + Obj(), ). Obj(), }, @@ -107,16 +105,14 @@ func TestMultiKueueAdapter(t *testing.T) { wantManagersTrainJobs: []kftrainerapi.TrainJob{ *baseTrainJobManagedByKueueBuilder.Clone(). JobsStatus( - kftrainerapi.JobStatus{ - Name: "replicated-job-1", - Ready: 1, - Succeeded: 1, - }, - kftrainerapi.JobStatus{ - Name: "replicated-job-2", - Ready: 3, - Succeeded: 0, - }, + testingtrainjob.MakeJobStatusWrapper("replicated-job-1"). + Ready(1). + Succeeded(1). + Obj(), + testingtrainjob.MakeJobStatusWrapper("replicated-job-2"). + Ready(3). + Succeeded(0). + Obj(), ). Obj(), }, @@ -125,16 +121,14 @@ func TestMultiKueueAdapter(t *testing.T) { Label(constants.PrebuiltWorkloadLabel, "wl1"). Label(kueue.MultiKueueOriginLabel, "origin1"). JobsStatus( - kftrainerapi.JobStatus{ - Name: "replicated-job-1", - Ready: 1, - Succeeded: 1, - }, - kftrainerapi.JobStatus{ - Name: "replicated-job-2", - Ready: 3, - Succeeded: 0, - }, + testingtrainjob.MakeJobStatusWrapper("replicated-job-1"). + Ready(1). + Succeeded(1). + Obj(), + testingtrainjob.MakeJobStatusWrapper("replicated-job-2"). + Ready(3). + Succeeded(0). + Obj(), ). Obj(), }, @@ -151,16 +145,14 @@ func TestMultiKueueAdapter(t *testing.T) { Label(kueue.MultiKueueOriginLabel, "origin1"). Suspend(true). JobsStatus( - kftrainerapi.JobStatus{ - Name: "replicated-job-1", - Ready: 1, - Succeeded: 1, - }, - kftrainerapi.JobStatus{ - Name: "replicated-job-2", - Ready: 3, - Succeeded: 0, - }, + testingtrainjob.MakeJobStatusWrapper("replicated-job-1"). + Ready(1). + Succeeded(1). + Obj(), + testingtrainjob.MakeJobStatusWrapper("replicated-job-2"). + Ready(3). + Succeeded(0). + Obj(), ). Obj(), }, @@ -179,16 +171,14 @@ func TestMultiKueueAdapter(t *testing.T) { Label(kueue.MultiKueueOriginLabel, "origin1"). Suspend(true). JobsStatus( - kftrainerapi.JobStatus{ - Name: "replicated-job-1", - Ready: 1, - Succeeded: 1, - }, - kftrainerapi.JobStatus{ - Name: "replicated-job-2", - Ready: 3, - Succeeded: 0, - }, + testingtrainjob.MakeJobStatusWrapper("replicated-job-1"). + Ready(1). + Succeeded(1). + Obj(), + testingtrainjob.MakeJobStatusWrapper("replicated-job-2"). + Ready(3). + Succeeded(0). + Obj(), ). Obj(), }, @@ -199,16 +189,14 @@ func TestMultiKueueAdapter(t *testing.T) { Label(constants.PrebuiltWorkloadLabel, "wl1"). Label(kueue.MultiKueueOriginLabel, "origin1"). JobsStatus( - kftrainerapi.JobStatus{ - Name: "replicated-job-1", - Ready: 1, - Succeeded: 1, - }, - kftrainerapi.JobStatus{ - Name: "replicated-job-2", - Ready: 3, - Succeeded: 0, - }, + testingtrainjob.MakeJobStatusWrapper("replicated-job-1"). + Ready(1). + Succeeded(1). + Obj(), + testingtrainjob.MakeJobStatusWrapper("replicated-job-2"). + Ready(3). + Succeeded(0). + Obj(), ). Obj(), }, diff --git a/pkg/controller/jobs/trainjob/trainjob_webhook.go b/pkg/controller/jobs/trainjob/trainjob_webhook.go index bea81f631f4..37f736f2d37 100644 --- a/pkg/controller/jobs/trainjob/trainjob_webhook.go +++ b/pkg/controller/jobs/trainjob/trainjob_webhook.go @@ -32,6 +32,7 @@ import ( controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobframework/webhook" + "sigs.k8s.io/kueue/pkg/features" ) type TrainJobWebhook struct { @@ -97,7 +98,11 @@ func (w *TrainJobWebhook) ValidateCreate(ctx context.Context, obj runtime.Object trainjob := fromObject(obj) log := ctrl.LoggerFrom(ctx).WithName("trainjob-webhook") log.Info("Validating create") - return nil, w.validateCreate(trainjob).ToAggregate() + validationErrs, err := w.validateCreate(ctx, trainjob) + if err != nil { + return nil, err + } + return nil, validationErrs.ToAggregate() } // ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type @@ -106,20 +111,55 @@ func (w *TrainJobWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj run newTrainJob := fromObject(newObj) log := ctrl.LoggerFrom(ctx).WithName("trainjob-webhook") log.Info("Validating update") - return nil, w.validateUpdate(oldTrainJob, newTrainJob).ToAggregate() + validationErrs, err := w.validateUpdate(ctx, oldTrainJob, newTrainJob) + if err != nil { + return nil, err + } + return nil, validationErrs.ToAggregate() } -func (w *TrainJobWebhook) validateUpdate(oldTrainJob, newTrainJob *TrainJob) field.ErrorList { +func (w *TrainJobWebhook) validateUpdate(ctx context.Context, oldTrainJob, newTrainJob *TrainJob) (field.ErrorList, error) { var allErrs field.ErrorList allErrs = append(allErrs, jobframework.ValidateJobOnUpdate(oldTrainJob, newTrainJob, w.queues.DefaultLocalQueueExist)...) - allErrs = append(allErrs, w.validateCreate(newTrainJob)...) - return allErrs + validationErrs, err := w.validateCreate(ctx, newTrainJob) + if err != nil { + return nil, err + } + allErrs = append(allErrs, validationErrs...) + return allErrs, nil } -func (w *TrainJobWebhook) validateCreate(trainjob *TrainJob) field.ErrorList { +func (w *TrainJobWebhook) validateCreate(ctx context.Context, trainjob *TrainJob) (field.ErrorList, error) { var allErrs field.ErrorList allErrs = append(allErrs, jobframework.ValidateJobOnCreate(trainjob)...) - return allErrs + if features.Enabled(features.TopologyAwareScheduling) { + validationErrs, err := w.validateTopologyRequest(ctx, trainjob) + if err != nil { + return nil, err + } + allErrs = append(allErrs, validationErrs...) + } + return allErrs, nil +} + +func (w *TrainJobWebhook) validateTopologyRequest(ctx context.Context, trainJob *TrainJob) (field.ErrorList, error) { + var allErrs field.ErrorList + + podSets, podSetsErr := podSets(ctx, trainJob) + for _, p := range podSets { + jobPath := field.NewPath("job").Key(string(p.Name)) + allErrs = append(allErrs, jobframework.ValidateTASPodSetRequest(jobPath, &p.Template.ObjectMeta)...) + allErrs = append(allErrs, jobframework.ValidateSliceSizeAnnotationUpperBound(jobPath, &p.Template.ObjectMeta, &p)...) + } + + if len(allErrs) > 0 { + for _, err := range allErrs { + err.Detail += `. Adjust either the "TrainJob.spec.podTemplateOverrides" or the "TrainingRuntime.Template" annotations for the corresponding Job` + } + return allErrs, nil + } + + return nil, podSetsErr } // ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type diff --git a/pkg/controller/jobs/trainjob/trainjob_webhook_test.go b/pkg/controller/jobs/trainjob/trainjob_webhook_test.go index 4392b364779..44c476e91a5 100644 --- a/pkg/controller/jobs/trainjob/trainjob_webhook_test.go +++ b/pkg/controller/jobs/trainjob/trainjob_webhook_test.go @@ -24,6 +24,9 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" @@ -32,6 +35,7 @@ import ( "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" ) @@ -45,30 +49,144 @@ var ( ) func TestValidateCreate(t *testing.T) { - testTrainJob := testingtrainjob.MakeTrainJob("trainjob", "ns").Suspend(false) + testCtr := testingtrainjob.MakeClusterTrainingRuntime("testCtr", + testingjobset.MakeJobSet("", "").ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: "node", + Replicas: 1, + Parallelism: 1, + }).Obj().Spec) + testTrainJob := testingtrainjob.MakeTrainJob("trainjob", "ns").RuntimeRef(kftrainerapi.RuntimeRef{ + APIGroup: ptr.To(kftrainerapi.GroupVersion.Group), + Name: "testCtr", + Kind: ptr.To(kftrainerapi.ClusterTrainingRuntimeKind), + }).Suspend(false) testcases := map[string]struct { - name string - trainJob *kftrainerapi.TrainJob - wantErr error + clusterTrainingRuntime *kftrainerapi.ClusterTrainingRuntime + trainJob *kftrainerapi.TrainJob + wantErr error + topologyAwareScheduling bool }{ "base": { - trainJob: testTrainJob.Clone().Queue("local-queue").Obj(), - wantErr: nil, + clusterTrainingRuntime: testCtr, + trainJob: testTrainJob.Clone().Queue("local-queue").Obj(), + wantErr: nil, }, "invalid queue-name label": { - trainJob: testTrainJob.Clone().Queue("queue_name").Obj(), - wantErr: field.ErrorList{field.Invalid(queueNameLabelPath, "queue_name", invalidRFC1123Message)}.ToAggregate(), + clusterTrainingRuntime: testCtr, + trainJob: testTrainJob.Clone().Queue("queue_name").Obj(), + wantErr: field.ErrorList{field.Invalid(queueNameLabelPath, "queue_name", invalidRFC1123Message)}.ToAggregate(), }, "with prebuilt workload": { - trainJob: testTrainJob.Clone().Queue("local-queue").Label(controllerconstants.PrebuiltWorkloadLabel, "prebuilt-workload").Obj(), - wantErr: nil, + clusterTrainingRuntime: testCtr, + trainJob: testTrainJob.Clone().Queue("local-queue").Label(controllerconstants.PrebuiltWorkloadLabel, "prebuilt-workload").Obj(), + wantErr: nil, + }, + "valid topology request in PodTemplateOverride": { + clusterTrainingRuntime: testCtr, + trainJob: testTrainJob.Clone().PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ + { + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ + {Name: "node"}, + }, + Metadata: &metav1.ObjectMeta{ + Annotations: map[string]string{ + kueue.PodSetRequiredTopologyAnnotation: "cloud.com/block", + }, + }, + }, + }).Obj(), + topologyAwareScheduling: true, + }, + "valid topology request in TrainingRuntime": { + clusterTrainingRuntime: testingtrainjob.MakeClusterTrainingRuntime("testCtr", + testingjobset.MakeJobSet("", "").ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: "node", + PodAnnotations: map[string]string{ + kueue.PodSetRequiredTopologyAnnotation: "cloud.com/block", + }, + }).Obj().Spec), + trainJob: testTrainJob.Clone().Obj(), + topologyAwareScheduling: true, + }, + "invalid topology request in TrainJob": { + clusterTrainingRuntime: testCtr, + trainJob: testTrainJob.Clone().PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ + { + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ + {Name: "node"}, + }, + Metadata: &metav1.ObjectMeta{ + Annotations: map[string]string{ + kueue.PodSetPreferredTopologyAnnotation: "cloud.com/block", + kueue.PodSetRequiredTopologyAnnotation: "cloud.com/block", + }, + }, + }, + }).Obj(), + wantErr: field.ErrorList{field.Invalid(field.NewPath("job[node].annotations"), + field.OmitValueType{}, `must not contain more than one topology annotation: ["kueue.x-k8s.io/podset-required-topology", `+ + `"kueue.x-k8s.io/podset-preferred-topology", "kueue.x-k8s.io/podset-unconstrained-topology"]`+ + `. Adjust either the "TrainJob.spec.podTemplateOverrides" or the "TrainingRuntime.Template" annotations for the corresponding Job`), + }.ToAggregate(), + topologyAwareScheduling: true, + }, + "invalid topology request in TrainingRuntime": { + clusterTrainingRuntime: testingtrainjob.MakeClusterTrainingRuntime("testCtr", + testingjobset.MakeJobSet("", "").ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: "node", + PodAnnotations: map[string]string{ + kueue.PodSetPreferredTopologyAnnotation: "cloud.com/block", + kueue.PodSetRequiredTopologyAnnotation: "cloud.com/block", + }, + }).Obj().Spec), + trainJob: testTrainJob.Clone().Obj(), + wantErr: field.ErrorList{field.Invalid(field.NewPath("job[node].annotations"), + field.OmitValueType{}, `must not contain more than one topology annotation: ["kueue.x-k8s.io/podset-required-topology", `+ + `"kueue.x-k8s.io/podset-preferred-topology", "kueue.x-k8s.io/podset-unconstrained-topology"]`+ + `. Adjust either the "TrainJob.spec.podTemplateOverrides" or the "TrainingRuntime.Template" annotations for the corresponding Job`), + }.ToAggregate(), + topologyAwareScheduling: true, + }, + "invalid slice topology request - slice size larger than number of podsets": { + clusterTrainingRuntime: testCtr, + trainJob: testTrainJob.Clone().PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ + { + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ + {Name: "node"}, + }, + Metadata: &metav1.ObjectMeta{ + Annotations: map[string]string{ + kueue.PodSetRequiredTopologyAnnotation: "cloud.com/block", + kueue.PodSetSliceRequiredTopologyAnnotation: "cloud.com/block", + kueue.PodSetSliceSizeAnnotation: "20", + }, + }, + }, + }).Obj(), + wantErr: field.ErrorList{ + field.Invalid(field.NewPath("job[node].annotations"). + Key("kueue.x-k8s.io/podset-slice-size"), "20", "must not be greater than pod set count 1"+ + `. Adjust either the "TrainJob.spec.podTemplateOverrides" or the "TrainingRuntime.Template" annotations for the corresponding Job`), + }.ToAggregate(), + topologyAwareScheduling: true, }, } - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - webhook := &TrainJobWebhook{} + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { ctx, _ := utiltesting.ContextWithLog(t) + webhook := &TrainJobWebhook{} + clientBuilder := utiltesting.NewClientBuilder(kftrainerapi.AddToScheme, jobsetapi.AddToScheme) + kClient := clientBuilder.WithObjects(tc.trainJob, tc.clusterTrainingRuntime).Build() + indexer := utiltesting.AsIndexer(clientBuilder) + if err := SetupIndexes(ctx, indexer); err != nil { + t.Fatalf("Could not setup indexes: %v", err) + } + recorder := record.NewBroadcaster().NewRecorder(kClient.Scheme(), corev1.EventSource{Component: "test"}) + _, _ = NewReconciler(ctx, kClient, indexer, recorder) _, gotErr := webhook.ValidateCreate(ctx, tc.trainJob) if diff := cmp.Diff(tc.wantErr, gotErr); diff != "" { diff --git a/pkg/util/testingjobs/trainjob/wrappers.go b/pkg/util/testingjobs/trainjob/wrappers.go index 87484567e10..2cde8015c7d 100644 --- a/pkg/util/testingjobs/trainjob/wrappers.go +++ b/pkg/util/testingjobs/trainjob/wrappers.go @@ -53,9 +53,9 @@ func (t *TrainJobWrapper) Clone() *TrainJobWrapper { return &TrainJobWrapper{TrainJob: *t.DeepCopy()} } -// PodSpecOverrides sets the custom pod spec overrides to be set to the TrainJob jobset jobs -func (t *TrainJobWrapper) PodSpecOverrides(overrides []kftrainerapi.PodSpecOverride) *TrainJobWrapper { - t.Spec.PodSpecOverrides = overrides +// PodTemplateOverrides sets the custom pod template overrides to be set to the TrainJob jobset jobs +func (t *TrainJobWrapper) PodTemplateOverrides(overrides []kftrainerapi.PodTemplateOverride) *TrainJobWrapper { + t.Spec.PodTemplateOverrides = overrides return t } @@ -172,3 +172,35 @@ func MakeTrainingRuntime(name, ns string, jobsetSpec jobsetapi.JobSetSpec) *kftr }, } } + +type JobStatusWrapper struct{ kftrainerapi.JobStatus } + +func MakeJobStatusWrapper(name string) *JobStatusWrapper { + return &JobStatusWrapper{kftrainerapi.JobStatus{ + Name: name, + Ready: ptr.To(int32(0)), + Succeeded: ptr.To(int32(0)), + Failed: ptr.To(int32(0)), + Active: ptr.To(int32(0)), + Suspended: ptr.To(int32(0)), + }} +} + +func (s *JobStatusWrapper) Active(v int32) *JobStatusWrapper { + s.JobStatus.Active = ptr.To(v) + return s +} + +func (s *JobStatusWrapper) Ready(v int32) *JobStatusWrapper { + s.JobStatus.Ready = ptr.To(v) + return s +} + +func (s *JobStatusWrapper) Succeeded(v int32) *JobStatusWrapper { + s.JobStatus.Succeeded = ptr.To(v) + return s +} + +func (s *JobStatusWrapper) Obj() kftrainerapi.JobStatus { + return s.JobStatus +} diff --git a/test/e2e/singlecluster/tas_test.go b/test/e2e/singlecluster/tas_test.go index c513c1f3cf4..bcc94ed0594 100644 --- a/test/e2e/singlecluster/tas_test.go +++ b/test/e2e/singlecluster/tas_test.go @@ -18,10 +18,13 @@ package e2e import ( "fmt" + "strconv" + kftrainerapi "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,11 +34,13 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" podcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/pod" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" + workloadtrainjob "sigs.k8s.io/kueue/pkg/controller/jobs/trainjob" "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" + testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" ) @@ -442,4 +447,122 @@ var _ = ginkgo.Describe("TopologyAwareScheduling", func() { }) }) }) + + ginkgo.When("Creating a TrainJob requesting TAS", func() { + var ( + topology *kueue.Topology + onDemandRF *kueue.ResourceFlavor + clusterQueue *kueue.ClusterQueue + localQueue *kueue.LocalQueue + ) + ginkgo.BeforeEach(func() { + topology = utiltestingapi.MakeDefaultOneLevelTopology("hostname") + util.MustCreate(ctx, k8sClient, topology) + + onDemandRF = utiltestingapi.MakeResourceFlavor("on-demand"). + NodeLabel("instance-type", "on-demand"). + TopologyName(topology.Name). + Obj() + + util.MustCreate(ctx, k8sClient, onDemandRF) + clusterQueue = utiltestingapi.MakeClusterQueue("cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("on-demand"). + Resource(corev1.ResourceCPU, "1"). + Resource(corev1.ResourceMemory, "1Gi"). + Obj(), + ). + Obj() + + util.MustCreate(ctx, k8sClient, clusterQueue) + util.ExpectClusterQueuesToBeActive(ctx, k8sClient, clusterQueue) + + localQueue = utiltestingapi.MakeLocalQueue("main", ns.Name).ClusterQueue("cluster-queue").Obj() + util.MustCreate(ctx, k8sClient, localQueue) + }) + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteAllTrainJobsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed()) + // Force remove workloads to be sure that cluster queue can be removed. + gomega.Expect(util.DeleteWorkloadsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed()) + gomega.Expect(util.DeleteObject(ctx, k8sClient, localQueue)).Should(gomega.Succeed()) + util.ExpectObjectToBeDeleted(ctx, k8sClient, clusterQueue, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, onDemandRF, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, topology, true) + }) + + ginkgo.It("should admit a TrainJob via TAS", func() { + trainjob := testingtrainjob.MakeTrainJob("trainjob-test", ns.Name). + RuntimeRefName("torch-distributed"). + Queue(localQueue.Name). + // Even if we override the image coming from the TrainingRuntime, we still need to set the command and args + TrainerImage(util.GetAgnHostImage(), []string{"/agnhost"}, util.BehaviorExitFast). + TrainerRequest(corev1.ResourceCPU, "500m"). + TrainerRequest(corev1.ResourceMemory, "200Mi"). + PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ + { + TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ + {Name: "node"}, + }, + Metadata: &metav1.ObjectMeta{ + Annotations: map[string]string{kueue.PodSetRequiredTopologyAnnotation: corev1.LabelHostname}, + }, + }, + }). + Obj() + + ginkgo.By("Creating the TrainJob", func() { + util.MustCreate(ctx, k8sClient, trainjob) + }) + + ginkgo.By("waiting for the TrainJob to be unsuspended", func() { + trainjobKey := client.ObjectKeyFromObject(trainjob) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, trainjobKey, trainjob)).To(gomega.Succeed()) + g.Expect(trainjob.Spec.Suspend).Should(gomega.Equal(ptr.To(false))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verify the TrainJob has nodeSelector set", func() { + firstKueueOverride, exists := trainjob.Annotations["kueue.x-k8s.io/trainjob-override-idx"] + gomega.Expect(exists).Should(gomega.BeTrue()) + firstKueueOverrideIdx, err := strconv.Atoi(firstKueueOverride) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + gomega.Expect(trainjob.Spec.PodTemplateOverrides[firstKueueOverrideIdx].Spec.NodeSelector).To(gomega.Equal( + map[string]string{ + "instance-type": "on-demand", + }, + )) + }) + + wlLookupKey := types.NamespacedName{Name: workloadtrainjob.GetWorkloadNameForTrainJob(trainjob.Name, trainjob.UID), Namespace: ns.Name} + createdWorkload := &kueue.Workload{} + + ginkgo.By(fmt.Sprintf("await for admission of workload %q and verify TopologyAssignment", wlLookupKey), func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) + g.Expect(createdWorkload.Status.Admission).ShouldNot(gomega.BeNil()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + gomega.Expect(createdWorkload.Status.Admission).ShouldNot(gomega.BeNil()) + gomega.Expect(createdWorkload.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(1)) + gomega.Expect(createdWorkload.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( + &kueue.TopologyAssignment{ + Levels: []string{corev1.LabelHostname}, + Domains: []kueue.TopologyDomainAssignment{{ + Count: 1, + Values: []string{"kind-worker"}, + }}, + }, + )) + }) + + ginkgo.By(fmt.Sprintf("verify the workload %q gets finished", wlLookupKey), func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) + g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) + g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) }) diff --git a/test/e2e/singlecluster/trainjob_test.go b/test/e2e/singlecluster/trainjob_test.go index b7502e898fe..d2a29db07e3 100644 --- a/test/e2e/singlecluster/trainjob_test.go +++ b/test/e2e/singlecluster/trainjob_test.go @@ -185,7 +185,7 @@ var _ = ginkgo.Describe("TrainJob", func() { }) ginkgo.By("Verify the trainjob has nodeSelector set", func() { - gomega.Expect(trainjob.Spec.PodSpecOverrides[0].NodeSelector).To(gomega.Equal( + gomega.Expect(trainjob.Spec.PodTemplateOverrides[0].Spec.NodeSelector).To(gomega.Equal( map[string]string{ "instance-type": "on-demand", }, diff --git a/test/e2e/tas/trainjob_test.go b/test/e2e/tas/trainjob_test.go new file mode 100644 index 00000000000..42a8dffc99f --- /dev/null +++ b/test/e2e/tas/trainjob_test.go @@ -0,0 +1,314 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tase2e + +import ( + "fmt" + + kftrainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" + + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + "sigs.k8s.io/kueue/pkg/util/testing" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" + testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" + "sigs.k8s.io/kueue/test/util" +) + +var _ = ginkgo.Describe("TopologyAwareScheduling for TrainJob", func() { + var ns *corev1.Namespace + ginkgo.BeforeEach(func() { + ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "e2e-tas-jobset-") + }) + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + util.ExpectAllPodsInNamespaceDeleted(ctx, k8sClient, ns) + }) + + ginkgo.When("Creating a TrainJob", func() { + var ( + topology *kueue.Topology + tasFlavor *kueue.ResourceFlavor + localQueue *kueue.LocalQueue + clusterQueue *kueue.ClusterQueue + ) + ginkgo.BeforeEach(func() { + topology = utiltestingapi.MakeDefaultThreeLevelTopology("datacenter") + util.MustCreate(ctx, k8sClient, topology) + + tasFlavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). + NodeLabel(tasNodeGroupLabel, instanceType).TopologyName(topology.Name).Obj() + util.MustCreate(ctx, k8sClient, tasFlavor) + clusterQueue = utiltestingapi.MakeClusterQueue("cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("tas-flavor"). + Resource(extraResource, "8"). + Obj(), + ). + Obj() + util.MustCreate(ctx, k8sClient, clusterQueue) + util.ExpectClusterQueuesToBeActive(ctx, k8sClient, clusterQueue) + + localQueue = utiltestingapi.MakeLocalQueue("main", ns.Name).ClusterQueue("cluster-queue").Obj() + util.MustCreate(ctx, k8sClient, localQueue) + }) + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteAllTrainJobsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed()) + gomega.Expect(util.DeleteAllTrainingRuntimesInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed()) + // Force remove workloads to be sure that cluster queue can be removed. + gomega.Expect(util.DeleteWorkloadsInNamespace(ctx, k8sClient, ns)).Should(gomega.Succeed()) + util.ExpectObjectToBeDeleted(ctx, k8sClient, localQueue, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, clusterQueue, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, tasFlavor, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, topology, true) + util.ExpectAllPodsInNamespaceDeleted(ctx, k8sClient, ns) + }) + + ginkgo.It("Should place pods based on the ranks-ordering", func() { + replicas := 3 + parallelism := 2 + numPods := replicas * parallelism + + trainingRuntime := testingtrainjob.MakeTrainingRuntime("test-trainingruntime", ns.Name, testingjobset.MakeJobSet("", ""). + ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: "node", + Image: util.GetAgnHostImage(), + Args: util.BehaviorExitFast, + Replicas: int32(replicas), + Parallelism: int32(parallelism), + Completions: int32(parallelism), + PodAnnotations: map[string]string{ + kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + }, + }). + RequestAndLimit("node", extraResource, "1").Obj().Spec) + + trainjob := testingtrainjob.MakeTrainJob("trainjob-test", ns.Name).RuntimeRef(kftrainer.RuntimeRef{ + APIGroup: ptr.To(kftrainer.GroupVersion.Group), + Name: "test-trainingruntime", + Kind: ptr.To(kftrainer.TrainingRuntimeKind), + }). + Queue(localQueue.Name). + Obj() + + util.MustCreate(ctx, k8sClient, trainingRuntime) + util.MustCreate(ctx, k8sClient, trainjob) + + ginkgo.By("JobSet is unsuspended", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainjob), trainjob)).To(gomega.Succeed()) + g.Expect(trainjob.Spec.Suspend).Should(gomega.Equal(ptr.To(false))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + pods := &corev1.PodList{} + ginkgo.By("ensure all pods are created", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name))).To(gomega.Succeed()) + g.Expect(pods.Items).Should(gomega.HaveLen(numPods)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("ensure all pods are scheduled", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermNotEqualSelector("spec.nodeName", ""), + } + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name), listOpts)).To(gomega.Succeed()) + g.Expect(pods.Items).Should(gomega.HaveLen(numPods)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verify the assignment of pods are as expected with rank-based ordering", func() { + gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name))).To(gomega.Succeed()) + gotAssignment := readRankAssignmentsFromTrainJobPods(pods.Items) + wantAssignment := map[string]string{ + "0/0": "kind-worker", + "0/1": "kind-worker2", + "1/0": "kind-worker3", + "1/1": "kind-worker4", + "2/0": "kind-worker5", + "2/1": "kind-worker6", + } + gomega.Expect(wantAssignment).Should(gomega.BeComparableTo(gotAssignment)) + }) + }) + + ginkgo.It("Should place pods in podset slices with two-level scheduling based on the ranks-ordering", func() { + replicas := 2 + parallelism := 3 + numPods := replicas * parallelism + trainingRuntime := testingtrainjob.MakeTrainingRuntime("test-trainingruntime", ns.Name, testingjobset.MakeJobSet("", ""). + ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: "node", + Image: util.GetAgnHostImage(), + Args: util.BehaviorExitFast, + Replicas: int32(replicas), + Parallelism: int32(parallelism), + Completions: int32(parallelism), + PodAnnotations: map[string]string{ + kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetSliceSizeAnnotation: "3", + }, + }). + RequestAndLimit("node", extraResource, "1").Obj().Spec) + + trainjob := testingtrainjob.MakeTrainJob("trainjob-test", ns.Name).RuntimeRef(kftrainer.RuntimeRef{ + APIGroup: ptr.To(kftrainer.GroupVersion.Group), + Name: "test-trainingruntime", + Kind: ptr.To(kftrainer.TrainingRuntimeKind), + }). + Queue(localQueue.Name). + Obj() + + util.MustCreate(ctx, k8sClient, trainingRuntime) + util.MustCreate(ctx, k8sClient, trainjob) + + ginkgo.By("TrainJob is unsuspended", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainjob), trainjob)).To(gomega.Succeed()) + g.Expect(trainjob.Spec.Suspend).Should(gomega.Equal(ptr.To(false))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + pods := &corev1.PodList{} + ginkgo.By("ensure all pods are created", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name))).To(gomega.Succeed()) + g.Expect(pods.Items).Should(gomega.HaveLen(numPods)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("ensure all pods are scheduled", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermNotEqualSelector("spec.nodeName", ""), + } + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name), listOpts)).To(gomega.Succeed()) + g.Expect(pods.Items).Should(gomega.HaveLen(numPods)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verify the assignment of pods are as expected with rank-based ordering", func() { + gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name))).To(gomega.Succeed()) + gotAssignment := readRankAssignmentsFromJobSetPods(pods.Items) + wantAssignment := map[string]string{ + "0/0": "kind-worker", + "0/1": "kind-worker2", + "0/2": "kind-worker3", + "1/0": "kind-worker5", + "1/1": "kind-worker6", + "1/2": "kind-worker7", + } + gomega.Expect(wantAssignment).Should(gomega.BeComparableTo(gotAssignment)) + }) + }) + + ginkgo.It("Should place pods in podset slices with slice-only scheduling based on the ranks-ordering", func() { + replicas := 2 + parallelism := 3 + numPods := replicas * parallelism + trainingRuntime := testingtrainjob.MakeTrainingRuntime("test-trainingruntime", ns.Name, testingjobset.MakeJobSet("", ""). + ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: "node", + Image: util.GetAgnHostImage(), + Args: util.BehaviorExitFast, + Replicas: int32(replicas), + Parallelism: int32(parallelism), + Completions: int32(parallelism), + PodAnnotations: map[string]string{ + kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetSliceSizeAnnotation: "3", + }, + }). + RequestAndLimit("node", extraResource, "1").Obj().Spec) + + trainjob := testingtrainjob.MakeTrainJob("trainjob-test", ns.Name).RuntimeRef(kftrainer.RuntimeRef{ + APIGroup: ptr.To(kftrainer.GroupVersion.Group), + Name: "test-trainingruntime", + Kind: ptr.To(kftrainer.TrainingRuntimeKind), + }). + Queue(localQueue.Name). + Obj() + + util.MustCreate(ctx, k8sClient, trainingRuntime) + util.MustCreate(ctx, k8sClient, trainjob) + + ginkgo.By("TrainJob is unsuspended", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainjob), trainjob)).To(gomega.Succeed()) + g.Expect(trainjob.Spec.Suspend).Should(gomega.Equal(ptr.To(false))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + pods := &corev1.PodList{} + ginkgo.By("ensure all pods are created", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name))).To(gomega.Succeed()) + g.Expect(pods.Items).Should(gomega.HaveLen(numPods)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("ensure all pods are scheduled", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermNotEqualSelector("spec.nodeName", ""), + } + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name), listOpts)).To(gomega.Succeed()) + g.Expect(pods.Items).Should(gomega.HaveLen(numPods)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verify the assignment of pods are as expected with rank-based ordering", func() { + gomega.Expect(k8sClient.List(ctx, pods, client.InNamespace(ns.Name))).To(gomega.Succeed()) + gotAssignment := readRankAssignmentsFromJobSetPods(pods.Items) + wantAssignment := map[string]string{ + "0/0": "kind-worker", + "0/1": "kind-worker2", + "0/2": "kind-worker3", + "1/0": "kind-worker5", + "1/1": "kind-worker6", + "1/2": "kind-worker7", + } + gomega.Expect(wantAssignment).Should(gomega.BeComparableTo(gotAssignment)) + }) + }) + }) +}) + +func readRankAssignmentsFromTrainJobPods(pods []corev1.Pod) map[string]string { + assignment := make(map[string]string, len(pods)) + for _, pod := range pods { + podIndex := pod.Labels[batchv1.JobCompletionIndexAnnotation] + jobIndex := pod.Labels[jobset.JobIndexKey] + key := fmt.Sprintf("%v/%v", jobIndex, podIndex) + assignment[key] = pod.Spec.NodeName + } + return assignment +} diff --git a/test/integration/multikueue/jobs_test.go b/test/integration/multikueue/jobs_test.go index cf81e0f1cbc..80bd38fe845 100644 --- a/test/integration/multikueue/jobs_test.go +++ b/test/integration/multikueue/jobs_test.go @@ -1598,7 +1598,9 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, gomega.Eventually(func(g gomega.Gomega) { createdTrainJob := kftrainer.TrainJob{} g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, client.ObjectKeyFromObject(trainJob), &createdTrainJob)).To(gomega.Succeed()) - createdTrainJob.Status.JobsStatus = []kftrainer.JobStatus{{Name: "foo"}} + createdTrainJob.Status.JobsStatus = []kftrainer.JobStatus{ + testingtrainjob.MakeJobStatusWrapper("foo").Obj(), + } g.Expect(worker2TestCluster.client.Status().Update(worker2TestCluster.ctx, &createdTrainJob)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { diff --git a/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go b/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go index 70018b80c7e..386a7362399 100644 --- a/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go @@ -34,6 +34,8 @@ import ( "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/trainjob" + "sigs.k8s.io/kueue/pkg/controller/tas" + tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" @@ -84,7 +86,7 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { } } -func managerAndSchedulerSetup(opts ...jobframework.Option) framework.ManagerSetup { +func managerAndSchedulerSetup(setupTASControllers bool, opts ...jobframework.Option) framework.ManagerSetup { return func(ctx context.Context, mgr manager.Manager) { managerSetup(opts...)(ctx, mgr) @@ -100,6 +102,14 @@ func managerAndSchedulerSetup(opts ...jobframework.Option) framework.ManagerSetu failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) + if setupTASControllers { + failedCtrl, err = tas.SetupControllers(mgr, queues, cCache, configuration) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "TAS controller", failedCtrl) + + err = tasindexer.SetupIndexes(ctx, mgr.GetFieldIndexer()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + sched := scheduler.New(queues, cCache, mgr.GetClient(), mgr.GetEventRecorderFor(constants.AdmissionName)) err = sched.Start(ctx) gomega.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go b/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go index b74e35f0ca1..d4f914fb90e 100644 --- a/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go @@ -19,16 +19,20 @@ package trainjob import ( "fmt" + "github.com/google/go-cmp/cmp/cmpopts" kftrainerapi "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" @@ -37,6 +41,7 @@ import ( "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" + testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" @@ -244,9 +249,9 @@ var _ = ginkgo.Describe("Trainjob controller", ginkgo.Ordered, ginkgo.ContinueOn ginkgo.By("mark the trainjob as active", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainJob), trainJob)).To(gomega.Succeed()) - trainJob.Status.JobsStatus = make([]kftrainerapi.JobStatus, 1) - trainJob.Status.JobsStatus[0].Active = 1 - trainJob.Status.JobsStatus[0].Name = "node" + trainJob.Status.JobsStatus = []kftrainerapi.JobStatus{ + testingtrainjob.MakeJobStatusWrapper("node").Active(1).Obj(), + } g.Expect(k8sClient.Status().Update(ctx, trainJob)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -275,7 +280,7 @@ var _ = ginkgo.Describe("Trainjob controller", ginkgo.Ordered, ginkgo.ContinueOn ginkgo.By("mark the trainjob as inactive", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainJob), trainJob)).To(gomega.Succeed()) - trainJob.Status.JobsStatus[0].Active = 0 + trainJob.Status.JobsStatus[0].Active = ptr.To(int32(0)) g.Expect(k8sClient.Status().Update(ctx, trainJob)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -351,7 +356,7 @@ var _ = ginkgo.Describe("TrainJob controller for workloads when only jobs with q var _ = ginkgo.Describe("TrainJob controller interacting with scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { ginkgo.BeforeAll(func() { - fwk.StartManager(ctx, cfg, managerAndSchedulerSetup()) + fwk.StartManager(ctx, cfg, managerAndSchedulerSetup(false)) }) ginkgo.AfterAll(func() { fwk.StopManager(ctx) @@ -426,11 +431,11 @@ var _ = ginkgo.Describe("TrainJob controller interacting with scheduler", ginkgo gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: trainJob.Name, Namespace: ns.Name}, createdTrainJob)).Should(gomega.Succeed()) g.Expect(*createdTrainJob.Spec.Suspend).Should(gomega.BeFalse()) - g.Expect(createdTrainJob.Spec.PodSpecOverrides).To(gomega.HaveLen(2)) - g.Expect(createdTrainJob.Spec.PodSpecOverrides[0].TargetJobs[0]).Should(gomega.Equal(kftrainerapi.PodSpecOverrideTargetJob{Name: "node-1"})) - g.Expect(createdTrainJob.Spec.PodSpecOverrides[1].TargetJobs[0]).Should(gomega.Equal(kftrainerapi.PodSpecOverrideTargetJob{Name: "node-2"})) - g.Expect(createdTrainJob.Spec.PodSpecOverrides[0].NodeSelector[instanceKey]).Should(gomega.Equal(spotUntaintedFlavor.Name)) - g.Expect(createdTrainJob.Spec.PodSpecOverrides[1].NodeSelector[instanceKey]).Should(gomega.Equal(onDemandFlavor.Name)) + g.Expect(createdTrainJob.Spec.PodTemplateOverrides).To(gomega.HaveLen(2)) + g.Expect(createdTrainJob.Spec.PodTemplateOverrides[0].TargetJobs[0]).Should(gomega.Equal(kftrainerapi.PodTemplateOverrideTargetJob{Name: "node-1"})) + g.Expect(createdTrainJob.Spec.PodTemplateOverrides[1].TargetJobs[0]).Should(gomega.Equal(kftrainerapi.PodTemplateOverrideTargetJob{Name: "node-2"})) + g.Expect(createdTrainJob.Spec.PodTemplateOverrides[0].Spec.NodeSelector[instanceKey]).Should(gomega.Equal(spotUntaintedFlavor.Name)) + g.Expect(createdTrainJob.Spec.PodTemplateOverrides[1].Spec.NodeSelector[instanceKey]).Should(gomega.Equal(onDemandFlavor.Name)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPendingWorkloadsMetric(clusterQueue, 0, 0) @@ -524,14 +529,8 @@ var _ = ginkgo.Describe("TrainJob controller interacting with scheduler", ginkgo createdTrainJob1 := &kftrainerapi.TrainJob{} gomega.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: trainJob1.Name, Namespace: ns.Name}, createdTrainJob1)).Should(gomega.Succeed()) createdTrainJob1.Status.JobsStatus = []kftrainerapi.JobStatus{ - { - Name: "node-1", - Succeeded: 2, - }, - { - Name: "node-2", - Succeeded: 1, - }, + testingtrainjob.MakeJobStatusWrapper("node-1").Succeeded(2).Obj(), + testingtrainjob.MakeJobStatusWrapper("node-2").Succeeded(1).Obj(), } gomega.Expect(k8sClient.Status().Update(ctx, createdTrainJob1)).Should(gomega.Succeed()) @@ -561,3 +560,170 @@ var _ = ginkgo.Describe("TrainJob controller interacting with scheduler", ginkgo }) }) }) + +var _ = ginkgo.Describe("TrainJob controller with TopologyAwareScheduling", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { + const ( + nodeGroupLabel = "node-group" + ) + + var ( + ns *corev1.Namespace + nodes []corev1.Node + topology *kueue.Topology + tasFlavor *kueue.ResourceFlavor + clusterQueue *kueue.ClusterQueue + localQueue *kueue.LocalQueue + ) + + ginkgo.BeforeAll(func() { + fwk.StartManager(ctx, cfg, managerAndSchedulerSetup(true)) + }) + + ginkgo.AfterAll(func() { + fwk.StopManager(ctx) + }) + + ginkgo.BeforeEach(func() { + ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "tas-jobset-") + + nodes = []corev1.Node{ + *testingnode.MakeNode("b1r1"). + Label(nodeGroupLabel, "tas"). + Label(testing.DefaultBlockTopologyLevel, "b1"). + Label(testing.DefaultRackTopologyLevel, "r1"). + StatusAllocatable(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("10"), + }). + Ready(). + Obj(), + } + util.CreateNodesWithStatus(ctx, k8sClient, nodes) + + topology = utiltestingapi.MakeDefaultTwoLevelTopology("default") + util.MustCreate(ctx, k8sClient, topology) + + tasFlavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). + NodeLabel(nodeGroupLabel, "tas"). + TopologyName("default").Obj() + util.MustCreate(ctx, k8sClient, tasFlavor) + + clusterQueue = utiltestingapi.MakeClusterQueue("cluster-queue"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas(tasFlavor.Name).Resource(corev1.ResourceCPU, "5").Obj()). + Obj() + util.MustCreate(ctx, k8sClient, clusterQueue) + util.ExpectClusterQueuesToBeActive(ctx, k8sClient, clusterQueue) + + localQueue = utiltestingapi.MakeLocalQueue("local-queue", ns.Name).ClusterQueue(clusterQueue.Name).Obj() + util.MustCreate(ctx, k8sClient, localQueue) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + util.ExpectObjectToBeDeleted(ctx, k8sClient, clusterQueue, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, tasFlavor, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, topology, true) + for _, node := range nodes { + util.ExpectObjectToBeDeleted(ctx, k8sClient, &node, true) + } + }) + + ginkgo.It("should admit workload which fits in a required topology domain", func() { + testJobSet := testingjobset.MakeJobSet("", "").ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: "node-1", + Replicas: 1, + Parallelism: 1, + Completions: 1, + PodAnnotations: map[string]string{ + kueue.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + }, + }, testingjobset.ReplicatedJobRequirements{ + Name: "node-2", + Replicas: 1, + Parallelism: 1, + Completions: 1, + PodAnnotations: map[string]string{ + kueue.PodSetPreferredTopologyAnnotation: testing.DefaultRackTopologyLevel, + }, + }, + ). + Request("node-1", corev1.ResourceCPU, "100m"). + Request("node-2", corev1.ResourceCPU, "100m"). + Obj() + testTr := testingtrainjob.MakeTrainingRuntime("test", ns.Name, testJobSet.Spec) + trainJob := testingtrainjob.MakeTrainJob("trainjob-test", ns.Name).RuntimeRef(kftrainerapi.RuntimeRef{ + APIGroup: ptr.To("trainer.kubeflow.org"), + Name: "test", + Kind: ptr.To(kftrainerapi.TrainingRuntimeKind), + }). + Queue(localQueue.Name). + Suspend(false). + Obj() + + ginkgo.By("creating a TrainJob", func() { + util.MustCreate(ctx, k8sClient, testTr) + util.MustCreate(ctx, k8sClient, trainJob) + }) + + wl := &kueue.Workload{} + wlLookupKey := types.NamespacedName{ + Name: workloadtrainjob.GetWorkloadNameForTrainJob(trainJob.Name, trainJob.UID), + Namespace: ns.Name, + } + + ginkgo.By("verify the workload is created", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey, wl)).Should(gomega.Succeed()) + g.Expect(wl.Spec.PodSets).Should(gomega.BeComparableTo([]kueue.PodSet{ + { + Name: "node-1", + Count: 1, + TopologyRequest: &kueue.PodSetTopologyRequest{ + Required: ptr.To(testing.DefaultBlockTopologyLevel), + PodIndexLabel: ptr.To(batchv1.JobCompletionIndexAnnotation), + SubGroupIndexLabel: ptr.To(jobsetapi.JobIndexKey), + SubGroupCount: ptr.To[int32](1), + }, + }, + { + Name: "node-2", + Count: 1, + TopologyRequest: &kueue.PodSetTopologyRequest{ + Preferred: ptr.To(testing.DefaultRackTopologyLevel), + PodIndexLabel: ptr.To(batchv1.JobCompletionIndexAnnotation), + SubGroupIndexLabel: ptr.To(jobsetapi.JobIndexKey), + SubGroupCount: ptr.To[int32](1), + }, + }, + }, cmpopts.IgnoreFields(kueue.PodSet{}, "Template"))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verify the workload is admitted", func() { + util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, wl) + util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 1) + }) + + ginkgo.By("verify admission for the workload", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey, wl)).Should(gomega.Succeed()) + g.Expect(wl.Status.Admission).ShouldNot(gomega.BeNil()) + g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(2)) + g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( + &kueue.TopologyAssignment{ + Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, + }, + )) + g.Expect(wl.Status.Admission.PodSetAssignments[1].TopologyAssignment).Should(gomega.BeComparableTo( + &kueue.TopologyAssignment{ + Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, + }, + )) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) +}) diff --git a/test/integration/singlecluster/webhook/jobs/trainjob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/trainjob_webhook_test.go index 79d05bbf1f6..2b57b4f090b 100644 --- a/test/integration/singlecluster/webhook/jobs/trainjob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/trainjob_webhook_test.go @@ -23,9 +23,13 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/kueue/pkg/constants" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" + "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadtrainjob "sigs.k8s.io/kueue/pkg/controller/jobs/trainjob" + testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" "sigs.k8s.io/kueue/test/util" ) @@ -35,7 +39,18 @@ var _ = ginkgo.Describe("Trainjob Webhook", func() { ginkgo.When("with manageJobsWithoutQueueName disabled", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { ginkgo.BeforeAll(func() { - fwk.StartManager(ctx, cfg, managerSetup(workloadtrainjob.SetupTrainJobWebhook)) + fwk.StartManager(ctx, cfg, managerSetup(func(mgr ctrl.Manager, opts ...jobframework.Option) error { + // Necessary to initialize the runtimes + if _, err := workloadtrainjob.NewReconciler( + ctx, + mgr.GetClient(), + mgr.GetFieldIndexer(), + mgr.GetEventRecorderFor(constants.JobControllerName), + opts...); err != nil { + return err + } + return workloadtrainjob.SetupTrainJobWebhook(mgr, opts...) + })) }) ginkgo.BeforeEach(func() { ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "trainjob-") @@ -48,12 +63,24 @@ var _ = ginkgo.Describe("Trainjob Webhook", func() { }) ginkgo.It("should succed creating the TrainJob", func() { - trainJob := testingtrainjob.MakeTrainJob("trainjob-test", ns.Name). + testJobSet := testingjobset.MakeJobSet("", "").ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: "node", + Replicas: 1, + }). + Obj() + testTr := testingtrainjob.MakeTrainingRuntime("test", ns.Name, testJobSet.Spec) + trainJob := testingtrainjob.MakeTrainJob("trainjob-test", ns.Name).RuntimeRef(kftrainerapi.RuntimeRef{ + APIGroup: ptr.To(kftrainerapi.GroupVersion.Group), + Name: "test", + Kind: ptr.To(kftrainerapi.TrainingRuntimeKind), + }). Queue("queue"). Suspend(false). Obj() ginkgo.By("by creating the TrainJob", func() { + util.MustCreate(ctx, k8sClient, testTr) util.MustCreate(ctx, k8sClient, trainJob) }) @@ -68,11 +95,23 @@ var _ = ginkgo.Describe("Trainjob Webhook", func() { }) ginkgo.It("should not suspend a TrainJob without queue", func() { - trainJob := testingtrainjob.MakeTrainJob("trainjob-test", ns.Name). + testJobSet := testingjobset.MakeJobSet("", "").ReplicatedJobs( + testingjobset.ReplicatedJobRequirements{ + Name: "node", + Replicas: 1, + }). + Obj() + testTr := testingtrainjob.MakeTrainingRuntime("test", ns.Name, testJobSet.Spec) + trainJob := testingtrainjob.MakeTrainJob("trainjob-test", ns.Name).RuntimeRef(kftrainerapi.RuntimeRef{ + APIGroup: ptr.To(kftrainerapi.GroupVersion.Group), + Name: "test", + Kind: ptr.To(kftrainerapi.TrainingRuntimeKind), + }). Suspend(false). Obj() ginkgo.By("by creating the TrainJob", func() { + util.MustCreate(ctx, k8sClient, testTr) util.MustCreate(ctx, k8sClient, trainJob) }) diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/trainingruntime_types.go b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/trainingruntime_types.go index 6c4478b4c6a..221e70d8c5c 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/trainingruntime_types.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/trainingruntime_types.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" jobsetv1alpha2 "sigs.k8s.io/jobset/api/jobset/v1alpha2" + volcanov1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" ) const ( @@ -44,11 +45,13 @@ const ( type ClusterTrainingRuntime struct { metav1.TypeMeta `json:",inline"` - // Standard object's metadata. + // metadata of the ClusterTrainingRuntime. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - // Specification of the desired ClusterTrainingRuntime. - Spec TrainingRuntimeSpec `json:"spec,omitempty"` + // spec of the ClusterTrainingRuntime. + // +optional + Spec TrainingRuntimeSpec `json:"spec,omitzero"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -78,11 +81,13 @@ type ClusterTrainingRuntimeList struct { type TrainingRuntime struct { metav1.TypeMeta `json:",inline"` - // Standard object's metadata. + // metadata of the TrainingRuntime. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - // Specification of the desired TrainingRuntime. - Spec TrainingRuntimeSpec `json:"spec,omitempty"` + // spec of the TrainingRuntime. + // +optional + Spec TrainingRuntimeSpec `json:"spec,omitempty,omitzero"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -101,24 +106,31 @@ type TrainingRuntimeList struct { } // TrainingRuntimeSpec represents a specification of the desired training runtime. +// +kubebuilder:validation:MinProperties=1 type TrainingRuntimeSpec struct { - // Configuration for the model training with ML-specific parameters. + // mlPolicy provides the ML-specific parameters for the model training. + // +optional MLPolicy *MLPolicy `json:"mlPolicy,omitempty"` - // Configuration for the PodGroup to enable gang-scheduling via supported plugins. + // podGroupPolicy defines the configuration for the PodGroup to enable gang-scheduling via supported plugins. + // +optional PodGroupPolicy *PodGroupPolicy `json:"podGroupPolicy,omitempty"` - // JobSet template which will be used by TrainJob. - Template JobSetTemplateSpec `json:"template"` + // template for the JobSet which will be used by TrainJob. + // +optional + Template JobSetTemplateSpec `json:"template,omitzero"` } // JobSetTemplateSpec represents a template of the desired JobSet. +// +kubebuilder:validation:MinProperties=1 type JobSetTemplateSpec struct { - // Metadata for custom JobSet's labels and annotations. + // metadata for custom JobSet's labels and annotations. // JobSet name and namespace is equal to the TrainJob's name and namespace. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - // Specification of the desired JobSet which will be created from TrainJob. + // spec of the desired JobSet which will be created from TrainJob. + // +optional Spec jobsetv1alpha2.JobSetSpec `json:"spec,omitempty"` } @@ -131,28 +143,40 @@ type PodGroupPolicy struct { // PodGroupPolicySource represents supported plugins for gang-scheduling. // Only one of its members may be specified. type PodGroupPolicySource struct { - // Coscheduling plugin from the Kubernetes scheduler-plugins for gang-scheduling. + // coscheduling plugin from the Kubernetes scheduler-plugins for gang-scheduling. + // +optional Coscheduling *CoschedulingPodGroupPolicySource `json:"coscheduling,omitempty"` - // TODO (andreyvelich): Add support for Volcano gang-scheduler. + // volcano plugin for gang-scheduling. + // +optional + Volcano *VolcanoPodGroupPolicySource `json:"volcano,omitempty"` } // CoschedulingPodGroupPolicySource represents configuration for coscheduling plugin. // The number of min members in the PodGroupSpec is always equal to the number of nodes. type CoschedulingPodGroupPolicySource struct { - // Time threshold to schedule PodGroup for gang-scheduling. + // scheduleTimeoutSeconds is the maximum duration to schedule PodGroup for gang-scheduling. // If the scheduling timeout is equal to 0, the default value is used. // Defaults to 60 seconds. // +kubebuilder:default=60 + // +optional ScheduleTimeoutSeconds *int32 `json:"scheduleTimeoutSeconds,omitempty"` } -// MLPolicy represents configuration for the model trining with ML-specific parameters. +// VolcanoPodGroupPolicySource represents configuration for the Volcano gang-scheduler. +type VolcanoPodGroupPolicySource struct { + // networkTopology defines the NetworkTopology config, this field works in conjunction with network topology feature and hyperNode CRD. + // +optional + NetworkTopology *volcanov1beta1.NetworkTopologySpec `json:"networkTopology,omitempty"` +} + +// MLPolicy represents configuration for the model training with ML-specific parameters. // +kubebuilder:validation:XValidation:rule="!(has(self.numNodes) && (has(self.torch) && has(self.torch.elasticPolicy)))", message="numNodes should not be set if torch.elasticPolicy is configured" // +kubebuilder:validation:XValidation:rule="!(has(self.torch) && has(self.mpi))", message="Only one of the policy can be configured" type MLPolicy struct { - // Number of training nodes. + // numNodes is the number of training nodes. // Defaults to 1. + // +optional NumNodes *int32 `json:"numNodes,omitempty"` // Configuration for the runtime-specific parameters, such as Torch or MPI. @@ -163,24 +187,28 @@ type MLPolicy struct { // MLPolicySource represents the runtime-specific configuration for various technologies. // One of the following specs can be set. type MLPolicySource struct { - // Configuration for the PyTorch runtime. + // torch defines the configuration for the PyTorch runtime. + // +optional Torch *TorchMLPolicySource `json:"torch,omitempty"` - // Configuration for the MPI Runtime. + // mpi defines the configuration for the MPI Runtime. + // +optional MPI *MPIMLPolicySource `json:"mpi,omitempty"` } // TorchMLPolicySource represents a PyTorch runtime configuration. type TorchMLPolicySource struct { - // Number of processes per node. + // numProcPerNode is the number of processes per node. // This value is inserted into the `--nproc-per-node` argument of the `torchrun` CLI. // Supported values: `auto`, `cpu`, `gpu`, or int value. // Defaults to `auto`. // +kubebuilder:default="auto" // +kubebuilder:validation:XValidation:rule="self > 0 || self in ['auto', 'cpu', 'gpu']", message="NumProcPerNode must be equal to auto, cpu, gpu, or int value" + // +optional NumProcPerNode *intstr.IntOrString `json:"numProcPerNode,omitempty"` - // Elastic policy for the PyTorch training. + // elasticPolicy defines the Elastic policy for the PyTorch training. + // +optional ElasticPolicy *TorchElasticPolicy `json:"elasticPolicy,omitempty"` } @@ -189,46 +217,54 @@ type TorchMLPolicySource struct { // is used to configure the `torchrun` CLI argument: `--nnodes=minNodes:maxNodes`. // Only `c10d` backend is supported for the Rendezvous communication. type TorchElasticPolicy struct { - // How many times the training job can be restarted. + // maxRestarts defines how many times the training job can be restarted. // This value is inserted into the `--max-restarts` argument of the `torchrun` CLI and // the `.spec.failurePolicy.maxRestarts` parameter of the training Job. + // +optional MaxRestarts *int32 `json:"maxRestarts,omitempty"` - // Lower limit for the number of nodes to which training job can scale down. + // minNodes is the lower limit for the number of nodes to which training job can scale down. + // +optional MinNodes *int32 `json:"minNodes,omitempty"` - // Upper limit for the number of nodes to which training job can scale up. + // maxNodes is the upper limit for the number of nodes to which training job can scale up. + // +optional MaxNodes *int32 `json:"maxNodes,omitempty"` - // Specification which are used to calculate the desired number of nodes. See the individual + // metrics which are used to calculate the desired number of nodes. See the individual // metric source types for more information about how each type of metric must respond. // The HPA will be created to perform auto-scaling. // +listType=atomic + // +optional Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` } // MPIMLPolicySource represents a MPI runtime configuration. type MPIMLPolicySource struct { - // Number of processes per node. + // numProcPerNode is the number of processes per node. // This value is equal to the number of slots for each node in the hostfile. // Defaults to 1. // +kubebuilder:default=1 + // +optional NumProcPerNode *int32 `json:"numProcPerNode,omitempty"` - // Implementation name for the MPI to create the appropriate hostfile. + // mpiImplementation is the name of the MPI implementation to create the appropriate hostfile. // Defaults to OpenMPI. // +kubebuilder:default=OpenMPI - // +kubebuilder:validation:Enum=OpenMPI + // +kubebuilder:validation:Enum=OpenMPI;"" + // +optional MPIImplementation *MPIImplementation `json:"mpiImplementation,omitempty"` - // Directory where SSH keys are mounted. + // sshAuthMountPath is the directory where SSH keys are mounted. // Defaults to /root/.ssh. // +kubebuilder:default=/root/.ssh + // +optional SSHAuthMountPath *string `json:"sshAuthMountPath,omitempty"` - // Whether to run training process on the launcher Job. + // runLauncherAsNode defines whether to run training process on the launcher Job. // Defaults to false. // +kubebuilder:default=false + // +optional RunLauncherAsNode *bool `json:"runLauncherAsNode,omitempty"` } diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/trainjob_types.go b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/trainjob_types.go index 6d0f9e122a0..580382ee308 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/trainjob_types.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/trainjob_types.go @@ -34,19 +34,24 @@ const ( // +kubebuilder:storageversion // +kubebuilder:printcolumn:name="State",type=string,JSONPath=`.status.conditions[-1:].type` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +// +kubebuilder:validation:XValidation:rule="self.metadata.name.matches('^[a-z]([-a-z0-9]*[a-z0-9])?$')", message="metadata.name must match RFC 1035 DNS label format" +// +kubebuilder:validation:XValidation:rule="size(self.metadata.name) <= 63", message="metadata.name must be no more than 63 characters" // TrainJob represents configuration of a training job. type TrainJob struct { metav1.TypeMeta `json:",inline"` - // Standard object's metadata. + // metadata of the TrainJob. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - // Specification of the desired TrainJob. - Spec TrainJobSpec `json:"spec,omitempty"` + // spec of the TrainJob. + // +optional + Spec TrainJobSpec `json:"spec,omitzero"` - // Current status of TrainJob. - Status TrainJobStatus `json:"status,omitempty"` + // status of TrainJob. + // +optional + Status TrainJobStatus `json:"status,omitzero"` } const ( @@ -91,35 +96,43 @@ type TrainJobList struct { // TrainJobSpec represents specification of the desired TrainJob. type TrainJobSpec struct { - // Reference to the training runtime. + // runtimeRef is the reference to the training runtime. // The field is immutable. // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="runtimeRef is immutable" - RuntimeRef RuntimeRef `json:"runtimeRef"` + // +required + RuntimeRef RuntimeRef `json:"runtimeRef,omitzero"` - // Configuration of the initializer. + // initializer defines the configuration of the initializer. + // +optional Initializer *Initializer `json:"initializer,omitempty"` - // Configuration of the trainer. + // trainer defines the configuration of the trainer. + // +optional Trainer *Trainer `json:"trainer,omitempty"` - // Labels to apply for the derivative JobSet and Jobs. + // labels to apply for the derivative JobSet and Jobs. // They will be merged with the TrainingRuntime values. + // +optional Labels map[string]string `json:"labels,omitempty"` - // Annotations to apply for the derivative JobSet and Jobs. + // annotations to apply for the derivative JobSet and Jobs. // They will be merged with the TrainingRuntime values. + // +optional Annotations map[string]string `json:"annotations,omitempty"` - // Custom overrides for the training runtime. + // podTemplateOverrides define the PodTemplateOverrides for the training runtime. + // When multiple overrides apply to the same targetJob, later entries in the slice override earlier field values. // +listType=atomic - PodSpecOverrides []PodSpecOverride `json:"podSpecOverrides,omitempty"` + // +optional + PodTemplateOverrides []PodTemplateOverride `json:"podTemplateOverrides,omitempty"` - // Whether the controller should suspend the running TrainJob. + // suspend defines whether to suspend the running TrainJob. // Defaults to false. // +kubebuilder:default=false + // +optional Suspend *bool `json:"suspend,omitempty"` - // ManagedBy is used to indicate the controller or entity that manages a TrainJob. + // managedBy is used to indicate the controller or entity that manages a TrainJob. // The value must be either an empty, `trainer.kubeflow.org/trainjob-controller` or // `kueue.x-k8s.io/multikueue`. The built-in TrainJob controller reconciles TrainJob which // don't have this field at all or the field value is the reserved string @@ -129,34 +142,41 @@ type TrainJobSpec struct { // +kubebuilder:default="trainer.kubeflow.org/trainjob-controller" // +kubebuilder:validation:XValidation:rule="self in ['trainer.kubeflow.org/trainjob-controller', 'kueue.x-k8s.io/multikueue']", message="ManagedBy must be trainer.kubeflow.org/trainjob-controller or kueue.x-k8s.io/multikueue if set" // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="ManagedBy value is immutable" + // +optional ManagedBy *string `json:"managedBy,omitempty"` } // RuntimeRef represents the reference to the existing training runtime. type RuntimeRef struct { - // Name of the runtime being referenced. + // name of the runtime being referenced. // When namespaced-scoped TrainingRuntime is used, the TrainJob must have // the same namespace as the deployed runtime. - Name string `json:"name"` + // +kubebuilder:validation:MinLength=1 + // +required + Name string `json:"name,omitempty"` - // APIGroup of the runtime being referenced. + // apiGroup of the runtime being referenced. // Defaults to `trainer.kubeflow.org`. // +kubebuilder:default="trainer.kubeflow.org" + // +optional APIGroup *string `json:"apiGroup,omitempty"` - // Kind of the runtime being referenced. + // kind of the runtime being referenced. // Defaults to ClusterTrainingRuntime. // +kubebuilder:default="ClusterTrainingRuntime" + // +optional Kind *string `json:"kind,omitempty"` } // Initializer represents the desired configuration for the dataset and model initialization. // It is used to initialize the assets (dataset and pre-trained model) and pre-process data. type Initializer struct { - // Configuration of the dataset initialization and pre-processing. + // dataset defines the configuration for the dataset initialization and pre-processing. + // +optional Dataset *DatasetInitializer `json:"dataset,omitempty"` - // Configuration of the pre-trained model initialization + // model defines the configuration for the pre-trained model initialization + // +optional Model *ModelInitializer `json:"model,omitempty"` } @@ -164,35 +184,41 @@ type Initializer struct { // The DatasetInitializer spec will override the runtime Job template // which contains this label: `trainer.kubeflow.org/trainjob-ancestor-step: dataset-initializer` type DatasetInitializer struct { - // Storage uri for the dataset provider. + // storageUri is the URI for the dataset provider. + // +optional StorageUri *string `json:"storageUri,omitempty"` - // List of environment variables to set in the dataset initializer container. + // env is the list of environment variables to set in the dataset initializer container. // These values will be merged with the TrainingRuntime's dataset initializer environments. // +listType=map // +listMapKey=name + // +optional Env []corev1.EnvVar `json:"env,omitempty"` - // Reference to the secret with credentials to download dataset. + // secretRef is the reference to the secret with credentials to download dataset. // Secret must be created in the TrainJob's namespace. + // +optional SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` } -// DatasetInitializer represents the desired configuration to initialize pre-trained model. -// The DatasetInitializer spec will override the runtime Job template +// ModelInitializer represents the desired configuration to initialize pre-trained model. +// The ModelInitializer spec will override the runtime Job template // which contains this label: `trainer.kubeflow.org/trainjob-ancestor-step: dataset-initializer` type ModelInitializer struct { - // Storage uri for the model provider. + // storageUri is the URI for the model provider. + // +optional StorageUri *string `json:"storageUri,omitempty"` - // List of environment variables to set in the model initializer container. + // env is the list of environment variables to set in the model initializer container. // These values will be merged with the TrainingRuntime's model initializer environments. // +listType=map // +listMapKey=name + // +optional Env []corev1.EnvVar `json:"env,omitempty"` - // Reference to the secret with credentials to download model. + // secretRef is the reference to the secret with credentials to download model. // Secret must be created in the TrainJob's namespace. + // +optional SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"` } @@ -200,135 +226,191 @@ type ModelInitializer struct { // The Trainer spec will override the runtime template // which contains this label: `trainer.kubeflow.org/trainjob-ancestor-step: trainer` type Trainer struct { - // Docker image for the training container. + // image is the container image for the training container. + // +optional Image *string `json:"image,omitempty"` - // Entrypoint commands for the training container. + // command for the entrypoint of the training container. // +listType=atomic + // +optional Command []string `json:"command,omitempty"` - // Arguments to the entrypoint for the training container. + // args for the entrypoint for the training container. // +listType=atomic + // +optional Args []string `json:"args,omitempty"` - // List of environment variables to set in the training container. + // env is the list of environment variables to set in the training container. // These values will be merged with the TrainingRuntime's trainer environments. // +listType=map // +listMapKey=name + // +optional Env []corev1.EnvVar `json:"env,omitempty"` - // Number of training nodes. + // numNodes is the number of training nodes. // TODO (andreyvelich): Do we want to support dynamic num of nodes in TrainJob for PyTorch elastic: `--nnodes=1:4` ? + // +optional NumNodes *int32 `json:"numNodes,omitempty"` - // Compute resources for each training node. + // resourcesPerNode defines the compute resources for each training node. + // +optional ResourcesPerNode *corev1.ResourceRequirements `json:"resourcesPerNode,omitempty"` - // Number of processes/workers/slots on every training node. + // numProcPerNode is the number of processes/workers/slots on every training node. // For the Torch runtime: `auto`, `cpu`, `gpu`, or int value can be set. // For the MPI runtime only int value can be set. + // +optional NumProcPerNode *intstr.IntOrString `json:"numProcPerNode,omitempty"` } -// PodSpecOverride represents the custom overrides that will be applied for the TrainJob's resources. -type PodSpecOverride struct { - // TrainJobs is the training job replicas in the training runtime template to apply the overrides. +// PodTemplateOverride represents a custom PodTemplateSpec override that will be applied to the TrainJob's training runtime. +type PodTemplateOverride struct { + // targetJobs is the list of replicated jobs in the training runtime template to apply the overrides. // +listType=atomic - TargetJobs []PodSpecOverrideTargetJob `json:"targetJobs"` + // +required + TargetJobs []PodTemplateOverrideTargetJob `json:"targetJobs,omitempty"` + + // metadata overrides the Pod template metadata. + // These values will be merged with the TrainingRuntime's Pod template metadata. + // +optional + Metadata *metav1.ObjectMeta `json:"metadata,omitempty"` + + // spec overrides the Pod template spec. + // These values will be merged with the TrainingRuntime's Pod template spec. + // +optional + Spec *PodTemplateSpecOverride `json:"spec,omitempty"` +} - // Override for the service account. +type PodTemplateOverrideTargetJob struct { + // name is the target training job name for which the PodTemplateSpec is overridden. + // +kubebuilder:validation:MinLength=1 + // +required + Name string `json:"name,omitempty"` +} + +// PodTemplateSpecOverride represents the spec overrides for Pod template. +type PodTemplateSpecOverride struct { + // serviceAccountName overrides the service account. + // +optional ServiceAccountName *string `json:"serviceAccountName,omitempty"` - // Override for the node selector to place Pod on the specific node. + // nodeSelector overrides the node selector to place Pod on the specific node. + // +optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` - // Override for the Pod's tolerations. + // affinity overrides for the Pod's affinity. + // +optional + Affinity *corev1.Affinity `json:"affinity,omitempty"` + + // tolerations overrides the Pod's tolerations. // +listType=atomic + // +optional Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - // Overrides for the Pod volume configurations. + // volumes overrides the Pod's volumes. // +listType=map // +listMapKey=name + // +optional Volumes []corev1.Volume `json:"volumes,omitempty"` - // Overrides for the init container in the target job templates. + // initContainers overrides the init container in the target job templates. // +listType=map // +listMapKey=name + // +optional InitContainers []ContainerOverride `json:"initContainers,omitempty"` - // Overrides for the containers in the target job templates. + // containers overrides for the containers in the target job templates. // +listType=map // +listMapKey=name + // +optional Containers []ContainerOverride `json:"containers,omitempty"` - // SchedulingGates overrides the scheduling gates of the Pods in the target job templates. + // schedulingGates overrides the scheduling gates of the Pods in the target job templates. // More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/ // +listType=map // +listMapKey=name + // +optional SchedulingGates []corev1.PodSchedulingGate `json:"schedulingGates,omitempty"` -} -type PodSpecOverrideTargetJob struct { - // Name is the target training job name for which the PodSpec is overridden. - Name string `json:"name"` + // imagePullSecrets overrides the image pull secrets for the Pods in the target job templates. + // +listType=map + // +listMapKey=name + // +optional + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` } // ContainerOverride represents parameters that can be overridden using PodSpecOverrides. type ContainerOverride struct { - // Name for the container. TrainingRuntime must have this container. - Name string `json:"name"` + // name for the container. TrainingRuntime must have this container. + // +kubebuilder:validation:MinLength=1 + // +required + Name string `json:"name,omitempty"` - // List of environment variables to set in the container. + // env is the list of environment variables to set in the container. // These values will be merged with the TrainingRuntime's environments. // These values can't be set for container with the name: `node`, `dataset-initializer`, or // `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs. // +listType=map // +listMapKey=name + // +optional Env []corev1.EnvVar `json:"env,omitempty"` - // Pod volumes to mount into the container's filesystem. + // volumeMounts are the volumes to mount into the container's filesystem. // +listType=map // +listMapKey=name + // +optional VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` } // TrainJobStatus represents the current status of TrainJob. +// +kubebuilder:validation:MinProperties=1 type TrainJobStatus struct { - // Conditions for the TrainJob. + // conditions for the TrainJob. // // +optional // +listType=map // +listMapKey=type - // +patchStrategy=merge - // +patchMergeKey=type - Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + Conditions []metav1.Condition `json:"conditions,omitempty"` - // JobsStatus tracks the child Jobs in TrainJob. + // jobsStatus tracks the child Jobs in TrainJob. // +listType=map // +listMapKey=name + // +optional JobsStatus []JobStatus `json:"jobsStatus,omitempty"` } type JobStatus struct { - // Name of the child Job. - Name string `json:"name"` + // name of the child Job. + // +kubebuilder:validation:MinLength=1 + // +required + Name string `json:"name,omitempty"` - // Ready is the number of child Jobs where the number of ready pods and completed pods + // ready is the number of child Jobs where the number of ready pods and completed pods // is greater than or equal to the total expected pod count for the child Job. - Ready int32 `json:"ready"` + // +kubebuilder:validation:Minimum=0 + // +required + Ready *int32 `json:"ready,omitempty"` - // Succeeded is the number of successfully completed child Jobs. - Succeeded int32 `json:"succeeded"` + // succeeded is the number of successfully completed child Jobs. + // +kubebuilder:validation:Minimum=0 + // +required + Succeeded *int32 `json:"succeeded,omitempty"` - // Failed is the number of failed child Jobs. - Failed int32 `json:"failed"` + // failed is the number of failed child Jobs. + // +kubebuilder:validation:Minimum=0 + // +required + Failed *int32 `json:"failed,omitempty"` - // Active is the number of child Jobs with at least 1 pod in a running or pending state + // active is the number of child Jobs with at least 1 pod in a running or pending state // which are not marked for deletion. - Active int32 `json:"active"` - - // Suspended is the number of child Jobs which are in a suspended state. - Suspended int32 `json:"suspended"` + // +kubebuilder:validation:Minimum=0 + // +required + Active *int32 `json:"active,omitempty"` + + // suspended is the number of child Jobs which are in a suspended state. + // +kubebuilder:validation:Minimum=0 + // +required + Suspended *int32 `json:"suspended,omitempty"` } func init() { diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.deepcopy.go b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.deepcopy.go index edf81854d3f..93602c41aa0 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" intstr "k8s.io/apimachinery/pkg/util/intstr" + v1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -218,6 +219,31 @@ func (in *JobSetTemplateSpec) DeepCopy() *JobSetTemplateSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *JobStatus) DeepCopyInto(out *JobStatus) { *out = *in + if in.Ready != nil { + in, out := &in.Ready, &out.Ready + *out = new(int32) + **out = **in + } + if in.Succeeded != nil { + in, out := &in.Succeeded, &out.Succeeded + *out = new(int32) + **out = **in + } + if in.Failed != nil { + in, out := &in.Failed, &out.Failed + *out = new(int32) + **out = **in + } + if in.Active != nil { + in, out := &in.Active, &out.Active + *out = new(int32) + **out = **in + } + if in.Suspended != nil { + in, out := &in.Suspended, &out.Suspended + *out = new(int32) + **out = **in + } return } @@ -373,6 +399,11 @@ func (in *PodGroupPolicySource) DeepCopyInto(out *PodGroupPolicySource) { *out = new(CoschedulingPodGroupPolicySource) (*in).DeepCopyInto(*out) } + if in.Volcano != nil { + in, out := &in.Volcano, &out.Volcano + *out = new(VolcanoPodGroupPolicySource) + (*in).DeepCopyInto(*out) + } return } @@ -387,13 +418,55 @@ func (in *PodGroupPolicySource) DeepCopy() *PodGroupPolicySource { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PodSpecOverride) DeepCopyInto(out *PodSpecOverride) { +func (in *PodTemplateOverride) DeepCopyInto(out *PodTemplateOverride) { *out = *in if in.TargetJobs != nil { in, out := &in.TargetJobs, &out.TargetJobs - *out = make([]PodSpecOverrideTargetJob, len(*in)) + *out = make([]PodTemplateOverrideTargetJob, len(*in)) copy(*out, *in) } + if in.Metadata != nil { + in, out := &in.Metadata, &out.Metadata + *out = new(metav1.ObjectMeta) + (*in).DeepCopyInto(*out) + } + if in.Spec != nil { + in, out := &in.Spec, &out.Spec + *out = new(PodTemplateSpecOverride) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodTemplateOverride. +func (in *PodTemplateOverride) DeepCopy() *PodTemplateOverride { + if in == nil { + return nil + } + out := new(PodTemplateOverride) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodTemplateOverrideTargetJob) DeepCopyInto(out *PodTemplateOverrideTargetJob) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodTemplateOverrideTargetJob. +func (in *PodTemplateOverrideTargetJob) DeepCopy() *PodTemplateOverrideTargetJob { + if in == nil { + return nil + } + out := new(PodTemplateOverrideTargetJob) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodTemplateSpecOverride) DeepCopyInto(out *PodTemplateSpecOverride) { + *out = *in if in.ServiceAccountName != nil { in, out := &in.ServiceAccountName, &out.ServiceAccountName *out = new(string) @@ -406,6 +479,11 @@ func (in *PodSpecOverride) DeepCopyInto(out *PodSpecOverride) { (*out)[key] = val } } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in)) @@ -439,31 +517,20 @@ func (in *PodSpecOverride) DeepCopyInto(out *PodSpecOverride) { *out = make([]v1.PodSchedulingGate, len(*in)) copy(*out, *in) } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpecOverride. -func (in *PodSpecOverride) DeepCopy() *PodSpecOverride { - if in == nil { - return nil + if in.ImagePullSecrets != nil { + in, out := &in.ImagePullSecrets, &out.ImagePullSecrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) } - out := new(PodSpecOverride) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PodSpecOverrideTargetJob) DeepCopyInto(out *PodSpecOverrideTargetJob) { - *out = *in return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpecOverrideTargetJob. -func (in *PodSpecOverrideTargetJob) DeepCopy() *PodSpecOverrideTargetJob { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodTemplateSpecOverride. +func (in *PodTemplateSpecOverride) DeepCopy() *PodTemplateSpecOverride { if in == nil { return nil } - out := new(PodSpecOverrideTargetJob) + out := new(PodTemplateSpecOverride) in.DeepCopyInto(out) return out } @@ -647,9 +714,9 @@ func (in *TrainJobSpec) DeepCopyInto(out *TrainJobSpec) { (*out)[key] = val } } - if in.PodSpecOverrides != nil { - in, out := &in.PodSpecOverrides, &out.PodSpecOverrides - *out = make([]PodSpecOverride, len(*in)) + if in.PodTemplateOverrides != nil { + in, out := &in.PodTemplateOverrides, &out.PodTemplateOverrides + *out = make([]PodTemplateOverride, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -690,7 +757,9 @@ func (in *TrainJobStatus) DeepCopyInto(out *TrainJobStatus) { if in.JobsStatus != nil { in, out := &in.JobsStatus, &out.JobsStatus *out = make([]JobStatus, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } return } @@ -844,3 +913,24 @@ func (in *TrainingRuntimeSpec) DeepCopy() *TrainingRuntimeSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolcanoPodGroupPolicySource) DeepCopyInto(out *VolcanoPodGroupPolicySource) { + *out = *in + if in.NetworkTopology != nil { + in, out := &in.NetworkTopology, &out.NetworkTopology + *out = new(v1beta1.NetworkTopologySpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolcanoPodGroupPolicySource. +func (in *VolcanoPodGroupPolicySource) DeepCopy() *VolcanoPodGroupPolicySource { + if in == nil { + return nil + } + out := new(VolcanoPodGroupPolicySource) + in.DeepCopyInto(out) + return out +} diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.defaults.go b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.defaults.go index fc71e00873f..4482b70d93b 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.defaults.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.defaults.go @@ -93,6 +93,17 @@ func SetObjectDefaults_ClusterTrainingRuntime(in *ClusterTrainingRuntime) { c.Protocol = "TCP" } } + for k := range b.Env { + c := &b.Env[k] + if c.ValueFrom != nil { + if c.ValueFrom.FileKeyRef != nil { + if c.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + c.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } + } if b.LivenessProbe != nil { if b.LivenessProbe.ProbeHandler.GRPC != nil { if b.LivenessProbe.ProbeHandler.GRPC.Service == nil { @@ -126,6 +137,17 @@ func SetObjectDefaults_ClusterTrainingRuntime(in *ClusterTrainingRuntime) { c.Protocol = "TCP" } } + for k := range b.Env { + c := &b.Env[k] + if c.ValueFrom != nil { + if c.ValueFrom.FileKeyRef != nil { + if c.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + c.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } + } if b.LivenessProbe != nil { if b.LivenessProbe.ProbeHandler.GRPC != nil { if b.LivenessProbe.ProbeHandler.GRPC.Service == nil { @@ -159,6 +181,17 @@ func SetObjectDefaults_ClusterTrainingRuntime(in *ClusterTrainingRuntime) { c.Protocol = "TCP" } } + for k := range b.EphemeralContainerCommon.Env { + c := &b.EphemeralContainerCommon.Env[k] + if c.ValueFrom != nil { + if c.ValueFrom.FileKeyRef != nil { + if c.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + c.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } + } if b.EphemeralContainerCommon.LivenessProbe != nil { if b.EphemeralContainerCommon.LivenessProbe.ProbeHandler.GRPC != nil { if b.EphemeralContainerCommon.LivenessProbe.ProbeHandler.GRPC.Service == nil { @@ -195,50 +228,121 @@ func SetObjectDefaults_ClusterTrainingRuntimeList(in *ClusterTrainingRuntimeList } func SetObjectDefaults_TrainJob(in *TrainJob) { - for i := range in.Spec.PodSpecOverrides { - a := &in.Spec.PodSpecOverrides[i] - for j := range a.Volumes { - b := &a.Volumes[j] - if b.VolumeSource.ISCSI != nil { - if b.VolumeSource.ISCSI.ISCSIInterface == "" { - b.VolumeSource.ISCSI.ISCSIInterface = "default" + if in.Spec.Initializer != nil { + if in.Spec.Initializer.Dataset != nil { + for i := range in.Spec.Initializer.Dataset.Env { + a := &in.Spec.Initializer.Dataset.Env[i] + if a.ValueFrom != nil { + if a.ValueFrom.FileKeyRef != nil { + if a.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + a.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } } } - if b.VolumeSource.RBD != nil { - if b.VolumeSource.RBD.RBDPool == "" { - b.VolumeSource.RBD.RBDPool = "rbd" - } - if b.VolumeSource.RBD.RadosUser == "" { - b.VolumeSource.RBD.RadosUser = "admin" + } + if in.Spec.Initializer.Model != nil { + for i := range in.Spec.Initializer.Model.Env { + a := &in.Spec.Initializer.Model.Env[i] + if a.ValueFrom != nil { + if a.ValueFrom.FileKeyRef != nil { + if a.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + a.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } } - if b.VolumeSource.RBD.Keyring == "" { - b.VolumeSource.RBD.Keyring = "/etc/ceph/keyring" + } + } + } + if in.Spec.Trainer != nil { + for i := range in.Spec.Trainer.Env { + a := &in.Spec.Trainer.Env[i] + if a.ValueFrom != nil { + if a.ValueFrom.FileKeyRef != nil { + if a.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + a.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } } } - if b.VolumeSource.AzureDisk != nil { - if b.VolumeSource.AzureDisk.CachingMode == nil { - ptrVar1 := v1.AzureDataDiskCachingMode(v1.AzureDataDiskCachingReadWrite) - b.VolumeSource.AzureDisk.CachingMode = &ptrVar1 + } + } + for i := range in.Spec.PodTemplateOverrides { + a := &in.Spec.PodTemplateOverrides[i] + if a.Spec != nil { + for j := range a.Spec.Volumes { + b := &a.Spec.Volumes[j] + if b.VolumeSource.ISCSI != nil { + if b.VolumeSource.ISCSI.ISCSIInterface == "" { + b.VolumeSource.ISCSI.ISCSIInterface = "default" + } } - if b.VolumeSource.AzureDisk.FSType == nil { - var ptrVar1 string = "ext4" - b.VolumeSource.AzureDisk.FSType = &ptrVar1 + if b.VolumeSource.RBD != nil { + if b.VolumeSource.RBD.RBDPool == "" { + b.VolumeSource.RBD.RBDPool = "rbd" + } + if b.VolumeSource.RBD.RadosUser == "" { + b.VolumeSource.RBD.RadosUser = "admin" + } + if b.VolumeSource.RBD.Keyring == "" { + b.VolumeSource.RBD.Keyring = "/etc/ceph/keyring" + } } - if b.VolumeSource.AzureDisk.ReadOnly == nil { - var ptrVar1 bool = false - b.VolumeSource.AzureDisk.ReadOnly = &ptrVar1 + if b.VolumeSource.AzureDisk != nil { + if b.VolumeSource.AzureDisk.CachingMode == nil { + ptrVar1 := v1.AzureDataDiskCachingMode(v1.AzureDataDiskCachingReadWrite) + b.VolumeSource.AzureDisk.CachingMode = &ptrVar1 + } + if b.VolumeSource.AzureDisk.FSType == nil { + var ptrVar1 string = "ext4" + b.VolumeSource.AzureDisk.FSType = &ptrVar1 + } + if b.VolumeSource.AzureDisk.ReadOnly == nil { + var ptrVar1 bool = false + b.VolumeSource.AzureDisk.ReadOnly = &ptrVar1 + } + if b.VolumeSource.AzureDisk.Kind == nil { + ptrVar1 := v1.AzureDataDiskKind(v1.AzureSharedBlobDisk) + b.VolumeSource.AzureDisk.Kind = &ptrVar1 + } } - if b.VolumeSource.AzureDisk.Kind == nil { - ptrVar1 := v1.AzureDataDiskKind(v1.AzureSharedBlobDisk) - b.VolumeSource.AzureDisk.Kind = &ptrVar1 + if b.VolumeSource.ScaleIO != nil { + if b.VolumeSource.ScaleIO.StorageMode == "" { + b.VolumeSource.ScaleIO.StorageMode = "ThinProvisioned" + } + if b.VolumeSource.ScaleIO.FSType == "" { + b.VolumeSource.ScaleIO.FSType = "xfs" + } } } - if b.VolumeSource.ScaleIO != nil { - if b.VolumeSource.ScaleIO.StorageMode == "" { - b.VolumeSource.ScaleIO.StorageMode = "ThinProvisioned" + for j := range a.Spec.InitContainers { + b := &a.Spec.InitContainers[j] + for k := range b.Env { + c := &b.Env[k] + if c.ValueFrom != nil { + if c.ValueFrom.FileKeyRef != nil { + if c.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + c.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } } - if b.VolumeSource.ScaleIO.FSType == "" { - b.VolumeSource.ScaleIO.FSType = "xfs" + } + for j := range a.Spec.Containers { + b := &a.Spec.Containers[j] + for k := range b.Env { + c := &b.Env[k] + if c.ValueFrom != nil { + if c.ValueFrom.FileKeyRef != nil { + if c.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + c.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } } } } @@ -308,6 +412,17 @@ func SetObjectDefaults_TrainingRuntime(in *TrainingRuntime) { c.Protocol = "TCP" } } + for k := range b.Env { + c := &b.Env[k] + if c.ValueFrom != nil { + if c.ValueFrom.FileKeyRef != nil { + if c.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + c.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } + } if b.LivenessProbe != nil { if b.LivenessProbe.ProbeHandler.GRPC != nil { if b.LivenessProbe.ProbeHandler.GRPC.Service == nil { @@ -341,6 +456,17 @@ func SetObjectDefaults_TrainingRuntime(in *TrainingRuntime) { c.Protocol = "TCP" } } + for k := range b.Env { + c := &b.Env[k] + if c.ValueFrom != nil { + if c.ValueFrom.FileKeyRef != nil { + if c.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + c.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } + } if b.LivenessProbe != nil { if b.LivenessProbe.ProbeHandler.GRPC != nil { if b.LivenessProbe.ProbeHandler.GRPC.Service == nil { @@ -374,6 +500,17 @@ func SetObjectDefaults_TrainingRuntime(in *TrainingRuntime) { c.Protocol = "TCP" } } + for k := range b.EphemeralContainerCommon.Env { + c := &b.EphemeralContainerCommon.Env[k] + if c.ValueFrom != nil { + if c.ValueFrom.FileKeyRef != nil { + if c.ValueFrom.FileKeyRef.Optional == nil { + var ptrVar1 bool = false + c.ValueFrom.FileKeyRef.Optional = &ptrVar1 + } + } + } + } if b.EphemeralContainerCommon.LivenessProbe != nil { if b.EphemeralContainerCommon.LivenessProbe.ProbeHandler.GRPC != nil { if b.EphemeralContainerCommon.LivenessProbe.ProbeHandler.GRPC.Service == nil { diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.openapi.go b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.openapi.go index 235b2a8235e..acafe1b57e5 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.openapi.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1/zz_generated.openapi.go @@ -44,8 +44,9 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ModelInitializer": schema_pkg_apis_trainer_v1alpha1_ModelInitializer(ref), "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodGroupPolicy": schema_pkg_apis_trainer_v1alpha1_PodGroupPolicy(ref), "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodGroupPolicySource": schema_pkg_apis_trainer_v1alpha1_PodGroupPolicySource(ref), - "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodSpecOverride": schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref), - "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodSpecOverrideTargetJob": schema_pkg_apis_trainer_v1alpha1_PodSpecOverrideTargetJob(ref), + "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverride": schema_pkg_apis_trainer_v1alpha1_PodTemplateOverride(ref), + "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverrideTargetJob": schema_pkg_apis_trainer_v1alpha1_PodTemplateOverrideTargetJob(ref), + "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateSpecOverride": schema_pkg_apis_trainer_v1alpha1_PodTemplateSpecOverride(ref), "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimeRef": schema_pkg_apis_trainer_v1alpha1_RuntimeRef(ref), "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TorchElasticPolicy": schema_pkg_apis_trainer_v1alpha1_TorchElasticPolicy(ref), "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TorchMLPolicySource": schema_pkg_apis_trainer_v1alpha1_TorchMLPolicySource(ref), @@ -57,6 +58,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntime": schema_pkg_apis_trainer_v1alpha1_TrainingRuntime(ref), "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntimeList": schema_pkg_apis_trainer_v1alpha1_TrainingRuntimeList(ref), "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntimeSpec": schema_pkg_apis_trainer_v1alpha1_TrainingRuntimeSpec(ref), + "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.VolcanoPodGroupPolicySource": schema_pkg_apis_trainer_v1alpha1_VolcanoPodGroupPolicySource(ref), "k8s.io/api/autoscaling/v2.ContainerResourceMetricSource": schema_k8sio_api_autoscaling_v2_ContainerResourceMetricSource(ref), "k8s.io/api/autoscaling/v2.ContainerResourceMetricStatus": schema_k8sio_api_autoscaling_v2_ContainerResourceMetricStatus(ref), "k8s.io/api/autoscaling/v2.CrossVersionObjectReference": schema_k8sio_api_autoscaling_v2_CrossVersionObjectReference(ref), @@ -127,9 +129,12 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/api/core/v1.ConfigMapProjection": schema_k8sio_api_core_v1_ConfigMapProjection(ref), "k8s.io/api/core/v1.ConfigMapVolumeSource": schema_k8sio_api_core_v1_ConfigMapVolumeSource(ref), "k8s.io/api/core/v1.Container": schema_k8sio_api_core_v1_Container(ref), + "k8s.io/api/core/v1.ContainerExtendedResourceRequest": schema_k8sio_api_core_v1_ContainerExtendedResourceRequest(ref), "k8s.io/api/core/v1.ContainerImage": schema_k8sio_api_core_v1_ContainerImage(ref), "k8s.io/api/core/v1.ContainerPort": schema_k8sio_api_core_v1_ContainerPort(ref), "k8s.io/api/core/v1.ContainerResizePolicy": schema_k8sio_api_core_v1_ContainerResizePolicy(ref), + "k8s.io/api/core/v1.ContainerRestartRule": schema_k8sio_api_core_v1_ContainerRestartRule(ref), + "k8s.io/api/core/v1.ContainerRestartRuleOnExitCodes": schema_k8sio_api_core_v1_ContainerRestartRuleOnExitCodes(ref), "k8s.io/api/core/v1.ContainerState": schema_k8sio_api_core_v1_ContainerState(ref), "k8s.io/api/core/v1.ContainerStateRunning": schema_k8sio_api_core_v1_ContainerStateRunning(ref), "k8s.io/api/core/v1.ContainerStateTerminated": schema_k8sio_api_core_v1_ContainerStateTerminated(ref), @@ -158,6 +163,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/api/core/v1.EventSource": schema_k8sio_api_core_v1_EventSource(ref), "k8s.io/api/core/v1.ExecAction": schema_k8sio_api_core_v1_ExecAction(ref), "k8s.io/api/core/v1.FCVolumeSource": schema_k8sio_api_core_v1_FCVolumeSource(ref), + "k8s.io/api/core/v1.FileKeySelector": schema_k8sio_api_core_v1_FileKeySelector(ref), "k8s.io/api/core/v1.FlexPersistentVolumeSource": schema_k8sio_api_core_v1_FlexPersistentVolumeSource(ref), "k8s.io/api/core/v1.FlexVolumeSource": schema_k8sio_api_core_v1_FlexVolumeSource(ref), "k8s.io/api/core/v1.FlockerVolumeSource": schema_k8sio_api_core_v1_FlockerVolumeSource(ref), @@ -211,6 +217,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/api/core/v1.NodeSelectorTerm": schema_k8sio_api_core_v1_NodeSelectorTerm(ref), "k8s.io/api/core/v1.NodeSpec": schema_k8sio_api_core_v1_NodeSpec(ref), "k8s.io/api/core/v1.NodeStatus": schema_k8sio_api_core_v1_NodeStatus(ref), + "k8s.io/api/core/v1.NodeSwapStatus": schema_k8sio_api_core_v1_NodeSwapStatus(ref), "k8s.io/api/core/v1.NodeSystemInfo": schema_k8sio_api_core_v1_NodeSystemInfo(ref), "k8s.io/api/core/v1.ObjectFieldSelector": schema_k8sio_api_core_v1_ObjectFieldSelector(ref), "k8s.io/api/core/v1.ObjectReference": schema_k8sio_api_core_v1_ObjectReference(ref), @@ -232,10 +239,12 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "k8s.io/api/core/v1.PodAffinityTerm": schema_k8sio_api_core_v1_PodAffinityTerm(ref), "k8s.io/api/core/v1.PodAntiAffinity": schema_k8sio_api_core_v1_PodAntiAffinity(ref), "k8s.io/api/core/v1.PodAttachOptions": schema_k8sio_api_core_v1_PodAttachOptions(ref), + "k8s.io/api/core/v1.PodCertificateProjection": schema_k8sio_api_core_v1_PodCertificateProjection(ref), "k8s.io/api/core/v1.PodCondition": schema_k8sio_api_core_v1_PodCondition(ref), "k8s.io/api/core/v1.PodDNSConfig": schema_k8sio_api_core_v1_PodDNSConfig(ref), "k8s.io/api/core/v1.PodDNSConfigOption": schema_k8sio_api_core_v1_PodDNSConfigOption(ref), "k8s.io/api/core/v1.PodExecOptions": schema_k8sio_api_core_v1_PodExecOptions(ref), + "k8s.io/api/core/v1.PodExtendedResourceClaimStatus": schema_k8sio_api_core_v1_PodExtendedResourceClaimStatus(ref), "k8s.io/api/core/v1.PodIP": schema_k8sio_api_core_v1_PodIP(ref), "k8s.io/api/core/v1.PodList": schema_k8sio_api_core_v1_PodList(ref), "k8s.io/api/core/v1.PodLogOptions": schema_k8sio_api_core_v1_PodLogOptions(ref), @@ -395,6 +404,22 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "sigs.k8s.io/jobset/api/jobset/v1alpha2.ReplicatedJobStatus": schema_jobset_api_jobset_v1alpha2_ReplicatedJobStatus(ref), "sigs.k8s.io/jobset/api/jobset/v1alpha2.StartupPolicy": schema_jobset_api_jobset_v1alpha2_StartupPolicy(ref), "sigs.k8s.io/jobset/api/jobset/v1alpha2.SuccessPolicy": schema_jobset_api_jobset_v1alpha2_SuccessPolicy(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Affinity": schema_pkg_apis_scheduling_v1beta1_Affinity(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Cluster": schema_pkg_apis_scheduling_v1beta1_Cluster(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Guarantee": schema_pkg_apis_scheduling_v1beta1_Guarantee(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.NetworkTopologySpec": schema_pkg_apis_scheduling_v1beta1_NetworkTopologySpec(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.NodeGroupAffinity": schema_pkg_apis_scheduling_v1beta1_NodeGroupAffinity(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.NodeGroupAntiAffinity": schema_pkg_apis_scheduling_v1beta1_NodeGroupAntiAffinity(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroup": schema_pkg_apis_scheduling_v1beta1_PodGroup(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupCondition": schema_pkg_apis_scheduling_v1beta1_PodGroupCondition(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupList": schema_pkg_apis_scheduling_v1beta1_PodGroupList(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupSpec": schema_pkg_apis_scheduling_v1beta1_PodGroupSpec(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupStatus": schema_pkg_apis_scheduling_v1beta1_PodGroupStatus(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Queue": schema_pkg_apis_scheduling_v1beta1_Queue(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.QueueList": schema_pkg_apis_scheduling_v1beta1_QueueList(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.QueueSpec": schema_pkg_apis_scheduling_v1beta1_QueueSpec(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.QueueStatus": schema_pkg_apis_scheduling_v1beta1_QueueStatus(ref), + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Reservation": schema_pkg_apis_scheduling_v1beta1_Reservation(ref), } } @@ -421,14 +446,14 @@ func schema_pkg_apis_trainer_v1alpha1_ClusterTrainingRuntime(ref common.Referenc }, "metadata": { SchemaProps: spec.SchemaProps{ - Description: "Standard object's metadata.", + Description: "metadata of the ClusterTrainingRuntime.", Default: map[string]interface{}{}, Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), }, }, "spec": { SchemaProps: spec.SchemaProps{ - Description: "Specification of the desired ClusterTrainingRuntime.", + Description: "spec of the ClusterTrainingRuntime.", Default: map[string]interface{}{}, Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntimeSpec"), }, @@ -501,8 +526,7 @@ func schema_pkg_apis_trainer_v1alpha1_ContainerOverride(ref common.ReferenceCall Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name for the container. TrainingRuntime must have this container.", - Default: "", + Description: "name for the container. TrainingRuntime must have this container.", Type: []string{"string"}, Format: "", }, @@ -517,7 +541,7 @@ func schema_pkg_apis_trainer_v1alpha1_ContainerOverride(ref common.ReferenceCall }, }, SchemaProps: spec.SchemaProps{ - Description: "List of environment variables to set in the container. These values will be merged with the TrainingRuntime's environments. These values can't be set for container with the name: `node`, `dataset-initializer`, or `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.", + Description: "env is the list of environment variables to set in the container. These values will be merged with the TrainingRuntime's environments. These values can't be set for container with the name: `node`, `dataset-initializer`, or `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -539,7 +563,7 @@ func schema_pkg_apis_trainer_v1alpha1_ContainerOverride(ref common.ReferenceCall }, }, SchemaProps: spec.SchemaProps{ - Description: "Pod volumes to mount into the container's filesystem.", + Description: "volumeMounts are the volumes to mount into the container's filesystem.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -569,7 +593,7 @@ func schema_pkg_apis_trainer_v1alpha1_CoschedulingPodGroupPolicySource(ref commo Properties: map[string]spec.Schema{ "scheduleTimeoutSeconds": { SchemaProps: spec.SchemaProps{ - Description: "Time threshold to schedule PodGroup for gang-scheduling. If the scheduling timeout is equal to 0, the default value is used. Defaults to 60 seconds.", + Description: "scheduleTimeoutSeconds is the maximum duration to schedule PodGroup for gang-scheduling. If the scheduling timeout is equal to 0, the default value is used. Defaults to 60 seconds.", Type: []string{"integer"}, Format: "int32", }, @@ -589,7 +613,7 @@ func schema_pkg_apis_trainer_v1alpha1_DatasetInitializer(ref common.ReferenceCal Properties: map[string]spec.Schema{ "storageUri": { SchemaProps: spec.SchemaProps{ - Description: "Storage uri for the dataset provider.", + Description: "storageUri is the URI for the dataset provider.", Type: []string{"string"}, Format: "", }, @@ -604,7 +628,7 @@ func schema_pkg_apis_trainer_v1alpha1_DatasetInitializer(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "List of environment variables to set in the dataset initializer container. These values will be merged with the TrainingRuntime's dataset initializer environments.", + Description: "env is the list of environment variables to set in the dataset initializer container. These values will be merged with the TrainingRuntime's dataset initializer environments.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -618,7 +642,7 @@ func schema_pkg_apis_trainer_v1alpha1_DatasetInitializer(ref common.ReferenceCal }, "secretRef": { SchemaProps: spec.SchemaProps{ - Description: "Reference to the secret with credentials to download dataset. Secret must be created in the TrainJob's namespace.", + Description: "secretRef is the reference to the secret with credentials to download dataset. Secret must be created in the TrainJob's namespace.", Ref: ref("k8s.io/api/core/v1.LocalObjectReference"), }, }, @@ -639,13 +663,13 @@ func schema_pkg_apis_trainer_v1alpha1_Initializer(ref common.ReferenceCallback) Properties: map[string]spec.Schema{ "dataset": { SchemaProps: spec.SchemaProps{ - Description: "Configuration of the dataset initialization and pre-processing.", + Description: "dataset defines the configuration for the dataset initialization and pre-processing.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.DatasetInitializer"), }, }, "model": { SchemaProps: spec.SchemaProps{ - Description: "Configuration of the pre-trained model initialization", + Description: "model defines the configuration for the pre-trained model initialization", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ModelInitializer"), }, }, @@ -666,14 +690,14 @@ func schema_pkg_apis_trainer_v1alpha1_JobSetTemplateSpec(ref common.ReferenceCal Properties: map[string]spec.Schema{ "metadata": { SchemaProps: spec.SchemaProps{ - Description: "Metadata for custom JobSet's labels and annotations. JobSet name and namespace is equal to the TrainJob's name and namespace.", + Description: "metadata for custom JobSet's labels and annotations. JobSet name and namespace is equal to the TrainJob's name and namespace.", Default: map[string]interface{}{}, Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), }, }, "spec": { SchemaProps: spec.SchemaProps{ - Description: "Specification of the desired JobSet which will be created from TrainJob.", + Description: "spec of the desired JobSet which will be created from TrainJob.", Default: map[string]interface{}{}, Ref: ref("sigs.k8s.io/jobset/api/jobset/v1alpha2.JobSetSpec"), }, @@ -694,48 +718,42 @@ func schema_pkg_apis_trainer_v1alpha1_JobStatus(ref common.ReferenceCallback) co Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the child Job.", - Default: "", + Description: "name of the child Job.", Type: []string{"string"}, Format: "", }, }, "ready": { SchemaProps: spec.SchemaProps{ - Description: "Ready is the number of child Jobs where the number of ready pods and completed pods is greater than or equal to the total expected pod count for the child Job.", - Default: 0, + Description: "ready is the number of child Jobs where the number of ready pods and completed pods is greater than or equal to the total expected pod count for the child Job.", Type: []string{"integer"}, Format: "int32", }, }, "succeeded": { SchemaProps: spec.SchemaProps{ - Description: "Succeeded is the number of successfully completed child Jobs.", - Default: 0, + Description: "succeeded is the number of successfully completed child Jobs.", Type: []string{"integer"}, Format: "int32", }, }, "failed": { SchemaProps: spec.SchemaProps{ - Description: "Failed is the number of failed child Jobs.", - Default: 0, + Description: "failed is the number of failed child Jobs.", Type: []string{"integer"}, Format: "int32", }, }, "active": { SchemaProps: spec.SchemaProps{ - Description: "Active is the number of child Jobs with at least 1 pod in a running or pending state which are not marked for deletion.", - Default: 0, + Description: "active is the number of child Jobs with at least 1 pod in a running or pending state which are not marked for deletion.", Type: []string{"integer"}, Format: "int32", }, }, "suspended": { SchemaProps: spec.SchemaProps{ - Description: "Suspended is the number of child Jobs which are in a suspended state.", - Default: 0, + Description: "suspended is the number of child Jobs which are in a suspended state.", Type: []string{"integer"}, Format: "int32", }, @@ -751,25 +769,25 @@ func schema_pkg_apis_trainer_v1alpha1_MLPolicy(ref common.ReferenceCallback) com return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "MLPolicy represents configuration for the model trining with ML-specific parameters.", + Description: "MLPolicy represents configuration for the model training with ML-specific parameters.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "numNodes": { SchemaProps: spec.SchemaProps{ - Description: "Number of training nodes. Defaults to 1.", + Description: "numNodes is the number of training nodes. Defaults to 1.", Type: []string{"integer"}, Format: "int32", }, }, "torch": { SchemaProps: spec.SchemaProps{ - Description: "Configuration for the PyTorch runtime.", + Description: "torch defines the configuration for the PyTorch runtime.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TorchMLPolicySource"), }, }, "mpi": { SchemaProps: spec.SchemaProps{ - Description: "Configuration for the MPI Runtime.", + Description: "mpi defines the configuration for the MPI Runtime.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.MPIMLPolicySource"), }, }, @@ -790,13 +808,13 @@ func schema_pkg_apis_trainer_v1alpha1_MLPolicySource(ref common.ReferenceCallbac Properties: map[string]spec.Schema{ "torch": { SchemaProps: spec.SchemaProps{ - Description: "Configuration for the PyTorch runtime.", + Description: "torch defines the configuration for the PyTorch runtime.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TorchMLPolicySource"), }, }, "mpi": { SchemaProps: spec.SchemaProps{ - Description: "Configuration for the MPI Runtime.", + Description: "mpi defines the configuration for the MPI Runtime.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.MPIMLPolicySource"), }, }, @@ -817,28 +835,28 @@ func schema_pkg_apis_trainer_v1alpha1_MPIMLPolicySource(ref common.ReferenceCall Properties: map[string]spec.Schema{ "numProcPerNode": { SchemaProps: spec.SchemaProps{ - Description: "Number of processes per node. This value is equal to the number of slots for each node in the hostfile. Defaults to 1.", + Description: "numProcPerNode is the number of processes per node. This value is equal to the number of slots for each node in the hostfile. Defaults to 1.", Type: []string{"integer"}, Format: "int32", }, }, "mpiImplementation": { SchemaProps: spec.SchemaProps{ - Description: "Implementation name for the MPI to create the appropriate hostfile. Defaults to OpenMPI.", + Description: "mpiImplementation is the name of the MPI implementation to create the appropriate hostfile. Defaults to OpenMPI.", Type: []string{"string"}, Format: "", }, }, "sshAuthMountPath": { SchemaProps: spec.SchemaProps{ - Description: "Directory where SSH keys are mounted. Defaults to /root/.ssh.", + Description: "sshAuthMountPath is the directory where SSH keys are mounted. Defaults to /root/.ssh.", Type: []string{"string"}, Format: "", }, }, "runLauncherAsNode": { SchemaProps: spec.SchemaProps{ - Description: "Whether to run training process on the launcher Job. Defaults to false.", + Description: "runLauncherAsNode defines whether to run training process on the launcher Job. Defaults to false.", Type: []string{"boolean"}, Format: "", }, @@ -853,12 +871,12 @@ func schema_pkg_apis_trainer_v1alpha1_ModelInitializer(ref common.ReferenceCallb return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "DatasetInitializer represents the desired configuration to initialize pre-trained model. The DatasetInitializer spec will override the runtime Job template which contains this label: `trainer.kubeflow.org/trainjob-ancestor-step: dataset-initializer`", + Description: "ModelInitializer represents the desired configuration to initialize pre-trained model. The ModelInitializer spec will override the runtime Job template which contains this label: `trainer.kubeflow.org/trainjob-ancestor-step: dataset-initializer`", Type: []string{"object"}, Properties: map[string]spec.Schema{ "storageUri": { SchemaProps: spec.SchemaProps{ - Description: "Storage uri for the model provider.", + Description: "storageUri is the URI for the model provider.", Type: []string{"string"}, Format: "", }, @@ -873,7 +891,7 @@ func schema_pkg_apis_trainer_v1alpha1_ModelInitializer(ref common.ReferenceCallb }, }, SchemaProps: spec.SchemaProps{ - Description: "List of environment variables to set in the model initializer container. These values will be merged with the TrainingRuntime's model initializer environments.", + Description: "env is the list of environment variables to set in the model initializer container. These values will be merged with the TrainingRuntime's model initializer environments.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -887,7 +905,7 @@ func schema_pkg_apis_trainer_v1alpha1_ModelInitializer(ref common.ReferenceCallb }, "secretRef": { SchemaProps: spec.SchemaProps{ - Description: "Reference to the secret with credentials to download model. Secret must be created in the TrainJob's namespace.", + Description: "secretRef is the reference to the secret with credentials to download model. Secret must be created in the TrainJob's namespace.", Ref: ref("k8s.io/api/core/v1.LocalObjectReference"), }, }, @@ -908,15 +926,21 @@ func schema_pkg_apis_trainer_v1alpha1_PodGroupPolicy(ref common.ReferenceCallbac Properties: map[string]spec.Schema{ "coscheduling": { SchemaProps: spec.SchemaProps{ - Description: "Coscheduling plugin from the Kubernetes scheduler-plugins for gang-scheduling.", + Description: "coscheduling plugin from the Kubernetes scheduler-plugins for gang-scheduling.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.CoschedulingPodGroupPolicySource"), }, }, + "volcano": { + SchemaProps: spec.SchemaProps{ + Description: "volcano plugin for gang-scheduling.", + Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.VolcanoPodGroupPolicySource"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.CoschedulingPodGroupPolicySource"}, + "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.CoschedulingPodGroupPolicySource", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.VolcanoPodGroupPolicySource"}, } } @@ -929,23 +953,29 @@ func schema_pkg_apis_trainer_v1alpha1_PodGroupPolicySource(ref common.ReferenceC Properties: map[string]spec.Schema{ "coscheduling": { SchemaProps: spec.SchemaProps{ - Description: "Coscheduling plugin from the Kubernetes scheduler-plugins for gang-scheduling.", + Description: "coscheduling plugin from the Kubernetes scheduler-plugins for gang-scheduling.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.CoschedulingPodGroupPolicySource"), }, }, + "volcano": { + SchemaProps: spec.SchemaProps{ + Description: "volcano plugin for gang-scheduling.", + Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.VolcanoPodGroupPolicySource"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.CoschedulingPodGroupPolicySource"}, + "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.CoschedulingPodGroupPolicySource", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.VolcanoPodGroupPolicySource"}, } } -func schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref common.ReferenceCallback) common.OpenAPIDefinition { +func schema_pkg_apis_trainer_v1alpha1_PodTemplateOverride(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "PodSpecOverride represents the custom overrides that will be applied for the TrainJob's resources.", + Description: "PodTemplateOverride represents a custom PodTemplateSpec override that will be applied to the TrainJob's training runtime.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "targetJobs": { @@ -955,28 +985,76 @@ func schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref common.ReferenceCallba }, }, SchemaProps: spec.SchemaProps{ - Description: "TrainJobs is the training job replicas in the training runtime template to apply the overrides.", + Description: "targetJobs is the list of replicated jobs in the training runtime template to apply the overrides.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodSpecOverrideTargetJob"), + Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverrideTargetJob"), }, }, }, }, }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "metadata overrides the Pod template metadata. These values will be merged with the TrainingRuntime's Pod template metadata.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Description: "spec overrides the Pod template spec. These values will be merged with the TrainingRuntime's Pod template spec.", + Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateSpecOverride"), + }, + }, + }, + Required: []string{"targetJobs"}, + }, + }, + Dependencies: []string{ + "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverrideTargetJob", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateSpecOverride", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_pkg_apis_trainer_v1alpha1_PodTemplateOverrideTargetJob(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "name is the target training job name for which the PodTemplateSpec is overridden.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"name"}, + }, + }, + } +} + +func schema_pkg_apis_trainer_v1alpha1_PodTemplateSpecOverride(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodTemplateSpecOverride represents the spec overrides for Pod template.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ "serviceAccountName": { SchemaProps: spec.SchemaProps{ - Description: "Override for the service account.", + Description: "serviceAccountName overrides the service account.", Type: []string{"string"}, Format: "", }, }, "nodeSelector": { SchemaProps: spec.SchemaProps{ - Description: "Override for the node selector to place Pod on the specific node.", + Description: "nodeSelector overrides the node selector to place Pod on the specific node.", Type: []string{"object"}, AdditionalProperties: &spec.SchemaOrBool{ Allows: true, @@ -990,6 +1068,12 @@ func schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref common.ReferenceCallba }, }, }, + "affinity": { + SchemaProps: spec.SchemaProps{ + Description: "affinity overrides for the Pod's affinity.", + Ref: ref("k8s.io/api/core/v1.Affinity"), + }, + }, "tolerations": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -997,7 +1081,7 @@ func schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref common.ReferenceCallba }, }, SchemaProps: spec.SchemaProps{ - Description: "Override for the Pod's tolerations.", + Description: "tolerations overrides the Pod's tolerations.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1019,7 +1103,7 @@ func schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref common.ReferenceCallba }, }, SchemaProps: spec.SchemaProps{ - Description: "Overrides for the Pod volume configurations.", + Description: "volumes overrides the Pod's volumes.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1041,7 +1125,7 @@ func schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref common.ReferenceCallba }, }, SchemaProps: spec.SchemaProps{ - Description: "Overrides for the init container in the target job templates.", + Description: "initContainers overrides the init container in the target job templates.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1063,7 +1147,7 @@ func schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref common.ReferenceCallba }, }, SchemaProps: spec.SchemaProps{ - Description: "Overrides for the containers in the target job templates.", + Description: "containers overrides for the containers in the target job templates.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1085,7 +1169,7 @@ func schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref common.ReferenceCallba }, }, SchemaProps: spec.SchemaProps{ - Description: "SchedulingGates overrides the scheduling gates of the Pods in the target job templates. More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/", + Description: "schedulingGates overrides the scheduling gates of the Pods in the target job templates. More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1097,33 +1181,33 @@ func schema_pkg_apis_trainer_v1alpha1_PodSpecOverride(ref common.ReferenceCallba }, }, }, - }, - Required: []string{"targetJobs"}, - }, - }, - Dependencies: []string{ - "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerOverride", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodSpecOverrideTargetJob", "k8s.io/api/core/v1.PodSchedulingGate", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.Volume"}, - } -} - -func schema_pkg_apis_trainer_v1alpha1_PodSpecOverrideTargetJob(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "name": { + "imagePullSecrets": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "name", + }, + "x-kubernetes-list-type": "map", + }, + }, SchemaProps: spec.SchemaProps{ - Description: "Name is the target training job name for which the PodSpec is overridden.", - Default: "", - Type: []string{"string"}, - Format: "", + Description: "imagePullSecrets overrides the image pull secrets for the Pods in the target job templates.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.LocalObjectReference"), + }, + }, + }, }, }, }, - Required: []string{"name"}, }, }, + Dependencies: []string{ + "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerOverride", "k8s.io/api/core/v1.Affinity", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PodSchedulingGate", "k8s.io/api/core/v1.Toleration", "k8s.io/api/core/v1.Volume"}, } } @@ -1136,22 +1220,21 @@ func schema_pkg_apis_trainer_v1alpha1_RuntimeRef(ref common.ReferenceCallback) c Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the runtime being referenced. When namespaced-scoped TrainingRuntime is used, the TrainJob must have the same namespace as the deployed runtime.", - Default: "", + Description: "name of the runtime being referenced. When namespaced-scoped TrainingRuntime is used, the TrainJob must have the same namespace as the deployed runtime.", Type: []string{"string"}, Format: "", }, }, "apiGroup": { SchemaProps: spec.SchemaProps{ - Description: "APIGroup of the runtime being referenced. Defaults to `trainer.kubeflow.org`.", + Description: "apiGroup of the runtime being referenced. Defaults to `trainer.kubeflow.org`.", Type: []string{"string"}, Format: "", }, }, "kind": { SchemaProps: spec.SchemaProps{ - Description: "Kind of the runtime being referenced. Defaults to ClusterTrainingRuntime.", + Description: "kind of the runtime being referenced. Defaults to ClusterTrainingRuntime.", Type: []string{"string"}, Format: "", }, @@ -1172,21 +1255,21 @@ func schema_pkg_apis_trainer_v1alpha1_TorchElasticPolicy(ref common.ReferenceCal Properties: map[string]spec.Schema{ "maxRestarts": { SchemaProps: spec.SchemaProps{ - Description: "How many times the training job can be restarted. This value is inserted into the `--max-restarts` argument of the `torchrun` CLI and the `.spec.failurePolicy.maxRestarts` parameter of the training Job.", + Description: "maxRestarts defines how many times the training job can be restarted. This value is inserted into the `--max-restarts` argument of the `torchrun` CLI and the `.spec.failurePolicy.maxRestarts` parameter of the training Job.", Type: []string{"integer"}, Format: "int32", }, }, "minNodes": { SchemaProps: spec.SchemaProps{ - Description: "Lower limit for the number of nodes to which training job can scale down.", + Description: "minNodes is the lower limit for the number of nodes to which training job can scale down.", Type: []string{"integer"}, Format: "int32", }, }, "maxNodes": { SchemaProps: spec.SchemaProps{ - Description: "Upper limit for the number of nodes to which training job can scale up.", + Description: "maxNodes is the upper limit for the number of nodes to which training job can scale up.", Type: []string{"integer"}, Format: "int32", }, @@ -1198,7 +1281,7 @@ func schema_pkg_apis_trainer_v1alpha1_TorchElasticPolicy(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "Specification which are used to calculate the desired number of nodes. See the individual metric source types for more information about how each type of metric must respond. The HPA will be created to perform auto-scaling.", + Description: "metrics which are used to calculate the desired number of nodes. See the individual metric source types for more information about how each type of metric must respond. The HPA will be created to perform auto-scaling.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1227,13 +1310,13 @@ func schema_pkg_apis_trainer_v1alpha1_TorchMLPolicySource(ref common.ReferenceCa Properties: map[string]spec.Schema{ "numProcPerNode": { SchemaProps: spec.SchemaProps{ - Description: "Number of processes per node. This value is inserted into the `--nproc-per-node` argument of the `torchrun` CLI. Supported values: `auto`, `cpu`, `gpu`, or int value. Defaults to `auto`.", + Description: "numProcPerNode is the number of processes per node. This value is inserted into the `--nproc-per-node` argument of the `torchrun` CLI. Supported values: `auto`, `cpu`, `gpu`, or int value. Defaults to `auto`.", Ref: ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"), }, }, "elasticPolicy": { SchemaProps: spec.SchemaProps{ - Description: "Elastic policy for the PyTorch training.", + Description: "elasticPolicy defines the Elastic policy for the PyTorch training.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TorchElasticPolicy"), }, }, @@ -1268,21 +1351,21 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJob(ref common.ReferenceCallback) com }, "metadata": { SchemaProps: spec.SchemaProps{ - Description: "Standard object's metadata.", + Description: "metadata of the TrainJob.", Default: map[string]interface{}{}, Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), }, }, "spec": { SchemaProps: spec.SchemaProps{ - Description: "Specification of the desired TrainJob.", + Description: "spec of the TrainJob.", Default: map[string]interface{}{}, Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainJobSpec"), }, }, "status": { SchemaProps: spec.SchemaProps{ - Description: "Current status of TrainJob.", + Description: "status of TrainJob.", Default: map[string]interface{}{}, Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainJobStatus"), }, @@ -1355,26 +1438,26 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJobSpec(ref common.ReferenceCallback) Properties: map[string]spec.Schema{ "runtimeRef": { SchemaProps: spec.SchemaProps{ - Description: "Reference to the training runtime. The field is immutable.", + Description: "runtimeRef is the reference to the training runtime. The field is immutable.", Default: map[string]interface{}{}, Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimeRef"), }, }, "initializer": { SchemaProps: spec.SchemaProps{ - Description: "Configuration of the initializer.", + Description: "initializer defines the configuration of the initializer.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Initializer"), }, }, "trainer": { SchemaProps: spec.SchemaProps{ - Description: "Configuration of the trainer.", + Description: "trainer defines the configuration of the trainer.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Trainer"), }, }, "labels": { SchemaProps: spec.SchemaProps{ - Description: "Labels to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.", + Description: "labels to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.", Type: []string{"object"}, AdditionalProperties: &spec.SchemaOrBool{ Allows: true, @@ -1390,7 +1473,7 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJobSpec(ref common.ReferenceCallback) }, "annotations": { SchemaProps: spec.SchemaProps{ - Description: "Annotations to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.", + Description: "annotations to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.", Type: []string{"object"}, AdditionalProperties: &spec.SchemaOrBool{ Allows: true, @@ -1404,20 +1487,20 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJobSpec(ref common.ReferenceCallback) }, }, }, - "podSpecOverrides": { + "podTemplateOverrides": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ "x-kubernetes-list-type": "atomic", }, }, SchemaProps: spec.SchemaProps{ - Description: "Custom overrides for the training runtime.", + Description: "podTemplateOverrides define the PodTemplateOverrides for the training runtime. When multiple overrides apply to the same targetJob, later entries in the slice override earlier field values.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodSpecOverride"), + Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverride"), }, }, }, @@ -1425,14 +1508,14 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJobSpec(ref common.ReferenceCallback) }, "suspend": { SchemaProps: spec.SchemaProps{ - Description: "Whether the controller should suspend the running TrainJob. Defaults to false.", + Description: "suspend defines whether to suspend the running TrainJob. Defaults to false.", Type: []string{"boolean"}, Format: "", }, }, "managedBy": { SchemaProps: spec.SchemaProps{ - Description: "ManagedBy is used to indicate the controller or entity that manages a TrainJob. The value must be either an empty, `trainer.kubeflow.org/trainjob-controller` or `kueue.x-k8s.io/multikueue`. The built-in TrainJob controller reconciles TrainJob which don't have this field at all or the field value is the reserved string `trainer.kubeflow.org/trainjob-controller`, but delegates reconciling TrainJobs with a 'kueue.x-k8s.io/multikueue' to the Kueue. The field is immutable. Defaults to `trainer.kubeflow.org/trainjob-controller`", + Description: "managedBy is used to indicate the controller or entity that manages a TrainJob. The value must be either an empty, `trainer.kubeflow.org/trainjob-controller` or `kueue.x-k8s.io/multikueue`. The built-in TrainJob controller reconciles TrainJob which don't have this field at all or the field value is the reserved string `trainer.kubeflow.org/trainjob-controller`, but delegates reconciling TrainJobs with a 'kueue.x-k8s.io/multikueue' to the Kueue. The field is immutable. Defaults to `trainer.kubeflow.org/trainjob-controller`", Type: []string{"string"}, Format: "", }, @@ -1442,7 +1525,7 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJobSpec(ref common.ReferenceCallback) }, }, Dependencies: []string{ - "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Initializer", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodSpecOverride", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimeRef", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Trainer"}, + "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Initializer", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverride", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimeRef", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Trainer"}, } } @@ -1459,13 +1542,11 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJobStatus(ref common.ReferenceCallbac "x-kubernetes-list-map-keys": []interface{}{ "type", }, - "x-kubernetes-list-type": "map", - "x-kubernetes-patch-merge-key": "type", - "x-kubernetes-patch-strategy": "merge", + "x-kubernetes-list-type": "map", }, }, SchemaProps: spec.SchemaProps{ - Description: "Conditions for the TrainJob.", + Description: "conditions for the TrainJob.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1487,7 +1568,7 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJobStatus(ref common.ReferenceCallbac }, }, SchemaProps: spec.SchemaProps{ - Description: "JobsStatus tracks the child Jobs in TrainJob.", + Description: "jobsStatus tracks the child Jobs in TrainJob.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1516,7 +1597,7 @@ func schema_pkg_apis_trainer_v1alpha1_Trainer(ref common.ReferenceCallback) comm Properties: map[string]spec.Schema{ "image": { SchemaProps: spec.SchemaProps{ - Description: "Docker image for the training container.", + Description: "image is the container image for the training container.", Type: []string{"string"}, Format: "", }, @@ -1528,7 +1609,7 @@ func schema_pkg_apis_trainer_v1alpha1_Trainer(ref common.ReferenceCallback) comm }, }, SchemaProps: spec.SchemaProps{ - Description: "Entrypoint commands for the training container.", + Description: "command for the entrypoint of the training container.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1548,7 +1629,7 @@ func schema_pkg_apis_trainer_v1alpha1_Trainer(ref common.ReferenceCallback) comm }, }, SchemaProps: spec.SchemaProps{ - Description: "Arguments to the entrypoint for the training container.", + Description: "args for the entrypoint for the training container.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1571,7 +1652,7 @@ func schema_pkg_apis_trainer_v1alpha1_Trainer(ref common.ReferenceCallback) comm }, }, SchemaProps: spec.SchemaProps{ - Description: "List of environment variables to set in the training container. These values will be merged with the TrainingRuntime's trainer environments.", + Description: "env is the list of environment variables to set in the training container. These values will be merged with the TrainingRuntime's trainer environments.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1585,20 +1666,20 @@ func schema_pkg_apis_trainer_v1alpha1_Trainer(ref common.ReferenceCallback) comm }, "numNodes": { SchemaProps: spec.SchemaProps{ - Description: "Number of training nodes.", + Description: "numNodes is the number of training nodes.", Type: []string{"integer"}, Format: "int32", }, }, "resourcesPerNode": { SchemaProps: spec.SchemaProps{ - Description: "Compute resources for each training node.", + Description: "resourcesPerNode defines the compute resources for each training node.", Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), }, }, "numProcPerNode": { SchemaProps: spec.SchemaProps{ - Description: "Number of processes/workers/slots on every training node. For the Torch runtime: `auto`, `cpu`, `gpu`, or int value can be set. For the MPI runtime only int value can be set.", + Description: "numProcPerNode is the number of processes/workers/slots on every training node. For the Torch runtime: `auto`, `cpu`, `gpu`, or int value can be set. For the MPI runtime only int value can be set.", Ref: ref("k8s.io/apimachinery/pkg/util/intstr.IntOrString"), }, }, @@ -1633,14 +1714,14 @@ func schema_pkg_apis_trainer_v1alpha1_TrainingRuntime(ref common.ReferenceCallba }, "metadata": { SchemaProps: spec.SchemaProps{ - Description: "Standard object's metadata.", + Description: "metadata of the TrainingRuntime.", Default: map[string]interface{}{}, Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), }, }, "spec": { SchemaProps: spec.SchemaProps{ - Description: "Specification of the desired TrainingRuntime.", + Description: "spec of the TrainingRuntime.", Default: map[string]interface{}{}, Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntimeSpec"), }, @@ -1713,25 +1794,24 @@ func schema_pkg_apis_trainer_v1alpha1_TrainingRuntimeSpec(ref common.ReferenceCa Properties: map[string]spec.Schema{ "mlPolicy": { SchemaProps: spec.SchemaProps{ - Description: "Configuration for the model training with ML-specific parameters.", + Description: "mlPolicy provides the ML-specific parameters for the model training.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.MLPolicy"), }, }, "podGroupPolicy": { SchemaProps: spec.SchemaProps{ - Description: "Configuration for the PodGroup to enable gang-scheduling via supported plugins.", + Description: "podGroupPolicy defines the configuration for the PodGroup to enable gang-scheduling via supported plugins.", Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodGroupPolicy"), }, }, "template": { SchemaProps: spec.SchemaProps{ - Description: "JobSet template which will be used by TrainJob.", + Description: "template for the JobSet which will be used by TrainJob.", Default: map[string]interface{}{}, Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSetTemplateSpec"), }, }, }, - Required: []string{"template"}, }, }, Dependencies: []string{ @@ -1739,6 +1819,27 @@ func schema_pkg_apis_trainer_v1alpha1_TrainingRuntimeSpec(ref common.ReferenceCa } } +func schema_pkg_apis_trainer_v1alpha1_VolcanoPodGroupPolicySource(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "VolcanoPodGroupPolicySource represents configuration for the Volcano gang-scheduler.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "networkTopology": { + SchemaProps: spec.SchemaProps{ + Description: "networkTopology defines the NetworkTopology config, this field works in conjunction with network topology feature and hyperNode CRD.", + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.NetworkTopologySpec"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.NetworkTopologySpec"}, + } +} + func schema_k8sio_api_autoscaling_v2_ContainerResourceMetricSource(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -1956,7 +2057,7 @@ func schema_k8sio_api_autoscaling_v2_HPAScalingRules(ref common.ReferenceCallbac return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "HPAScalingRules configures the scaling behavior for one direction. These Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.", + Description: "HPAScalingRules configures the scaling behavior for one direction via scaling Policy Rules and a configurable metric tolerance.\n\nScaling Policy Rules are applied after calculating DesiredReplicas from metrics for the HPA. They can limit the scaling velocity by specifying scaling policies. They can prevent flapping by specifying the stabilization window, so that the number of replicas is not set instantly, instead, the safest value from the stabilization window is chosen.\n\nThe tolerance is applied to the metric values and prevents scaling too eagerly for small metric variations. (Note that setting a tolerance requires enabling the alpha HPAConfigurableTolerance feature gate.)", Type: []string{"object"}, Properties: map[string]spec.Schema{ "stabilizationWindowSeconds": { @@ -1980,7 +2081,7 @@ func schema_k8sio_api_autoscaling_v2_HPAScalingRules(ref common.ReferenceCallbac }, }, SchemaProps: spec.SchemaProps{ - Description: "policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid", + Description: "policies is a list of potential scaling polices which can be used during scaling. If not set, use the default values: - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. - For scale down: allow all pods to be removed in a 15s window.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -1992,11 +2093,17 @@ func schema_k8sio_api_autoscaling_v2_HPAScalingRules(ref common.ReferenceCallbac }, }, }, + "tolerance": { + SchemaProps: spec.SchemaProps{ + Description: "tolerance is the tolerance on the ratio between the current and desired metric value under which no updates are made to the desired number of replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not set, the default cluster-wide tolerance is applied (by default 10%).\n\nFor example, if autoscaling is configured with a memory consumption target of 100Mi, and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be triggered when the actual consumption falls below 95Mi or exceeds 101Mi.\n\nThis is an alpha field and requires enabling the HPAConfigurableTolerance feature gate.", + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/autoscaling/v2.HPAScalingPolicy"}, + "k8s.io/api/autoscaling/v2.HPAScalingPolicy", "k8s.io/apimachinery/pkg/api/resource.Quantity"}, } } @@ -3157,27 +3264,27 @@ func schema_k8sio_api_batch_v1_JobSpec(ref common.ReferenceCallback) common.Open }, "successPolicy": { SchemaProps: spec.SchemaProps{ - Description: "successPolicy specifies the policy when the Job can be declared as succeeded. If empty, the default behavior applies - the Job is declared as succeeded only when the number of succeeded pods equals to the completions. When the field is specified, it must be immutable and works only for the Indexed Jobs. Once the Job meets the SuccessPolicy, the lingering pods are terminated.\n\nThis field is beta-level. To use this field, you must enable the `JobSuccessPolicy` feature gate (enabled by default).", + Description: "successPolicy specifies the policy when the Job can be declared as succeeded. If empty, the default behavior applies - the Job is declared as succeeded only when the number of succeeded pods equals to the completions. When the field is specified, it must be immutable and works only for the Indexed Jobs. Once the Job meets the SuccessPolicy, the lingering pods are terminated.", Ref: ref("k8s.io/api/batch/v1.SuccessPolicy"), }, }, "backoffLimit": { SchemaProps: spec.SchemaProps{ - Description: "Specifies the number of retries before marking this job failed. Defaults to 6", + Description: "Specifies the number of retries before marking this job failed. Defaults to 6, unless backoffLimitPerIndex (only Indexed Job) is specified. When backoffLimitPerIndex is specified, backoffLimit defaults to 2147483647.", Type: []string{"integer"}, Format: "int32", }, }, "backoffLimitPerIndex": { SchemaProps: spec.SchemaProps{ - Description: "Specifies the limit for the number of retries within an index before marking this index as failed. When enabled the number of failures per index is kept in the pod's batch.kubernetes.io/job-index-failure-count annotation. It can only be set when Job's completionMode=Indexed, and the Pod's restart policy is Never. The field is immutable. This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default).", + Description: "Specifies the limit for the number of retries within an index before marking this index as failed. When enabled the number of failures per index is kept in the pod's batch.kubernetes.io/job-index-failure-count annotation. It can only be set when Job's completionMode=Indexed, and the Pod's restart policy is Never. The field is immutable.", Type: []string{"integer"}, Format: "int32", }, }, "maxFailedIndexes": { SchemaProps: spec.SchemaProps{ - Description: "Specifies the maximal number of failed indexes before marking the Job as failed, when backoffLimitPerIndex is set. Once the number of failed indexes exceeds this number the entire Job is marked as Failed and its execution is terminated. When left as null the job continues execution of all of its indexes and is marked with the `Complete` Job condition. It can only be specified when backoffLimitPerIndex is set. It can be null or up to completions. It is required and must be less than or equal to 10^4 when is completions greater than 10^5. This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default).", + Description: "Specifies the maximal number of failed indexes before marking the Job as failed, when backoffLimitPerIndex is set. Once the number of failed indexes exceeds this number the entire Job is marked as Failed and its execution is terminated. When left as null the job continues execution of all of its indexes and is marked with the `Complete` Job condition. It can only be specified when backoffLimitPerIndex is set. It can be null or up to completions. It is required and must be less than or equal to 10^4 when is completions greater than 10^5.", Type: []string{"integer"}, Format: "int32", }, @@ -3226,7 +3333,7 @@ func schema_k8sio_api_batch_v1_JobSpec(ref common.ReferenceCallback) common.Open }, "podReplacementPolicy": { SchemaProps: spec.SchemaProps{ - Description: "podReplacementPolicy specifies when to create replacement Pods. Possible values are: - TerminatingOrFailed means that we recreate pods\n when they are terminating (has a metadata.deletionTimestamp) or failed.\n- Failed means to wait until a previously created Pod is fully terminated (has phase\n Failed or Succeeded) before creating a replacement Pod.\n\nWhen using podFailurePolicy, Failed is the the only allowed value. TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. This is on by default.\n\nPossible enum values:\n - `\"Failed\"` means to wait until a previously created Pod is fully terminated (has phase Failed or Succeeded) before creating a replacement Pod.\n - `\"TerminatingOrFailed\"` means that we recreate pods when they are terminating (has a metadata.deletionTimestamp) or failed.", + Description: "podReplacementPolicy specifies when to create replacement Pods. Possible values are: - TerminatingOrFailed means that we recreate pods\n when they are terminating (has a metadata.deletionTimestamp) or failed.\n- Failed means to wait until a previously created Pod is fully terminated (has phase\n Failed or Succeeded) before creating a replacement Pod.\n\nWhen using podFailurePolicy, Failed is the the only allowed value. TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use.\n\nPossible enum values:\n - `\"Failed\"` means to wait until a previously created Pod is fully terminated (has phase Failed or Succeeded) before creating a replacement Pod.\n - `\"TerminatingOrFailed\"` means that we recreate pods when they are terminating (has a metadata.deletionTimestamp) or failed.", Type: []string{"string"}, Format: "", Enum: []interface{}{"Failed", "TerminatingOrFailed"}, @@ -3325,7 +3432,7 @@ func schema_k8sio_api_batch_v1_JobStatus(ref common.ReferenceCallback) common.Op }, "failedIndexes": { SchemaProps: spec.SchemaProps{ - Description: "FailedIndexes holds the failed indexes when spec.backoffLimitPerIndex is set. The indexes are represented in the text format analogous as for the `completedIndexes` field, ie. they are kept as decimal integers separated by commas. The numbers are listed in increasing order. Three or more consecutive numbers are compressed and represented by the first and last element of the series, separated by a hyphen. For example, if the failed indexes are 1, 3, 4, 5 and 7, they are represented as \"1,3-5,7\". The set of failed indexes cannot overlap with the set of completed indexes.\n\nThis field is beta-level. It can be used when the `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default).", + Description: "FailedIndexes holds the failed indexes when spec.backoffLimitPerIndex is set. The indexes are represented in the text format analogous as for the `completedIndexes` field, ie. they are kept as decimal integers separated by commas. The numbers are listed in increasing order. Three or more consecutive numbers are compressed and represented by the first and last element of the series, separated by a hyphen. For example, if the failed indexes are 1, 3, 4, 5 and 7, they are represented as \"1,3-5,7\". The set of failed indexes cannot overlap with the set of completed indexes.", Type: []string{"string"}, Format: "", }, @@ -3504,7 +3611,7 @@ func schema_k8sio_api_batch_v1_PodFailurePolicyRule(ref common.ReferenceCallback Properties: map[string]spec.Schema{ "action": { SchemaProps: spec.SchemaProps{ - Description: "Specifies the action taken on a pod failure when the requirements are satisfied. Possible values are:\n\n- FailJob: indicates that the pod's job is marked as Failed and all\n running pods are terminated.\n- FailIndex: indicates that the pod's index is marked as Failed and will\n not be restarted.\n This value is beta-level. It can be used when the\n `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default).\n- Ignore: indicates that the counter towards the .backoffLimit is not\n incremented and a replacement pod is created.\n- Count: indicates that the pod is handled in the default way - the\n counter towards the .backoffLimit is incremented.\nAdditional values are considered to be added in the future. Clients should react to an unknown action by skipping the rule.\n\nPossible enum values:\n - `\"Count\"` This is an action which might be taken on a pod failure - the pod failure is handled in the default way - the counter towards .backoffLimit, represented by the job's .status.failed field, is incremented.\n - `\"FailIndex\"` This is an action which might be taken on a pod failure - mark the Job's index as failed to avoid restarts within this index. This action can only be used when backoffLimitPerIndex is set. This value is beta-level.\n - `\"FailJob\"` This is an action which might be taken on a pod failure - mark the pod's job as Failed and terminate all running pods.\n - `\"Ignore\"` This is an action which might be taken on a pod failure - the counter towards .backoffLimit, represented by the job's .status.failed field, is not incremented and a replacement pod is created.", + Description: "Specifies the action taken on a pod failure when the requirements are satisfied. Possible values are:\n\n- FailJob: indicates that the pod's job is marked as Failed and all\n running pods are terminated.\n- FailIndex: indicates that the pod's index is marked as Failed and will\n not be restarted.\n- Ignore: indicates that the counter towards the .backoffLimit is not\n incremented and a replacement pod is created.\n- Count: indicates that the pod is handled in the default way - the\n counter towards the .backoffLimit is incremented.\nAdditional values are considered to be added in the future. Clients should react to an unknown action by skipping the rule.\n\nPossible enum values:\n - `\"Count\"` This is an action which might be taken on a pod failure - the pod failure is handled in the default way - the counter towards .backoffLimit, represented by the job's .status.failed field, is incremented.\n - `\"FailIndex\"` This is an action which might be taken on a pod failure - mark the Job's index as failed to avoid restarts within this index. This action can only be used when backoffLimitPerIndex is set.\n - `\"FailJob\"` This is an action which might be taken on a pod failure - mark the pod's job as Failed and terminate all running pods.\n - `\"Ignore\"` This is an action which might be taken on a pod failure - the counter towards .backoffLimit, represented by the job's .status.failed field, is not incremented and a replacement pod is created.", Default: "", Type: []string{"string"}, Format: "", @@ -3559,7 +3666,7 @@ func schema_k8sio_api_batch_v1_SuccessPolicy(ref common.ReferenceCallback) commo }, }, SchemaProps: spec.SchemaProps{ - Description: "rules represents the list of alternative rules for the declaring the Jobs as successful before `.status.succeeded >= .spec.completions`. Once any of the rules are met, the \"SucceededCriteriaMet\" condition is added, and the lingering pods are removed. The terminal state for such a Job has the \"Complete\" condition. Additionally, these rules are evaluated in order; Once the Job meets one of the rules, other rules are ignored. At most 20 elements are allowed.", + Description: "rules represents the list of alternative rules for the declaring the Jobs as successful before `.status.succeeded >= .spec.completions`. Once any of the rules are met, the \"SuccessCriteriaMet\" condition is added, and the lingering pods are removed. The terminal state for such a Job has the \"Complete\" condition. Additionally, these rules are evaluated in order; Once the Job meets one of the rules, other rules are ignored. At most 20 elements are allowed.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -5143,7 +5250,7 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope }, }, SchemaProps: spec.SchemaProps{ - Description: "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", + Description: "List of sources to populate environment variables in the container. The keys defined within a source may consist of any printable ASCII characters except '='. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -5207,11 +5314,30 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope }, "restartPolicy": { SchemaProps: spec.SchemaProps{ - Description: "RestartPolicy defines the restart behavior of individual containers in a pod. This field may only be set for init containers, and the only allowed value is \"Always\". For non-init containers or when this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. Setting the RestartPolicy as \"Always\" for the init container will have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy \"Always\" will be shut down. This lifecycle differs from normal init containers and is often referred to as a \"sidecar\" container. Although this init container still starts in the init container sequence, it does not wait for the container to complete before proceeding to the next init container. Instead, the next init container starts immediately after this init container is started, or after any startupProbe has successfully completed.", + Description: "RestartPolicy defines the restart behavior of individual containers in a pod. This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. Additionally, setting the RestartPolicy as \"Always\" for the init container will have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy \"Always\" will be shut down. This lifecycle differs from normal init containers and is often referred to as a \"sidecar\" container. Although this init container still starts in the init container sequence, it does not wait for the container to complete before proceeding to the next init container. Instead, the next init container starts immediately after this init container is started, or after any startupProbe has successfully completed.", Type: []string{"string"}, Format: "", }, }, + "restartPolicyRules": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Represents a list of rules to be checked to determine if the container should be restarted on exit. The rules are evaluated in order. Once a rule matches a container exit condition, the remaining rules are ignored. If no rule matches the container exit condition, the Container-level restart policy determines the whether the container is restarted or not. Constraints on the rules: - At most 20 rules are allowed. - Rules can have the same action. - Identical rules are not forbidden in validations. When rules are specified, container MUST set RestartPolicy explicitly even it if matches the Pod's RestartPolicy.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.ContainerRestartRule"), + }, + }, + }, + }, + }, "volumeMounts": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -5339,7 +5465,45 @@ func schema_k8sio_api_core_v1_Container(ref common.ReferenceCallback) common.Ope }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, + "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.ContainerRestartRule", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, + } +} + +func schema_k8sio_api_core_v1_ContainerExtendedResourceRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ContainerExtendedResourceRequest has the mapping of container name, extended resource name to the device request name.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "containerName": { + SchemaProps: spec.SchemaProps{ + Description: "The name of the container requesting resources.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "resourceName": { + SchemaProps: spec.SchemaProps{ + Description: "The name of the extended resource in that container which gets backed by DRA.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "requestName": { + SchemaProps: spec.SchemaProps{ + Description: "The name of the request in the special ResourceClaim which corresponds to the extended resource.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"containerName", "resourceName", "requestName"}, + }, + }, } } @@ -5465,6 +5629,76 @@ func schema_k8sio_api_core_v1_ContainerResizePolicy(ref common.ReferenceCallback } } +func schema_k8sio_api_core_v1_ContainerRestartRule(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ContainerRestartRule describes how a container exit is handled.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "action": { + SchemaProps: spec.SchemaProps{ + Description: "Specifies the action taken on a container exit if the requirements are satisfied. The only possible value is \"Restart\" to restart the container.", + Type: []string{"string"}, + Format: "", + }, + }, + "exitCodes": { + SchemaProps: spec.SchemaProps{ + Description: "Represents the exit codes to check on container exits.", + Ref: ref("k8s.io/api/core/v1.ContainerRestartRuleOnExitCodes"), + }, + }, + }, + Required: []string{"action"}, + }, + }, + Dependencies: []string{ + "k8s.io/api/core/v1.ContainerRestartRuleOnExitCodes"}, + } +} + +func schema_k8sio_api_core_v1_ContainerRestartRuleOnExitCodes(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ContainerRestartRuleOnExitCodes describes the condition for handling an exited container based on its exit codes.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "operator": { + SchemaProps: spec.SchemaProps{ + Description: "Represents the relationship between the container exit code(s) and the specified values. Possible values are: - In: the requirement is satisfied if the container exit code is in the\n set of specified values.\n- NotIn: the requirement is satisfied if the container exit code is\n not in the set of specified values.", + Type: []string{"string"}, + Format: "", + }, + }, + "values": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Specifies the set of values to check for container exit codes. At most 255 elements are allowed.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + }, + Required: []string{"operator"}, + }, + }, + } +} + func schema_k8sio_api_core_v1_ContainerState(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -5759,6 +5993,14 @@ func schema_k8sio_api_core_v1_ContainerStatus(ref common.ReferenceCallback) comm }, }, }, + "stopSignal": { + SchemaProps: spec.SchemaProps{ + Description: "StopSignal reports the effective stop signal for this container\n\nPossible enum values:\n - `\"SIGABRT\"`\n - `\"SIGALRM\"`\n - `\"SIGBUS\"`\n - `\"SIGCHLD\"`\n - `\"SIGCLD\"`\n - `\"SIGCONT\"`\n - `\"SIGFPE\"`\n - `\"SIGHUP\"`\n - `\"SIGILL\"`\n - `\"SIGINT\"`\n - `\"SIGIO\"`\n - `\"SIGIOT\"`\n - `\"SIGKILL\"`\n - `\"SIGPIPE\"`\n - `\"SIGPOLL\"`\n - `\"SIGPROF\"`\n - `\"SIGPWR\"`\n - `\"SIGQUIT\"`\n - `\"SIGRTMAX\"`\n - `\"SIGRTMAX-1\"`\n - `\"SIGRTMAX-10\"`\n - `\"SIGRTMAX-11\"`\n - `\"SIGRTMAX-12\"`\n - `\"SIGRTMAX-13\"`\n - `\"SIGRTMAX-14\"`\n - `\"SIGRTMAX-2\"`\n - `\"SIGRTMAX-3\"`\n - `\"SIGRTMAX-4\"`\n - `\"SIGRTMAX-5\"`\n - `\"SIGRTMAX-6\"`\n - `\"SIGRTMAX-7\"`\n - `\"SIGRTMAX-8\"`\n - `\"SIGRTMAX-9\"`\n - `\"SIGRTMIN\"`\n - `\"SIGRTMIN+1\"`\n - `\"SIGRTMIN+10\"`\n - `\"SIGRTMIN+11\"`\n - `\"SIGRTMIN+12\"`\n - `\"SIGRTMIN+13\"`\n - `\"SIGRTMIN+14\"`\n - `\"SIGRTMIN+15\"`\n - `\"SIGRTMIN+2\"`\n - `\"SIGRTMIN+3\"`\n - `\"SIGRTMIN+4\"`\n - `\"SIGRTMIN+5\"`\n - `\"SIGRTMIN+6\"`\n - `\"SIGRTMIN+7\"`\n - `\"SIGRTMIN+8\"`\n - `\"SIGRTMIN+9\"`\n - `\"SIGSEGV\"`\n - `\"SIGSTKFLT\"`\n - `\"SIGSTOP\"`\n - `\"SIGSYS\"`\n - `\"SIGTERM\"`\n - `\"SIGTRAP\"`\n - `\"SIGTSTP\"`\n - `\"SIGTTIN\"`\n - `\"SIGTTOU\"`\n - `\"SIGURG\"`\n - `\"SIGUSR1\"`\n - `\"SIGUSR2\"`\n - `\"SIGVTALRM\"`\n - `\"SIGWINCH\"`\n - `\"SIGXCPU\"`\n - `\"SIGXFSZ\"`", + Type: []string{"string"}, + Format: "", + Enum: []interface{}{"SIGABRT", "SIGALRM", "SIGBUS", "SIGCHLD", "SIGCLD", "SIGCONT", "SIGFPE", "SIGHUP", "SIGILL", "SIGINT", "SIGIO", "SIGIOT", "SIGKILL", "SIGPIPE", "SIGPOLL", "SIGPROF", "SIGPWR", "SIGQUIT", "SIGRTMAX", "SIGRTMAX-1", "SIGRTMAX-10", "SIGRTMAX-11", "SIGRTMAX-12", "SIGRTMAX-13", "SIGRTMAX-14", "SIGRTMAX-2", "SIGRTMAX-3", "SIGRTMAX-4", "SIGRTMAX-5", "SIGRTMAX-6", "SIGRTMAX-7", "SIGRTMAX-8", "SIGRTMAX-9", "SIGRTMIN", "SIGRTMIN+1", "SIGRTMIN+10", "SIGRTMIN+11", "SIGRTMIN+12", "SIGRTMIN+13", "SIGRTMIN+14", "SIGRTMIN+15", "SIGRTMIN+2", "SIGRTMIN+3", "SIGRTMIN+4", "SIGRTMIN+5", "SIGRTMIN+6", "SIGRTMIN+7", "SIGRTMIN+8", "SIGRTMIN+9", "SIGSEGV", "SIGSTKFLT", "SIGSTOP", "SIGSYS", "SIGTERM", "SIGTRAP", "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGUSR1", "SIGUSR2", "SIGVTALRM", "SIGWINCH", "SIGXCPU", "SIGXFSZ"}, + }, + }, }, Required: []string{"name", "ready", "restartCount", "image", "imageID"}, }, @@ -5961,7 +6203,7 @@ func schema_k8sio_api_core_v1_EndpointAddress(ref common.ReferenceCallback) comm return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "EndpointAddress is a tuple that describes single IP address.", + Description: "EndpointAddress is a tuple that describes single IP address. Deprecated: This API is deprecated in v1.33+.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "ip": { @@ -6010,7 +6252,7 @@ func schema_k8sio_api_core_v1_EndpointPort(ref common.ReferenceCallback) common. return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "EndpointPort is a tuple that describes a single port.", + Description: "EndpointPort is a tuple that describes a single port. Deprecated: This API is deprecated in v1.33+.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "name": { @@ -6059,7 +6301,7 @@ func schema_k8sio_api_core_v1_EndpointSubset(ref common.ReferenceCallback) commo return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "EndpointSubset is a group of addresses with a common set of ports. The expanded set of endpoints is the Cartesian product of Addresses x Ports. For example, given:\n\n\t{\n\t Addresses: [{\"ip\": \"10.10.1.1\"}, {\"ip\": \"10.10.2.2\"}],\n\t Ports: [{\"name\": \"a\", \"port\": 8675}, {\"name\": \"b\", \"port\": 309}]\n\t}\n\nThe resulting set of endpoints can be viewed as:\n\n\ta: [ 10.10.1.1:8675, 10.10.2.2:8675 ],\n\tb: [ 10.10.1.1:309, 10.10.2.2:309 ]", + Description: "EndpointSubset is a group of addresses with a common set of ports. The expanded set of endpoints is the Cartesian product of Addresses x Ports. For example, given:\n\n\t{\n\t Addresses: [{\"ip\": \"10.10.1.1\"}, {\"ip\": \"10.10.2.2\"}],\n\t Ports: [{\"name\": \"a\", \"port\": 8675}, {\"name\": \"b\", \"port\": 309}]\n\t}\n\nThe resulting set of endpoints can be viewed as:\n\n\ta: [ 10.10.1.1:8675, 10.10.2.2:8675 ],\n\tb: [ 10.10.1.1:309, 10.10.2.2:309 ]\n\nDeprecated: This API is deprecated in v1.33+.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "addresses": { @@ -6131,7 +6373,7 @@ func schema_k8sio_api_core_v1_Endpoints(ref common.ReferenceCallback) common.Ope return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "Endpoints is a collection of endpoints that implement the actual service. Example:\n\n\t Name: \"mysvc\",\n\t Subsets: [\n\t {\n\t Addresses: [{\"ip\": \"10.10.1.1\"}, {\"ip\": \"10.10.2.2\"}],\n\t Ports: [{\"name\": \"a\", \"port\": 8675}, {\"name\": \"b\", \"port\": 309}]\n\t },\n\t {\n\t Addresses: [{\"ip\": \"10.10.3.3\"}],\n\t Ports: [{\"name\": \"a\", \"port\": 93}, {\"name\": \"b\", \"port\": 76}]\n\t },\n\t]", + Description: "Endpoints is a collection of endpoints that implement the actual service. Example:\n\n\t Name: \"mysvc\",\n\t Subsets: [\n\t {\n\t Addresses: [{\"ip\": \"10.10.1.1\"}, {\"ip\": \"10.10.2.2\"}],\n\t Ports: [{\"name\": \"a\", \"port\": 8675}, {\"name\": \"b\", \"port\": 309}]\n\t },\n\t {\n\t Addresses: [{\"ip\": \"10.10.3.3\"}],\n\t Ports: [{\"name\": \"a\", \"port\": 93}, {\"name\": \"b\", \"port\": 76}]\n\t },\n\t]\n\nEndpoints is a legacy API and does not contain information about all Service features. Use discoveryv1.EndpointSlice for complete information about Service endpoints.\n\nDeprecated: This API is deprecated in v1.33+. Use discoveryv1.EndpointSlice.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { @@ -6186,7 +6428,7 @@ func schema_k8sio_api_core_v1_EndpointsList(ref common.ReferenceCallback) common return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "EndpointsList is a list of endpoints.", + Description: "EndpointsList is a list of endpoints. Deprecated: This API is deprecated in v1.33+.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { @@ -6237,12 +6479,12 @@ func schema_k8sio_api_core_v1_EnvFromSource(ref common.ReferenceCallback) common return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "EnvFromSource represents the source of a set of ConfigMaps", + Description: "EnvFromSource represents the source of a set of ConfigMaps or Secrets", Type: []string{"object"}, Properties: map[string]spec.Schema{ "prefix": { SchemaProps: spec.SchemaProps{ - Description: "An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER.", + Description: "Optional text to prepend to the name of each environment variable. May consist of any printable ASCII characters except '='.", Type: []string{"string"}, Format: "", }, @@ -6276,7 +6518,7 @@ func schema_k8sio_api_core_v1_EnvVar(ref common.ReferenceCallback) common.OpenAP Properties: map[string]spec.Schema{ "name": { SchemaProps: spec.SchemaProps{ - Description: "Name of the environment variable. Must be a C_IDENTIFIER.", + Description: "Name of the environment variable. May consist of any printable ASCII characters except '='.", Default: "", Type: []string{"string"}, Format: "", @@ -6335,11 +6577,17 @@ func schema_k8sio_api_core_v1_EnvVarSource(ref common.ReferenceCallback) common. Ref: ref("k8s.io/api/core/v1.SecretKeySelector"), }, }, + "fileKeyRef": { + SchemaProps: spec.SchemaProps{ + Description: "FileKeyRef selects a key of the env file. Requires the EnvFiles feature gate to be enabled.", + Ref: ref("k8s.io/api/core/v1.FileKeySelector"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ConfigMapKeySelector", "k8s.io/api/core/v1.ObjectFieldSelector", "k8s.io/api/core/v1.ResourceFieldSelector", "k8s.io/api/core/v1.SecretKeySelector"}, + "k8s.io/api/core/v1.ConfigMapKeySelector", "k8s.io/api/core/v1.FileKeySelector", "k8s.io/api/core/v1.ObjectFieldSelector", "k8s.io/api/core/v1.ResourceFieldSelector", "k8s.io/api/core/v1.SecretKeySelector"}, } } @@ -6444,7 +6692,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c }, }, SchemaProps: spec.SchemaProps{ - Description: "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", + Description: "List of sources to populate environment variables in the container. The keys defined within a source may consist of any printable ASCII characters except '='. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -6508,11 +6756,30 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c }, "restartPolicy": { SchemaProps: spec.SchemaProps{ - Description: "Restart policy for the container to manage the restart behavior of each container within a pod. This may only be set for init containers. You cannot set this field on ephemeral containers.", + Description: "Restart policy for the container to manage the restart behavior of each container within a pod. You cannot set this field on ephemeral containers.", Type: []string{"string"}, Format: "", }, }, + "restartPolicyRules": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Represents a list of rules to be checked to determine if the container should be restarted on exit. You cannot set this field on ephemeral containers.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.ContainerRestartRule"), + }, + }, + }, + }, + }, "volumeMounts": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -6647,7 +6914,7 @@ func schema_k8sio_api_core_v1_EphemeralContainer(ref common.ReferenceCallback) c }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, + "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.ContainerRestartRule", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, } } @@ -6752,7 +7019,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb }, }, SchemaProps: spec.SchemaProps{ - Description: "List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", + Description: "List of sources to populate environment variables in the container. The keys defined within a source may consist of any printable ASCII characters except '='. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -6816,15 +7083,34 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb }, "restartPolicy": { SchemaProps: spec.SchemaProps{ - Description: "Restart policy for the container to manage the restart behavior of each container within a pod. This may only be set for init containers. You cannot set this field on ephemeral containers.", + Description: "Restart policy for the container to manage the restart behavior of each container within a pod. You cannot set this field on ephemeral containers.", Type: []string{"string"}, Format: "", }, }, - "volumeMounts": { + "restartPolicyRules": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ - "x-kubernetes-list-map-keys": []interface{}{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Represents a list of rules to be checked to determine if the container should be restarted on exit. You cannot set this field on ephemeral containers.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.ContainerRestartRule"), + }, + }, + }, + }, + }, + "volumeMounts": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ "mountPath", }, "x-kubernetes-list-type": "map", @@ -6948,7 +7234,7 @@ func schema_k8sio_api_core_v1_EphemeralContainerCommon(ref common.ReferenceCallb }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, + "k8s.io/api/core/v1.ContainerPort", "k8s.io/api/core/v1.ContainerResizePolicy", "k8s.io/api/core/v1.ContainerRestartRule", "k8s.io/api/core/v1.EnvFromSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.Lifecycle", "k8s.io/api/core/v1.Probe", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecurityContext", "k8s.io/api/core/v1.VolumeDevice", "k8s.io/api/core/v1.VolumeMount"}, } } @@ -7318,6 +7604,57 @@ func schema_k8sio_api_core_v1_FCVolumeSource(ref common.ReferenceCallback) commo } } +func schema_k8sio_api_core_v1_FileKeySelector(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "FileKeySelector selects a key of the env file.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "volumeName": { + SchemaProps: spec.SchemaProps{ + Description: "The name of the volume mount containing the env file.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "path": { + SchemaProps: spec.SchemaProps{ + Description: "The path within the volume from which to select the file. Must be relative and may not contain the '..' path or start with '..'.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "key": { + SchemaProps: spec.SchemaProps{ + Description: "The key within the env file. An invalid key will prevent the pod from starting. The keys defined within a source may consist of any printable ASCII characters except '='. During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "optional": { + SchemaProps: spec.SchemaProps{ + Description: "Specify whether the file or its key must be defined. If the file or key does not exist, then the env var is not published. If optional is set to true and the specified key does not exist, the environment variable will not be set in the Pod's containers.\n\nIf optional is set to false and the specified key does not exist, an error will be returned during Pod creation.", + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"volumeName", "path", "key"}, + }, + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-map-type": "atomic", + }, + }, + }, + } +} + func schema_k8sio_api_core_v1_FlexPersistentVolumeSource(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -7627,7 +7964,7 @@ func schema_k8sio_api_core_v1_GlusterfsVolumeSource(ref common.ReferenceCallback Properties: map[string]spec.Schema{ "endpoints": { SchemaProps: spec.SchemaProps{ - Description: "endpoints is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", + Description: "endpoints is the endpoint name that details Glusterfs topology.", Default: "", Type: []string{"string"}, Format: "", @@ -8144,6 +8481,14 @@ func schema_k8sio_api_core_v1_Lifecycle(ref common.ReferenceCallback) common.Ope Ref: ref("k8s.io/api/core/v1.LifecycleHandler"), }, }, + "stopSignal": { + SchemaProps: spec.SchemaProps{ + Description: "StopSignal defines which signal will be sent to a container when it is being stopped. If not specified, the default is defined by the container runtime in use. StopSignal can only be set for Pods with a non-empty .spec.os.name\n\nPossible enum values:\n - `\"SIGABRT\"`\n - `\"SIGALRM\"`\n - `\"SIGBUS\"`\n - `\"SIGCHLD\"`\n - `\"SIGCLD\"`\n - `\"SIGCONT\"`\n - `\"SIGFPE\"`\n - `\"SIGHUP\"`\n - `\"SIGILL\"`\n - `\"SIGINT\"`\n - `\"SIGIO\"`\n - `\"SIGIOT\"`\n - `\"SIGKILL\"`\n - `\"SIGPIPE\"`\n - `\"SIGPOLL\"`\n - `\"SIGPROF\"`\n - `\"SIGPWR\"`\n - `\"SIGQUIT\"`\n - `\"SIGRTMAX\"`\n - `\"SIGRTMAX-1\"`\n - `\"SIGRTMAX-10\"`\n - `\"SIGRTMAX-11\"`\n - `\"SIGRTMAX-12\"`\n - `\"SIGRTMAX-13\"`\n - `\"SIGRTMAX-14\"`\n - `\"SIGRTMAX-2\"`\n - `\"SIGRTMAX-3\"`\n - `\"SIGRTMAX-4\"`\n - `\"SIGRTMAX-5\"`\n - `\"SIGRTMAX-6\"`\n - `\"SIGRTMAX-7\"`\n - `\"SIGRTMAX-8\"`\n - `\"SIGRTMAX-9\"`\n - `\"SIGRTMIN\"`\n - `\"SIGRTMIN+1\"`\n - `\"SIGRTMIN+10\"`\n - `\"SIGRTMIN+11\"`\n - `\"SIGRTMIN+12\"`\n - `\"SIGRTMIN+13\"`\n - `\"SIGRTMIN+14\"`\n - `\"SIGRTMIN+15\"`\n - `\"SIGRTMIN+2\"`\n - `\"SIGRTMIN+3\"`\n - `\"SIGRTMIN+4\"`\n - `\"SIGRTMIN+5\"`\n - `\"SIGRTMIN+6\"`\n - `\"SIGRTMIN+7\"`\n - `\"SIGRTMIN+8\"`\n - `\"SIGRTMIN+9\"`\n - `\"SIGSEGV\"`\n - `\"SIGSTKFLT\"`\n - `\"SIGSTOP\"`\n - `\"SIGSYS\"`\n - `\"SIGTERM\"`\n - `\"SIGTRAP\"`\n - `\"SIGTSTP\"`\n - `\"SIGTTIN\"`\n - `\"SIGTTOU\"`\n - `\"SIGURG\"`\n - `\"SIGUSR1\"`\n - `\"SIGUSR2\"`\n - `\"SIGVTALRM\"`\n - `\"SIGWINCH\"`\n - `\"SIGXCPU\"`\n - `\"SIGXFSZ\"`", + Type: []string{"string"}, + Format: "", + Enum: []interface{}{"SIGABRT", "SIGALRM", "SIGBUS", "SIGCHLD", "SIGCLD", "SIGCONT", "SIGFPE", "SIGHUP", "SIGILL", "SIGINT", "SIGIO", "SIGIOT", "SIGKILL", "SIGPIPE", "SIGPOLL", "SIGPROF", "SIGPWR", "SIGQUIT", "SIGRTMAX", "SIGRTMAX-1", "SIGRTMAX-10", "SIGRTMAX-11", "SIGRTMAX-12", "SIGRTMAX-13", "SIGRTMAX-14", "SIGRTMAX-2", "SIGRTMAX-3", "SIGRTMAX-4", "SIGRTMAX-5", "SIGRTMAX-6", "SIGRTMAX-7", "SIGRTMAX-8", "SIGRTMAX-9", "SIGRTMIN", "SIGRTMIN+1", "SIGRTMIN+10", "SIGRTMIN+11", "SIGRTMIN+12", "SIGRTMIN+13", "SIGRTMIN+14", "SIGRTMIN+15", "SIGRTMIN+2", "SIGRTMIN+3", "SIGRTMIN+4", "SIGRTMIN+5", "SIGRTMIN+6", "SIGRTMIN+7", "SIGRTMIN+8", "SIGRTMIN+9", "SIGSEGV", "SIGSTKFLT", "SIGSTOP", "SIGSYS", "SIGTERM", "SIGTRAP", "SIGTSTP", "SIGTTIN", "SIGTTOU", "SIGURG", "SIGUSR1", "SIGUSR2", "SIGVTALRM", "SIGWINCH", "SIGXCPU", "SIGXFSZ"}, + }, + }, }, }, }, @@ -9820,6 +10165,26 @@ func schema_k8sio_api_core_v1_NodeStatus(ref common.ReferenceCallback) common.Op } } +func schema_k8sio_api_core_v1_NodeSwapStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "NodeSwapStatus represents swap memory information.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "capacity": { + SchemaProps: spec.SchemaProps{ + Description: "Total amount of swap memory in bytes.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + }, + }, + }, + } +} + func schema_k8sio_api_core_v1_NodeSystemInfo(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -9907,10 +10272,18 @@ func schema_k8sio_api_core_v1_NodeSystemInfo(ref common.ReferenceCallback) commo Format: "", }, }, + "swap": { + SchemaProps: spec.SchemaProps{ + Description: "Swap Info reported by the node.", + Ref: ref("k8s.io/api/core/v1.NodeSwapStatus"), + }, + }, }, Required: []string{"machineID", "systemUUID", "bootID", "kernelVersion", "osImage", "containerRuntimeVersion", "kubeletVersion", "kubeProxyVersion", "operatingSystem", "architecture"}, }, }, + Dependencies: []string{ + "k8s.io/api/core/v1.NodeSwapStatus"}, } } @@ -10301,7 +10674,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimSpec(ref common.ReferenceCall }, "volumeAttributesClassName": { SchemaProps: spec.SchemaProps{ - Description: "volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass will be applied to the claim but it's not allowed to reset this field to empty string once it is set. If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass will be set by the persistentvolume controller if it exists. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default).", + Description: "volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, it can be changed after the claim is created. An empty string or nil value indicates that no VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/", Type: []string{"string"}, Format: "", }, @@ -10426,14 +10799,14 @@ func schema_k8sio_api_core_v1_PersistentVolumeClaimStatus(ref common.ReferenceCa }, "currentVolumeAttributesClassName": { SchemaProps: spec.SchemaProps{ - Description: "currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim This is a beta field and requires enabling VolumeAttributesClass feature (off by default).", + Description: "currentVolumeAttributesClassName is the current name of the VolumeAttributesClass the PVC is using. When unset, there is no VolumeAttributeClass applied to this PersistentVolumeClaim", Type: []string{"string"}, Format: "", }, }, "modifyVolumeStatus": { SchemaProps: spec.SchemaProps{ - Description: "ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted. This is a beta field and requires enabling VolumeAttributesClass feature (off by default).", + Description: "ModifyVolumeStatus represents the status object of ControllerModifyVolume operation. When this is unset, there is no ModifyVolume operation being attempted.", Ref: ref("k8s.io/api/core/v1.ModifyVolumeStatus"), }, }, @@ -10938,7 +11311,7 @@ func schema_k8sio_api_core_v1_PersistentVolumeSpec(ref common.ReferenceCallback) }, "volumeAttributesClassName": { SchemaProps: spec.SchemaProps{ - Description: "Name of VolumeAttributesClass to which this persistent volume belongs. Empty value is not allowed. When this field is not set, it indicates that this volume does not belong to any VolumeAttributesClass. This field is mutable and can be changed by the CSI driver after a volume has been updated successfully to a new class. For an unbound PersistentVolume, the volumeAttributesClassName will be matched with unbound PersistentVolumeClaims during the binding process. This is a beta field and requires enabling VolumeAttributesClass feature (off by default).", + Description: "Name of VolumeAttributesClass to which this persistent volume belongs. Empty value is not allowed. When this field is not set, it indicates that this volume does not belong to any VolumeAttributesClass. This field is mutable and can be changed by the CSI driver after a volume has been updated successfully to a new class. For an unbound PersistentVolume, the volumeAttributesClassName will be matched with unbound PersistentVolumeClaims during the binding process.", Type: []string{"string"}, Format: "", }, @@ -11180,7 +11553,7 @@ func schema_k8sio_api_core_v1_PodAffinityTerm(ref common.ReferenceCallback) comm }, }, SchemaProps: spec.SchemaProps{ - Description: "MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).", + Description: "MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -11200,7 +11573,7 @@ func schema_k8sio_api_core_v1_PodAffinityTerm(ref common.ReferenceCallback) comm }, }, SchemaProps: spec.SchemaProps{ - Description: "MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default).", + Description: "MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -11255,7 +11628,7 @@ func schema_k8sio_api_core_v1_PodAntiAffinity(ref common.ReferenceCallback) comm }, }, SchemaProps: spec.SchemaProps{ - Description: "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", + Description: "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and subtracting \"weight\" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -11337,6 +11710,62 @@ func schema_k8sio_api_core_v1_PodAttachOptions(ref common.ReferenceCallback) com } } +func schema_k8sio_api_core_v1_PodCertificateProjection(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodCertificateProjection provides a private key and X.509 certificate in the pod filesystem.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "signerName": { + SchemaProps: spec.SchemaProps{ + Description: "Kubelet's generated CSRs will be addressed to this signer.", + Type: []string{"string"}, + Format: "", + }, + }, + "keyType": { + SchemaProps: spec.SchemaProps{ + Description: "The type of keypair Kubelet will generate for the pod.\n\nValid values are \"RSA3072\", \"RSA4096\", \"ECDSAP256\", \"ECDSAP384\", \"ECDSAP521\", and \"ED25519\".", + Type: []string{"string"}, + Format: "", + }, + }, + "maxExpirationSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "maxExpirationSeconds is the maximum lifetime permitted for the certificate.\n\nKubelet copies this value verbatim into the PodCertificateRequests it generates for this projection.\n\nIf omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver will reject values shorter than 3600 (1 hour). The maximum allowable value is 7862400 (91 days).\n\nThe signer implementation is then free to issue a certificate with any lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 seconds (1 hour). This constraint is enforced by kube-apiserver. `kubernetes.io` signers will never issue certificates with a lifetime longer than 24 hours.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "credentialBundlePath": { + SchemaProps: spec.SchemaProps{ + Description: "Write the credential bundle at this path in the projected volume.\n\nThe credential bundle is a single file that contains multiple PEM blocks. The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private key.\n\nThe remaining blocks are CERTIFICATE blocks, containing the issued certificate chain from the signer (leaf and any intermediates).\n\nUsing credentialBundlePath lets your Pod's application code make a single atomic read that retrieves a consistent key and certificate chain. If you project them to separate files, your application code will need to additionally check that the leaf certificate was issued to the key.", + Type: []string{"string"}, + Format: "", + }, + }, + "keyPath": { + SchemaProps: spec.SchemaProps{ + Description: "Write the key at this path in the projected volume.\n\nMost applications should use credentialBundlePath. When using keyPath and certificateChainPath, your application needs to check that the key and leaf certificate are consistent, because it is possible to read the files mid-rotation.", + Type: []string{"string"}, + Format: "", + }, + }, + "certificateChainPath": { + SchemaProps: spec.SchemaProps{ + Description: "Write the certificate chain at this path in the projected volume.\n\nMost applications should use credentialBundlePath. When using keyPath and certificateChainPath, your application needs to check that the key and leaf certificate are consistent, because it is possible to read the files mid-rotation.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"signerName", "keyType"}, + }, + }, + } +} + func schema_k8sio_api_core_v1_PodCondition(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -11352,6 +11781,13 @@ func schema_k8sio_api_core_v1_PodCondition(ref common.ReferenceCallback) common. Format: "", }, }, + "observedGeneration": { + SchemaProps: spec.SchemaProps{ + Description: "If set, this represents the .metadata.generation that the pod condition was set based upon. This is an alpha field. Enable PodObservedGenerationTracking to be able to use this field.", + Type: []string{"integer"}, + Format: "int64", + }, + }, "status": { SchemaProps: spec.SchemaProps{ Description: "Status is the status of the condition. Can be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions", @@ -11579,6 +12015,49 @@ func schema_k8sio_api_core_v1_PodExecOptions(ref common.ReferenceCallback) commo } } +func schema_k8sio_api_core_v1_PodExtendedResourceClaimStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodExtendedResourceClaimStatus is stored in the PodStatus for the extended resource requests backed by DRA. It stores the generated name for the corresponding special ResourceClaim created by the scheduler.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "requestMappings": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "RequestMappings identifies the mapping of to device request in the generated ResourceClaim.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/api/core/v1.ContainerExtendedResourceRequest"), + }, + }, + }, + }, + }, + "resourceClaimName": { + SchemaProps: spec.SchemaProps{ + Description: "ResourceClaimName is the name of the ResourceClaim that was generated for the Pod in the namespace of the Pod.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"requestMappings", "resourceClaimName"}, + }, + }, + Dependencies: []string{ + "k8s.io/api/core/v1.ContainerExtendedResourceRequest"}, + } +} + func schema_k8sio_api_core_v1_PodIP(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -12155,7 +12634,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, }, SchemaProps: spec.SchemaProps{ - Description: "List of initialization containers belonging to the pod. Init containers are executed in order prior to containers being started. If any init container fails, the pod is considered to have failed and is handled according to its restartPolicy. The name for an init container or normal container must be unique among all containers. Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. The resourceRequirements of an init container are taken into account during scheduling by finding the highest request/limit for each resource type, and then using the max of of that value or the sum of the normal containers. Limits are applied to init containers in a similar fashion. Init containers cannot currently be added or removed. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/", + Description: "List of initialization containers belonging to the pod. Init containers are executed in order prior to containers being started. If any init container fails, the pod is considered to have failed and is handled according to its restartPolicy. The name for an init container or normal container must be unique among all containers. Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. The resourceRequirements of an init container are taken into account during scheduling by finding the highest request/limit for each resource type, and then using the max of that value or the sum of the normal containers. Limits are applied to init containers in a similar fashion. Init containers cannot currently be added or removed. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -12296,7 +12775,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "hostNetwork": { SchemaProps: spec.SchemaProps{ - Description: "Host networking requested for this pod. Use the host's network namespace. If this option is set, the ports that will be used must be specified. Default to false.", + Description: "Host networking requested for this pod. Use the host's network namespace. When using HostNetwork you should specify ports so the scheduler is aware. When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. Default to false.", Type: []string{"boolean"}, Format: "", }, @@ -12531,7 +13010,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "os": { SchemaProps: spec.SchemaProps{ - Description: "Specifies the OS of the containers in the pod. Some pod and container fields are restricted if this is set.\n\nIf the OS field is set to linux, the following fields must be unset: -securityContext.windowsOptions\n\nIf the OS field is set to windows, following fields must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers - spec.securityContext.appArmorProfile - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy - spec.securityContext.sysctls - spec.shareProcessNamespace - spec.securityContext.runAsUser - spec.securityContext.runAsGroup - spec.securityContext.supplementalGroups - spec.securityContext.supplementalGroupsPolicy - spec.containers[*].securityContext.appArmorProfile - spec.containers[*].securityContext.seLinuxOptions - spec.containers[*].securityContext.seccompProfile - spec.containers[*].securityContext.capabilities - spec.containers[*].securityContext.readOnlyRootFilesystem - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - spec.containers[*].securityContext.runAsGroup", + Description: "Specifies the OS of the containers in the pod. Some pod and container fields are restricted if this is set.\n\nIf the OS field is set to linux, the following fields must be unset: -securityContext.windowsOptions\n\nIf the OS field is set to windows, following fields must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers - spec.resources - spec.securityContext.appArmorProfile - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy - spec.securityContext.sysctls - spec.shareProcessNamespace - spec.securityContext.runAsUser - spec.securityContext.runAsGroup - spec.securityContext.supplementalGroups - spec.securityContext.supplementalGroupsPolicy - spec.containers[*].securityContext.appArmorProfile - spec.containers[*].securityContext.seLinuxOptions - spec.containers[*].securityContext.seccompProfile - spec.containers[*].securityContext.capabilities - spec.containers[*].securityContext.readOnlyRootFilesystem - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - spec.containers[*].securityContext.runAsGroup", Ref: ref("k8s.io/api/core/v1.PodOS"), }, }, @@ -12592,10 +13071,17 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "resources": { SchemaProps: spec.SchemaProps{ - Description: "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\" and \"memory\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate.", + Description: "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate.", Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), }, }, + "hostnameOverride": { + SchemaProps: spec.SchemaProps{ + Description: "HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. This field only specifies the pod's hostname and does not affect its DNS records. When this field is set to a non-empty string: - It takes precedence over the values set in `hostname` and `subdomain`. - The Pod's hostname will be set to this value. - `setHostnameAsFQDN` must be nil or set to false. - `hostNetwork` must be set to false.\n\nThis field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. Requires the HostnameOverride feature gate to be enabled.", + Type: []string{"string"}, + Format: "", + }, + }, }, Required: []string{"containers"}, }, @@ -12612,6 +13098,13 @@ func schema_k8sio_api_core_v1_PodStatus(ref common.ReferenceCallback) common.Ope Description: "PodStatus represents information about the status of a pod. Status may trail the actual state of a system, especially if the node that hosts the pod cannot contact the control plane.", Type: []string{"object"}, Properties: map[string]spec.Schema{ + "observedGeneration": { + SchemaProps: spec.SchemaProps{ + Description: "If set, this represents the .metadata.generation that the pod status was set based upon. This is an alpha field. Enable PodObservedGenerationTracking to be able to use this field.", + Type: []string{"integer"}, + Format: "int64", + }, + }, "phase": { SchemaProps: spec.SchemaProps{ Description: "The phase of a Pod is a simple, high-level summary of where the Pod is in its lifecycle. The conditions array, the reason and message fields, and the individual container status arrays contain more detail about the pod's status. There are five possible phase values:\n\nPending: The pod has been accepted by the Kubernetes system, but one or more of the container images has not been created. This includes time before being scheduled as well as time spent downloading images over the network, which could take a while. Running: The pod has been bound to a node, and all of the containers have been created. At least one container is still running, or is in the process of starting or restarting. Succeeded: All containers in the pod have terminated in success, and will not be restarted. Failed: All containers in the pod have terminated, and at least one container has terminated in failure. The container either exited with non-zero status or was terminated by the system. Unknown: For some reason the state of the pod could not be obtained, typically due to an error in communicating with the host of the pod.\n\nMore info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-phase\n\nPossible enum values:\n - `\"Failed\"` means that all containers in the pod have terminated, and at least one container has terminated in a failure (exited with a non-zero exit code or was stopped by the system).\n - `\"Pending\"` means the pod has been accepted by the system, but one or more of the containers has not been started. This includes time before being bound to a node, as well as time spent pulling images onto the host.\n - `\"Running\"` means the pod has been bound to a node and all of the containers have been started. At least one container is still running or is in the process of being restarted.\n - `\"Succeeded\"` means that all containers in the pod have voluntarily terminated with a container exit code of 0, and the system is not going to restart any of these containers.\n - `\"Unknown\"` means that for some reason the state of the pod could not be obtained, typically due to an error in communicating with the host of the pod. Deprecated: It isn't being set since 2015 (74da3b14b0c0f658b3bb8d2def5094686d0e9095)", @@ -12797,7 +13290,7 @@ func schema_k8sio_api_core_v1_PodStatus(ref common.ReferenceCallback) common.Ope }, "resize": { SchemaProps: spec.SchemaProps{ - Description: "Status of resources resize desired for pod's containers. It is empty if no resources resize is pending. Any changes to container resources will automatically set this to \"Proposed\"", + Description: "Status of resources resize desired for pod's containers. It is empty if no resources resize is pending. Any changes to container resources will automatically set this to \"Proposed\" Deprecated: Resize status is moved to two pod conditions PodResizePending and PodResizeInProgress. PodResizePending will track states where the spec has been resized, but the Kubelet has not yet allocated the resources. PodResizeInProgress will track in-progress resizes, and should be present whenever allocated resources != acknowledged resources.", Type: []string{"string"}, Format: "", }, @@ -12826,11 +13319,17 @@ func schema_k8sio_api_core_v1_PodStatus(ref common.ReferenceCallback) common.Ope }, }, }, + "extendedResourceClaimStatus": { + SchemaProps: spec.SchemaProps{ + Description: "Status of extended resource claim backed by DRA.", + Ref: ref("k8s.io/api/core/v1.PodExtendedResourceClaimStatus"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ContainerStatus", "k8s.io/api/core/v1.HostIP", "k8s.io/api/core/v1.PodCondition", "k8s.io/api/core/v1.PodIP", "k8s.io/api/core/v1.PodResourceClaimStatus", "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, + "k8s.io/api/core/v1.ContainerStatus", "k8s.io/api/core/v1.HostIP", "k8s.io/api/core/v1.PodCondition", "k8s.io/api/core/v1.PodExtendedResourceClaimStatus", "k8s.io/api/core/v1.PodIP", "k8s.io/api/core/v1.PodResourceClaimStatus", "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, } } @@ -13758,6 +14257,7 @@ func schema_k8sio_api_core_v1_ReplicationControllerSpec(ref common.ReferenceCall "replicas": { SchemaProps: spec.SchemaProps{ Description: "Replicas is the number of desired replicas. This is a pointer to distinguish between explicit zero and unspecified. Defaults to 1. More info: https://kubernetes.io/docs/concepts/workloads/controllers/replicationcontroller#what-is-a-replicationcontroller", + Default: 1, Type: []string{"integer"}, Format: "int32", }, @@ -13765,6 +14265,7 @@ func schema_k8sio_api_core_v1_ReplicationControllerSpec(ref common.ReferenceCall "minReadySeconds": { SchemaProps: spec.SchemaProps{ Description: "Minimum number of seconds for which a newly created pod should be ready without any of its container crashing, for it to be considered available. Defaults to 0 (pod will be considered available as soon as it is ready)", + Default: 0, Type: []string{"integer"}, Format: "int32", }, @@ -14117,7 +14618,7 @@ func schema_k8sio_api_core_v1_ResourceQuotaSpec(ref common.ReferenceCallback) co Default: "", Type: []string{"string"}, Format: "", - Enum: []interface{}{"BestEffort", "CrossNamespacePodAffinity", "NotBestEffort", "NotTerminating", "PriorityClass", "Terminating"}, + Enum: []interface{}{"BestEffort", "CrossNamespacePodAffinity", "NotBestEffort", "NotTerminating", "PriorityClass", "Terminating", "VolumeAttributesClass"}, }, }, }, @@ -14225,7 +14726,7 @@ func schema_k8sio_api_core_v1_ResourceRequirements(ref common.ReferenceCallback) }, }, SchemaProps: spec.SchemaProps{ - Description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container.\n\nThis is an alpha field and requires enabling the DynamicResourceAllocation feature gate.\n\nThis field is immutable. It can only be set for containers.", + Description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container.\n\nThis field depends on the DynamicResourceAllocation feature gate.\n\nThis field is immutable. It can only be set for containers.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -14558,11 +15059,11 @@ func schema_k8sio_api_core_v1_ScopedResourceSelectorRequirement(ref common.Refer Properties: map[string]spec.Schema{ "scopeName": { SchemaProps: spec.SchemaProps{ - Description: "The name of the scope that the selector applies to.\n\nPossible enum values:\n - `\"BestEffort\"` Match all pod objects that have best effort quality of service\n - `\"CrossNamespacePodAffinity\"` Match all pod objects that have cross-namespace pod (anti)affinity mentioned.\n - `\"NotBestEffort\"` Match all pod objects that do not have best effort quality of service\n - `\"NotTerminating\"` Match all pod objects where spec.activeDeadlineSeconds is nil\n - `\"PriorityClass\"` Match all pod objects that have priority class mentioned\n - `\"Terminating\"` Match all pod objects where spec.activeDeadlineSeconds >=0", + Description: "The name of the scope that the selector applies to.\n\nPossible enum values:\n - `\"BestEffort\"` Match all pod objects that have best effort quality of service\n - `\"CrossNamespacePodAffinity\"` Match all pod objects that have cross-namespace pod (anti)affinity mentioned.\n - `\"NotBestEffort\"` Match all pod objects that do not have best effort quality of service\n - `\"NotTerminating\"` Match all pod objects where spec.activeDeadlineSeconds is nil\n - `\"PriorityClass\"` Match all pod objects that have priority class mentioned\n - `\"Terminating\"` Match all pod objects where spec.activeDeadlineSeconds >=0\n - `\"VolumeAttributesClass\"` Match all pvc objects that have volume attributes class mentioned.", Default: "", Type: []string{"string"}, Format: "", - Enum: []interface{}{"BestEffort", "CrossNamespacePodAffinity", "NotBestEffort", "NotTerminating", "PriorityClass", "Terminating"}, + Enum: []interface{}{"BestEffort", "CrossNamespacePodAffinity", "NotBestEffort", "NotTerminating", "PriorityClass", "Terminating", "VolumeAttributesClass"}, }, }, "operator": { @@ -15711,7 +16212,7 @@ func schema_k8sio_api_core_v1_ServiceSpec(ref common.ReferenceCallback) common.O }, "trafficDistribution": { SchemaProps: spec.SchemaProps{ - Description: "TrafficDistribution offers a way to express preferences for how traffic is distributed to Service endpoints. Implementations can use this field as a hint, but are not required to guarantee strict adherence. If the field is not set, the implementation will apply its default routing strategy. If set to \"PreferClose\", implementations should prioritize endpoints that are topologically close (e.g., same zone). This is a beta field and requires enabling ServiceTrafficDistribution feature.", + Description: "TrafficDistribution offers a way to express preferences for how traffic is distributed to Service endpoints. Implementations can use this field as a hint, but are not required to guarantee strict adherence. If the field is not set, the implementation will apply its default routing strategy. If set to \"PreferClose\", implementations should prioritize endpoints that are in the same zone.", Type: []string{"string"}, Format: "", }, @@ -16003,7 +16504,7 @@ func schema_k8sio_api_core_v1_Taint(ref common.ReferenceCallback) common.OpenAPI }, "timeAdded": { SchemaProps: spec.SchemaProps{ - Description: "TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints.", + Description: "TimeAdded represents the time at which the taint was added.", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), }, }, @@ -16194,7 +16695,7 @@ func schema_k8sio_api_core_v1_TopologySpreadConstraint(ref common.ReferenceCallb }, "nodeAffinityPolicy": { SchemaProps: spec.SchemaProps{ - Description: "NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector when calculating pod topology spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.\n\nIf this value is nil, the behavior is equivalent to the Honor policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.\n\nPossible enum values:\n - `\"Honor\"` means use this scheduling directive when calculating pod topology spread skew.\n - `\"Ignore\"` means ignore this scheduling directive when calculating pod topology spread skew.", + Description: "NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector when calculating pod topology spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.\n\nIf this value is nil, the behavior is equivalent to the Honor policy.\n\nPossible enum values:\n - `\"Honor\"` means use this scheduling directive when calculating pod topology spread skew.\n - `\"Ignore\"` means ignore this scheduling directive when calculating pod topology spread skew.", Type: []string{"string"}, Format: "", Enum: []interface{}{"Honor", "Ignore"}, @@ -16202,7 +16703,7 @@ func schema_k8sio_api_core_v1_TopologySpreadConstraint(ref common.ReferenceCallb }, "nodeTaintsPolicy": { SchemaProps: spec.SchemaProps{ - Description: "NodeTaintsPolicy indicates how we will treat node taints when calculating pod topology spread skew. Options are: - Honor: nodes without taints, along with tainted nodes for which the incoming pod has a toleration, are included. - Ignore: node taints are ignored. All nodes are included.\n\nIf this value is nil, the behavior is equivalent to the Ignore policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag.\n\nPossible enum values:\n - `\"Honor\"` means use this scheduling directive when calculating pod topology spread skew.\n - `\"Ignore\"` means ignore this scheduling directive when calculating pod topology spread skew.", + Description: "NodeTaintsPolicy indicates how we will treat node taints when calculating pod topology spread skew. Options are: - Honor: nodes without taints, along with tainted nodes for which the incoming pod has a toleration, are included. - Ignore: node taints are ignored. All nodes are included.\n\nIf this value is nil, the behavior is equivalent to the Ignore policy.\n\nPossible enum values:\n - `\"Honor\"` means use this scheduling directive when calculating pod topology spread skew.\n - `\"Ignore\"` means ignore this scheduling directive when calculating pod topology spread skew.", Type: []string{"string"}, Format: "", Enum: []interface{}{"Honor", "Ignore"}, @@ -16382,13 +16883,13 @@ func schema_k8sio_api_core_v1_Volume(ref common.ReferenceCallback) common.OpenAP }, "iscsi": { SchemaProps: spec.SchemaProps{ - Description: "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md", + Description: "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi", Ref: ref("k8s.io/api/core/v1.ISCSIVolumeSource"), }, }, "glusterfs": { SchemaProps: spec.SchemaProps{ - Description: "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. More info: https://examples.k8s.io/volumes/glusterfs/README.md", + Description: "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported.", Ref: ref("k8s.io/api/core/v1.GlusterfsVolumeSource"), }, }, @@ -16400,7 +16901,7 @@ func schema_k8sio_api_core_v1_Volume(ref common.ReferenceCallback) common.OpenAP }, "rbd": { SchemaProps: spec.SchemaProps{ - Description: "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. More info: https://examples.k8s.io/volumes/rbd/README.md", + Description: "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported.", Ref: ref("k8s.io/api/core/v1.RBDVolumeSource"), }, }, @@ -16514,7 +17015,7 @@ func schema_k8sio_api_core_v1_Volume(ref common.ReferenceCallback) common.OpenAP }, "image": { SchemaProps: spec.SchemaProps{ - Description: "image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. The volume is resolved at pod startup depending on which PullPolicy value is provided:\n\n- Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails.\n\nThe volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. The volume will be mounted read-only (ro) and non-executable files (noexec). Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type.", + Description: "image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. The volume is resolved at pod startup depending on which PullPolicy value is provided:\n\n- Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails.\n\nThe volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. The volume will be mounted read-only (ro) and non-executable files (noexec). Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type.", Ref: ref("k8s.io/api/core/v1.ImageVolumeSource"), }, }, @@ -16725,11 +17226,17 @@ func schema_k8sio_api_core_v1_VolumeProjection(ref common.ReferenceCallback) com Ref: ref("k8s.io/api/core/v1.ClusterTrustBundleProjection"), }, }, + "podCertificate": { + SchemaProps: spec.SchemaProps{ + Description: "Projects an auto-rotating credential bundle (private key and certificate chain) that the pod can use either as a TLS client or server.\n\nKubelet generates a private key and uses it to send a PodCertificateRequest to the named signer. Once the signer approves the request and issues a certificate chain, Kubelet writes the key and certificate chain to the pod filesystem. The pod does not start until certificates have been issued for each podCertificate projected volume source in its spec.\n\nKubelet will begin trying to rotate the certificate at the time indicated by the signer using the PodCertificateRequest.Status.BeginRefreshAt timestamp.\n\nKubelet can write a single file, indicated by the credentialBundlePath field, or separate files, indicated by the keyPath and certificateChainPath fields.\n\nThe credential bundle is a single file in PEM format. The first PEM entry is the private key (in PKCS#8 format), and the remaining PEM entries are the certificate chain issued by the signer (typically, signers will return their certificate chain in leaf-to-root order).\n\nPrefer using the credential bundle format, since your application code can read it atomically. If you use keyPath and certificateChainPath, your application must make two separate file reads. If these coincide with a certificate rotation, it is possible that the private key and leaf certificate you read may not correspond to each other. Your application will need to check for this condition, and re-read until they are consistent.\n\nThe named signer controls chooses the format of the certificate it issues; consult the signer implementation's documentation to learn how to use the certificates it issues.", + Ref: ref("k8s.io/api/core/v1.PodCertificateProjection"), + }, + }, }, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.ClusterTrustBundleProjection", "k8s.io/api/core/v1.ConfigMapProjection", "k8s.io/api/core/v1.DownwardAPIProjection", "k8s.io/api/core/v1.SecretProjection", "k8s.io/api/core/v1.ServiceAccountTokenProjection"}, + "k8s.io/api/core/v1.ClusterTrustBundleProjection", "k8s.io/api/core/v1.ConfigMapProjection", "k8s.io/api/core/v1.DownwardAPIProjection", "k8s.io/api/core/v1.PodCertificateProjection", "k8s.io/api/core/v1.SecretProjection", "k8s.io/api/core/v1.ServiceAccountTokenProjection"}, } } @@ -16827,13 +17334,13 @@ func schema_k8sio_api_core_v1_VolumeSource(ref common.ReferenceCallback) common. }, "iscsi": { SchemaProps: spec.SchemaProps{ - Description: "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md", + Description: "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi", Ref: ref("k8s.io/api/core/v1.ISCSIVolumeSource"), }, }, "glusterfs": { SchemaProps: spec.SchemaProps{ - Description: "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. More info: https://examples.k8s.io/volumes/glusterfs/README.md", + Description: "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported.", Ref: ref("k8s.io/api/core/v1.GlusterfsVolumeSource"), }, }, @@ -16845,7 +17352,7 @@ func schema_k8sio_api_core_v1_VolumeSource(ref common.ReferenceCallback) common. }, "rbd": { SchemaProps: spec.SchemaProps{ - Description: "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. More info: https://examples.k8s.io/volumes/rbd/README.md", + Description: "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported.", Ref: ref("k8s.io/api/core/v1.RBDVolumeSource"), }, }, @@ -16959,7 +17466,7 @@ func schema_k8sio_api_core_v1_VolumeSource(ref common.ReferenceCallback) common. }, "image": { SchemaProps: spec.SchemaProps{ - Description: "image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. The volume is resolved at pod startup depending on which PullPolicy value is provided:\n\n- Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails.\n\nThe volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. The volume will be mounted read-only (ro) and non-executable files (noexec). Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type.", + Description: "image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. The volume is resolved at pod startup depending on which PullPolicy value is provided:\n\n- Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails.\n\nThe volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. The volume will be mounted read-only (ro) and non-executable files (noexec). Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type.", Ref: ref("k8s.io/api/core/v1.ImageVolumeSource"), }, }, @@ -19785,7 +20292,7 @@ func schema_jobset_api_jobset_v1alpha2_FailurePolicyRule(ref common.ReferenceCal return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "FailurePolicyRule defines a FailurePolicyAction to be executed if a child job fails due to a reason listed in OnJobFailureReasons.", + Description: "FailurePolicyRule defines a FailurePolicyAction to be executed if a child job fails due to a reason listed in OnJobFailureReasons and a message pattern listed in OnJobFailureMessagePatterns. The rule must match both the job failure reason and the job failure message. The rules are evaluated in order and the first matching rule is executed.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "name": { @@ -19806,7 +20313,22 @@ func schema_jobset_api_jobset_v1alpha2_FailurePolicyRule(ref common.ReferenceCal }, "onJobFailureReasons": { SchemaProps: spec.SchemaProps{ - Description: "The requirement on the job failure reasons. The requirement is satisfied if at least one reason matches the list. The rules are evaluated in order, and the first matching rule is executed. An empty list applies the rule to any job failure reason.", + Description: "The requirement on the job failure reasons. The requirement is satisfied if at least one reason matches the list. An empty list matches any job failure reason.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "onJobFailureMessagePatterns": { + SchemaProps: spec.SchemaProps{ + Description: "The requirement on the job failure message. The requirement is satisfied if at least one pattern (regex) matches the job failure message. An empty list matches any job failure message. The syntax of the regular expressions accepted is the same general syntax used by Perl, Python, and other languages. More precisely, it is the syntax accepted by RE2 and described at https://golang.org/s/re2syntax, except for \\C. For an overview of the syntax, see https://pkg.go.dev/regexp/syntax.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -20061,6 +20583,7 @@ func schema_jobset_api_jobset_v1alpha2_JobSetStatus(ref common.ReferenceCallback "restarts": { SchemaProps: spec.SchemaProps{ Description: "Restarts tracks the number of times the JobSet has restarted (i.e. recreated in case of RecreateAll policy).", + Default: 0, Type: []string{"integer"}, Format: "int32", }, @@ -20074,7 +20597,7 @@ func schema_jobset_api_jobset_v1alpha2_JobSetStatus(ref common.ReferenceCallback }, "terminalState": { SchemaProps: spec.SchemaProps{ - Description: "TerminalState the state of the JobSet when it finishes execution. It can be either Complete or Failed. Otherwise, it is empty by default.", + Description: "TerminalState the state of the JobSet when it finishes execution. It can be either Completed or Failed. Otherwise, it is empty by default.", Type: []string{"string"}, Format: "", }, @@ -20156,6 +20679,13 @@ func schema_jobset_api_jobset_v1alpha2_ReplicatedJob(ref common.ReferenceCallbac Format: "", }, }, + "groupName": { + SchemaProps: spec.SchemaProps{ + Description: "GroupName defines the name of the group this ReplicatedJob belongs to. Defaults to \"default\"", + Type: []string{"string"}, + Format: "", + }, + }, "template": { SchemaProps: spec.SchemaProps{ Description: "Template defines the template of the Job that will be created.", @@ -20324,3 +20854,815 @@ func schema_jobset_api_jobset_v1alpha2_SuccessPolicy(ref common.ReferenceCallbac }, } } + +func schema_pkg_apis_scheduling_v1beta1_Affinity(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Affinity is a group of affinity scheduling rules.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "nodeGroupAffinity": { + SchemaProps: spec.SchemaProps{ + Description: "Describes nodegroup affinity scheduling rules for the queue(e.g. putting pods of the queue in the nodes of the nodegroup)", + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.NodeGroupAffinity"), + }, + }, + "nodeGroupAntiAffinity": { + SchemaProps: spec.SchemaProps{ + Description: "Describes nodegroup anti-affinity scheduling rules for the queue(e.g. avoid putting pods of the queue in the nodes of the nodegroup).", + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.NodeGroupAntiAffinity"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.NodeGroupAffinity", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.NodeGroupAntiAffinity"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_Cluster(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "CluterSpec represents the template of Cluster", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "weight": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int32", + }, + }, + "capacity": { + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_Guarantee(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Guarantee represents configuration of queue resource reservation", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "resource": { + SchemaProps: spec.SchemaProps{ + Description: "The amount of cluster resource reserved for queue. Just set either `percentage` or `resource`", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_NetworkTopologySpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "mode": { + SchemaProps: spec.SchemaProps{ + Description: "Mode specifies the mode of the network topology constrain.", + Type: []string{"string"}, + Format: "", + }, + }, + "highestTierAllowed": { + SchemaProps: spec.SchemaProps{ + Description: "HighestTierAllowed specifies the highest tier that a job allowed to cross when scheduling.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_scheduling_v1beta1_NodeGroupAffinity(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "requiredDuringSchedulingIgnoredDuringExecution": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "preferredDuringSchedulingIgnoredDuringExecution": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_scheduling_v1beta1_NodeGroupAntiAffinity(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "requiredDuringSchedulingIgnoredDuringExecution": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "preferredDuringSchedulingIgnoredDuringExecution": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_scheduling_v1beta1_PodGroup(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodGroup is a collection of Pod; used for batch workload.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Description: "Specification of the desired behavior of the pod group. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupSpec"), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "Status represents the current information about a pod group. This data may not be up to date.", + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupStatus"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupSpec", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupStatus"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_PodGroupCondition(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodGroupCondition contains details for the current state of this pod group.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "type": { + SchemaProps: spec.SchemaProps{ + Description: "Type is the type of the condition", + Type: []string{"string"}, + Format: "", + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "Status is the status of the condition.", + Type: []string{"string"}, + Format: "", + }, + }, + "transitionID": { + SchemaProps: spec.SchemaProps{ + Description: "The ID of condition transition.", + Type: []string{"string"}, + Format: "", + }, + }, + "lastTransitionTime": { + SchemaProps: spec.SchemaProps{ + Description: "Last time the phase transitioned from another to current phase.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), + }, + }, + "reason": { + SchemaProps: spec.SchemaProps{ + Description: "Unique, one-word, CamelCase reason for the phase's last transition.", + Type: []string{"string"}, + Format: "", + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "Human-readable message indicating details about last transition.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_PodGroupList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodGroupList is a collection of pod groups.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "Standard list metadata More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Description: "items is the list of PodGroup", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroup"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroup"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_PodGroupSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodGroupSpec represents the template of a pod group.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "minMember": { + SchemaProps: spec.SchemaProps{ + Description: "MinMember defines the minimal number of members/tasks to run the pod group; if there's not enough resources to start all tasks, the scheduler will not start anyone.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "minTaskMember": { + SchemaProps: spec.SchemaProps{ + Description: "MinTaskMember defines the minimal number of pods to run each task in the pod group; if there's not enough resources to start each task, the scheduler will not start anyone.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + "queue": { + SchemaProps: spec.SchemaProps{ + Description: "Queue defines the queue to allocate resource for PodGroup; if queue does not exist, the PodGroup will not be scheduled. Defaults to `default` Queue with the lowest weight.", + Type: []string{"string"}, + Format: "", + }, + }, + "priorityClassName": { + SchemaProps: spec.SchemaProps{ + Description: "If specified, indicates the PodGroup's priority. \"system-node-critical\" and \"system-cluster-critical\" are two special keywords which indicate the highest priorities with the former being the highest priority. Any other name must be defined by creating a PriorityClass object with that name. If not specified, the PodGroup priority will be default or zero if there is no default.", + Type: []string{"string"}, + Format: "", + }, + }, + "minResources": { + SchemaProps: spec.SchemaProps{ + Description: "MinResources defines the minimal resource of members/tasks to run the pod group; if there's not enough resources to start all tasks, the scheduler will not start anyone.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + "networkTopology": { + SchemaProps: spec.SchemaProps{ + Description: "NetworkTopology defines the NetworkTopology config, this field works in conjunction with network topology feature and hyperNode CRD.", + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.NetworkTopologySpec"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.NetworkTopologySpec"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_PodGroupStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodGroupStatus represents the current state of a pod group.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "phase": { + SchemaProps: spec.SchemaProps{ + Description: "Current phase of PodGroup.", + Type: []string{"string"}, + Format: "", + }, + }, + "conditions": { + SchemaProps: spec.SchemaProps{ + Description: "The conditions of PodGroup.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupCondition"), + }, + }, + }, + }, + }, + "running": { + SchemaProps: spec.SchemaProps{ + Description: "The number of actively running pods.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "succeeded": { + SchemaProps: spec.SchemaProps{ + Description: "The number of pods which reached phase Succeeded.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "failed": { + SchemaProps: spec.SchemaProps{ + Description: "The number of pods which reached phase Failed.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + Dependencies: []string{ + "volcano.sh/apis/pkg/apis/scheduling/v1beta1.PodGroupCondition"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_Queue(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Queue is a queue of PodGroup.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Description: "Specification of the desired behavior of the queue. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.QueueSpec"), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "The status of queue.", + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.QueueStatus"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.QueueSpec", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.QueueStatus"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_QueueList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "QueueList is a collection of queues.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "Standard list metadata More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Description: "items is the list of PodGroup", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.Queue"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Queue"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_QueueSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "QueueSpec represents the template of Queue.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "weight": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int32", + }, + }, + "capability": { + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + "reclaimable": { + SchemaProps: spec.SchemaProps{ + Description: "Reclaimable indicate whether the queue can be reclaimed by other queue", + Type: []string{"boolean"}, + Format: "", + }, + }, + "extendClusters": { + SchemaProps: spec.SchemaProps{ + Description: "extendCluster indicate the jobs in this Queue will be dispatched to these clusters.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.Cluster"), + }, + }, + }, + }, + }, + "guarantee": { + SchemaProps: spec.SchemaProps{ + Description: "Guarantee indicate configuration about resource reservation", + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.Guarantee"), + }, + }, + "affinity": { + SchemaProps: spec.SchemaProps{ + Description: "If specified, the pod owned by the queue will be scheduled with constraint", + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.Affinity"), + }, + }, + "type": { + SchemaProps: spec.SchemaProps{ + Description: "Type define the type of queue", + Type: []string{"string"}, + Format: "", + }, + }, + "parent": { + SchemaProps: spec.SchemaProps{ + Description: "Parent define the parent of queue", + Type: []string{"string"}, + Format: "", + }, + }, + "deserved": { + SchemaProps: spec.SchemaProps{ + Description: "The amount of resources configured by the user. This part of resource can be shared with other queues and reclaimed back.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + "priority": { + SchemaProps: spec.SchemaProps{ + Description: "Priority define the priority of queue. Higher values are prioritized for scheduling and considered later during reclamation.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Affinity", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Cluster", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Guarantee"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_QueueStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "QueueStatus represents the status of Queue.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "state": { + SchemaProps: spec.SchemaProps{ + Description: "State is state of queue", + Type: []string{"string"}, + Format: "", + }, + }, + "unknown": { + SchemaProps: spec.SchemaProps{ + Description: "The number of 'Unknown' PodGroup in this queue.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "pending": { + SchemaProps: spec.SchemaProps{ + Description: "The number of 'Pending' PodGroup in this queue.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "running": { + SchemaProps: spec.SchemaProps{ + Description: "The number of 'Running' PodGroup in this queue.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "inqueue": { + SchemaProps: spec.SchemaProps{ + Description: "The number of `Inqueue` PodGroup in this queue.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "completed": { + SchemaProps: spec.SchemaProps{ + Description: "The number of `Completed` PodGroup in this queue.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "reservation": { + SchemaProps: spec.SchemaProps{ + Description: "Reservation is the profile of resource reservation for queue", + Default: map[string]interface{}{}, + Ref: ref("volcano.sh/apis/pkg/apis/scheduling/v1beta1.Reservation"), + }, + }, + "allocated": { + SchemaProps: spec.SchemaProps{ + Description: "Allocated is allocated resources in queue", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity", "volcano.sh/apis/pkg/apis/scheduling/v1beta1.Reservation"}, + } +} + +func schema_pkg_apis_scheduling_v1beta1_Reservation(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Reservation represents current condition about resource reservation", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "nodes": { + SchemaProps: spec.SchemaProps{ + Description: "Nodes are Locked nodes for queue", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "resource": { + SchemaProps: spec.SchemaProps{ + Description: "Resource is a list of total idle resource in locked nodes.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + } +} diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/apply/apply.go b/vendor/github.com/kubeflow/trainer/v2/pkg/apply/apply.go index 4f4655fcb1a..933dba35408 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/apply/apply.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/apply/apply.go @@ -34,12 +34,6 @@ var ( errorRequestedFieldPathNotFound = errors.New("requested field path not found") ) -func UpsertEnvVar(envVars *[]corev1ac.EnvVarApplyConfiguration, envVar ...corev1ac.EnvVarApplyConfiguration) { - for _, e := range envVar { - upsert(envVars, e, byEnvVarName) - } -} - func UpsertEnvVars(envVars *[]corev1ac.EnvVarApplyConfiguration, upEnvVars ...corev1ac.EnvVarApplyConfiguration) { for _, e := range upEnvVars { upsert(envVars, e, byEnvVarName) diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/constants/constants.go b/vendor/github.com/kubeflow/trainer/v2/pkg/constants/constants.go index b2523cdd08d..19b4a1daebf 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/constants/constants.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/constants/constants.go @@ -24,6 +24,15 @@ const ( // trainer.kubeflow.org/trainjob-ancestor-step: trainer - trainJob.spec.trainer LabelTrainJobAncestor string = "trainer.kubeflow.org/trainjob-ancestor-step" + // LabelSupport indicates support status for a runtime, e.g. "deprecated". + LabelSupport string = "trainer.kubeflow.org/support" + + // SupportDeprecated indicates the runtime is deprecated when used with LabelSupport. + SupportDeprecated string = "deprecated" + + // RuntimeDeprecationPolicyURL is the URL to the runtime deprecation policy documentation. + RuntimeDeprecationPolicyURL string = "https://www.kubeflow.org/docs/components/trainer/operator-guides/runtime/#runtime-deprecation-policy" + // DatasetInitializer is the name of the Job, volume mount, container, and label value for the dataset initializer. DatasetInitializer string = "dataset-initializer" @@ -146,12 +155,42 @@ const ( // TorchTuneFullFinetuneMultiNodesConfigSuffix is the config suffix for the multi node distributed full finetune. TorchTuneFullFinetuneMultiNodesConfigSuffix string = "_full_multinode" + // TorchTuneLoRAFinetuneSingleDevice Recipe is the recipe for the single device LoRA finetune. + TorchTuneLoRAFinetuneSingleDevice string = "lora_finetune_single_device" + + // TorchTuneLoRAFinetuneSingleDeviceConfigSuffix is the config suffix for the single device LoRA finetune. + TorchTuneLoRAFinetuneSingleDeviceConfigSuffix string = "_lora_single_device" + + // TorchTuneQLoRAFinetuneSingleDeviceConfigSuffix is the config suffix for the single device QLoRA finetune. + TorchTuneQLoRAFinetuneSingleDeviceConfigSuffix string = "_qlora_single_device" + + // TorchTuneLoRAFinetuneDistributed Recipe is the recipe for the distributed LoRA finetune. + TorchTuneLoRAFinetuneDistributed string = "lora_finetune_distributed" + + // TorchTuneLoRAFinetuneDistributedConfigSuffix is the config suffix for the distributed LoRA finetune. + TorchTuneLoRAFinetuneDistributedConfigSuffix string = "_lora" + + // TorchTuneQLoRAFinetuneDistributedConfigSuffix is the config suffix for the distributed QLoRA finetune. + TorchTuneQLoRAFinetuneDistributedConfigSuffix string = "_qlora" + + // TorchTuneLoraAttnModules is the config item name for the LoRA attention modules. + TorchTuneLoraAttnModules string = "model.lora_attn_modules" + + // TorchTuneQuantizeBase is the config item name for the quantization base. + TorchTuneQuantizeBase string = "model.quantize_base" + + // TorchTuneUseDora is the config item name for using DoRA. + TorchTuneUseDora string = "model.use_dora" + // TorchTuneModelOutputDir is the config item name for the model output directory. TorchTuneModelOutputDir string = "output_dir" // TorchTuneTokenizerPath is the config item name for the tokenizer path. TorchTuneTokenizerPath string = "tokenizer.path" + // TorchTuneTokenizerMergeFile is the config item name for the tokenizer merge file. + TorchTuneTokenizerMergeFile string = "tokenizer.merges_file" + // TorchTuneCheckpointerDir is the config item name for the checkpointer directory. TorchTuneCheckpointDir string = "checkpointer.checkpoint_dir" ) @@ -160,11 +199,14 @@ const ( // TORCHTUNE_MODEL_LLAMA3_2_1B is the model name for the Llama3.2 1B Instruct model. TORCHTUNE_MODEL_LLAMA3_2_1B = "llama3_2/1B" - // TORCHTUNE_MODEL_LLAMA3_2_7B is the model name for the Llama3.2 7B Instruct model. - TORCHTUNE_MODEL_LLAMA3_2_7B = "llama3_2/7B" + // TORCHTUNE_MODEL_LLAMA3_2_3B is the model name for the Llama3.2 3B Instruct model. + TORCHTUNE_MODEL_LLAMA3_2_3B = "llama3_2/3B" // TORCHTUNE_MODEL_LLAMA3_3_70B is the model name for the Llama3.3 70B Instruct model. TORCHTUNE_MODEL_LLAMA3_3_70B = "llama3_3/70B" + + // TORCHTUNE_MODEL_QWEN2_1.5B is the model name for the Qwen2.5 1.5B model. + TORCHTUNE_MODEL_QWEN2_5_1_5B = "qwen2_5/1.5B" ) var ( @@ -178,11 +220,11 @@ var ( ResourceInUseFinalizer = fmt.Sprintf("%s/resource-in-use", trainer.GroupVersion.Group) // TorchTuneSupportedPretrainedModels supported pretrained models for TorchTune Trainer. - TorchTuneSupportedPretrainedModels = sets.New(TORCHTUNE_MODEL_LLAMA3_2_1B, TORCHTUNE_MODEL_LLAMA3_2_7B, TORCHTUNE_MODEL_LLAMA3_3_70B) + TorchTuneSupportedPretrainedModels = sets.New(TORCHTUNE_MODEL_LLAMA3_2_1B, TORCHTUNE_MODEL_LLAMA3_2_3B, TORCHTUNE_MODEL_LLAMA3_3_70B, TORCHTUNE_MODEL_QWEN2_5_1_5B) // TorchTuneEntrypoint is the entrypoint for the torchtune. TorchTuneEntrypoint = []string{"tune", "run"} // TorchTuneImmutableConfigs is the set of immutable configs for the TorchTune Trainer. - TorchTuneImmutableConfigs = sets.New(TorchTuneModelOutputDir, TorchTuneTokenizerPath, TorchTuneCheckpointDir) + TorchTuneImmutableConfigs = sets.New(TorchTuneModelOutputDir, TorchTuneTokenizerPath, TorchTuneCheckpointDir, TorchTuneTokenizerMergeFile) ) diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/core/clustertrainingruntime.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/core/clustertrainingruntime.go index 16214203e5e..2e3c653e742 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/core/clustertrainingruntime.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/core/clustertrainingruntime.go @@ -21,14 +21,15 @@ import ( "errors" "fmt" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" + "github.com/kubeflow/trainer/v2/pkg/constants" "github.com/kubeflow/trainer/v2/pkg/runtime" + trainingruntime "github.com/kubeflow/trainer/v2/pkg/util/trainingruntime" ) var ( @@ -71,8 +72,8 @@ func (r *ClusterTrainingRuntime) RuntimeInfo( return r.TrainingRuntime.RuntimeInfo(trainJob, runtimeTemplateSpec, mlPolicy, podGroupPolicy) } -func (r *ClusterTrainingRuntime) TerminalCondition(ctx context.Context, trainJob *trainer.TrainJob) (*metav1.Condition, error) { - return r.TrainingRuntime.TerminalCondition(ctx, trainJob) +func (r *ClusterTrainingRuntime) TrainJobStatus(ctx context.Context, trainJob *trainer.TrainJob) (*trainer.TrainJobStatus, error) { + return r.TrainingRuntime.TrainJobStatus(ctx, trainJob) } func (r *ClusterTrainingRuntime) EventHandlerRegistrars() []runtime.ReconcilerBuilder { @@ -89,6 +90,18 @@ func (r *ClusterTrainingRuntime) ValidateObjects(ctx context.Context, old, new * fmt.Sprintf("%v: specified clusterTrainingRuntime must be created before the TrainJob is created", err)), } } + var warnings admission.Warnings + if trainingruntime.IsSupportDeprecated(clusterTrainingRuntime.Labels) { + warnings = append(warnings, fmt.Sprintf( + "Referenced ClusterTrainingRuntime \"%s\" is deprecated and will be removed in a future release of Kubeflow Trainer. See runtime deprecation policy: %s", + clusterTrainingRuntime.Name, + constants.RuntimeDeprecationPolicyURL, + )) + } info, _ := r.newRuntimeInfo(new, clusterTrainingRuntime.Spec.Template, clusterTrainingRuntime.Spec.MLPolicy, clusterTrainingRuntime.Spec.PodGroupPolicy) - return r.framework.RunCustomValidationPlugins(ctx, info, old, new) + fwWarnings, errs := r.framework.RunCustomValidationPlugins(ctx, info, old, new) + if len(fwWarnings) != 0 { + warnings = append(warnings, fwWarnings...) + } + return warnings, errs } diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/core/trainingruntime.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/core/trainingruntime.go index 89ffac2a7f9..0f03d467e4b 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/core/trainingruntime.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/core/trainingruntime.go @@ -138,7 +138,7 @@ func (r *TrainingRuntime) newRuntimeInfo( // The JobSetTemplateSpec annotations are overridden by the TrainJob Annotations (.spec.annotations). propagationAnnotations[k] = v } - err := r.mergePodSpecOverrides(trainJob, &jobSetTemplateSpec) + err := r.mergePodTemplateOverrides(trainJob, &jobSetTemplateSpec) if err != nil { return nil, err } @@ -187,32 +187,51 @@ func (r *TrainingRuntime) newRuntimeInfo( return runtime.NewInfo(opts...), nil } -func (r *TrainingRuntime) mergePodSpecOverrides(trainJob *trainer.TrainJob, jobSetTemplateSpec *trainer.JobSetTemplateSpec) error { - for _, podSpecOverride := range trainJob.Spec.PodSpecOverrides { +func (r *TrainingRuntime) mergePodTemplateOverrides(trainJob *trainer.TrainJob, jobSetTemplateSpec *trainer.JobSetTemplateSpec) error { + for _, podTemplateOverride := range trainJob.Spec.PodTemplateOverrides { for i, job := range jobSetTemplateSpec.Spec.ReplicatedJobs { - if !slices.ContainsFunc(podSpecOverride.TargetJobs, func(targetJob trainer.PodSpecOverrideTargetJob) bool { + if !slices.ContainsFunc(podTemplateOverride.TargetJobs, func(targetJob trainer.PodTemplateOverrideTargetJob) bool { return targetJob.Name == job.Name }) { continue } - patch, err := json.Marshal(podSpecOverride) + + podTemplatePatch := map[string]any{} + if podTemplateOverride.Metadata != nil { + metadata := map[string]any{} + if podTemplateOverride.Metadata.Labels != nil { + metadata["labels"] = podTemplateOverride.Metadata.Labels + } + if podTemplateOverride.Metadata.Annotations != nil { + metadata["annotations"] = podTemplateOverride.Metadata.Annotations + } + if len(metadata) > 0 { + podTemplatePatch["metadata"] = metadata + } + } + + if podTemplateOverride.Spec != nil { + podTemplatePatch["spec"] = podTemplateOverride.Spec + } + + // Apply a strategic merge patch against the full PodTemplateSpec + source, err := json.Marshal(job.Template.Spec.Template) if err != nil { return err } - source, err := json.Marshal(job.Template.Spec.Template.Spec) + patch, err := json.Marshal(podTemplatePatch) if err != nil { return err } - merged, err := strategicpatch.StrategicMergePatch(source, patch, corev1.PodSpec{}) + merged, err := strategicpatch.StrategicMergePatch(source, patch, corev1.PodTemplateSpec{}) if err != nil { return err } - spec := corev1.PodSpec{} - err = json.Unmarshal(merged, &spec) - if err != nil { + mergedTemplate := corev1.PodTemplateSpec{} + if err := json.Unmarshal(merged, &mergedTemplate); err != nil { return err } - jobSetTemplateSpec.Spec.ReplicatedJobs[i].Template.Spec.Template.Spec = spec + jobSetTemplateSpec.Spec.ReplicatedJobs[i].Template.Spec.Template = mergedTemplate } } return nil @@ -230,7 +249,7 @@ func syncPodSets(info *runtime.Info) { } apply.UpsertVolumes(&jsSpec.ReplicatedJobs[psIdx].Template.Spec.Template.Spec.Volumes, ps.Volumes...) for containerIdx, container := range ps.Containers { - apply.UpsertEnvVar( + apply.UpsertEnvVars( &jsSpec.ReplicatedJobs[psIdx].Template.Spec.Template.Spec.Containers[containerIdx].Env, container.Env..., ) @@ -246,8 +265,8 @@ func syncPodSets(info *runtime.Info) { } } -func (r *TrainingRuntime) TerminalCondition(ctx context.Context, trainJob *trainer.TrainJob) (*metav1.Condition, error) { - return r.framework.RunTerminalConditionPlugins(ctx, trainJob) +func (r *TrainingRuntime) TrainJobStatus(ctx context.Context, trainJob *trainer.TrainJob) (*trainer.TrainJobStatus, error) { + return r.framework.RunTrainJobStatusPlugin(ctx, trainJob) } func (r *TrainingRuntime) EventHandlerRegistrars() []runtime.ReconcilerBuilder { diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/core/framework.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/core/framework.go index 551164e3659..dc654d3aeae 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/core/framework.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/core/framework.go @@ -20,7 +20,6 @@ import ( "context" "errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -29,9 +28,10 @@ import ( "github.com/kubeflow/trainer/v2/pkg/runtime" "github.com/kubeflow/trainer/v2/pkg/runtime/framework" fwkplugins "github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins" + index "github.com/kubeflow/trainer/v2/pkg/runtime/indexer" ) -var errorTooManyTerminalConditionPlugin = errors.New("too many TerminalCondition plugins are registered") +var errorTooManyTrainJobStatusPlugin = errors.New("too many TrainJobStatus plugins are registered") type Framework struct { registry fwkplugins.Registry @@ -42,7 +42,7 @@ type Framework struct { watchExtensionPlugins []framework.WatchExtensionPlugin podNetworkPlugins []framework.PodNetworkPlugin componentBuilderPlugins []framework.ComponentBuilderPlugin - terminalConditionPlugins []framework.TerminalConditionPlugin + trainJobStatusPlugin framework.TrainJobStatusPlugin } func New(ctx context.Context, c client.Client, r fwkplugins.Registry, indexer client.FieldIndexer) (*Framework, error) { @@ -50,6 +50,9 @@ func New(ctx context.Context, c client.Client, r fwkplugins.Registry, indexer cl registry: r, } plugins := make(map[string]framework.Plugin, len(r)) + if err := f.SetupRuntimeClassIndexer(ctx, indexer); err != nil { + return nil, err + } for name, factory := range r { plugin, err := factory(ctx, c, indexer) @@ -75,8 +78,11 @@ func New(ctx context.Context, c client.Client, r fwkplugins.Registry, indexer cl if p, ok := plugin.(framework.ComponentBuilderPlugin); ok { f.componentBuilderPlugins = append(f.componentBuilderPlugins, p) } - if p, ok := plugin.(framework.TerminalConditionPlugin); ok { - f.terminalConditionPlugins = append(f.terminalConditionPlugins, p) + if p, ok := plugin.(framework.TrainJobStatusPlugin); ok { + if f.trainJobStatusPlugin != nil { + return nil, errorTooManyTrainJobStatusPlugin + } + f.trainJobStatusPlugin = p } } f.plugins = plugins @@ -137,13 +143,9 @@ func (f *Framework) RunComponentBuilderPlugins(ctx context.Context, info *runtim return objs, nil } -func (f *Framework) RunTerminalConditionPlugins(ctx context.Context, trainJob *trainer.TrainJob) (*metav1.Condition, error) { - // TODO (tenzen-y): Once we provide the Configuration API, we should validate which plugin should have terminalCondition execution points. - if len(f.terminalConditionPlugins) > 1 { - return nil, errorTooManyTerminalConditionPlugin - } - if len(f.terminalConditionPlugins) != 0 { - return f.terminalConditionPlugins[0].TerminalCondition(ctx, trainJob) +func (f *Framework) RunTrainJobStatusPlugin(ctx context.Context, trainJob *trainer.TrainJob) (*trainer.TrainJobStatus, error) { + if f.trainJobStatusPlugin != nil { + return f.trainJobStatusPlugin.Status(ctx, trainJob) } return nil, nil } @@ -151,3 +153,17 @@ func (f *Framework) RunTerminalConditionPlugins(ctx context.Context, trainJob *t func (f *Framework) WatchExtensionPlugins() []framework.WatchExtensionPlugin { return f.watchExtensionPlugins } + +func (f *Framework) SetupRuntimeClassIndexer(ctx context.Context, indexer client.FieldIndexer) error { + if err := indexer.IndexField(ctx, &trainer.TrainingRuntime{}, + index.TrainingRuntimeContainerRuntimeClassKey, + index.IndexTrainingRuntimeContainerRuntimeClass); err != nil { + return index.ErrorCanNotSetupTrainingRuntimeRuntimeClassIndexer + } + if err := indexer.IndexField(ctx, &trainer.ClusterTrainingRuntime{}, + index.ClusterTrainingRuntimeContainerRuntimeClassKey, + index.IndexClusterTrainingRuntimeContainerRuntimeClass); err != nil { + return index.ErrorCanNotSetupClusterTrainingRuntimeRuntimeClassIndexer + } + return nil +} diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/interface.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/interface.go index 676548ebe7e..703379c1f1f 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/interface.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/interface.go @@ -19,7 +19,6 @@ package framework import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -61,7 +60,7 @@ type ComponentBuilderPlugin interface { Build(ctx context.Context, info *runtime.Info, trainJob *trainer.TrainJob) ([]any, error) } -type TerminalConditionPlugin interface { +type TrainJobStatusPlugin interface { Plugin - TerminalCondition(ctx context.Context, trainJob *trainer.TrainJob) (*metav1.Condition, error) + Status(ctx context.Context, trainJob *trainer.TrainJob) (*trainer.TrainJobStatus, error) } diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/coscheduling/coscheduling.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/coscheduling/coscheduling.go index d2a1305608b..abcfd4634cd 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/coscheduling/coscheduling.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/coscheduling/coscheduling.go @@ -18,8 +18,6 @@ package coscheduling import ( "context" - "errors" - "fmt" "slices" "github.com/go-logr/logr" @@ -47,6 +45,7 @@ import ( trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" "github.com/kubeflow/trainer/v2/pkg/runtime" "github.com/kubeflow/trainer/v2/pkg/runtime/framework" + index "github.com/kubeflow/trainer/v2/pkg/runtime/indexer" runtimeindexer "github.com/kubeflow/trainer/v2/pkg/runtime/indexer" ) @@ -61,24 +60,13 @@ var _ framework.EnforcePodGroupPolicyPlugin = (*CoScheduling)(nil) var _ framework.WatchExtensionPlugin = (*CoScheduling)(nil) var _ framework.ComponentBuilderPlugin = (*CoScheduling)(nil) -var ( - ErrorCanNotSetupTrainingRuntimeRuntimeClassIndexer = errors.New("setting index on runtimeClass for TrainingRuntime") - ErrorCanNotSetupClusterTrainingRuntimeRuntimeClassIndexer = errors.New("setting index on runtimeClass for ClusterTrainingRuntime") -) - const Name = "CoScheduling" // +kubebuilder:rbac:groups=scheduling.x-k8s.io,resources=podgroups,verbs=create;get;list;watch;update;patch +// +kubebuilder:rbac:groups=node.k8s.io,resources=runtimeclasses,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=limitranges,verbs=get;list;watch -func New(ctx context.Context, client client.Client, indexer client.FieldIndexer) (framework.Plugin, error) { - if err := indexer.IndexField(ctx, &trainer.TrainingRuntime{}, TrainingRuntimeContainerRuntimeClassKey, - IndexTrainingRuntimeContainerRuntimeClass); err != nil { - return nil, fmt.Errorf("%w: %w", ErrorCanNotSetupTrainingRuntimeRuntimeClassIndexer, err) - } - if err := indexer.IndexField(ctx, &trainer.ClusterTrainingRuntime{}, ClusterTrainingRuntimeContainerRuntimeClassKey, - IndexClusterTrainingRuntimeContainerRuntimeClass); err != nil { - return nil, fmt.Errorf("%w: %w", ErrorCanNotSetupClusterTrainingRuntimeRuntimeClassIndexer, err) - } +func New(_ context.Context, client client.Client, _ client.FieldIndexer) (framework.Plugin, error) { return &CoScheduling{ client: client, restMapper: client.RESTMapper(), @@ -91,7 +79,7 @@ func (c *CoScheduling) Name() string { } func (c *CoScheduling) EnforcePodGroupPolicy(info *runtime.Info, trainJob *trainer.TrainJob) error { - if info == nil || info.RuntimePolicy.PodGroupPolicy == nil || trainJob == nil { + if info == nil || info.RuntimePolicy.PodGroupPolicy == nil || info.RuntimePolicy.PodGroupPolicy.Coscheduling == nil || trainJob == nil { return nil } @@ -185,11 +173,11 @@ func (h *PodGroupRuntimeClassHandler) Generic(context.Context, event.TypedGeneri func (h *PodGroupRuntimeClassHandler) queueSuspendedTrainJobs(ctx context.Context, runtimeClass *nodev1.RuntimeClass, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { var trainingRuntimes trainer.TrainingRuntimeList - if err := h.client.List(ctx, &trainingRuntimes, client.MatchingFields{TrainingRuntimeContainerRuntimeClassKey: runtimeClass.Name}); err != nil { + if err := h.client.List(ctx, &trainingRuntimes, client.MatchingFields{index.TrainingRuntimeContainerRuntimeClassKey: runtimeClass.Name}); err != nil { return err } var clusterTrainingRuntimes trainer.ClusterTrainingRuntimeList - if err := h.client.List(ctx, &clusterTrainingRuntimes, client.MatchingFields{ClusterTrainingRuntimeContainerRuntimeClassKey: runtimeClass.Name}); err != nil { + if err := h.client.List(ctx, &clusterTrainingRuntimes, client.MatchingFields{index.ClusterTrainingRuntimeContainerRuntimeClassKey: runtimeClass.Name}); err != nil { return err } diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/coscheduling/indexer.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/coscheduling/indexer.go deleted file mode 100644 index 846d8cd47bf..00000000000 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/coscheduling/indexer.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2024 The Kubeflow Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package coscheduling - -import ( - "sigs.k8s.io/controller-runtime/pkg/client" - - trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" -) - -var ( - TrainingRuntimeContainerRuntimeClassKey = ".trainingRuntimeSpec.jobSetTemplateSpec.replicatedJobs.podTemplateSpec.runtimeClassName" - ClusterTrainingRuntimeContainerRuntimeClassKey = ".clusterTrainingRuntimeSpec.jobSetTemplateSpec.replicatedJobs.podTemplateSpec.runtimeClassName" -) - -func IndexTrainingRuntimeContainerRuntimeClass(obj client.Object) []string { - runtime, ok := obj.(*trainer.TrainingRuntime) - if !ok { - return nil - } - var runtimeClasses []string - for _, rJob := range runtime.Spec.Template.Spec.ReplicatedJobs { - if rJob.Template.Spec.Template.Spec.RuntimeClassName != nil { - runtimeClasses = append(runtimeClasses, *rJob.Template.Spec.Template.Spec.RuntimeClassName) - } - } - return runtimeClasses -} - -func IndexClusterTrainingRuntimeContainerRuntimeClass(obj client.Object) []string { - clRuntime, ok := obj.(*trainer.ClusterTrainingRuntime) - if !ok { - return nil - } - var runtimeClasses []string - for _, rJob := range clRuntime.Spec.Template.Spec.ReplicatedJobs { - if rJob.Template.Spec.Template.Spec.RuntimeClassName != nil { - runtimeClasses = append(runtimeClasses, *rJob.Template.Spec.Template.Spec.RuntimeClassName) - } - } - return runtimeClasses -} diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset/builder.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset/builder.go index e3cf615a18c..d5d8e1ca86f 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset/builder.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset/builder.go @@ -56,7 +56,7 @@ func (b *Builder) Initializer(trainJob *trainer.TrainJob) *Builder { env := &b.Spec.ReplicatedJobs[i].Template.Spec.Template.Spec.Containers[j].Env // Update the dataset initializer envs. if storageUri := trainJob.Spec.Initializer.Dataset.StorageUri; storageUri != nil { - apply.UpsertEnvVar(env, *corev1ac.EnvVar(). + apply.UpsertEnvVars(env, *corev1ac.EnvVar(). WithName(jobsetplgconsts.InitializerEnvStorageUri). WithValue(*storageUri)) } @@ -82,7 +82,7 @@ func (b *Builder) Initializer(trainJob *trainer.TrainJob) *Builder { env := &b.Spec.ReplicatedJobs[i].Template.Spec.Template.Spec.Containers[j].Env // Update the model initializer envs. if storageUri := trainJob.Spec.Initializer.Model.StorageUri; storageUri != nil { - apply.UpsertEnvVar(env, *corev1ac.EnvVar(). + apply.UpsertEnvVars(env, *corev1ac.EnvVar(). WithName(jobsetplgconsts.InitializerEnvStorageUri). WithValue(*storageUri)) } @@ -101,21 +101,26 @@ func (b *Builder) Initializer(trainJob *trainer.TrainJob) *Builder { return b } +// isRunLauncherAsNode returns true if runLauncherAsNode is set to true in the MPI policy. +func (b *Builder) isRunLauncherAsNode(info *runtime.Info) bool { + return info.RuntimePolicy.MLPolicySource != nil && + info.RuntimePolicy.MLPolicySource.MPI != nil && + info.RuntimePolicy.MLPolicySource.MPI.RunLauncherAsNode != nil && + *info.RuntimePolicy.MLPolicySource.MPI.RunLauncherAsNode +} + // Trainer updates JobSet values for the trainer Job. func (b *Builder) Trainer(info *runtime.Info, trainJob *trainer.TrainJob) *Builder { for i, rJob := range b.Spec.ReplicatedJobs { + ancestor := "" jobMetadata := rJob.Template.ObjectMetaApplyConfiguration - if jobMetadata == nil || jobMetadata.Labels == nil { - continue + if jobMetadata != nil && jobMetadata.Labels != nil { + ancestor = jobMetadata.Labels[constants.LabelTrainJobAncestor] } - if ancestor, ok := jobMetadata.Labels[constants.LabelTrainJobAncestor]; ok && ancestor == constants.AncestorTrainer { + if ancestor == constants.AncestorTrainer { // TODO: Support multiple replicas ('.template.spec.replicatedJobs[*].replicas') for replicated Jobs. // REF: https://github.com/kubeflow/trainer/issues/2318 b.Spec.ReplicatedJobs[i].Replicas = ptr.To[int32](1) - // Update the Parallelism and Completions values for the Trainer Job. - b.Spec.ReplicatedJobs[i].Template.Spec.Parallelism = info.FindPodSetByAncestor(constants.AncestorTrainer).Count - b.Spec.ReplicatedJobs[i].Template.Spec.Completions = info.FindPodSetByAncestor(constants.AncestorTrainer).Count - // Update values for the Trainer container. for j, container := range rJob.Template.Spec.Template.Spec.Containers { if *container.Name == constants.Node { @@ -130,6 +135,16 @@ func (b *Builder) Trainer(info *runtime.Info, trainJob *trainer.TrainJob) *Build if args := jobTrainer.Args; args != nil { b.Spec.ReplicatedJobs[i].Template.Spec.Template.Spec.Containers[j].Args = args } + } + } + } + } + if ancestor == constants.AncestorTrainer || b.isRunLauncherAsNode(info) && *rJob.Name == constants.Node { + // TODO (andreyvelich): For MPI we should apply container resources to the Node ReplicatedJob also. + // Eventually, we should find better way to propagate resources from TrainJob to JobSet. + for j, container := range rJob.Template.Spec.Template.Spec.Containers { + if *container.Name == constants.Node { + if jobTrainer := trainJob.Spec.Trainer; jobTrainer != nil { if resourcesPerNode := jobTrainer.ResourcesPerNode; resourcesPerNode != nil && (resourcesPerNode.Limits != nil || resourcesPerNode.Requests != nil) { requirements := corev1ac.ResourceRequirements() @@ -163,6 +178,13 @@ func (b *Builder) PodLabels(labels map[string]string) *Builder { return b } +func (b *Builder) PodAnnotations(annotations map[string]string) *Builder { + for i := range b.Spec.ReplicatedJobs { + b.Spec.ReplicatedJobs[i].Template.Spec.Template.WithAnnotations(annotations) + } + return b +} + func (b *Builder) Suspend(suspend *bool) *Builder { b.Spec.Suspend = suspend return b diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset/jobset.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset/jobset.go index 0e57bfb986b..982e8d0af00 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset/jobset.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset/jobset.go @@ -48,8 +48,8 @@ import ( ) var ( - runtimeRefPath = field.NewPath("spec").Child("runtimeRef") - podSpecOverridePath = field.NewPath("spec").Child("podSpecOverrides") + runtimeRefPath = field.NewPath("spec").Child("runtimeRef") + podTemplateOverridePath = field.NewPath("spec").Child("podTemplateOverrides") ) type JobSet struct { @@ -62,7 +62,7 @@ type JobSet struct { var _ framework.WatchExtensionPlugin = (*JobSet)(nil) var _ framework.PodNetworkPlugin = (*JobSet)(nil) var _ framework.ComponentBuilderPlugin = (*JobSet)(nil) -var _ framework.TerminalConditionPlugin = (*JobSet)(nil) +var _ framework.TrainJobStatusPlugin = (*JobSet)(nil) var _ framework.CustomValidationPlugin = (*JobSet)(nil) const Name = constants.JobSetKind @@ -118,45 +118,39 @@ func (j *JobSet) Validate(ctx context.Context, info *runtime.Info, oldObj, newOb } } - allErrs = append(allErrs, j.checkPodSpecOverridesImmutability(ctx, oldObj, newObj)...) + allErrs = append(allErrs, j.checkPodTemplateOverridesImmutability(ctx, oldObj, newObj)...) // TODO (andreyvelich): Validate Volumes, VolumeMounts, and Tolerations. - targetJobNames := sets.New[string]() - for _, podSpecOverride := range newObj.Spec.PodSpecOverrides { - // Validate that there are no duplicate target job names within the same PodSpecOverride - for _, targetJob := range podSpecOverride.TargetJobs { - if targetJobNames.Has(targetJob.Name) { - allErrs = append(allErrs, field.Duplicate(podSpecOverridePath, targetJob.Name)) - } - targetJobNames.Insert(targetJob.Name) - } - - for _, targetJob := range podSpecOverride.TargetJobs { + for _, podTemplateOverride := range newObj.Spec.PodTemplateOverrides { + for _, targetJob := range podTemplateOverride.TargetJobs { containers, ok := rJobContainerNames[targetJob.Name] if !ok { - allErrs = append(allErrs, field.Invalid(podSpecOverridePath, newObj.Spec.PodSpecOverrides, "must not have targetJob that doesn't exist in the runtime job template")) + allErrs = append(allErrs, field.Invalid(podTemplateOverridePath, newObj.Spec.PodTemplateOverrides, "must not have targetJob that doesn't exist in the runtime job template")) } - for _, overrideContainer := range podSpecOverride.InitContainers { - if !containers.Has(overrideContainer.Name) { - allErrs = append(allErrs, field.Invalid(podSpecOverridePath, newObj.Spec.PodSpecOverrides, fmt.Sprintf("must not have initContainer that doesn't exist in the runtime job %s", targetJob.Name))) + if podTemplateOverride.Spec != nil { + for _, overrideContainer := range podTemplateOverride.Spec.InitContainers { + if !containers.Has(overrideContainer.Name) { + allErrs = append(allErrs, field.Invalid(podTemplateOverridePath, newObj.Spec.PodTemplateOverrides, fmt.Sprintf("must not have initContainer that doesn't exist in the runtime job %s", targetJob.Name))) + } } - } - for _, overrideContainer := range podSpecOverride.Containers { - if !containers.Has(overrideContainer.Name) { - allErrs = append(allErrs, field.Invalid(podSpecOverridePath, newObj.Spec.PodSpecOverrides, fmt.Sprintf("must not have container that doesn't exist in the runtime job %s", targetJob.Name))) - // Trainer and Initializer APIs should be used to set TrainJob envs for the reserved containers. - } else if len(overrideContainer.Env) > 0 && (overrideContainer.Name == constants.DatasetInitializer || overrideContainer.Name == constants.ModelInitializer || overrideContainer.Name == constants.Node) { - allErrs = append(allErrs, field.Invalid(podSpecOverridePath, newObj.Spec.PodSpecOverrides, - fmt.Sprintf("must not have envs for the %s, %s, %s containers", constants.DatasetInitializer, constants.ModelInitializer, constants.Node))) + for _, overrideContainer := range podTemplateOverride.Spec.Containers { + if !containers.Has(overrideContainer.Name) { + allErrs = append(allErrs, field.Invalid(podTemplateOverridePath, newObj.Spec.PodTemplateOverrides, fmt.Sprintf("must not have container that doesn't exist in the runtime job %s", targetJob.Name))) + // Trainer and Initializer APIs should be used to set TrainJob envs for the reserved containers. + } else if len(overrideContainer.Env) > 0 && (overrideContainer.Name == constants.DatasetInitializer || overrideContainer.Name == constants.ModelInitializer || overrideContainer.Name == constants.Node) { + allErrs = append(allErrs, field.Invalid(podTemplateOverridePath, newObj.Spec.PodTemplateOverrides, + fmt.Sprintf("must not have envs for the %s, %s, %s containers", constants.DatasetInitializer, constants.ModelInitializer, constants.Node))) + } } } } + } return nil, allErrs } -func (j *JobSet) checkPodSpecOverridesImmutability(ctx context.Context, oldObj, newObj *trainer.TrainJob) field.ErrorList { +func (j *JobSet) checkPodTemplateOverridesImmutability(ctx context.Context, oldObj, newObj *trainer.TrainJob) field.ErrorList { var allErrs field.ErrorList if oldObj == nil { @@ -165,13 +159,13 @@ func (j *JobSet) checkPodSpecOverridesImmutability(ctx context.Context, oldObj, } jobSet := &jobsetv1alpha2.JobSet{} - changed := !equality.Semantic.DeepEqual(oldObj.Spec.PodSpecOverrides, newObj.Spec.PodSpecOverrides) + changed := !equality.Semantic.DeepEqual(oldObj.Spec.PodTemplateOverrides, newObj.Spec.PodTemplateOverrides) suspended := ptr.Equal(newObj.Spec.Suspend, ptr.To(true)) if changed { if !suspended { - allErrs = append(allErrs, field.Forbidden(podSpecOverridePath, "PodSpecOverrides can only be modified when the TrainJob is suspended")) + allErrs = append(allErrs, field.Forbidden(podTemplateOverridePath, "PodTemplateOverrides can only be modified when the TrainJob is suspended")) } else if err := j.client.Get(ctx, client.ObjectKeyFromObject(newObj), jobSet); client.IgnoreNotFound(err) != nil { - allErrs = append(allErrs, field.InternalError(podSpecOverridePath, err)) + allErrs = append(allErrs, field.InternalError(podTemplateOverridePath, err)) } else { // If the JobSet exists, check whether it's inactive // so changes won't have side effects on the JobSet's Pods @@ -180,8 +174,8 @@ func (j *JobSet) checkPodSpecOverridesImmutability(ctx context.Context, oldObj, // from unsuspended state. for _, replicatedJob := range jobSet.Status.ReplicatedJobsStatus { if replicatedJob.Active > 0 { - allErrs = append(allErrs, field.Forbidden(podSpecOverridePath, - fmt.Sprintf("PodSpecOverrides cannot be modified when the JobSet's ReplicatedJob %s is still active", replicatedJob.Name))) + allErrs = append(allErrs, field.Forbidden(podTemplateOverridePath, + fmt.Sprintf("PodTemplateOverrides cannot be modified when the JobSet's ReplicatedJob %s is still active", replicatedJob.Name))) } } } @@ -275,6 +269,7 @@ func (j *JobSet) Build(ctx context.Context, info *runtime.Info, trainJob *traine Initializer(trainJob). Trainer(info, trainJob). PodLabels(info.Scheduler.PodLabels). + PodAnnotations(info.Scheduler.PodAnnotations). Suspend(trainJob.Spec.Suspend). Build(). WithOwnerReferences(metav1ac.OwnerReference(). @@ -288,18 +283,34 @@ func (j *JobSet) Build(ctx context.Context, info *runtime.Info, trainJob *traine return []any{jobSet}, nil } -func (j *JobSet) TerminalCondition(ctx context.Context, trainJob *trainer.TrainJob) (*metav1.Condition, error) { +func (j *JobSet) Status(ctx context.Context, trainJob *trainer.TrainJob) (*trainer.TrainJobStatus, error) { jobSet := &jobsetv1alpha2.JobSet{} if err := j.client.Get(ctx, client.ObjectKeyFromObject(trainJob), jobSet); err != nil { return nil, err } + status := trainJob.Status.DeepCopy() + if completed := meta.FindStatusCondition(jobSet.Status.Conditions, string(jobsetv1alpha2.JobSetCompleted)); completed != nil && completed.Status == metav1.ConditionTrue { completed.Type = trainer.TrainJobComplete - return completed, nil + meta.SetStatusCondition(&status.Conditions, *completed) } if failed := meta.FindStatusCondition(jobSet.Status.Conditions, string(jobsetv1alpha2.JobSetFailed)); failed != nil && failed.Status == metav1.ConditionTrue { failed.Type = trainer.TrainJobFailed - return failed, nil + meta.SetStatusCondition(&status.Conditions, *failed) + } + + var statuses []trainer.JobStatus + for _, status := range jobSet.Status.ReplicatedJobsStatus { + statuses = append(statuses, trainer.JobStatus{ + Name: status.Name, + Ready: &status.Ready, + Succeeded: &status.Succeeded, + Failed: &status.Failed, + Active: &status.Active, + Suspended: &status.Suspended, + }) } - return nil, nil + status.JobsStatus = statuses + + return status, nil } diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/mpi/mpi.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/mpi/mpi.go index ed32344da34..053fd38e98a 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/mpi/mpi.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/mpi/mpi.go @@ -113,8 +113,6 @@ func (m *MPI) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) er if trainJob.Spec.Trainer != nil && trainJob.Spec.Trainer.NumNodes != nil { if node := info.FindPodSetByName(constants.Node); node != nil && node.Count != nil { if ptr.Deref(info.RuntimePolicy.MLPolicySource.MPI.RunLauncherAsNode, false) { - // TODO: We should implement more strong validations for the MPIRuntime with runLauncherAsNode. - // REF: https://github.com/kubeflow/trainer/issues/2550 // When runLauncherAsNode is enabled, 1 nodes should be allocated to launcher. *node.Count = max(*trainJob.Spec.Trainer.NumNodes-1, 1) } else { @@ -125,6 +123,15 @@ func (m *MPI) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) er if trainJob.Spec.Trainer != nil && trainJob.Spec.Trainer.NumProcPerNode != nil { info.RuntimePolicy.MLPolicySource.MPI.NumProcPerNode = ptr.To(int32(trainJob.Spec.Trainer.NumProcPerNode.IntValue())) + // If numProcPerNode is set to 1 in runtime, we make it equal to number of GPUs. + } else if *info.RuntimePolicy.MLPolicySource.MPI.NumProcPerNode == 1 { + resourcesPerNode := ptr.Deref(runtime.ExtractResourcePerNodeFromRuntime(info), corev1.ResourceRequirements{}) + if jobTrainer := trainJob.Spec.Trainer; jobTrainer != nil && jobTrainer.ResourcesPerNode != nil { + resourcesPerNode = ptr.Deref(jobTrainer.ResourcesPerNode, corev1.ResourceRequirements{}) + } + if gpuQ := runtime.GetNumGPUPerNode(&resourcesPerNode); gpuQ > 1 { + info.RuntimePolicy.MLPolicySource.MPI.NumProcPerNode = ptr.To(int32(gpuQ)) + } } // Add Secret and ConfigMap volumes to the Info object diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/registry.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/registry.go index 3944a63a66f..64445abadeb 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/registry.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/registry.go @@ -27,6 +27,7 @@ import ( "github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/mpi" "github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/plainml" "github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch" + "github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/volcano" ) type Registry map[string]func(ctx context.Context, client client.Client, indexer client.FieldIndexer) (framework.Plugin, error) @@ -34,6 +35,7 @@ type Registry map[string]func(ctx context.Context, client client.Client, indexer func NewRegistry() Registry { return Registry{ coscheduling.Name: coscheduling.New, + volcano.Name: volcano.New, mpi.Name: mpi.New, plainml.Name: plainml.New, torch.Name: torch.New, diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch/torch.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch/torch.go index d096f3485bd..0aef2c75966 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch/torch.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch/torch.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "slices" - "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -30,7 +29,6 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - jobsetv1alpha2ac "sigs.k8s.io/jobset/client-go/applyconfiguration/jobset/v1alpha2" trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" "github.com/kubeflow/trainer/v2/pkg/apply" @@ -88,21 +86,10 @@ func (t *Torch) Validate(_ context.Context, runtimeInfo *runtime.Info, _, newObj // Check supported pretrained models for torchtune. // TODO(Electronic-Waste): Add more validation for torchtune when we support more arguments. if slices.Equal(newObj.Spec.Trainer.Command, constants.TorchTuneEntrypoint) { - runtimeRefNamePath := specPath.Child("runtimeRef").Child("name") - model := getModelFromRuntimeRef(newObj.Spec.RuntimeRef.Name) - - if !constants.TorchTuneSupportedPretrainedModels.Has(model) { - allErrs = append(allErrs, field.Invalid(runtimeRefNamePath, newObj.Spec.RuntimeRef.Name, fmt.Sprintf("must have a supported pretrained model, invalid model configured: %v", model))) - } - - numNodesRefPath := specPath.Child("trainer").Child("numNodes") - numNodes := *newObj.Spec.Trainer.NumNodes - if numNodes > 1 && model != constants.TORCHTUNE_MODEL_LLAMA3_3_70B { - allErrs = append(allErrs, field.Invalid(numNodesRefPath, numNodes, fmt.Sprintf("must be 1 for %v model", model))) - } + _, torchTuneErrs := validateTorchTune(runtimeInfo, newObj) + allErrs = append(allErrs, torchTuneErrs...) } } - return nil, allErrs } @@ -123,37 +110,15 @@ func (t *Torch) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) numProcPerNode = ptr.Deref(trainJob.Spec.Trainer.NumProcPerNode, intstr.FromString("auto")) } + // Determine numProcPerNode based on the resourcesPerNode. + resourcesPerNode := ptr.Deref(runtime.ExtractResourcePerNodeFromRuntime(info), corev1.ResourceRequirements{}) if jobTrainer := trainJob.Spec.Trainer; jobTrainer != nil && jobTrainer.ResourcesPerNode != nil { - var ( - shouldUseCPU func(resources corev1.ResourceList) bool - fallbackNumProcPerNode intstr.IntOrString - ) - switch numProcPerNode.String() { - case "auto": - shouldUseCPU = func(resources corev1.ResourceList) bool { - for resName := range resources { - if strings.Contains(strings.ToLower(resName.String()), "gpu") { - return false - } - } - return true - } - fallbackNumProcPerNode = intstr.FromString("auto") - case "cpu": - shouldUseCPU = func(resources corev1.ResourceList) bool { - _, ok := resources[corev1.ResourceCPU] - return ok - } - fallbackNumProcPerNode = intstr.FromInt32(1) - default: - shouldUseCPU = func(corev1.ResourceList) bool { return false } - fallbackNumProcPerNode = numProcPerNode - } - nppNode, usedCPU := calculateNumProcPerNode(fallbackNumProcPerNode, jobTrainer.ResourcesPerNode.Limits, shouldUseCPU) - if !usedCPU { - nppNode, _ = calculateNumProcPerNode(fallbackNumProcPerNode, jobTrainer.ResourcesPerNode.Requests, shouldUseCPU) - } - numProcPerNode = nppNode + resourcesPerNode = ptr.Deref(jobTrainer.ResourcesPerNode, corev1.ResourceRequirements{}) + } + gpuQ := runtime.GetNumGPUPerNode(&resourcesPerNode) + // If numProcPerNode is "cpu" or no GPU is set in resource, we calculate numProcPerNode based on CPU. + if numProcPerNode.String() == "cpu" || numProcPerNode.String() == "auto" && gpuQ == 0 { + numProcPerNode = intstr.FromInt(max(1, getNumCPUPerNode(&resourcesPerNode))) } // Update envs for Info object. @@ -167,7 +132,7 @@ func (t *Torch) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) // Add PyTorch distributed "PET_" values for torchrun and torchtune. // TODO (andreyvelich): We should validate that envs from different plugins don't conflict with each other. // Ref: https://github.com/kubeflow/trainer/pull/2308#discussion_r1823229940 - apply.UpsertEnvVar(&trainerContainer.Env, + apply.UpsertEnvVars(&trainerContainer.Env, *corev1ac.EnvVar(). WithName(constants.TorchEnvNumNodes). WithValue(fmt.Sprintf("%d", ptr.Deref(ptr.Deref(trainerPS, runtime.PodSet{}).Count, 1))), @@ -183,7 +148,7 @@ func (t *Torch) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) if !slices.Equal(trainJob.Spec.Trainer.Command, constants.TorchTuneEntrypoint) { // Add PET_MASTER_ADDR and PET_MASTER_PORT envs for torchrun. - apply.UpsertEnvVar(&trainerContainer.Env, + apply.UpsertEnvVars(&trainerContainer.Env, *corev1ac.EnvVar(). WithName(constants.TorchEnvMasterAddr). WithValue(fmt.Sprintf("%s-%s-0-0.%s", trainJob.Name, constants.Node, trainJob.Name)), @@ -198,7 +163,7 @@ func (t *Torch) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) // Rendezvous backend is only enabled for multi-nodes or multi-devices training. var newCommand []string numNodes := ptr.Deref(ptr.Deref(trainerPS, runtime.PodSet{}).Count, 1) - if numNodes > 1 || !(numProcPerNode.Type == intstr.Int && numProcPerNode.IntVal == 1) { + if numNodes > 1 || numProcPerNode.Type == intstr.Int && numProcPerNode.IntVal > 1 || numProcPerNode.Type == intstr.String && gpuQ > 1 { newCommand = append(newCommand, fmt.Sprintf("%s=%s-%s-0-0.%s:%d", constants.TorchTuneArgRdzvEndpoint, @@ -208,12 +173,7 @@ func (t *Torch) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) } // 2. Get the recipe and config from old args and append them to newCommand. - recipe, config := getRecipeAndConfig( - numNodes, - numProcPerNode, - getModelFromRuntimeRef(trainJob.Spec.RuntimeRef.Name), - trainJob.Spec.Trainer.Args, - ) + recipe, config := getRecipeAndConfig(numNodes, numProcPerNode, gpuQ, trainJob) newCommand = append(newCommand, recipe, constants.TorchTuneArgConfig, config) // 3. Extract output directory, tokenizer path and model mount path from (Cluster)TrainingRuntime. @@ -228,71 +188,17 @@ func (t *Torch) EnforceMLPolicy(info *runtime.Info, trainJob *trainer.TrainJob) return nil } -// calculateNumProcPerNode calculates the number of processes per node based on the provided resources. -// It returns the calculated number of processes per node and a boolean indicating whether CPU resources were used. -func calculateNumProcPerNode( - fallbackNumProcPerNode intstr.IntOrString, resources corev1.ResourceList, shouldUseCPU func(resources corev1.ResourceList) bool, -) (intstr.IntOrString, bool) { - var defaultCPU int32 = 1 - if resources != nil { - if shouldUseCPU(resources) { - cpuQ := resources[corev1.ResourceCPU] - return intstr.FromInt32(max(defaultCPU, int32(cpuQ.Value()))), true - } - return fallbackNumProcPerNode, false - } - return intstr.FromInt32(defaultCPU), false -} - -// getRecipeAndConfig returns the recipe and config file name based on the number of nodes, -// number of processes per node, model name, and command line arguments. -func getRecipeAndConfig(numNodes int32, numProcPerNode intstr.IntOrString, model string, _ []string) (string, string) { - recipe := constants.TorchTuneFullFinetuneDistributed - suffix := constants.TorchTuneFullFinetuneMultiDevicesConfigSuffix - if numNodes == 1 && numProcPerNode.Type == intstr.Int && numProcPerNode.IntVal == 1 { - recipe = constants.TorchTuneFullFinetuneSingleDevice - suffix = constants.TorchTuneFullFinetuneSingleDeviceConfigSuffix - } else if numNodes > 1 { - suffix = constants.TorchTuneFullFinetuneMultiNodesConfigSuffix +// getNumCPUPerNode calculates the number of CPU processes per node based on the provided resources. +func getNumCPUPerNode(res *corev1.ResourceRequirements) int { + if res == nil { + return 0 } - - return recipe, fmt.Sprintf("%s%s", model, suffix) -} - -// extractOverridesFromRuntime extracts overrides from the TorchTune Trainer Node. -func extractOverridesFromRuntime(info *runtime.Info) []string { - overrides := []string{} - jobSetSpec, ok := runtime.TemplateSpecApply[jobsetv1alpha2ac.JobSetSpecApplyConfiguration](info) - if !ok { - return overrides - } - - for _, rJob := range jobSetSpec.ReplicatedJobs { - jobMetadata := rJob.Template.ObjectMetaApplyConfiguration - if jobMetadata == nil || jobMetadata.Labels == nil { - continue - } - if ancestor, ok := jobMetadata.Labels[constants.LabelTrainJobAncestor]; ok && ancestor == constants.AncestorTrainer { - for _, container := range rJob.Template.Spec.Template.Spec.Containers { - if container.Name != nil && *container.Name == constants.Node { - for _, command := range container.Command { - if constants.TorchTuneImmutableConfigs.Has(strings.Split(command, "=")[0]) { - overrides = append(overrides, command) - } - } - } - } + limitCpuQ, requestCpuQ := res.Limits.Cpu(), res.Requests.Cpu() + if requestCpuQ == nil || requestCpuQ.IsZero() { + if limitCpuQ != nil { + return int(limitCpuQ.Value()) } + return 0 } - - return overrides -} - -func getModelFromRuntimeRef(runtimeRefName string) string { - fields := strings.Split(runtimeRefName, "-") - if len(fields) != 3 { - return "" - } - - return fmt.Sprintf("%s/%s", strings.ReplaceAll(fields[1], ".", "_"), strings.ToUpper(fields[2])) + return int(requestCpuQ.Value()) } diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch/torchtune.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch/torchtune.go new file mode 100644 index 00000000000..87d69a29534 --- /dev/null +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch/torchtune.go @@ -0,0 +1,164 @@ +/* +Copyright 2025 The Kubeflow Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package torch + +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + jobsetv1alpha2ac "sigs.k8s.io/jobset/client-go/applyconfiguration/jobset/v1alpha2" + + trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" + "github.com/kubeflow/trainer/v2/pkg/constants" + "github.com/kubeflow/trainer/v2/pkg/runtime" +) + +func validateTorchTune(runtimeInfo *runtime.Info, newObj *trainer.TrainJob) (admission.Warnings, field.ErrorList) { + var allErrs field.ErrorList + specPath := field.NewPath("spec") + + runtimeRefNamePath := specPath.Child("runtimeRef").Child("name") + model := getModelFromRuntimeRef(newObj.Spec.RuntimeRef.Name) + + if !constants.TorchTuneSupportedPretrainedModels.Has(model) { + allErrs = append(allErrs, field.Invalid(runtimeRefNamePath, newObj.Spec.RuntimeRef.Name, fmt.Sprintf("must have a supported pretrained model, invalid model configured: %v", model))) + } + + numNodesRefPath := specPath.Child("trainer").Child("numNodes") + numNodes := *newObj.Spec.Trainer.NumNodes + if numNodes > 1 && model != constants.TORCHTUNE_MODEL_LLAMA3_3_70B { + allErrs = append(allErrs, field.Invalid(numNodesRefPath, numNodes, fmt.Sprintf("must be 1 for %v model", model))) + } + + numProcPerNodeRefPath := specPath.Child("trainer").Child("numProcPerNode") + numProcPerNode := *newObj.Spec.Trainer.NumProcPerNode + resourcesPerNode := ptr.Deref(runtime.ExtractResourcePerNodeFromRuntime(runtimeInfo), corev1.ResourceRequirements{}) + if jobTrainer := newObj.Spec.Trainer; jobTrainer != nil && jobTrainer.ResourcesPerNode != nil { + resourcesPerNode = ptr.Deref(jobTrainer.ResourcesPerNode, corev1.ResourceRequirements{}) + } + _, config := getRecipeAndConfig(numNodes, numProcPerNode, runtime.GetNumGPUPerNode(&resourcesPerNode), newObj) + if strings.Contains(config, constants.TorchTuneQLoRAFinetuneDistributedConfigSuffix) { + if model == constants.TORCHTUNE_MODEL_QWEN2_5_1_5B { + allErrs = append(allErrs, field.Invalid(runtimeRefNamePath, newObj.Spec.RuntimeRef.Name, fmt.Sprintf("QLoRA is not supported for %v model", model))) + } + resourcePerNodeRefPath := specPath.Child("trainer").Child("resourcesPerNode") + if !strings.Contains(config, constants.TorchTuneQLoRAFinetuneSingleDeviceConfigSuffix) && + (model == constants.TORCHTUNE_MODEL_LLAMA3_2_1B || model == constants.TORCHTUNE_MODEL_LLAMA3_2_3B) { + allErrs = append( + allErrs, + field.Invalid(numProcPerNodeRefPath, numProcPerNode, fmt.Sprintf("must be auto or 1 for %v model when using QLoRA", model)), + field.Invalid(resourcePerNodeRefPath, newObj.Spec.Trainer.ResourcesPerNode, fmt.Sprintf("must have gpu resource with value 1 for %v model when using QLoRA", model)), + ) + } + } + return nil, allErrs +} + +// getRecipeAndConfig returns the recipe and config file name based on the number of nodes, +// number of processes per node, gpu count, resource per node, model name, and command line arguments. +func getRecipeAndConfig(numNodes int32, numProcPerNode intstr.IntOrString, gpuQ int, trainJob *trainer.TrainJob) (string, string) { + recipe := constants.TorchTuneFullFinetuneDistributed + suffix := constants.TorchTuneFullFinetuneMultiDevicesConfigSuffix + if numNodes == 1 && (numProcPerNode.Type == intstr.Int && numProcPerNode.IntVal == 1 || gpuQ == 1) { + if isUseQLoraFinetune(trainJob.Spec.Trainer.Args) { + recipe = constants.TorchTuneLoRAFinetuneSingleDevice + suffix = constants.TorchTuneQLoRAFinetuneSingleDeviceConfigSuffix + } else if isLoraConfigEnabled(trainJob.Spec.Trainer.Args) { + recipe = constants.TorchTuneLoRAFinetuneSingleDevice + suffix = constants.TorchTuneLoRAFinetuneSingleDeviceConfigSuffix + } else { + recipe = constants.TorchTuneFullFinetuneSingleDevice + suffix = constants.TorchTuneFullFinetuneSingleDeviceConfigSuffix + } + } else if numNodes == 1 && isUseQLoraFinetune(trainJob.Spec.Trainer.Args) { + recipe = constants.TorchTuneLoRAFinetuneDistributed + suffix = constants.TorchTuneQLoRAFinetuneDistributedConfigSuffix + } else if numNodes == 1 && isLoraConfigEnabled(trainJob.Spec.Trainer.Args) { + recipe = constants.TorchTuneLoRAFinetuneDistributed + suffix = constants.TorchTuneLoRAFinetuneDistributedConfigSuffix + } else if numNodes > 1 { + suffix = constants.TorchTuneFullFinetuneMultiNodesConfigSuffix + } + + return recipe, fmt.Sprintf("%s%s", getModelFromRuntimeRef(trainJob.Spec.RuntimeRef.Name), suffix) +} + +// isLoraConfigEnabled checks if we enables LoraConfig. +func isLoraConfigEnabled(args []string) bool { + for _, arg := range args { + if strings.Contains(arg, constants.TorchTuneLoraAttnModules) { + return true + } + } + return false +} + +// isUseQLoraFinetune checks if QLoRA fine-tuning should be used. +func isUseQLoraFinetune(args []string) bool { + hasQuantizeBase := false + for _, arg := range args { + switch { + case strings.Contains(arg, constants.TorchTuneUseDora): + // If Dora is enabled, no need to continue + return false + case strings.Contains(arg, constants.TorchTuneQuantizeBase): + hasQuantizeBase = true + } + } + return hasQuantizeBase +} + +// extractOverridesFromRuntime extracts overrides from the TorchTune Trainer Node. +func extractOverridesFromRuntime(info *runtime.Info) []string { + overrides := []string{} + jobSetSpec, ok := runtime.TemplateSpecApply[jobsetv1alpha2ac.JobSetSpecApplyConfiguration](info) + if !ok { + return overrides + } + + for _, rJob := range jobSetSpec.ReplicatedJobs { + jobMetadata := rJob.Template.ObjectMetaApplyConfiguration + if jobMetadata == nil || jobMetadata.Labels == nil { + continue + } + if ancestor, ok := jobMetadata.Labels[constants.LabelTrainJobAncestor]; ok && ancestor == constants.AncestorTrainer { + for _, container := range rJob.Template.Spec.Template.Spec.Containers { + if container.Name != nil && *container.Name == constants.Node { + for _, command := range container.Command { + if constants.TorchTuneImmutableConfigs.Has(strings.Split(command, "=")[0]) { + overrides = append(overrides, command) + } + } + } + } + } + } + return overrides +} + +func getModelFromRuntimeRef(runtimeRefName string) string { + fields := strings.Split(runtimeRefName, "-") + if len(fields) != 3 { + return "" + } + return fmt.Sprintf("%s/%s", strings.ReplaceAll(fields[1], ".", "_"), strings.ToUpper(fields[2])) +} diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/volcano/volcano.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/volcano/volcano.go new file mode 100644 index 00000000000..376e6515af4 --- /dev/null +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/volcano/volcano.go @@ -0,0 +1,359 @@ +/* +Copyright 2025 The Kubeflow Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volcano + +import ( + "context" + "fmt" + "slices" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" + schedulingv1 "k8s.io/api/scheduling/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + apiruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" + metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "sigs.k8s.io/jobset/client-go/applyconfiguration/jobset/v1alpha2" + volcanov1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" + volcanov1beta1ac "volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1" + + trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" + "github.com/kubeflow/trainer/v2/pkg/runtime" + "github.com/kubeflow/trainer/v2/pkg/runtime/framework" + index "github.com/kubeflow/trainer/v2/pkg/runtime/indexer" + runtimeindexer "github.com/kubeflow/trainer/v2/pkg/runtime/indexer" +) + +type Volcano struct { + client client.Client + restMapper meta.RESTMapper + scheme *apiruntime.Scheme + logger logr.Logger +} + +var _ framework.EnforcePodGroupPolicyPlugin = (*Volcano)(nil) +var _ framework.ComponentBuilderPlugin = (*Volcano)(nil) +var _ framework.WatchExtensionPlugin = (*Volcano)(nil) + +const Name = "Volcano" + +// +kubebuilder:rbac:groups=scheduling.volcano.sh,resources=podgroups,verbs=create;get;list;watch;update;patch +// +kubebuilder:rbac:groups=node.k8s.io,resources=runtimeclasses,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=limitranges,verbs=get;list;watch + +func New(_ context.Context, client client.Client, _ client.FieldIndexer) (framework.Plugin, error) { + return &Volcano{ + client: client, + restMapper: client.RESTMapper(), + scheme: client.Scheme(), + }, nil +} + +func (v *Volcano) Name() string { + return Name +} + +func (v *Volcano) Validate(ctx context.Context, info *runtime.Info, _, newObj *trainer.TrainJob) (admission.Warnings, field.ErrorList) { + var allErrs field.ErrorList + + if info == nil || info.RuntimePolicy.PodGroupPolicy == nil || info.RuntimePolicy.PodGroupPolicy.Volcano == nil || newObj == nil { + return nil, allErrs + } + + specPath := field.NewPath("spec") + + // Validate queue (must not be empty if explicitly set) + if queue, ok := info.Annotations[volcanov1beta1.QueueNameAnnotationKey]; ok { + if queue == "" { + allErrs = append(allErrs, field.Invalid(specPath.Child("annotations").Key(volcanov1beta1.QueueNameAnnotationKey), + queue, "Volcano queue name must not be empty")) + } + } + + // Validate priorityClassName from the Pod template + jobSetSpec, ok := runtime.TemplateSpecApply[v1alpha2.JobSetSpecApplyConfiguration](info) + if ok && jobSetSpec != nil { + for _, rj := range jobSetSpec.ReplicatedJobs { + if rj.Template != nil && rj.Template.Spec != nil && rj.Template.Spec.Template != nil && rj.Template.Spec.Template.Spec != nil { + priorityClassName := rj.Template.Spec.Template.Spec.PriorityClassName + if priorityClassName != nil { + pcName := *priorityClassName + // Skip two special keywords which indicate the highest priorities + if pcName == "system-cluster-critical" || pcName == "system-node-critical" { + return nil, allErrs + } + // Any other name must be defined by creating a PriorityClass object with that name. + var pc schedulingv1.PriorityClass + if err := v.client.Get(ctx, types.NamespacedName{Name: pcName}, &pc); err != nil { + allErrs = append(allErrs, field.Invalid(specPath.Child("templateSpec").Child("priorityClassName"), + pcName, fmt.Sprintf("PriorityClass %q doesn't exist: %v", pcName, err))) + } + } + } + } + } + + return nil, allErrs +} + +func (v *Volcano) EnforcePodGroupPolicy(info *runtime.Info, trainJob *trainer.TrainJob) error { + if info == nil || info.RuntimePolicy.PodGroupPolicy == nil || trainJob == nil || info.RuntimePolicy.PodGroupPolicy.Volcano == nil { + return nil + } + if info.Scheduler.PodAnnotations == nil { + info.Scheduler.PodAnnotations = map[string]string{} + } + info.Scheduler.PodAnnotations[volcanov1beta1.KubeGroupNameAnnotationKey] = trainJob.Name + return nil +} + +func (v *Volcano) Build(ctx context.Context, info *runtime.Info, trainJob *trainer.TrainJob) ([]any, error) { + if info == nil || info.RuntimePolicy.PodGroupPolicy == nil || info.RuntimePolicy.PodGroupPolicy.Volcano == nil || trainJob == nil { + return nil, nil + } + + // Do not update the PodGroup if it already exists and the TrainJob is not suspended + oldPodGroup := &volcanov1beta1.PodGroup{} + if err := v.client.Get(ctx, client.ObjectKeyFromObject(trainJob), oldPodGroup); err != nil { + if !apierrors.IsNotFound(err) { + return nil, err + } + oldPodGroup = nil + } + if oldPodGroup != nil && !ptr.Deref(trainJob.Spec.Suspend, false) { + return nil, nil + } + + volcanoSpec := info.RuntimePolicy.PodGroupPolicy.Volcano + + // Aggregate pod resource requests + var totalMembers int32 + totalResources := make(corev1.ResourceList) + for _, ps := range info.TemplateSpec.PodSets { + count := *ps.Count + totalMembers += count + for resName, quantity := range ps.SinglePodRequests { + quantity.Mul(int64(count)) + current := totalResources[resName] + current.Add(quantity) + totalResources[resName] = current + } + } + pg := volcanov1beta1ac.PodGroup(trainJob.Name, trainJob.Namespace). + WithSpec(volcanov1beta1ac.PodGroupSpec(). + WithMinMember(totalMembers). + WithMinResources(totalResources)) + + // Configure queue via annotations `scheduling.volcano.sh/queue-name`. + // The field is initially set in TrainingRuntime, but can be overridden by the TrainJob. + if queue, ok := info.Annotations[volcanov1beta1.QueueNameAnnotationKey]; ok { + pg.Spec.WithQueue(queue) + } + + // Configure priorityClassName from the Pod template + jobSetSpec, ok := runtime.TemplateSpecApply[v1alpha2.JobSetSpecApplyConfiguration](info) + if ok && jobSetSpec != nil { + for _, rj := range jobSetSpec.ReplicatedJobs { + if rj.Template != nil && rj.Template.Spec != nil && rj.Template.Spec.Template != nil && rj.Template.Spec.Template.Spec != nil { + priorityClassName := rj.Template.Spec.Template.Spec.PriorityClassName + if priorityClassName != nil { + pg.Spec.WithPriorityClassName(*priorityClassName) + } + } + } + } + + if volcanoSpec.NetworkTopology != nil { + pg.Spec.WithNetworkTopology(volcanov1beta1ac.NetworkTopologySpec(). + WithMode(volcanoSpec.NetworkTopology.Mode). + WithHighestTierAllowed(*volcanoSpec.NetworkTopology.HighestTierAllowed)) + } + + pg.WithOwnerReferences(metav1ac.OwnerReference(). + WithAPIVersion(trainer.GroupVersion.String()). + WithKind(trainer.TrainJobKind). + WithName(trainJob.Name). + WithUID(trainJob.UID). + WithController(true). + WithBlockOwnerDeletion(true)) + + return []any{pg}, nil +} + +type PodGroupRuntimeClassHandler struct { + client client.Client +} + +var _ handler.TypedEventHandler[*nodev1.RuntimeClass, reconcile.Request] = (*PodGroupRuntimeClassHandler)(nil) + +func (h *PodGroupRuntimeClassHandler) Create(ctx context.Context, e event.TypedCreateEvent[*nodev1.RuntimeClass], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + containerRuntimeClass := e.Object + log := ctrl.LoggerFrom(ctx).WithValues("runtimeClass", klog.KObj(containerRuntimeClass)) + if err := h.queueSuspendedTrainJobs(ctx, containerRuntimeClass, q); err != nil { + log.Error(err, "could not queue suspended TrainJob to reconcile queue") + } +} + +func (h *PodGroupRuntimeClassHandler) Update(ctx context.Context, e event.TypedUpdateEvent[*nodev1.RuntimeClass], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + newContainerRuntimeClass := e.ObjectNew + log := ctrl.LoggerFrom(ctx).WithValues("runtimeClass", klog.KObj(newContainerRuntimeClass)) + if err := h.queueSuspendedTrainJobs(ctx, newContainerRuntimeClass, q); err != nil { + log.Error(err, "could not queue suspended TrainJob to reconcile queue") + } +} + +func (h *PodGroupRuntimeClassHandler) Delete(ctx context.Context, e event.TypedDeleteEvent[*nodev1.RuntimeClass], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + containerRuntimeClass := e.Object + log := ctrl.LoggerFrom(ctx).WithValues("runtimeClass", klog.KObj(containerRuntimeClass)) + if err := h.queueSuspendedTrainJobs(ctx, containerRuntimeClass, q); err != nil { + log.Error(err, "could not queue suspended TrainJob to reconcile queue") + } +} + +func (h *PodGroupRuntimeClassHandler) Generic(context.Context, event.TypedGenericEvent[*nodev1.RuntimeClass], workqueue.TypedRateLimitingInterface[reconcile.Request]) { +} + +func (h *PodGroupRuntimeClassHandler) queueSuspendedTrainJobs(ctx context.Context, runtimeClass *nodev1.RuntimeClass, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + var trainingRuntimes trainer.TrainingRuntimeList + if err := h.client.List(ctx, &trainingRuntimes, client.MatchingFields{index.TrainingRuntimeContainerRuntimeClassKey: runtimeClass.Name}); err != nil { + return err + } + var clusterTrainingRuntimes trainer.ClusterTrainingRuntimeList + if err := h.client.List(ctx, &clusterTrainingRuntimes, client.MatchingFields{index.ClusterTrainingRuntimeContainerRuntimeClassKey: runtimeClass.Name}); err != nil { + return err + } + + var trainJobs []trainer.TrainJob + for _, trainingRuntime := range trainingRuntimes.Items { + var trainJobsWithTrainingRuntime trainer.TrainJobList + err := h.client.List(ctx, &trainJobsWithTrainingRuntime, client.MatchingFields{runtimeindexer.TrainJobRuntimeRefKey: trainingRuntime.Name}) + if err != nil { + return err + } + trainJobs = append(trainJobs, trainJobsWithTrainingRuntime.Items...) + } + for _, clusterTrainingRuntime := range clusterTrainingRuntimes.Items { + var trainJobsWithClTrainingRuntime trainer.TrainJobList + err := h.client.List(ctx, &trainJobsWithClTrainingRuntime, client.MatchingFields{runtimeindexer.TrainJobClusterRuntimeRefKey: clusterTrainingRuntime.Name}) + if err != nil { + return err + } + trainJobs = append(trainJobs, trainJobsWithClTrainingRuntime.Items...) + } + trainJobs = slices.CompactFunc(trainJobs, func(a, b trainer.TrainJob) bool { + return a.Name == b.Name + }) + for _, trainJob := range trainJobs { + if ptr.Deref(trainJob.Spec.Suspend, false) { + q.Add(reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&trainJob)}) + } + } + return nil +} + +type PodGroupLimitRangeHandler struct { + client client.Client +} + +var _ handler.TypedEventHandler[*corev1.LimitRange, reconcile.Request] = (*PodGroupLimitRangeHandler)(nil) + +func (h *PodGroupLimitRangeHandler) Create(ctx context.Context, e event.TypedCreateEvent[*corev1.LimitRange], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + limitRange := e.Object + log := ctrl.LoggerFrom(ctx).WithValues("limitRange", klog.KObj(limitRange)) + if err := h.queueSuspendedTrainJob(ctx, limitRange.Namespace, q); err != nil { + log.Error(err, "could not queue suspended TrainJob to reconcile queue") + } +} + +func (h *PodGroupLimitRangeHandler) Update(ctx context.Context, e event.TypedUpdateEvent[*corev1.LimitRange], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + newLimitRange := e.ObjectNew + log := ctrl.LoggerFrom(ctx).WithValues("limitRange", klog.KObj(newLimitRange)) + if err := h.queueSuspendedTrainJob(ctx, newLimitRange.Namespace, q); err != nil { + log.Error(err, "could not queue suspended TrainJob to reconcile queue") + } +} + +func (h *PodGroupLimitRangeHandler) Delete(ctx context.Context, e event.TypedDeleteEvent[*corev1.LimitRange], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + limitRange := e.Object + log := ctrl.LoggerFrom(ctx).WithValues("limitRange", klog.KObj(limitRange)) + if err := h.queueSuspendedTrainJob(ctx, limitRange.Namespace, q); err != nil { + log.Error(err, "could not queue suspended TrainJob to reconcile queue") + } +} + +func (h *PodGroupLimitRangeHandler) Generic(context.Context, event.TypedGenericEvent[*corev1.LimitRange], workqueue.TypedRateLimitingInterface[reconcile.Request]) { +} + +func (h *PodGroupLimitRangeHandler) queueSuspendedTrainJob(ctx context.Context, ns string, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + var trainJobs trainer.TrainJobList + if err := h.client.List(ctx, &trainJobs, client.InNamespace(ns)); err != nil { + return err + } + for _, trainJob := range trainJobs.Items { + if ptr.Deref(trainJob.Spec.Suspend, false) { + q.Add(reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&trainJob)}) + } + } + return nil +} + +func (v *Volcano) ReconcilerBuilders() []runtime.ReconcilerBuilder { + if _, err := v.restMapper.RESTMapping( + schema.GroupKind{Group: volcanov1beta1.SchemeGroupVersion.Group, Kind: "PodGroup"}, + volcanov1beta1.SchemeGroupVersion.Version, + ); err != nil { + v.logger.Error(err, "PodGroup CRDs must be installed in advance") + return nil + } + return []runtime.ReconcilerBuilder{ + func(b *builder.Builder, cl client.Client, cache cache.Cache) *builder.Builder { + return b.Watches( + &volcanov1beta1.PodGroup{}, + handler.EnqueueRequestForOwner( + v.client.Scheme(), v.client.RESTMapper(), &trainer.TrainJob{}, handler.OnlyControllerOwner(), + ), + ) + }, + func(b *builder.Builder, cl client.Client, cache cache.Cache) *builder.Builder { + return b.WatchesRawSource(source.TypedKind[*corev1.LimitRange, reconcile.Request](cache, &corev1.LimitRange{}, &PodGroupLimitRangeHandler{ + client: cl, + })) + }, + func(b *builder.Builder, cl client.Client, cache cache.Cache) *builder.Builder { + return b.WatchesRawSource(source.TypedKind[*nodev1.RuntimeClass, reconcile.Request](cache, &nodev1.RuntimeClass{}, &PodGroupRuntimeClassHandler{ + client: cl, + })) + }, + } +} diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/indexer/indexer.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/indexer/indexer.go index 036b75f9a3f..b421708a434 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/indexer/indexer.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/indexer/indexer.go @@ -17,6 +17,8 @@ limitations under the License. package indexer import ( + "errors" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,6 +31,41 @@ const ( TrainJobClusterRuntimeRefKey = ".spec.runtimeRef.kind=clusterTrainingRuntime" ) +var ( + TrainingRuntimeContainerRuntimeClassKey = ".trainingRuntimeSpec.jobSetTemplateSpec.replicatedJobs.podTemplateSpec.runtimeClassName" + ClusterTrainingRuntimeContainerRuntimeClassKey = ".clusterTrainingRuntimeSpec.jobSetTemplateSpec.replicatedJobs.podTemplateSpec.runtimeClassName" + ErrorCanNotSetupTrainingRuntimeRuntimeClassIndexer = errors.New("setting index on runtimeClass for TrainingRuntime") + ErrorCanNotSetupClusterTrainingRuntimeRuntimeClassIndexer = errors.New("setting index on runtimeClass for ClusterTrainingRuntime") +) + +func IndexTrainingRuntimeContainerRuntimeClass(obj client.Object) []string { + runtime, ok := obj.(*trainer.TrainingRuntime) + if !ok { + return nil + } + var runtimeClasses []string + for _, rJob := range runtime.Spec.Template.Spec.ReplicatedJobs { + if rJob.Template.Spec.Template.Spec.RuntimeClassName != nil { + runtimeClasses = append(runtimeClasses, *rJob.Template.Spec.Template.Spec.RuntimeClassName) + } + } + return runtimeClasses +} + +func IndexClusterTrainingRuntimeContainerRuntimeClass(obj client.Object) []string { + clRuntime, ok := obj.(*trainer.ClusterTrainingRuntime) + if !ok { + return nil + } + var runtimeClasses []string + for _, rJob := range clRuntime.Spec.Template.Spec.ReplicatedJobs { + if rJob.Template.Spec.Template.Spec.RuntimeClassName != nil { + runtimeClasses = append(runtimeClasses, *rJob.Template.Spec.Template.Spec.RuntimeClassName) + } + } + return runtimeClasses +} + func IndexTrainJobTrainingRuntime(obj client.Object) []string { trainJob, ok := obj.(*trainer.TrainJob) if !ok { diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/interface.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/interface.go index 6f49bace314..3e831b8fa61 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/interface.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/interface.go @@ -19,7 +19,6 @@ package runtime import ( "context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -38,7 +37,7 @@ type Runtime interface { NewObjects(ctx context.Context, trainJob *trainer.TrainJob) ([]any, error) RuntimeInfo(trainJob *trainer.TrainJob, runtimeTemplateSpec any, mlPolicy *trainer.MLPolicy, podGroupPolicy *trainer.PodGroupPolicy) (*Info, error) - TerminalCondition(ctx context.Context, trainJob *trainer.TrainJob) (*metav1.Condition, error) + TrainJobStatus(ctx context.Context, trainJob *trainer.TrainJob) (*trainer.TrainJobStatus, error) EventHandlerRegistrars() []ReconcilerBuilder ValidateObjects(ctx context.Context, old, new *trainer.TrainJob) (admission.Warnings, field.ErrorList) } diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/runtime.go b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/runtime.go index 71b5c533f51..ec7ed3bdd44 100644 --- a/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/runtime.go +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/runtime/runtime.go @@ -20,14 +20,17 @@ import ( "iter" "maps" "slices" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" corev1ac "k8s.io/client-go/applyconfigurations/core/v1" resourcehelpers "k8s.io/component-helpers/resource" "k8s.io/utils/ptr" + jobsetv1alpha2ac "sigs.k8s.io/jobset/client-go/applyconfiguration/jobset/v1alpha2" trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" + "github.com/kubeflow/trainer/v2/pkg/constants" ) var ( @@ -87,7 +90,8 @@ type Container struct { // TODO (andreyvelich): Potentially, we can add ScheduleTimeoutSeconds to the Scheduler for consistency. type Scheduler struct { - PodLabels map[string]string + PodLabels map[string]string + PodAnnotations map[string]string } type InfoOptions struct { @@ -239,3 +243,50 @@ func RuntimeRefToRuntimeRegistryKey(runtimeRef trainer.RuntimeRef) string { Kind: ptr.Deref(runtimeRef.Kind, ""), }.String() } + +// ExtractResourcePerNodeFromRuntime extracts the Trainer resource per node from the Info object. +func ExtractResourcePerNodeFromRuntime(info *Info) *corev1.ResourceRequirements { + if jobSetSpec, ok := TemplateSpecApply[jobsetv1alpha2ac.JobSetSpecApplyConfiguration](info); ok { + for _, rJob := range jobSetSpec.ReplicatedJobs { + if rJob.Name != nil && *rJob.Name == constants.Node || rJob.Template.Labels[constants.LabelTrainJobAncestor] == constants.AncestorTrainer { + for _, container := range rJob.Template.Spec.Template.Spec.Containers { + if container.Name != nil && *container.Name == constants.Node && container.Resources != nil { + res := &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{}, + Requests: corev1.ResourceList{}, + } + if container.Resources.Limits != nil { + res.Limits = *container.Resources.Limits + } + if container.Resources.Requests != nil { + res.Requests = *container.Resources.Requests + } + return res + } + } + } + } + } + return nil +} + +// GetNumGPUPerNode returns the GPU count if found in container resources. +func GetNumGPUPerNode(res *corev1.ResourceRequirements) int { + if res == nil { + return 0 + } + gpuQ := numGPU(res.Requests) + if limitGpuQ := numGPU(res.Limits); gpuQ == 0 && limitGpuQ > 0 { + gpuQ = limitGpuQ + } + return gpuQ +} + +func numGPU(resourcePerNode corev1.ResourceList) int { + for resName, resQ := range resourcePerNode { + if strings.Contains(strings.ToLower(resName.String()), "gpu") { + return int(resQ.Value()) + } + } + return 0 +} diff --git a/vendor/github.com/kubeflow/trainer/v2/pkg/util/trainingruntime/trainingruntime.go b/vendor/github.com/kubeflow/trainer/v2/pkg/util/trainingruntime/trainingruntime.go new file mode 100644 index 00000000000..84d5d61a6f7 --- /dev/null +++ b/vendor/github.com/kubeflow/trainer/v2/pkg/util/trainingruntime/trainingruntime.go @@ -0,0 +1,12 @@ +package trainingruntime + +import "github.com/kubeflow/trainer/v2/pkg/constants" + +// IsSupportDeprecated returns true if TrainingRuntime labels indicate support=deprecated. +func IsSupportDeprecated(labels map[string]string) bool { + if labels == nil { + return false + } + val, ok := labels[constants.LabelSupport] + return ok && val == constants.SupportDeprecated +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 3b555377a1f..0844ea8db26 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -193,7 +193,7 @@ github.com/json-iterator/go # github.com/kubeflow/mpi-operator v0.6.0 ## explicit; go 1.23 github.com/kubeflow/mpi-operator/pkg/apis/kubeflow/v2beta1 -# github.com/kubeflow/trainer/v2 v2.0.1 +# github.com/kubeflow/trainer/v2 v2.1.0-rc.0 ## explicit; go 1.24.0 github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1 github.com/kubeflow/trainer/v2/pkg/apply @@ -209,7 +209,9 @@ github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset/constants github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/mpi github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/plainml github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch +github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/volcano github.com/kubeflow/trainer/v2/pkg/runtime/indexer +github.com/kubeflow/trainer/v2/pkg/util/trainingruntime github.com/kubeflow/trainer/v2/pkg/util/trainjob # github.com/kubeflow/training-operator v1.9.3 ## explicit; go 1.23 @@ -1755,3 +1757,8 @@ sigs.k8s.io/structured-merge-diff/v6/value ## explicit; go 1.22 sigs.k8s.io/yaml sigs.k8s.io/yaml/kyaml +# volcano.sh/apis v1.12.2 +## explicit; go 1.23.0 +volcano.sh/apis/pkg/apis/scheduling +volcano.sh/apis/pkg/apis/scheduling/v1beta1 +volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1 diff --git a/vendor/volcano.sh/apis/LICENSE b/vendor/volcano.sh/apis/LICENSE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/vendor/volcano.sh/apis/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/doc.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/doc.go new file mode 100644 index 00000000000..4bad97492ce --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/doc.go @@ -0,0 +1,17 @@ +/* +Copyright 2019 The Volcano Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package + +// Package scheduling is the internal version of the API. +package scheduling diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/register.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/register.go new file mode 100644 index 00000000000..5eaaa8325cc --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/register.go @@ -0,0 +1,42 @@ +/* +Copyright 2019 The Volcano Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) + +// GroupName is the group name used in this package. +const GroupName = "scheduling.volcano.sh" + +// SchemeGroupVersion is the group version used to register these objects. +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: runtime.APIVersionInternal} + +// addKnownTypes adds the set of types defined in this package to the supplied scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &PodGroup{}, + &PodGroupList{}, + &Queue{}, + &QueueList{}, + ) + + return nil +} diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go new file mode 100644 index 00000000000..d275d21b25c --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go @@ -0,0 +1,405 @@ +/* +Copyright 2019 The Volcano Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PodGroupPhase is the phase of a pod group at the current time. +type PodGroupPhase string + +// QueueState is state type of queue. +type QueueState string + +const ( + // QueueStateOpen indicate `Open` state of queue + QueueStateOpen QueueState = "Open" + // QueueStateClosed indicate `Closed` state of queue + QueueStateClosed QueueState = "Closed" + // QueueStateClosing indicate `Closing` state of queue + QueueStateClosing QueueState = "Closing" + // QueueStateUnknown indicate `Unknown` state of queue + QueueStateUnknown QueueState = "Unknown" +) + +// These are the valid phase of podGroups. +const ( + // PodGroupPending means the pod group has been accepted by the system, but scheduler can not allocate + // enough resources to it. + PodGroupPending PodGroupPhase = "Pending" + + // PodGroupRunning means `spec.minMember` pods of PodGroup has been in running phase. + PodGroupRunning PodGroupPhase = "Running" + + // PodGroupUnknown means part of `spec.minMember` pods are running but the other part can not + // be scheduled, e.g. not enough resource; scheduler will wait for related controller to recover it. + PodGroupUnknown PodGroupPhase = "Unknown" + + // PodGroupInqueue means controllers can start to create pods, + // is a new state between PodGroupPending and PodGroupRunning + PodGroupInqueue PodGroupPhase = "Inqueue" + + // PodGroupCompleted means all the pods of PodGroup are completed + PodGroupCompleted PodGroupPhase = "Completed" +) + +type PodGroupConditionType string + +const ( + // PodGroupUnschedulableType is Unschedulable event type + PodGroupUnschedulableType PodGroupConditionType = "Unschedulable" + + // PodGroupScheduled is scheduled event type + PodGroupScheduled PodGroupConditionType = "Scheduled" +) + +type PodGroupConditionDetail string + +const ( + // PodGroupReady is that PodGroup has reached scheduling restriction + PodGroupReady PodGroupConditionDetail = "pod group is ready" + // PodGroupNotReady is that PodGroup has not yet reached the scheduling restriction + PodGroupNotReady PodGroupConditionDetail = "pod group is not ready" +) + +// PodGroupCondition contains details for the current state of this pod group. +type PodGroupCondition struct { + // Type is the type of the condition + Type PodGroupConditionType + + // Status is the status of the condition. + Status v1.ConditionStatus + + // The ID of condition transition. + TransitionID string + + // Last time the phase transitioned from another to current phase. + // +optional + LastTransitionTime metav1.Time + + // Unique, one-word, CamelCase reason for the phase's last transition. + // +optional + Reason string + + // Human-readable message indicating details about last transition. + // +optional + Message string +} + +const ( + // PodFailedReason is probed if pod of PodGroup failed + PodFailedReason string = "PodFailed" + + // PodDeletedReason is probed if pod of PodGroup deleted + PodDeletedReason string = "PodDeleted" + + // NotEnoughResourcesReason is probed if there're not enough resources to schedule pods + NotEnoughResourcesReason string = "NotEnoughResources" + + // NotEnoughPodsReason is probed if there're not enough tasks compared to `spec.minMember` + NotEnoughPodsReason string = "NotEnoughTasks" +) + +// QueueEvent represent the phase of queue. +type QueueEvent string + +const ( + // QueueOutOfSyncEvent is triggered if PodGroup/Queue were updated + QueueOutOfSyncEvent QueueEvent = "OutOfSync" + // QueueCommandIssuedEvent is triggered if a command is raised by user + QueueCommandIssuedEvent QueueEvent = "CommandIssued" +) + +// QueueAction is the action that queue controller will take according to the event. +type QueueAction string + +const ( + // SyncQueueAction is the action to sync queue status. + SyncQueueAction QueueAction = "SyncQueue" + // OpenQueueAction is the action to open queue + OpenQueueAction QueueAction = "OpenQueue" + // CloseQueueAction is the action to close queue + CloseQueueAction QueueAction = "CloseQueue" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// PodGroup is a collection of Pod; used for batch workload. +// +kubebuilder:printcolumn:name="STATUS",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="minMember",type=integer,JSONPath=`.spec.minMember` +// +kubebuilder:printcolumn:name="RUNNINGS",type=integer,JSONPath=`.status.running` +// +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=`.metadata.creationTimestamp` +// +kubebuilder:printcolumn:name="QUEUE",type=string,priority=1,JSONPath=`.spec.queue` +type PodGroup struct { + metav1.TypeMeta + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ObjectMeta + + // Specification of the desired behavior of the pod group. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status + // +optional + Spec PodGroupSpec + + // Status represents the current information about a pod group. + // This data may not be up to date. + // +optional + Status PodGroupStatus +} + +// PodGroupSpec represents the template of a pod group. +type PodGroupSpec struct { + // MinMember defines the minimal number of members/tasks to run the pod group; + // if there's not enough resources to start all tasks, the scheduler + // will not start anyone. + MinMember int32 + + // MinTaskMember defines the minimal number of pods to run for each task in the pod group; + // if there's not enough resources to start each task, the scheduler + // will not start anyone. + MinTaskMember map[string]int32 + + // Queue defines the queue to allocate resource for PodGroup; if queue does not exist, + // the PodGroup will not be scheduled. + // +kubebuilder:default:="default" + Queue string + + // If specified, indicates the PodGroup's priority. "system-node-critical" and + // "system-cluster-critical" are two special keywords which indicate the + // highest priorities with the former being the highest priority. Any other + // name must be defined by creating a PriorityClass object with that name. + // If not specified, the PodGroup priority will be default or zero if there is no + // default. + // +optional + PriorityClassName string + + // MinResources defines the minimal resource of members/tasks to run the pod group; + // if there's not enough resources to start all tasks, the scheduler + // will not start anyone. + MinResources *v1.ResourceList + + // NetworkTopology defines the NetworkTopology config, this field works in conjunction with network topology feature and hyperNode CRD. + // +optional + NetworkTopology *NetworkTopologySpec +} + +// NetworkTopologyMode represents the networkTopology mode, valid values are "hard" and "soft". +// +kubebuilder:validation:Enum=hard;soft +type NetworkTopologyMode string + +const ( + // HardNetworkTopologyMode represents a strict network topology constraint that jobs must adhere to. + HardNetworkTopologyMode NetworkTopologyMode = "hard" + + // SoftNetworkTopologyMode represents a flexible network topology constraint that + // allows jobs to cross network boundaries under certain conditions. + SoftNetworkTopologyMode NetworkTopologyMode = "soft" +) + +type NetworkTopologySpec struct { + // Mode specifies the mode of the network topology constrain. + // +kubebuilder:default=hard + // +optional + Mode NetworkTopologyMode + + // HighestTierAllowed specifies the highest tier that a job allowed to cross when scheduling. + // +kubebuilder:default=1 + // +optional + HighestTierAllowed *int +} + +// PodGroupStatus represents the current state of a pod group. +type PodGroupStatus struct { + // Current phase of PodGroup. + Phase PodGroupPhase + + // The conditions of PodGroup. + // +optional + Conditions []PodGroupCondition + + // The number of actively running pods. + // +optional + Running int32 + + // The number of pods which reached phase Succeeded. + // +optional + Succeeded int32 + + // The number of pods which reached phase Failed. + // +optional + Failed int32 +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// PodGroupList is a collection of pod groups. +type PodGroupList struct { + metav1.TypeMeta + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ListMeta + + // items is the list of PodGroup + Items []PodGroup +} + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Queue is a queue of PodGroup. +type Queue struct { + metav1.TypeMeta + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ObjectMeta + + // Specification of the desired behavior of the queue. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status + // +optional + Spec QueueSpec + + // The status of queue. + // +optional + Status QueueStatus +} + +// Guarantee represents configuration of queue resource reservation +type Guarantee struct { + // The amount of cluster resource reserved for queue. Just set either `percentage` or `resource` + // +optional + Resource v1.ResourceList `json:"resource,omitempty" protobuf:"bytes,3,opt,name=resource"` +} + +// Reservation represents current condition about resource reservation +type Reservation struct { + // Nodes are Locked nodes for queue + // +optional + Nodes []string `json:"nodes,omitempty" protobuf:"bytes,1,opt,name=nodes"` + // Resource is a list of total idle resource in locked nodes. + // +optional + Resource v1.ResourceList `json:"resource,omitempty" protobuf:"bytes,2,opt,name=resource"` +} + +// QueueStatus represents the status of Queue. +type QueueStatus struct { + // State is status of queue + State QueueState + + // The number of 'Unknown' PodGroup in this queue. + Unknown int32 + // The number of 'Pending' PodGroup in this queue. + Pending int32 + // The number of 'Running' PodGroup in this queue. + Running int32 + // The number of `Inqueue` PodGroup in this queue. + Inqueue int32 + // The number of `Completed` PodGroup in this queue. + Completed int32 + + // Reservation is the profile of resource reservation for queue + Reservation Reservation + + // Allocated is allocated resources in queue + // +optional + Allocated v1.ResourceList +} + +// CluterSpec represents the template of Cluster +type Cluster struct { + Name string + Weight int32 + Capacity v1.ResourceList +} + +// Affinity is a group of affinity scheduling rules. +type Affinity struct { + // Describes nodegroup affinity scheduling rules for the queue. + // +optional + NodeGroupAffinity *NodeGroupAffinity `json:"nodeGroupAffinity,omitempty" protobuf:"bytes,1,opt,name=nodeGroupAffinity"` + + // Describes nodegroup affinity scheduling rules for the queue. + // +optional + NodeGroupAntiAffinity *NodeGroupAntiAffinity `json:"nodeGroupAntiAffinity,omitempty" protobuf:"bytes,2,opt,name=nodeGroupAntiAffinity"` +} + +type NodeGroupAffinity struct { + // +optional + RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"` + // +optional + PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"` +} + +type NodeGroupAntiAffinity struct { + // +optional + RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"` + // +optional + PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"` +} + +// QueueSpec represents the template of Queue. +type QueueSpec struct { + Weight int32 + Capability v1.ResourceList + + // Depreicated: replaced by status.State + State QueueState + // Reclaimable indicate whether the queue can be reclaimed by other queue + Reclaimable *bool + + // extendCluster indicate the jobs in this Queue will be dispatched to these clusters. + ExtendClusters []Cluster + + // Guarantee indicate configuration about resource reservation + Guarantee Guarantee `json:"guarantee,omitempty" protobuf:"bytes,4,opt,name=guarantee"` + + // If specified, the queue's scheduling constraints + // +optional + Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,6,opt,name=affinity"` + + // Type define the type of queue + Type string `json:"type,omitempty" protobuf:"bytes,7,opt,name=type"` + + // Parent define the parent of queue + // +optional + Parent string `json:"parent,omitempty" protobuf:"bytes,8,opt,name=parent"` + + // The amount of resources configured by the user. This part of resource can be shared with other queues and reclaimed back. + // +optional + Deserved v1.ResourceList `json:"deserved,omitempty" protobuf:"bytes,9,opt,name=deserved"` + + // Priority define the priority of queue. Higher values are prioritized for scheduling and considered later during reclamation. + // +optional + Priority int32 `json:"priority,omitempty" protobuf:"bytes,10,opt,name=priority"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// QueueList is a collection of queues. +type QueueList struct { + metav1.TypeMeta + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ListMeta + + // items is the list of PodGroup + Items []Queue +} diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/conversion.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/conversion.go new file mode 100644 index 00000000000..e903517bd2e --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/conversion.go @@ -0,0 +1,34 @@ +/* +Copyright 2020 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + unsafe "unsafe" + + v1 "k8s.io/api/core/v1" + conversion "k8s.io/apimachinery/pkg/conversion" + scheduling "volcano.sh/apis/pkg/apis/scheduling" +) + +func Convert_scheduling_QueueSpec_To_v1beta1_QueueSpec(in *scheduling.QueueSpec, out *QueueSpec, s conversion.Scope) error { + out.Weight = in.Weight + out.Capability = *(*v1.ResourceList)(unsafe.Pointer(&in.Capability)) + out.Reclaimable = (*bool)(unsafe.Pointer(in.Reclaimable)) + out.Parent = in.Parent + + return nil +} diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/doc.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/doc.go new file mode 100644 index 00000000000..620c13774f3 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/doc.go @@ -0,0 +1,24 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta1 contains API Schema definitions for the scheduling v1beta1 API group +// +kubebuilder:object:generate=true +// +groupName=scheduling.volcano.sh +// +k8s:deepcopy-gen=package +// +k8s:openapi-gen=true +// +k8s:conversion-gen=volcano.sh/apis/pkg/apis/scheduling + +package v1beta1 diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go new file mode 100644 index 00000000000..02ffe6ba34d --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go @@ -0,0 +1,73 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +// AnnotationPrefix is the key prefix to pick annotations from upper resource to podgroup +const AnnotationPrefix = "volcano.sh/" + +// JobWaitingTime is the key of sla plugin to set maximum waiting time +// that a job could stay Pending in service level agreement +const JobWaitingTime = "volcano.sh/sla-waiting-time" + +const KubeHierarchyAnnotationKey = "volcano.sh/hierarchy" + +const KubeHierarchyWeightAnnotationKey = "volcano.sh/hierarchy-weights" + +// KubeGroupNameAnnotationKey is the annotation key of Pod to identify +// which PodGroup it belongs to. +const KubeGroupNameAnnotationKey = "scheduling.k8s.io/group-name" + +// VolcanoGroupNameAnnotationKey is the annotation key of Pod to identify +// which PodGroup it belongs to. +const VolcanoGroupNameAnnotationKey = GroupName + "/group-name" + +// VolcanoGroupMinMemberAnnotationKey is the annotation key of Pod controllers (e.g. Deployment) to identify +// the minimum member of PodGroup. +const VolcanoGroupMinMemberAnnotationKey = GroupName + "/group-min-member" + +// VolcanoGroupMinResourcesAnnotationKey is the annotation key of PodGroup's PodGroup.Spec.MinResources +// which PodGroup it belongs to. +const VolcanoGroupMinResourcesAnnotationKey = GroupName + "/group-min-resources" + +// QueueNameAnnotationKey is the annotation key of Pod to identify +// which queue it belongs to. +const QueueNameAnnotationKey = GroupName + "/queue-name" + +// PodPreemptable is the key of preemptable +const PodPreemptable = "volcano.sh/preemptable" + +// CooldownTime is the key of cooldown-time, value's format "600s","10m" +// Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +const CooldownTime = "volcano.sh/cooldown-time" + +// RevocableZone is the key of revocable-zone +const RevocableZone = "volcano.sh/revocable-zone" + +// JDBMinAvailable is the key of min available pod number +const JDBMinAvailable = "volcano.sh/jdb-min-available" + +// JDBMaxUnavailable is the key of max unavailable pod number +const JDBMaxUnavailable = "volcano.sh/jdb-max-unavailable" + +// NumaPolicyKey is the key of pod numa-topology policy +const NumaPolicyKey = "volcano.sh/numa-topology-policy" + +// TopologyDecisionAnnotation is the key of topology decision about pod request resource +const TopologyDecisionAnnotation = "volcano.sh/topology-decision" + +// PodQosLevel is the key of pod qos level +const PodQosLevel = "volcano.sh/qos-level" diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/register.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/register.go new file mode 100644 index 00000000000..43b6a435e57 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/register.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + localSchemeBuilder = &SchemeBuilder + AddToScheme = SchemeBuilder.AddToScheme +) + +const ( + // GroupName is the group name used in this package. + GroupName = "scheduling.volcano.sh" + + // GroupVersion is the version of scheduling group + GroupVersion = "v1beta1" +) + +// SchemeGroupVersion is the group version used to register these objects. +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: GroupVersion} + +// Resource takes an unqualified resource and returns a Group-qualified GroupResource. +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +// addKnownTypes adds the set of types defined in this package to the supplied scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &PodGroup{}, + &PodGroupList{}, + &Queue{}, + &QueueList{}, + ) + + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go new file mode 100644 index 00000000000..5d9146a0e6f --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go @@ -0,0 +1,428 @@ +/* +Copyright 2019 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// QueueState is state type of queue. +type QueueState string + +const ( + // QueueStateOpen indicate `Open` state of queue + QueueStateOpen QueueState = "Open" + // QueueStateClosed indicate `Closed` state of queue + QueueStateClosed QueueState = "Closed" + // QueueStateClosing indicate `Closing` state of queue + QueueStateClosing QueueState = "Closing" + // QueueStateUnknown indicate `Unknown` state of queue + QueueStateUnknown QueueState = "Unknown" +) + +// PodGroupPhase is the phase of a pod group at the current time. +type PodGroupPhase string + +// These are the valid phase of podGroups. +const ( + // PodGroupPending means the pod group has been accepted by the system, but scheduler can not allocate + // enough resources to it. + PodGroupPending PodGroupPhase = "Pending" + + // PodGroupRunning means `spec.minMember` pods of PodGroup has been in running phase. + PodGroupRunning PodGroupPhase = "Running" + + // PodGroupUnknown means part of `spec.minMember` pods are running but the other part can not + // be scheduled, e.g. not enough resource; scheduler will wait for related controller to recover it. + PodGroupUnknown PodGroupPhase = "Unknown" + + // PodGroupInqueue means controllers can start to create pods, + // is a new state between PodGroupPending and PodGroupRunning + PodGroupInqueue PodGroupPhase = "Inqueue" + + // PodGroupCompleted means all the pods of PodGroup are completed + PodGroupCompleted PodGroupPhase = "Completed" +) + +type PodGroupConditionType string + +const ( + // PodGroupUnschedulableType is Unschedulable event type + PodGroupUnschedulableType PodGroupConditionType = "Unschedulable" + + // PodGroupScheduled is scheduled event type + PodGroupScheduled PodGroupConditionType = "Scheduled" +) + +type PodGroupConditionDetail string + +const ( + // PodGroupReady is that PodGroup has reached scheduling restriction + PodGroupReady PodGroupConditionDetail = "pod group is ready" + // PodGroupNotReady is that PodGroup has not yet reached the scheduling restriction + PodGroupNotReady PodGroupConditionDetail = "pod group is not ready" +) + +// PodGroupCondition contains details for the current state of this pod group. +type PodGroupCondition struct { + // Type is the type of the condition + Type PodGroupConditionType `json:"type,omitempty" protobuf:"bytes,1,opt,name=type"` + + // Status is the status of the condition. + Status v1.ConditionStatus `json:"status,omitempty" protobuf:"bytes,2,opt,name=status"` + + // The ID of condition transition. + TransitionID string `json:"transitionID,omitempty" protobuf:"bytes,3,opt,name=transitionID"` + + // Last time the phase transitioned from another to current phase. + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,4,opt,name=lastTransitionTime"` + + // Unique, one-word, CamelCase reason for the phase's last transition. + // +optional + Reason string `json:"reason,omitempty" protobuf:"bytes,5,opt,name=reason"` + + // Human-readable message indicating details about last transition. + // +optional + Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"` +} + +const ( + // PodFailedReason is probed if pod of PodGroup failed + PodFailedReason string = "PodFailed" + + // PodDeletedReason is probed if pod of PodGroup deleted + PodDeletedReason string = "PodDeleted" + + // NotEnoughResourcesReason is probed if there're not enough resources to schedule pods + NotEnoughResourcesReason string = "NotEnoughResources" + + // NotEnoughPodsReason is probed if there're not enough tasks compared to `spec.minMember` + NotEnoughPodsReason string = "NotEnoughTasks" + + // NotEnoughPodsOfTaskReason is probed if there're not enough pods of task compared to `spec.minTaskMember` + NotEnoughPodsOfTaskReason string = "NotEnoughPodsOfTask" +) + +// QueueEvent represent the phase of queue. +type QueueEvent string + +const ( + // QueueOutOfSyncEvent is triggered if PodGroup/Queue were updated + QueueOutOfSyncEvent QueueEvent = "OutOfSync" + // QueueCommandIssuedEvent is triggered if a command is raised by user + QueueCommandIssuedEvent QueueEvent = "CommandIssued" +) + +// QueueAction is the action that queue controller will take according to the event. +type QueueAction string + +const ( + // SyncQueueAction is the action to sync queue status. + SyncQueueAction QueueAction = "SyncQueue" + // OpenQueueAction is the action to open queue + OpenQueueAction QueueAction = "OpenQueue" + // CloseQueueAction is the action to close queue + CloseQueueAction QueueAction = "CloseQueue" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=podgroups,shortName=pg;podgroup-v1beta1 +// +kubebuilder:printcolumn:name="STATUS",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="minMember",type=integer,JSONPath=`.spec.minMember` +// +kubebuilder:printcolumn:name="RUNNINGS",type=integer,JSONPath=`.status.running` +// +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=`.metadata.creationTimestamp` +// +kubebuilder:printcolumn:name="QUEUE",type=string,priority=1,JSONPath=`.spec.queue` + +// PodGroup is a collection of Pod; used for batch workload. +type PodGroup struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Specification of the desired behavior of the pod group. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status + // +optional + Spec PodGroupSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` + + // Status represents the current information about a pod group. + // This data may not be up to date. + // +optional + Status PodGroupStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"` +} + +// PodGroupSpec represents the template of a pod group. +type PodGroupSpec struct { + // MinMember defines the minimal number of members/tasks to run the pod group; + // if there's not enough resources to start all tasks, the scheduler + // will not start anyone. + MinMember int32 `json:"minMember,omitempty" protobuf:"bytes,1,opt,name=minMember"` + + // MinTaskMember defines the minimal number of pods to run each task in the pod group; + // if there's not enough resources to start each task, the scheduler + // will not start anyone. + MinTaskMember map[string]int32 `json:"minTaskMember,omitempty" protobuf:"bytes,1,opt,name=minTaskMember"` + + // Queue defines the queue to allocate resource for PodGroup; if queue does not exist, + // the PodGroup will not be scheduled. Defaults to `default` Queue with the lowest weight. + // +optional + // +kubebuilder:default:="default" + Queue string `json:"queue,omitempty" protobuf:"bytes,2,opt,name=queue"` + + // If specified, indicates the PodGroup's priority. "system-node-critical" and + // "system-cluster-critical" are two special keywords which indicate the + // highest priorities with the former being the highest priority. Any other + // name must be defined by creating a PriorityClass object with that name. + // If not specified, the PodGroup priority will be default or zero if there is no + // default. + // +optional + PriorityClassName string `json:"priorityClassName,omitempty" protobuf:"bytes,3,opt,name=priorityClassName"` + + // MinResources defines the minimal resource of members/tasks to run the pod group; + // if there's not enough resources to start all tasks, the scheduler + // will not start anyone. + MinResources *v1.ResourceList `json:"minResources,omitempty" protobuf:"bytes,4,opt,name=minResources"` + + // NetworkTopology defines the NetworkTopology config, this field works in conjunction with network topology feature and hyperNode CRD. + // +optional + NetworkTopology *NetworkTopologySpec `json:"networkTopology,omitempty" protobuf:"bytes,5,opt,name=networkTopology"` +} + +type NetworkTopologyMode string + +const ( + // HardNetworkTopologyMode represents a strict network topology constraint that jobs must adhere to. + HardNetworkTopologyMode NetworkTopologyMode = "hard" + + // SoftNetworkTopologyMode represents a flexible network topology constraint that allows jobs + // to cross network boundaries under certain conditions. + SoftNetworkTopologyMode NetworkTopologyMode = "soft" +) + +type NetworkTopologySpec struct { + // Mode specifies the mode of the network topology constrain. + // +kubebuilder:validation:Enum=hard;soft + // +kubebuilder:default=hard + // +optional + Mode NetworkTopologyMode `json:"mode,omitempty" protobuf:"bytes,1,opt,name=mode"` + + // HighestTierAllowed specifies the highest tier that a job allowed to cross when scheduling. + // +kubebuilder:default=1 + // +optional + HighestTierAllowed *int `json:"highestTierAllowed,omitempty" protobuf:"bytes,2,opt,name=highestTierAllowed"` +} + +// PodGroupStatus represents the current state of a pod group. +type PodGroupStatus struct { + // Current phase of PodGroup. + Phase PodGroupPhase `json:"phase,omitempty" protobuf:"bytes,1,opt,name=phase"` + + // The conditions of PodGroup. + // +optional + Conditions []PodGroupCondition `json:"conditions,omitempty" protobuf:"bytes,2,opt,name=conditions"` + + // The number of actively running pods. + // +optional + Running int32 `json:"running,omitempty" protobuf:"bytes,3,opt,name=running"` + + // The number of pods which reached phase Succeeded. + // +optional + Succeeded int32 `json:"succeeded,omitempty" protobuf:"bytes,4,opt,name=succeeded"` + + // The number of pods which reached phase Failed. + // +optional + Failed int32 `json:"failed,omitempty" protobuf:"bytes,5,opt,name=failed"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// PodGroupList is a collection of pod groups. +type PodGroupList struct { + metav1.TypeMeta `json:",inline"` + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // items is the list of PodGroup + Items []PodGroup `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +const ( + // DefaultQueue constant stores the name of the queue as "default" + DefaultQueue = "default" +) + +// +genclient +// +genclient:nonNamespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=queues,scope=Cluster,shortName=q;queue-v1beta1 +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="PARENT",type=string,JSONPath=`.spec.parent` + +// Queue is a queue of PodGroup. +type Queue struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // Specification of the desired behavior of the queue. + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status + // +optional + Spec QueueSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"` + + // The status of queue. + // +optional + Status QueueStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"` +} + +// Guarantee represents configuration of queue resource reservation +type Guarantee struct { + // The amount of cluster resource reserved for queue. Just set either `percentage` or `resource` + // +optional + Resource v1.ResourceList `json:"resource,omitempty" protobuf:"bytes,3,opt,name=resource"` +} + +// Reservation represents current condition about resource reservation +type Reservation struct { + // Nodes are Locked nodes for queue + // +optional + Nodes []string `json:"nodes,omitempty" protobuf:"bytes,1,opt,name=nodes"` + // Resource is a list of total idle resource in locked nodes. + // +optional + Resource v1.ResourceList `json:"resource,omitempty" protobuf:"bytes,2,opt,name=resource"` +} + +// QueueStatus represents the status of Queue. +type QueueStatus struct { + // State is state of queue + State QueueState `json:"state,omitempty" protobuf:"bytes,1,opt,name=state"` + + // The number of 'Unknown' PodGroup in this queue. + Unknown int32 `json:"unknown,omitempty" protobuf:"bytes,2,opt,name=unknown"` + // The number of 'Pending' PodGroup in this queue. + Pending int32 `json:"pending,omitempty" protobuf:"bytes,3,opt,name=pending"` + // The number of 'Running' PodGroup in this queue. + Running int32 `json:"running,omitempty" protobuf:"bytes,4,opt,name=running"` + // The number of `Inqueue` PodGroup in this queue. + Inqueue int32 `json:"inqueue,omitempty" protobuf:"bytes,5,opt,name=inqueue"` + // The number of `Completed` PodGroup in this queue. + Completed int32 `json:"completed,omitempty" protobuf:"bytes,6,opt,name=completed"` + + // Reservation is the profile of resource reservation for queue + Reservation Reservation `json:"reservation,omitempty" protobuf:"bytes,7,opt,name=reservation"` + + // Allocated is allocated resources in queue + // +optional + Allocated v1.ResourceList `json:"allocated" protobuf:"bytes,8,opt,name=allocated"` +} + +// CluterSpec represents the template of Cluster +type Cluster struct { + // +optional + Name string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"` + // +optional + Weight int32 `json:"weight,omitempty" protobuf:"bytes,2,opt,name=weight"` + // +optional + Capacity v1.ResourceList `json:"capacity,omitempty" protobuf:"bytes,3,opt,name=capacity"` +} + +// Affinity is a group of affinity scheduling rules. +type Affinity struct { + // Describes nodegroup affinity scheduling rules for the queue(e.g. putting pods of the queue in the nodes of the nodegroup) + // +optional + NodeGroupAffinity *NodeGroupAffinity `json:"nodeGroupAffinity,omitempty" protobuf:"bytes,1,opt,name=nodeGroupAffinity"` + + // Describes nodegroup anti-affinity scheduling rules for the queue(e.g. avoid putting pods of the queue in the nodes of the nodegroup). + // +optional + NodeGroupAntiAffinity *NodeGroupAntiAffinity `json:"nodeGroupAntiAffinity,omitempty" protobuf:"bytes,2,opt,name=nodeGroupAntiAffinity"` +} + +type NodeGroupAffinity struct { + // +optional + RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"` + // +optional + PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"` +} + +type NodeGroupAntiAffinity struct { + // +optional + RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"` + // +optional + PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"` +} + +// QueueSpec represents the template of Queue. +type QueueSpec struct { + // +optional + // +kubebuilder:default:=1 + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Weight int32 `json:"weight,omitempty" protobuf:"bytes,1,opt,name=weight"` + + // +optional + Capability v1.ResourceList `json:"capability,omitempty" protobuf:"bytes,2,opt,name=capability"` + + // Reclaimable indicate whether the queue can be reclaimed by other queue + Reclaimable *bool `json:"reclaimable,omitempty" protobuf:"bytes,3,opt,name=reclaimable"` + + // extendCluster indicate the jobs in this Queue will be dispatched to these clusters. + ExtendClusters []Cluster `json:"extendClusters,omitempty" protobuf:"bytes,4,opt,name=extendClusters"` + + // Guarantee indicate configuration about resource reservation + Guarantee Guarantee `json:"guarantee,omitempty" protobuf:"bytes,5,opt,name=guarantee"` + + // If specified, the pod owned by the queue will be scheduled with constraint + // +optional + Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,6,opt,name=affinity"` + + // Type define the type of queue + Type string `json:"type,omitempty" protobuf:"bytes,7,opt,name=type"` + // Parent define the parent of queue + // +optional + Parent string `json:"parent,omitempty" protobuf:"bytes,8,opt,name=parent"` + + // The amount of resources configured by the user. This part of resource can be shared with other queues and reclaimed back. + // +optional + Deserved v1.ResourceList `json:"deserved,omitempty" protobuf:"bytes,9,opt,name=deserved"` + + // Priority define the priority of queue. Higher values are prioritized for scheduling and considered later during reclamation. + // +optional + Priority int32 `json:"priority,omitempty" protobuf:"bytes,10,opt,name=priority"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// QueueList is a collection of queues. +type QueueList struct { + metav1.TypeMeta `json:",inline"` + // Standard list metadata + // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + // +optional + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + // items is the list of PodGroup + Items []Queue `json:"items" protobuf:"bytes,2,rep,name=items"` +} diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go new file mode 100644 index 00000000000..b4260c6a3cd --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go @@ -0,0 +1,646 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by conversion-gen. DO NOT EDIT. + +package v1beta1 + +import ( + unsafe "unsafe" + + v1 "k8s.io/api/core/v1" + conversion "k8s.io/apimachinery/pkg/conversion" + runtime "k8s.io/apimachinery/pkg/runtime" + scheduling "volcano.sh/apis/pkg/apis/scheduling" +) + +func init() { + localSchemeBuilder.Register(RegisterConversions) +} + +// RegisterConversions adds conversion functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*Affinity)(nil), (*scheduling.Affinity)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Affinity_To_scheduling_Affinity(a.(*Affinity), b.(*scheduling.Affinity), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.Affinity)(nil), (*Affinity)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_Affinity_To_v1beta1_Affinity(a.(*scheduling.Affinity), b.(*Affinity), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Cluster)(nil), (*scheduling.Cluster)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Cluster_To_scheduling_Cluster(a.(*Cluster), b.(*scheduling.Cluster), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.Cluster)(nil), (*Cluster)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_Cluster_To_v1beta1_Cluster(a.(*scheduling.Cluster), b.(*Cluster), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Guarantee)(nil), (*scheduling.Guarantee)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Guarantee_To_scheduling_Guarantee(a.(*Guarantee), b.(*scheduling.Guarantee), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.Guarantee)(nil), (*Guarantee)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_Guarantee_To_v1beta1_Guarantee(a.(*scheduling.Guarantee), b.(*Guarantee), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*NetworkTopologySpec)(nil), (*scheduling.NetworkTopologySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_NetworkTopologySpec_To_scheduling_NetworkTopologySpec(a.(*NetworkTopologySpec), b.(*scheduling.NetworkTopologySpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.NetworkTopologySpec)(nil), (*NetworkTopologySpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_NetworkTopologySpec_To_v1beta1_NetworkTopologySpec(a.(*scheduling.NetworkTopologySpec), b.(*NetworkTopologySpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*NodeGroupAffinity)(nil), (*scheduling.NodeGroupAffinity)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_NodeGroupAffinity_To_scheduling_NodeGroupAffinity(a.(*NodeGroupAffinity), b.(*scheduling.NodeGroupAffinity), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.NodeGroupAffinity)(nil), (*NodeGroupAffinity)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_NodeGroupAffinity_To_v1beta1_NodeGroupAffinity(a.(*scheduling.NodeGroupAffinity), b.(*NodeGroupAffinity), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*NodeGroupAntiAffinity)(nil), (*scheduling.NodeGroupAntiAffinity)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_NodeGroupAntiAffinity_To_scheduling_NodeGroupAntiAffinity(a.(*NodeGroupAntiAffinity), b.(*scheduling.NodeGroupAntiAffinity), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.NodeGroupAntiAffinity)(nil), (*NodeGroupAntiAffinity)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_NodeGroupAntiAffinity_To_v1beta1_NodeGroupAntiAffinity(a.(*scheduling.NodeGroupAntiAffinity), b.(*NodeGroupAntiAffinity), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodGroup)(nil), (*scheduling.PodGroup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodGroup_To_scheduling_PodGroup(a.(*PodGroup), b.(*scheduling.PodGroup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.PodGroup)(nil), (*PodGroup)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_PodGroup_To_v1beta1_PodGroup(a.(*scheduling.PodGroup), b.(*PodGroup), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodGroupCondition)(nil), (*scheduling.PodGroupCondition)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodGroupCondition_To_scheduling_PodGroupCondition(a.(*PodGroupCondition), b.(*scheduling.PodGroupCondition), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.PodGroupCondition)(nil), (*PodGroupCondition)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_PodGroupCondition_To_v1beta1_PodGroupCondition(a.(*scheduling.PodGroupCondition), b.(*PodGroupCondition), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodGroupList)(nil), (*scheduling.PodGroupList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodGroupList_To_scheduling_PodGroupList(a.(*PodGroupList), b.(*scheduling.PodGroupList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.PodGroupList)(nil), (*PodGroupList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_PodGroupList_To_v1beta1_PodGroupList(a.(*scheduling.PodGroupList), b.(*PodGroupList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodGroupSpec)(nil), (*scheduling.PodGroupSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodGroupSpec_To_scheduling_PodGroupSpec(a.(*PodGroupSpec), b.(*scheduling.PodGroupSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.PodGroupSpec)(nil), (*PodGroupSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_PodGroupSpec_To_v1beta1_PodGroupSpec(a.(*scheduling.PodGroupSpec), b.(*PodGroupSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodGroupStatus)(nil), (*scheduling.PodGroupStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodGroupStatus_To_scheduling_PodGroupStatus(a.(*PodGroupStatus), b.(*scheduling.PodGroupStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.PodGroupStatus)(nil), (*PodGroupStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_PodGroupStatus_To_v1beta1_PodGroupStatus(a.(*scheduling.PodGroupStatus), b.(*PodGroupStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Queue)(nil), (*scheduling.Queue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Queue_To_scheduling_Queue(a.(*Queue), b.(*scheduling.Queue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.Queue)(nil), (*Queue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_Queue_To_v1beta1_Queue(a.(*scheduling.Queue), b.(*Queue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*QueueList)(nil), (*scheduling.QueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_QueueList_To_scheduling_QueueList(a.(*QueueList), b.(*scheduling.QueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.QueueList)(nil), (*QueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_QueueList_To_v1beta1_QueueList(a.(*scheduling.QueueList), b.(*QueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*QueueSpec)(nil), (*scheduling.QueueSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_QueueSpec_To_scheduling_QueueSpec(a.(*QueueSpec), b.(*scheduling.QueueSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*QueueStatus)(nil), (*scheduling.QueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_QueueStatus_To_scheduling_QueueStatus(a.(*QueueStatus), b.(*scheduling.QueueStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.QueueStatus)(nil), (*QueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_QueueStatus_To_v1beta1_QueueStatus(a.(*scheduling.QueueStatus), b.(*QueueStatus), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Reservation)(nil), (*scheduling.Reservation)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Reservation_To_scheduling_Reservation(a.(*Reservation), b.(*scheduling.Reservation), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.Reservation)(nil), (*Reservation)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_Reservation_To_v1beta1_Reservation(a.(*scheduling.Reservation), b.(*Reservation), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*scheduling.QueueSpec)(nil), (*QueueSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_QueueSpec_To_v1beta1_QueueSpec(a.(*scheduling.QueueSpec), b.(*QueueSpec), scope) + }); err != nil { + return err + } + return nil +} + +func autoConvert_v1beta1_Affinity_To_scheduling_Affinity(in *Affinity, out *scheduling.Affinity, s conversion.Scope) error { + out.NodeGroupAffinity = (*scheduling.NodeGroupAffinity)(unsafe.Pointer(in.NodeGroupAffinity)) + out.NodeGroupAntiAffinity = (*scheduling.NodeGroupAntiAffinity)(unsafe.Pointer(in.NodeGroupAntiAffinity)) + return nil +} + +// Convert_v1beta1_Affinity_To_scheduling_Affinity is an autogenerated conversion function. +func Convert_v1beta1_Affinity_To_scheduling_Affinity(in *Affinity, out *scheduling.Affinity, s conversion.Scope) error { + return autoConvert_v1beta1_Affinity_To_scheduling_Affinity(in, out, s) +} + +func autoConvert_scheduling_Affinity_To_v1beta1_Affinity(in *scheduling.Affinity, out *Affinity, s conversion.Scope) error { + out.NodeGroupAffinity = (*NodeGroupAffinity)(unsafe.Pointer(in.NodeGroupAffinity)) + out.NodeGroupAntiAffinity = (*NodeGroupAntiAffinity)(unsafe.Pointer(in.NodeGroupAntiAffinity)) + return nil +} + +// Convert_scheduling_Affinity_To_v1beta1_Affinity is an autogenerated conversion function. +func Convert_scheduling_Affinity_To_v1beta1_Affinity(in *scheduling.Affinity, out *Affinity, s conversion.Scope) error { + return autoConvert_scheduling_Affinity_To_v1beta1_Affinity(in, out, s) +} + +func autoConvert_v1beta1_Cluster_To_scheduling_Cluster(in *Cluster, out *scheduling.Cluster, s conversion.Scope) error { + out.Name = in.Name + out.Weight = in.Weight + out.Capacity = *(*v1.ResourceList)(unsafe.Pointer(&in.Capacity)) + return nil +} + +// Convert_v1beta1_Cluster_To_scheduling_Cluster is an autogenerated conversion function. +func Convert_v1beta1_Cluster_To_scheduling_Cluster(in *Cluster, out *scheduling.Cluster, s conversion.Scope) error { + return autoConvert_v1beta1_Cluster_To_scheduling_Cluster(in, out, s) +} + +func autoConvert_scheduling_Cluster_To_v1beta1_Cluster(in *scheduling.Cluster, out *Cluster, s conversion.Scope) error { + out.Name = in.Name + out.Weight = in.Weight + out.Capacity = *(*v1.ResourceList)(unsafe.Pointer(&in.Capacity)) + return nil +} + +// Convert_scheduling_Cluster_To_v1beta1_Cluster is an autogenerated conversion function. +func Convert_scheduling_Cluster_To_v1beta1_Cluster(in *scheduling.Cluster, out *Cluster, s conversion.Scope) error { + return autoConvert_scheduling_Cluster_To_v1beta1_Cluster(in, out, s) +} + +func autoConvert_v1beta1_Guarantee_To_scheduling_Guarantee(in *Guarantee, out *scheduling.Guarantee, s conversion.Scope) error { + out.Resource = *(*v1.ResourceList)(unsafe.Pointer(&in.Resource)) + return nil +} + +// Convert_v1beta1_Guarantee_To_scheduling_Guarantee is an autogenerated conversion function. +func Convert_v1beta1_Guarantee_To_scheduling_Guarantee(in *Guarantee, out *scheduling.Guarantee, s conversion.Scope) error { + return autoConvert_v1beta1_Guarantee_To_scheduling_Guarantee(in, out, s) +} + +func autoConvert_scheduling_Guarantee_To_v1beta1_Guarantee(in *scheduling.Guarantee, out *Guarantee, s conversion.Scope) error { + out.Resource = *(*v1.ResourceList)(unsafe.Pointer(&in.Resource)) + return nil +} + +// Convert_scheduling_Guarantee_To_v1beta1_Guarantee is an autogenerated conversion function. +func Convert_scheduling_Guarantee_To_v1beta1_Guarantee(in *scheduling.Guarantee, out *Guarantee, s conversion.Scope) error { + return autoConvert_scheduling_Guarantee_To_v1beta1_Guarantee(in, out, s) +} + +func autoConvert_v1beta1_NetworkTopologySpec_To_scheduling_NetworkTopologySpec(in *NetworkTopologySpec, out *scheduling.NetworkTopologySpec, s conversion.Scope) error { + out.Mode = scheduling.NetworkTopologyMode(in.Mode) + out.HighestTierAllowed = (*int)(unsafe.Pointer(in.HighestTierAllowed)) + return nil +} + +// Convert_v1beta1_NetworkTopologySpec_To_scheduling_NetworkTopologySpec is an autogenerated conversion function. +func Convert_v1beta1_NetworkTopologySpec_To_scheduling_NetworkTopologySpec(in *NetworkTopologySpec, out *scheduling.NetworkTopologySpec, s conversion.Scope) error { + return autoConvert_v1beta1_NetworkTopologySpec_To_scheduling_NetworkTopologySpec(in, out, s) +} + +func autoConvert_scheduling_NetworkTopologySpec_To_v1beta1_NetworkTopologySpec(in *scheduling.NetworkTopologySpec, out *NetworkTopologySpec, s conversion.Scope) error { + out.Mode = NetworkTopologyMode(in.Mode) + out.HighestTierAllowed = (*int)(unsafe.Pointer(in.HighestTierAllowed)) + return nil +} + +// Convert_scheduling_NetworkTopologySpec_To_v1beta1_NetworkTopologySpec is an autogenerated conversion function. +func Convert_scheduling_NetworkTopologySpec_To_v1beta1_NetworkTopologySpec(in *scheduling.NetworkTopologySpec, out *NetworkTopologySpec, s conversion.Scope) error { + return autoConvert_scheduling_NetworkTopologySpec_To_v1beta1_NetworkTopologySpec(in, out, s) +} + +func autoConvert_v1beta1_NodeGroupAffinity_To_scheduling_NodeGroupAffinity(in *NodeGroupAffinity, out *scheduling.NodeGroupAffinity, s conversion.Scope) error { + out.RequiredDuringSchedulingIgnoredDuringExecution = *(*[]string)(unsafe.Pointer(&in.RequiredDuringSchedulingIgnoredDuringExecution)) + out.PreferredDuringSchedulingIgnoredDuringExecution = *(*[]string)(unsafe.Pointer(&in.PreferredDuringSchedulingIgnoredDuringExecution)) + return nil +} + +// Convert_v1beta1_NodeGroupAffinity_To_scheduling_NodeGroupAffinity is an autogenerated conversion function. +func Convert_v1beta1_NodeGroupAffinity_To_scheduling_NodeGroupAffinity(in *NodeGroupAffinity, out *scheduling.NodeGroupAffinity, s conversion.Scope) error { + return autoConvert_v1beta1_NodeGroupAffinity_To_scheduling_NodeGroupAffinity(in, out, s) +} + +func autoConvert_scheduling_NodeGroupAffinity_To_v1beta1_NodeGroupAffinity(in *scheduling.NodeGroupAffinity, out *NodeGroupAffinity, s conversion.Scope) error { + out.RequiredDuringSchedulingIgnoredDuringExecution = *(*[]string)(unsafe.Pointer(&in.RequiredDuringSchedulingIgnoredDuringExecution)) + out.PreferredDuringSchedulingIgnoredDuringExecution = *(*[]string)(unsafe.Pointer(&in.PreferredDuringSchedulingIgnoredDuringExecution)) + return nil +} + +// Convert_scheduling_NodeGroupAffinity_To_v1beta1_NodeGroupAffinity is an autogenerated conversion function. +func Convert_scheduling_NodeGroupAffinity_To_v1beta1_NodeGroupAffinity(in *scheduling.NodeGroupAffinity, out *NodeGroupAffinity, s conversion.Scope) error { + return autoConvert_scheduling_NodeGroupAffinity_To_v1beta1_NodeGroupAffinity(in, out, s) +} + +func autoConvert_v1beta1_NodeGroupAntiAffinity_To_scheduling_NodeGroupAntiAffinity(in *NodeGroupAntiAffinity, out *scheduling.NodeGroupAntiAffinity, s conversion.Scope) error { + out.RequiredDuringSchedulingIgnoredDuringExecution = *(*[]string)(unsafe.Pointer(&in.RequiredDuringSchedulingIgnoredDuringExecution)) + out.PreferredDuringSchedulingIgnoredDuringExecution = *(*[]string)(unsafe.Pointer(&in.PreferredDuringSchedulingIgnoredDuringExecution)) + return nil +} + +// Convert_v1beta1_NodeGroupAntiAffinity_To_scheduling_NodeGroupAntiAffinity is an autogenerated conversion function. +func Convert_v1beta1_NodeGroupAntiAffinity_To_scheduling_NodeGroupAntiAffinity(in *NodeGroupAntiAffinity, out *scheduling.NodeGroupAntiAffinity, s conversion.Scope) error { + return autoConvert_v1beta1_NodeGroupAntiAffinity_To_scheduling_NodeGroupAntiAffinity(in, out, s) +} + +func autoConvert_scheduling_NodeGroupAntiAffinity_To_v1beta1_NodeGroupAntiAffinity(in *scheduling.NodeGroupAntiAffinity, out *NodeGroupAntiAffinity, s conversion.Scope) error { + out.RequiredDuringSchedulingIgnoredDuringExecution = *(*[]string)(unsafe.Pointer(&in.RequiredDuringSchedulingIgnoredDuringExecution)) + out.PreferredDuringSchedulingIgnoredDuringExecution = *(*[]string)(unsafe.Pointer(&in.PreferredDuringSchedulingIgnoredDuringExecution)) + return nil +} + +// Convert_scheduling_NodeGroupAntiAffinity_To_v1beta1_NodeGroupAntiAffinity is an autogenerated conversion function. +func Convert_scheduling_NodeGroupAntiAffinity_To_v1beta1_NodeGroupAntiAffinity(in *scheduling.NodeGroupAntiAffinity, out *NodeGroupAntiAffinity, s conversion.Scope) error { + return autoConvert_scheduling_NodeGroupAntiAffinity_To_v1beta1_NodeGroupAntiAffinity(in, out, s) +} + +func autoConvert_v1beta1_PodGroup_To_scheduling_PodGroup(in *PodGroup, out *scheduling.PodGroup, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_PodGroupSpec_To_scheduling_PodGroupSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta1_PodGroupStatus_To_scheduling_PodGroupStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_PodGroup_To_scheduling_PodGroup is an autogenerated conversion function. +func Convert_v1beta1_PodGroup_To_scheduling_PodGroup(in *PodGroup, out *scheduling.PodGroup, s conversion.Scope) error { + return autoConvert_v1beta1_PodGroup_To_scheduling_PodGroup(in, out, s) +} + +func autoConvert_scheduling_PodGroup_To_v1beta1_PodGroup(in *scheduling.PodGroup, out *PodGroup, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_scheduling_PodGroupSpec_To_v1beta1_PodGroupSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_scheduling_PodGroupStatus_To_v1beta1_PodGroupStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_scheduling_PodGroup_To_v1beta1_PodGroup is an autogenerated conversion function. +func Convert_scheduling_PodGroup_To_v1beta1_PodGroup(in *scheduling.PodGroup, out *PodGroup, s conversion.Scope) error { + return autoConvert_scheduling_PodGroup_To_v1beta1_PodGroup(in, out, s) +} + +func autoConvert_v1beta1_PodGroupCondition_To_scheduling_PodGroupCondition(in *PodGroupCondition, out *scheduling.PodGroupCondition, s conversion.Scope) error { + out.Type = scheduling.PodGroupConditionType(in.Type) + out.Status = v1.ConditionStatus(in.Status) + out.TransitionID = in.TransitionID + out.LastTransitionTime = in.LastTransitionTime + out.Reason = in.Reason + out.Message = in.Message + return nil +} + +// Convert_v1beta1_PodGroupCondition_To_scheduling_PodGroupCondition is an autogenerated conversion function. +func Convert_v1beta1_PodGroupCondition_To_scheduling_PodGroupCondition(in *PodGroupCondition, out *scheduling.PodGroupCondition, s conversion.Scope) error { + return autoConvert_v1beta1_PodGroupCondition_To_scheduling_PodGroupCondition(in, out, s) +} + +func autoConvert_scheduling_PodGroupCondition_To_v1beta1_PodGroupCondition(in *scheduling.PodGroupCondition, out *PodGroupCondition, s conversion.Scope) error { + out.Type = PodGroupConditionType(in.Type) + out.Status = v1.ConditionStatus(in.Status) + out.TransitionID = in.TransitionID + out.LastTransitionTime = in.LastTransitionTime + out.Reason = in.Reason + out.Message = in.Message + return nil +} + +// Convert_scheduling_PodGroupCondition_To_v1beta1_PodGroupCondition is an autogenerated conversion function. +func Convert_scheduling_PodGroupCondition_To_v1beta1_PodGroupCondition(in *scheduling.PodGroupCondition, out *PodGroupCondition, s conversion.Scope) error { + return autoConvert_scheduling_PodGroupCondition_To_v1beta1_PodGroupCondition(in, out, s) +} + +func autoConvert_v1beta1_PodGroupList_To_scheduling_PodGroupList(in *PodGroupList, out *scheduling.PodGroupList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]scheduling.PodGroup)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_PodGroupList_To_scheduling_PodGroupList is an autogenerated conversion function. +func Convert_v1beta1_PodGroupList_To_scheduling_PodGroupList(in *PodGroupList, out *scheduling.PodGroupList, s conversion.Scope) error { + return autoConvert_v1beta1_PodGroupList_To_scheduling_PodGroupList(in, out, s) +} + +func autoConvert_scheduling_PodGroupList_To_v1beta1_PodGroupList(in *scheduling.PodGroupList, out *PodGroupList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]PodGroup)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_scheduling_PodGroupList_To_v1beta1_PodGroupList is an autogenerated conversion function. +func Convert_scheduling_PodGroupList_To_v1beta1_PodGroupList(in *scheduling.PodGroupList, out *PodGroupList, s conversion.Scope) error { + return autoConvert_scheduling_PodGroupList_To_v1beta1_PodGroupList(in, out, s) +} + +func autoConvert_v1beta1_PodGroupSpec_To_scheduling_PodGroupSpec(in *PodGroupSpec, out *scheduling.PodGroupSpec, s conversion.Scope) error { + out.MinMember = in.MinMember + out.MinTaskMember = *(*map[string]int32)(unsafe.Pointer(&in.MinTaskMember)) + out.Queue = in.Queue + out.PriorityClassName = in.PriorityClassName + out.MinResources = (*v1.ResourceList)(unsafe.Pointer(in.MinResources)) + out.NetworkTopology = (*scheduling.NetworkTopologySpec)(unsafe.Pointer(in.NetworkTopology)) + return nil +} + +// Convert_v1beta1_PodGroupSpec_To_scheduling_PodGroupSpec is an autogenerated conversion function. +func Convert_v1beta1_PodGroupSpec_To_scheduling_PodGroupSpec(in *PodGroupSpec, out *scheduling.PodGroupSpec, s conversion.Scope) error { + return autoConvert_v1beta1_PodGroupSpec_To_scheduling_PodGroupSpec(in, out, s) +} + +func autoConvert_scheduling_PodGroupSpec_To_v1beta1_PodGroupSpec(in *scheduling.PodGroupSpec, out *PodGroupSpec, s conversion.Scope) error { + out.MinMember = in.MinMember + out.MinTaskMember = *(*map[string]int32)(unsafe.Pointer(&in.MinTaskMember)) + out.Queue = in.Queue + out.PriorityClassName = in.PriorityClassName + out.MinResources = (*v1.ResourceList)(unsafe.Pointer(in.MinResources)) + out.NetworkTopology = (*NetworkTopologySpec)(unsafe.Pointer(in.NetworkTopology)) + return nil +} + +// Convert_scheduling_PodGroupSpec_To_v1beta1_PodGroupSpec is an autogenerated conversion function. +func Convert_scheduling_PodGroupSpec_To_v1beta1_PodGroupSpec(in *scheduling.PodGroupSpec, out *PodGroupSpec, s conversion.Scope) error { + return autoConvert_scheduling_PodGroupSpec_To_v1beta1_PodGroupSpec(in, out, s) +} + +func autoConvert_v1beta1_PodGroupStatus_To_scheduling_PodGroupStatus(in *PodGroupStatus, out *scheduling.PodGroupStatus, s conversion.Scope) error { + out.Phase = scheduling.PodGroupPhase(in.Phase) + out.Conditions = *(*[]scheduling.PodGroupCondition)(unsafe.Pointer(&in.Conditions)) + out.Running = in.Running + out.Succeeded = in.Succeeded + out.Failed = in.Failed + return nil +} + +// Convert_v1beta1_PodGroupStatus_To_scheduling_PodGroupStatus is an autogenerated conversion function. +func Convert_v1beta1_PodGroupStatus_To_scheduling_PodGroupStatus(in *PodGroupStatus, out *scheduling.PodGroupStatus, s conversion.Scope) error { + return autoConvert_v1beta1_PodGroupStatus_To_scheduling_PodGroupStatus(in, out, s) +} + +func autoConvert_scheduling_PodGroupStatus_To_v1beta1_PodGroupStatus(in *scheduling.PodGroupStatus, out *PodGroupStatus, s conversion.Scope) error { + out.Phase = PodGroupPhase(in.Phase) + out.Conditions = *(*[]PodGroupCondition)(unsafe.Pointer(&in.Conditions)) + out.Running = in.Running + out.Succeeded = in.Succeeded + out.Failed = in.Failed + return nil +} + +// Convert_scheduling_PodGroupStatus_To_v1beta1_PodGroupStatus is an autogenerated conversion function. +func Convert_scheduling_PodGroupStatus_To_v1beta1_PodGroupStatus(in *scheduling.PodGroupStatus, out *PodGroupStatus, s conversion.Scope) error { + return autoConvert_scheduling_PodGroupStatus_To_v1beta1_PodGroupStatus(in, out, s) +} + +func autoConvert_v1beta1_Queue_To_scheduling_Queue(in *Queue, out *scheduling.Queue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_QueueSpec_To_scheduling_QueueSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_v1beta1_QueueStatus_To_scheduling_QueueStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_Queue_To_scheduling_Queue is an autogenerated conversion function. +func Convert_v1beta1_Queue_To_scheduling_Queue(in *Queue, out *scheduling.Queue, s conversion.Scope) error { + return autoConvert_v1beta1_Queue_To_scheduling_Queue(in, out, s) +} + +func autoConvert_scheduling_Queue_To_v1beta1_Queue(in *scheduling.Queue, out *Queue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_scheduling_QueueSpec_To_v1beta1_QueueSpec(&in.Spec, &out.Spec, s); err != nil { + return err + } + if err := Convert_scheduling_QueueStatus_To_v1beta1_QueueStatus(&in.Status, &out.Status, s); err != nil { + return err + } + return nil +} + +// Convert_scheduling_Queue_To_v1beta1_Queue is an autogenerated conversion function. +func Convert_scheduling_Queue_To_v1beta1_Queue(in *scheduling.Queue, out *Queue, s conversion.Scope) error { + return autoConvert_scheduling_Queue_To_v1beta1_Queue(in, out, s) +} + +func autoConvert_v1beta1_QueueList_To_scheduling_QueueList(in *QueueList, out *scheduling.QueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]scheduling.Queue, len(*in)) + for i := range *in { + if err := Convert_v1beta1_Queue_To_scheduling_Queue(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } + return nil +} + +// Convert_v1beta1_QueueList_To_scheduling_QueueList is an autogenerated conversion function. +func Convert_v1beta1_QueueList_To_scheduling_QueueList(in *QueueList, out *scheduling.QueueList, s conversion.Scope) error { + return autoConvert_v1beta1_QueueList_To_scheduling_QueueList(in, out, s) +} + +func autoConvert_scheduling_QueueList_To_v1beta1_QueueList(in *scheduling.QueueList, out *QueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Queue, len(*in)) + for i := range *in { + if err := Convert_scheduling_Queue_To_v1beta1_Queue(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } + return nil +} + +// Convert_scheduling_QueueList_To_v1beta1_QueueList is an autogenerated conversion function. +func Convert_scheduling_QueueList_To_v1beta1_QueueList(in *scheduling.QueueList, out *QueueList, s conversion.Scope) error { + return autoConvert_scheduling_QueueList_To_v1beta1_QueueList(in, out, s) +} + +func autoConvert_v1beta1_QueueSpec_To_scheduling_QueueSpec(in *QueueSpec, out *scheduling.QueueSpec, s conversion.Scope) error { + out.Weight = in.Weight + out.Capability = *(*v1.ResourceList)(unsafe.Pointer(&in.Capability)) + out.Reclaimable = (*bool)(unsafe.Pointer(in.Reclaimable)) + out.ExtendClusters = *(*[]scheduling.Cluster)(unsafe.Pointer(&in.ExtendClusters)) + if err := Convert_v1beta1_Guarantee_To_scheduling_Guarantee(&in.Guarantee, &out.Guarantee, s); err != nil { + return err + } + out.Affinity = (*scheduling.Affinity)(unsafe.Pointer(in.Affinity)) + out.Type = in.Type + out.Parent = in.Parent + out.Deserved = *(*v1.ResourceList)(unsafe.Pointer(&in.Deserved)) + out.Priority = in.Priority + return nil +} + +// Convert_v1beta1_QueueSpec_To_scheduling_QueueSpec is an autogenerated conversion function. +func Convert_v1beta1_QueueSpec_To_scheduling_QueueSpec(in *QueueSpec, out *scheduling.QueueSpec, s conversion.Scope) error { + return autoConvert_v1beta1_QueueSpec_To_scheduling_QueueSpec(in, out, s) +} + +func autoConvert_scheduling_QueueSpec_To_v1beta1_QueueSpec(in *scheduling.QueueSpec, out *QueueSpec, s conversion.Scope) error { + out.Weight = in.Weight + out.Capability = *(*v1.ResourceList)(unsafe.Pointer(&in.Capability)) + // WARNING: in.State requires manual conversion: does not exist in peer-type + out.Reclaimable = (*bool)(unsafe.Pointer(in.Reclaimable)) + out.ExtendClusters = *(*[]Cluster)(unsafe.Pointer(&in.ExtendClusters)) + if err := Convert_scheduling_Guarantee_To_v1beta1_Guarantee(&in.Guarantee, &out.Guarantee, s); err != nil { + return err + } + out.Affinity = (*Affinity)(unsafe.Pointer(in.Affinity)) + out.Type = in.Type + out.Parent = in.Parent + out.Deserved = *(*v1.ResourceList)(unsafe.Pointer(&in.Deserved)) + out.Priority = in.Priority + return nil +} + +func autoConvert_v1beta1_QueueStatus_To_scheduling_QueueStatus(in *QueueStatus, out *scheduling.QueueStatus, s conversion.Scope) error { + out.State = scheduling.QueueState(in.State) + out.Unknown = in.Unknown + out.Pending = in.Pending + out.Running = in.Running + out.Inqueue = in.Inqueue + out.Completed = in.Completed + if err := Convert_v1beta1_Reservation_To_scheduling_Reservation(&in.Reservation, &out.Reservation, s); err != nil { + return err + } + out.Allocated = *(*v1.ResourceList)(unsafe.Pointer(&in.Allocated)) + return nil +} + +// Convert_v1beta1_QueueStatus_To_scheduling_QueueStatus is an autogenerated conversion function. +func Convert_v1beta1_QueueStatus_To_scheduling_QueueStatus(in *QueueStatus, out *scheduling.QueueStatus, s conversion.Scope) error { + return autoConvert_v1beta1_QueueStatus_To_scheduling_QueueStatus(in, out, s) +} + +func autoConvert_scheduling_QueueStatus_To_v1beta1_QueueStatus(in *scheduling.QueueStatus, out *QueueStatus, s conversion.Scope) error { + out.State = QueueState(in.State) + out.Unknown = in.Unknown + out.Pending = in.Pending + out.Running = in.Running + out.Inqueue = in.Inqueue + out.Completed = in.Completed + if err := Convert_scheduling_Reservation_To_v1beta1_Reservation(&in.Reservation, &out.Reservation, s); err != nil { + return err + } + out.Allocated = *(*v1.ResourceList)(unsafe.Pointer(&in.Allocated)) + return nil +} + +// Convert_scheduling_QueueStatus_To_v1beta1_QueueStatus is an autogenerated conversion function. +func Convert_scheduling_QueueStatus_To_v1beta1_QueueStatus(in *scheduling.QueueStatus, out *QueueStatus, s conversion.Scope) error { + return autoConvert_scheduling_QueueStatus_To_v1beta1_QueueStatus(in, out, s) +} + +func autoConvert_v1beta1_Reservation_To_scheduling_Reservation(in *Reservation, out *scheduling.Reservation, s conversion.Scope) error { + out.Nodes = *(*[]string)(unsafe.Pointer(&in.Nodes)) + out.Resource = *(*v1.ResourceList)(unsafe.Pointer(&in.Resource)) + return nil +} + +// Convert_v1beta1_Reservation_To_scheduling_Reservation is an autogenerated conversion function. +func Convert_v1beta1_Reservation_To_scheduling_Reservation(in *Reservation, out *scheduling.Reservation, s conversion.Scope) error { + return autoConvert_v1beta1_Reservation_To_scheduling_Reservation(in, out, s) +} + +func autoConvert_scheduling_Reservation_To_v1beta1_Reservation(in *scheduling.Reservation, out *Reservation, s conversion.Scope) error { + out.Nodes = *(*[]string)(unsafe.Pointer(&in.Nodes)) + out.Resource = *(*v1.ResourceList)(unsafe.Pointer(&in.Resource)) + return nil +} + +// Convert_scheduling_Reservation_To_v1beta1_Reservation is an autogenerated conversion function. +func Convert_scheduling_Reservation_To_v1beta1_Reservation(in *scheduling.Reservation, out *Reservation, s conversion.Scope) error { + return autoConvert_scheduling_Reservation_To_v1beta1_Reservation(in, out, s) +} diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.deepcopy.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 00000000000..996c5f56b72 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,473 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by deepcopy-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + resource "k8s.io/apimachinery/pkg/api/resource" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Affinity) DeepCopyInto(out *Affinity) { + *out = *in + if in.NodeGroupAffinity != nil { + in, out := &in.NodeGroupAffinity, &out.NodeGroupAffinity + *out = new(NodeGroupAffinity) + (*in).DeepCopyInto(*out) + } + if in.NodeGroupAntiAffinity != nil { + in, out := &in.NodeGroupAntiAffinity, &out.NodeGroupAntiAffinity + *out = new(NodeGroupAntiAffinity) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Affinity. +func (in *Affinity) DeepCopy() *Affinity { + if in == nil { + return nil + } + out := new(Affinity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Cluster) DeepCopyInto(out *Cluster) { + *out = *in + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster. +func (in *Cluster) DeepCopy() *Cluster { + if in == nil { + return nil + } + out := new(Cluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Guarantee) DeepCopyInto(out *Guarantee) { + *out = *in + if in.Resource != nil { + in, out := &in.Resource, &out.Resource + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Guarantee. +func (in *Guarantee) DeepCopy() *Guarantee { + if in == nil { + return nil + } + out := new(Guarantee) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkTopologySpec) DeepCopyInto(out *NetworkTopologySpec) { + *out = *in + if in.HighestTierAllowed != nil { + in, out := &in.HighestTierAllowed, &out.HighestTierAllowed + *out = new(int) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkTopologySpec. +func (in *NetworkTopologySpec) DeepCopy() *NetworkTopologySpec { + if in == nil { + return nil + } + out := new(NetworkTopologySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeGroupAffinity) DeepCopyInto(out *NodeGroupAffinity) { + *out = *in + if in.RequiredDuringSchedulingIgnoredDuringExecution != nil { + in, out := &in.RequiredDuringSchedulingIgnoredDuringExecution, &out.RequiredDuringSchedulingIgnoredDuringExecution + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PreferredDuringSchedulingIgnoredDuringExecution != nil { + in, out := &in.PreferredDuringSchedulingIgnoredDuringExecution, &out.PreferredDuringSchedulingIgnoredDuringExecution + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeGroupAffinity. +func (in *NodeGroupAffinity) DeepCopy() *NodeGroupAffinity { + if in == nil { + return nil + } + out := new(NodeGroupAffinity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeGroupAntiAffinity) DeepCopyInto(out *NodeGroupAntiAffinity) { + *out = *in + if in.RequiredDuringSchedulingIgnoredDuringExecution != nil { + in, out := &in.RequiredDuringSchedulingIgnoredDuringExecution, &out.RequiredDuringSchedulingIgnoredDuringExecution + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PreferredDuringSchedulingIgnoredDuringExecution != nil { + in, out := &in.PreferredDuringSchedulingIgnoredDuringExecution, &out.PreferredDuringSchedulingIgnoredDuringExecution + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeGroupAntiAffinity. +func (in *NodeGroupAntiAffinity) DeepCopy() *NodeGroupAntiAffinity { + if in == nil { + return nil + } + out := new(NodeGroupAntiAffinity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroup) DeepCopyInto(out *PodGroup) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroup. +func (in *PodGroup) DeepCopy() *PodGroup { + if in == nil { + return nil + } + out := new(PodGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PodGroup) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupCondition) DeepCopyInto(out *PodGroupCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupCondition. +func (in *PodGroupCondition) DeepCopy() *PodGroupCondition { + if in == nil { + return nil + } + out := new(PodGroupCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupList) DeepCopyInto(out *PodGroupList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PodGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupList. +func (in *PodGroupList) DeepCopy() *PodGroupList { + if in == nil { + return nil + } + out := new(PodGroupList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PodGroupList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupSpec) DeepCopyInto(out *PodGroupSpec) { + *out = *in + if in.MinTaskMember != nil { + in, out := &in.MinTaskMember, &out.MinTaskMember + *out = make(map[string]int32, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.MinResources != nil { + in, out := &in.MinResources, &out.MinResources + *out = new(v1.ResourceList) + if **in != nil { + in, out := *in, *out + *out = make(map[v1.ResourceName]resource.Quantity, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + } + if in.NetworkTopology != nil { + in, out := &in.NetworkTopology, &out.NetworkTopology + *out = new(NetworkTopologySpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupSpec. +func (in *PodGroupSpec) DeepCopy() *PodGroupSpec { + if in == nil { + return nil + } + out := new(PodGroupSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupStatus) DeepCopyInto(out *PodGroupStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]PodGroupCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupStatus. +func (in *PodGroupStatus) DeepCopy() *PodGroupStatus { + if in == nil { + return nil + } + out := new(PodGroupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Queue) DeepCopyInto(out *Queue) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Queue. +func (in *Queue) DeepCopy() *Queue { + if in == nil { + return nil + } + out := new(Queue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Queue) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueList) DeepCopyInto(out *QueueList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Queue, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueList. +func (in *QueueList) DeepCopy() *QueueList { + if in == nil { + return nil + } + out := new(QueueList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *QueueList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueSpec) DeepCopyInto(out *QueueSpec) { + *out = *in + if in.Capability != nil { + in, out := &in.Capability, &out.Capability + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Reclaimable != nil { + in, out := &in.Reclaimable, &out.Reclaimable + *out = new(bool) + **out = **in + } + if in.ExtendClusters != nil { + in, out := &in.ExtendClusters, &out.ExtendClusters + *out = make([]Cluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Guarantee.DeepCopyInto(&out.Guarantee) + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(Affinity) + (*in).DeepCopyInto(*out) + } + if in.Deserved != nil { + in, out := &in.Deserved, &out.Deserved + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueSpec. +func (in *QueueSpec) DeepCopy() *QueueSpec { + if in == nil { + return nil + } + out := new(QueueSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueStatus) DeepCopyInto(out *QueueStatus) { + *out = *in + in.Reservation.DeepCopyInto(&out.Reservation) + if in.Allocated != nil { + in, out := &in.Allocated, &out.Allocated + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueStatus. +func (in *QueueStatus) DeepCopy() *QueueStatus { + if in == nil { + return nil + } + out := new(QueueStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Reservation) DeepCopyInto(out *Reservation) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Resource != nil { + in, out := &in.Resource, &out.Resource + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Reservation. +func (in *Reservation) DeepCopy() *Reservation { + if in == nil { + return nil + } + out := new(Reservation) + in.DeepCopyInto(out) + return out +} diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/zz_generated.deepcopy.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/zz_generated.deepcopy.go new file mode 100644 index 00000000000..2e0e999e1c2 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/zz_generated.deepcopy.go @@ -0,0 +1,473 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by deepcopy-gen. DO NOT EDIT. + +package scheduling + +import ( + v1 "k8s.io/api/core/v1" + resource "k8s.io/apimachinery/pkg/api/resource" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Affinity) DeepCopyInto(out *Affinity) { + *out = *in + if in.NodeGroupAffinity != nil { + in, out := &in.NodeGroupAffinity, &out.NodeGroupAffinity + *out = new(NodeGroupAffinity) + (*in).DeepCopyInto(*out) + } + if in.NodeGroupAntiAffinity != nil { + in, out := &in.NodeGroupAntiAffinity, &out.NodeGroupAntiAffinity + *out = new(NodeGroupAntiAffinity) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Affinity. +func (in *Affinity) DeepCopy() *Affinity { + if in == nil { + return nil + } + out := new(Affinity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Cluster) DeepCopyInto(out *Cluster) { + *out = *in + if in.Capacity != nil { + in, out := &in.Capacity, &out.Capacity + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster. +func (in *Cluster) DeepCopy() *Cluster { + if in == nil { + return nil + } + out := new(Cluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Guarantee) DeepCopyInto(out *Guarantee) { + *out = *in + if in.Resource != nil { + in, out := &in.Resource, &out.Resource + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Guarantee. +func (in *Guarantee) DeepCopy() *Guarantee { + if in == nil { + return nil + } + out := new(Guarantee) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkTopologySpec) DeepCopyInto(out *NetworkTopologySpec) { + *out = *in + if in.HighestTierAllowed != nil { + in, out := &in.HighestTierAllowed, &out.HighestTierAllowed + *out = new(int) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkTopologySpec. +func (in *NetworkTopologySpec) DeepCopy() *NetworkTopologySpec { + if in == nil { + return nil + } + out := new(NetworkTopologySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeGroupAffinity) DeepCopyInto(out *NodeGroupAffinity) { + *out = *in + if in.RequiredDuringSchedulingIgnoredDuringExecution != nil { + in, out := &in.RequiredDuringSchedulingIgnoredDuringExecution, &out.RequiredDuringSchedulingIgnoredDuringExecution + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PreferredDuringSchedulingIgnoredDuringExecution != nil { + in, out := &in.PreferredDuringSchedulingIgnoredDuringExecution, &out.PreferredDuringSchedulingIgnoredDuringExecution + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeGroupAffinity. +func (in *NodeGroupAffinity) DeepCopy() *NodeGroupAffinity { + if in == nil { + return nil + } + out := new(NodeGroupAffinity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeGroupAntiAffinity) DeepCopyInto(out *NodeGroupAntiAffinity) { + *out = *in + if in.RequiredDuringSchedulingIgnoredDuringExecution != nil { + in, out := &in.RequiredDuringSchedulingIgnoredDuringExecution, &out.RequiredDuringSchedulingIgnoredDuringExecution + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PreferredDuringSchedulingIgnoredDuringExecution != nil { + in, out := &in.PreferredDuringSchedulingIgnoredDuringExecution, &out.PreferredDuringSchedulingIgnoredDuringExecution + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeGroupAntiAffinity. +func (in *NodeGroupAntiAffinity) DeepCopy() *NodeGroupAntiAffinity { + if in == nil { + return nil + } + out := new(NodeGroupAntiAffinity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroup) DeepCopyInto(out *PodGroup) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroup. +func (in *PodGroup) DeepCopy() *PodGroup { + if in == nil { + return nil + } + out := new(PodGroup) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PodGroup) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupCondition) DeepCopyInto(out *PodGroupCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupCondition. +func (in *PodGroupCondition) DeepCopy() *PodGroupCondition { + if in == nil { + return nil + } + out := new(PodGroupCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupList) DeepCopyInto(out *PodGroupList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PodGroup, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupList. +func (in *PodGroupList) DeepCopy() *PodGroupList { + if in == nil { + return nil + } + out := new(PodGroupList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PodGroupList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupSpec) DeepCopyInto(out *PodGroupSpec) { + *out = *in + if in.MinTaskMember != nil { + in, out := &in.MinTaskMember, &out.MinTaskMember + *out = make(map[string]int32, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.MinResources != nil { + in, out := &in.MinResources, &out.MinResources + *out = new(v1.ResourceList) + if **in != nil { + in, out := *in, *out + *out = make(map[v1.ResourceName]resource.Quantity, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + } + if in.NetworkTopology != nil { + in, out := &in.NetworkTopology, &out.NetworkTopology + *out = new(NetworkTopologySpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupSpec. +func (in *PodGroupSpec) DeepCopy() *PodGroupSpec { + if in == nil { + return nil + } + out := new(PodGroupSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupStatus) DeepCopyInto(out *PodGroupStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]PodGroupCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupStatus. +func (in *PodGroupStatus) DeepCopy() *PodGroupStatus { + if in == nil { + return nil + } + out := new(PodGroupStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Queue) DeepCopyInto(out *Queue) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Queue. +func (in *Queue) DeepCopy() *Queue { + if in == nil { + return nil + } + out := new(Queue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Queue) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueList) DeepCopyInto(out *QueueList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Queue, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueList. +func (in *QueueList) DeepCopy() *QueueList { + if in == nil { + return nil + } + out := new(QueueList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *QueueList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueSpec) DeepCopyInto(out *QueueSpec) { + *out = *in + if in.Capability != nil { + in, out := &in.Capability, &out.Capability + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Reclaimable != nil { + in, out := &in.Reclaimable, &out.Reclaimable + *out = new(bool) + **out = **in + } + if in.ExtendClusters != nil { + in, out := &in.ExtendClusters, &out.ExtendClusters + *out = make([]Cluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Guarantee.DeepCopyInto(&out.Guarantee) + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(Affinity) + (*in).DeepCopyInto(*out) + } + if in.Deserved != nil { + in, out := &in.Deserved, &out.Deserved + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueSpec. +func (in *QueueSpec) DeepCopy() *QueueSpec { + if in == nil { + return nil + } + out := new(QueueSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueStatus) DeepCopyInto(out *QueueStatus) { + *out = *in + in.Reservation.DeepCopyInto(&out.Reservation) + if in.Allocated != nil { + in, out := &in.Allocated, &out.Allocated + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueStatus. +func (in *QueueStatus) DeepCopy() *QueueStatus { + if in == nil { + return nil + } + out := new(QueueStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Reservation) DeepCopyInto(out *Reservation) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Resource != nil { + in, out := &in.Resource, &out.Resource + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Reservation. +func (in *Reservation) DeepCopy() *Reservation { + if in == nil { + return nil + } + out := new(Reservation) + in.DeepCopyInto(out) + return out +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/affinity.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/affinity.go new file mode 100644 index 00000000000..5d141a9e766 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/affinity.go @@ -0,0 +1,47 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// AffinityApplyConfiguration represents a declarative configuration of the Affinity type for use +// with apply. +type AffinityApplyConfiguration struct { + NodeGroupAffinity *NodeGroupAffinityApplyConfiguration `json:"nodeGroupAffinity,omitempty"` + NodeGroupAntiAffinity *NodeGroupAntiAffinityApplyConfiguration `json:"nodeGroupAntiAffinity,omitempty"` +} + +// AffinityApplyConfiguration constructs a declarative configuration of the Affinity type for use with +// apply. +func Affinity() *AffinityApplyConfiguration { + return &AffinityApplyConfiguration{} +} + +// WithNodeGroupAffinity sets the NodeGroupAffinity field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the NodeGroupAffinity field is set to the value of the last call. +func (b *AffinityApplyConfiguration) WithNodeGroupAffinity(value *NodeGroupAffinityApplyConfiguration) *AffinityApplyConfiguration { + b.NodeGroupAffinity = value + return b +} + +// WithNodeGroupAntiAffinity sets the NodeGroupAntiAffinity field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the NodeGroupAntiAffinity field is set to the value of the last call. +func (b *AffinityApplyConfiguration) WithNodeGroupAntiAffinity(value *NodeGroupAntiAffinityApplyConfiguration) *AffinityApplyConfiguration { + b.NodeGroupAntiAffinity = value + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/cluster.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/cluster.go new file mode 100644 index 00000000000..1594d0f65cd --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/cluster.go @@ -0,0 +1,60 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" +) + +// ClusterApplyConfiguration represents a declarative configuration of the Cluster type for use +// with apply. +type ClusterApplyConfiguration struct { + Name *string `json:"name,omitempty"` + Weight *int32 `json:"weight,omitempty"` + Capacity *v1.ResourceList `json:"capacity,omitempty"` +} + +// ClusterApplyConfiguration constructs a declarative configuration of the Cluster type for use with +// apply. +func Cluster() *ClusterApplyConfiguration { + return &ClusterApplyConfiguration{} +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *ClusterApplyConfiguration) WithName(value string) *ClusterApplyConfiguration { + b.Name = &value + return b +} + +// WithWeight sets the Weight field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Weight field is set to the value of the last call. +func (b *ClusterApplyConfiguration) WithWeight(value int32) *ClusterApplyConfiguration { + b.Weight = &value + return b +} + +// WithCapacity sets the Capacity field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Capacity field is set to the value of the last call. +func (b *ClusterApplyConfiguration) WithCapacity(value v1.ResourceList) *ClusterApplyConfiguration { + b.Capacity = &value + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/guarantee.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/guarantee.go new file mode 100644 index 00000000000..5efa54ec915 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/guarantee.go @@ -0,0 +1,42 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" +) + +// GuaranteeApplyConfiguration represents a declarative configuration of the Guarantee type for use +// with apply. +type GuaranteeApplyConfiguration struct { + Resource *v1.ResourceList `json:"resource,omitempty"` +} + +// GuaranteeApplyConfiguration constructs a declarative configuration of the Guarantee type for use with +// apply. +func Guarantee() *GuaranteeApplyConfiguration { + return &GuaranteeApplyConfiguration{} +} + +// WithResource sets the Resource field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Resource field is set to the value of the last call. +func (b *GuaranteeApplyConfiguration) WithResource(value v1.ResourceList) *GuaranteeApplyConfiguration { + b.Resource = &value + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/networktopologyspec.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/networktopologyspec.go new file mode 100644 index 00000000000..887393ad393 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/networktopologyspec.go @@ -0,0 +1,51 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" +) + +// NetworkTopologySpecApplyConfiguration represents a declarative configuration of the NetworkTopologySpec type for use +// with apply. +type NetworkTopologySpecApplyConfiguration struct { + Mode *v1beta1.NetworkTopologyMode `json:"mode,omitempty"` + HighestTierAllowed *int `json:"highestTierAllowed,omitempty"` +} + +// NetworkTopologySpecApplyConfiguration constructs a declarative configuration of the NetworkTopologySpec type for use with +// apply. +func NetworkTopologySpec() *NetworkTopologySpecApplyConfiguration { + return &NetworkTopologySpecApplyConfiguration{} +} + +// WithMode sets the Mode field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Mode field is set to the value of the last call. +func (b *NetworkTopologySpecApplyConfiguration) WithMode(value v1beta1.NetworkTopologyMode) *NetworkTopologySpecApplyConfiguration { + b.Mode = &value + return b +} + +// WithHighestTierAllowed sets the HighestTierAllowed field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the HighestTierAllowed field is set to the value of the last call. +func (b *NetworkTopologySpecApplyConfiguration) WithHighestTierAllowed(value int) *NetworkTopologySpecApplyConfiguration { + b.HighestTierAllowed = &value + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/nodegroupaffinity.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/nodegroupaffinity.go new file mode 100644 index 00000000000..ddb9f1ab8bb --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/nodegroupaffinity.go @@ -0,0 +1,51 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// NodeGroupAffinityApplyConfiguration represents a declarative configuration of the NodeGroupAffinity type for use +// with apply. +type NodeGroupAffinityApplyConfiguration struct { + RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty"` + PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty"` +} + +// NodeGroupAffinityApplyConfiguration constructs a declarative configuration of the NodeGroupAffinity type for use with +// apply. +func NodeGroupAffinity() *NodeGroupAffinityApplyConfiguration { + return &NodeGroupAffinityApplyConfiguration{} +} + +// WithRequiredDuringSchedulingIgnoredDuringExecution adds the given value to the RequiredDuringSchedulingIgnoredDuringExecution field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the RequiredDuringSchedulingIgnoredDuringExecution field. +func (b *NodeGroupAffinityApplyConfiguration) WithRequiredDuringSchedulingIgnoredDuringExecution(values ...string) *NodeGroupAffinityApplyConfiguration { + for i := range values { + b.RequiredDuringSchedulingIgnoredDuringExecution = append(b.RequiredDuringSchedulingIgnoredDuringExecution, values[i]) + } + return b +} + +// WithPreferredDuringSchedulingIgnoredDuringExecution adds the given value to the PreferredDuringSchedulingIgnoredDuringExecution field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the PreferredDuringSchedulingIgnoredDuringExecution field. +func (b *NodeGroupAffinityApplyConfiguration) WithPreferredDuringSchedulingIgnoredDuringExecution(values ...string) *NodeGroupAffinityApplyConfiguration { + for i := range values { + b.PreferredDuringSchedulingIgnoredDuringExecution = append(b.PreferredDuringSchedulingIgnoredDuringExecution, values[i]) + } + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/nodegroupantiaffinity.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/nodegroupantiaffinity.go new file mode 100644 index 00000000000..6b4cca2803f --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/nodegroupantiaffinity.go @@ -0,0 +1,51 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// NodeGroupAntiAffinityApplyConfiguration represents a declarative configuration of the NodeGroupAntiAffinity type for use +// with apply. +type NodeGroupAntiAffinityApplyConfiguration struct { + RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty"` + PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty"` +} + +// NodeGroupAntiAffinityApplyConfiguration constructs a declarative configuration of the NodeGroupAntiAffinity type for use with +// apply. +func NodeGroupAntiAffinity() *NodeGroupAntiAffinityApplyConfiguration { + return &NodeGroupAntiAffinityApplyConfiguration{} +} + +// WithRequiredDuringSchedulingIgnoredDuringExecution adds the given value to the RequiredDuringSchedulingIgnoredDuringExecution field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the RequiredDuringSchedulingIgnoredDuringExecution field. +func (b *NodeGroupAntiAffinityApplyConfiguration) WithRequiredDuringSchedulingIgnoredDuringExecution(values ...string) *NodeGroupAntiAffinityApplyConfiguration { + for i := range values { + b.RequiredDuringSchedulingIgnoredDuringExecution = append(b.RequiredDuringSchedulingIgnoredDuringExecution, values[i]) + } + return b +} + +// WithPreferredDuringSchedulingIgnoredDuringExecution adds the given value to the PreferredDuringSchedulingIgnoredDuringExecution field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the PreferredDuringSchedulingIgnoredDuringExecution field. +func (b *NodeGroupAntiAffinityApplyConfiguration) WithPreferredDuringSchedulingIgnoredDuringExecution(values ...string) *NodeGroupAntiAffinityApplyConfiguration { + for i := range values { + b.PreferredDuringSchedulingIgnoredDuringExecution = append(b.PreferredDuringSchedulingIgnoredDuringExecution, values[i]) + } + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroup.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroup.go new file mode 100644 index 00000000000..37fef63dcb7 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroup.go @@ -0,0 +1,224 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +// PodGroupApplyConfiguration represents a declarative configuration of the PodGroup type for use +// with apply. +type PodGroupApplyConfiguration struct { + v1.TypeMetaApplyConfiguration `json:",inline"` + *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Spec *PodGroupSpecApplyConfiguration `json:"spec,omitempty"` + Status *PodGroupStatusApplyConfiguration `json:"status,omitempty"` +} + +// PodGroup constructs a declarative configuration of the PodGroup type for use with +// apply. +func PodGroup(name, namespace string) *PodGroupApplyConfiguration { + b := &PodGroupApplyConfiguration{} + b.WithName(name) + b.WithNamespace(namespace) + b.WithKind("PodGroup") + b.WithAPIVersion("scheduling.volcano.sh/v1beta1") + return b +} + +// WithKind sets the Kind field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Kind field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithKind(value string) *PodGroupApplyConfiguration { + b.TypeMetaApplyConfiguration.Kind = &value + return b +} + +// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the APIVersion field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithAPIVersion(value string) *PodGroupApplyConfiguration { + b.TypeMetaApplyConfiguration.APIVersion = &value + return b +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithName(value string) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Name = &value + return b +} + +// WithGenerateName sets the GenerateName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the GenerateName field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithGenerateName(value string) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.GenerateName = &value + return b +} + +// WithNamespace sets the Namespace field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Namespace field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithNamespace(value string) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Namespace = &value + return b +} + +// WithUID sets the UID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UID field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithUID(value types.UID) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.UID = &value + return b +} + +// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceVersion field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithResourceVersion(value string) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.ResourceVersion = &value + return b +} + +// WithGeneration sets the Generation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Generation field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithGeneration(value int64) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Generation = &value + return b +} + +// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CreationTimestamp field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithCreationTimestamp(value metav1.Time) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.CreationTimestamp = &value + return b +} + +// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionTimestamp field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionTimestamp = &value + return b +} + +// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionGracePeriodSeconds = &value + return b +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *PodGroupApplyConfiguration) WithLabels(entries map[string]string) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Labels == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *PodGroupApplyConfiguration) WithAnnotations(entries map[string]string) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Annotations == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Annotations[k] = v + } + return b +} + +// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the OwnerReferences field. +func (b *PodGroupApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + if values[i] == nil { + panic("nil value passed to WithOwnerReferences") + } + b.ObjectMetaApplyConfiguration.OwnerReferences = append(b.ObjectMetaApplyConfiguration.OwnerReferences, *values[i]) + } + return b +} + +// WithFinalizers adds the given value to the Finalizers field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Finalizers field. +func (b *PodGroupApplyConfiguration) WithFinalizers(values ...string) *PodGroupApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + b.ObjectMetaApplyConfiguration.Finalizers = append(b.ObjectMetaApplyConfiguration.Finalizers, values[i]) + } + return b +} + +func (b *PodGroupApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { + if b.ObjectMetaApplyConfiguration == nil { + b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} + } +} + +// WithSpec sets the Spec field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Spec field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithSpec(value *PodGroupSpecApplyConfiguration) *PodGroupApplyConfiguration { + b.Spec = value + return b +} + +// WithStatus sets the Status field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Status field is set to the value of the last call. +func (b *PodGroupApplyConfiguration) WithStatus(value *PodGroupStatusApplyConfiguration) *PodGroupApplyConfiguration { + b.Status = value + return b +} + +// GetName retrieves the value of the Name field in the declarative configuration. +func (b *PodGroupApplyConfiguration) GetName() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Name +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupcondition.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupcondition.go new file mode 100644 index 00000000000..5b2de0a2492 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupcondition.go @@ -0,0 +1,89 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" +) + +// PodGroupConditionApplyConfiguration represents a declarative configuration of the PodGroupCondition type for use +// with apply. +type PodGroupConditionApplyConfiguration struct { + Type *schedulingv1beta1.PodGroupConditionType `json:"type,omitempty"` + Status *v1.ConditionStatus `json:"status,omitempty"` + TransitionID *string `json:"transitionID,omitempty"` + LastTransitionTime *metav1.Time `json:"lastTransitionTime,omitempty"` + Reason *string `json:"reason,omitempty"` + Message *string `json:"message,omitempty"` +} + +// PodGroupConditionApplyConfiguration constructs a declarative configuration of the PodGroupCondition type for use with +// apply. +func PodGroupCondition() *PodGroupConditionApplyConfiguration { + return &PodGroupConditionApplyConfiguration{} +} + +// WithType sets the Type field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Type field is set to the value of the last call. +func (b *PodGroupConditionApplyConfiguration) WithType(value schedulingv1beta1.PodGroupConditionType) *PodGroupConditionApplyConfiguration { + b.Type = &value + return b +} + +// WithStatus sets the Status field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Status field is set to the value of the last call. +func (b *PodGroupConditionApplyConfiguration) WithStatus(value v1.ConditionStatus) *PodGroupConditionApplyConfiguration { + b.Status = &value + return b +} + +// WithTransitionID sets the TransitionID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the TransitionID field is set to the value of the last call. +func (b *PodGroupConditionApplyConfiguration) WithTransitionID(value string) *PodGroupConditionApplyConfiguration { + b.TransitionID = &value + return b +} + +// WithLastTransitionTime sets the LastTransitionTime field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the LastTransitionTime field is set to the value of the last call. +func (b *PodGroupConditionApplyConfiguration) WithLastTransitionTime(value metav1.Time) *PodGroupConditionApplyConfiguration { + b.LastTransitionTime = &value + return b +} + +// WithReason sets the Reason field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Reason field is set to the value of the last call. +func (b *PodGroupConditionApplyConfiguration) WithReason(value string) *PodGroupConditionApplyConfiguration { + b.Reason = &value + return b +} + +// WithMessage sets the Message field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Message field is set to the value of the last call. +func (b *PodGroupConditionApplyConfiguration) WithMessage(value string) *PodGroupConditionApplyConfiguration { + b.Message = &value + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupspec.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupspec.go new file mode 100644 index 00000000000..95bcdb2894d --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupspec.go @@ -0,0 +1,93 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" +) + +// PodGroupSpecApplyConfiguration represents a declarative configuration of the PodGroupSpec type for use +// with apply. +type PodGroupSpecApplyConfiguration struct { + MinMember *int32 `json:"minMember,omitempty"` + MinTaskMember map[string]int32 `json:"minTaskMember,omitempty"` + Queue *string `json:"queue,omitempty"` + PriorityClassName *string `json:"priorityClassName,omitempty"` + MinResources *v1.ResourceList `json:"minResources,omitempty"` + NetworkTopology *NetworkTopologySpecApplyConfiguration `json:"networkTopology,omitempty"` +} + +// PodGroupSpecApplyConfiguration constructs a declarative configuration of the PodGroupSpec type for use with +// apply. +func PodGroupSpec() *PodGroupSpecApplyConfiguration { + return &PodGroupSpecApplyConfiguration{} +} + +// WithMinMember sets the MinMember field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the MinMember field is set to the value of the last call. +func (b *PodGroupSpecApplyConfiguration) WithMinMember(value int32) *PodGroupSpecApplyConfiguration { + b.MinMember = &value + return b +} + +// WithMinTaskMember puts the entries into the MinTaskMember field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the MinTaskMember field, +// overwriting an existing map entries in MinTaskMember field with the same key. +func (b *PodGroupSpecApplyConfiguration) WithMinTaskMember(entries map[string]int32) *PodGroupSpecApplyConfiguration { + if b.MinTaskMember == nil && len(entries) > 0 { + b.MinTaskMember = make(map[string]int32, len(entries)) + } + for k, v := range entries { + b.MinTaskMember[k] = v + } + return b +} + +// WithQueue sets the Queue field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Queue field is set to the value of the last call. +func (b *PodGroupSpecApplyConfiguration) WithQueue(value string) *PodGroupSpecApplyConfiguration { + b.Queue = &value + return b +} + +// WithPriorityClassName sets the PriorityClassName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the PriorityClassName field is set to the value of the last call. +func (b *PodGroupSpecApplyConfiguration) WithPriorityClassName(value string) *PodGroupSpecApplyConfiguration { + b.PriorityClassName = &value + return b +} + +// WithMinResources sets the MinResources field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the MinResources field is set to the value of the last call. +func (b *PodGroupSpecApplyConfiguration) WithMinResources(value v1.ResourceList) *PodGroupSpecApplyConfiguration { + b.MinResources = &value + return b +} + +// WithNetworkTopology sets the NetworkTopology field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the NetworkTopology field is set to the value of the last call. +func (b *PodGroupSpecApplyConfiguration) WithNetworkTopology(value *NetworkTopologySpecApplyConfiguration) *PodGroupSpecApplyConfiguration { + b.NetworkTopology = value + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupstatus.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupstatus.go new file mode 100644 index 00000000000..acd9c96d76a --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/podgroupstatus.go @@ -0,0 +1,83 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" +) + +// PodGroupStatusApplyConfiguration represents a declarative configuration of the PodGroupStatus type for use +// with apply. +type PodGroupStatusApplyConfiguration struct { + Phase *schedulingv1beta1.PodGroupPhase `json:"phase,omitempty"` + Conditions []PodGroupConditionApplyConfiguration `json:"conditions,omitempty"` + Running *int32 `json:"running,omitempty"` + Succeeded *int32 `json:"succeeded,omitempty"` + Failed *int32 `json:"failed,omitempty"` +} + +// PodGroupStatusApplyConfiguration constructs a declarative configuration of the PodGroupStatus type for use with +// apply. +func PodGroupStatus() *PodGroupStatusApplyConfiguration { + return &PodGroupStatusApplyConfiguration{} +} + +// WithPhase sets the Phase field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Phase field is set to the value of the last call. +func (b *PodGroupStatusApplyConfiguration) WithPhase(value schedulingv1beta1.PodGroupPhase) *PodGroupStatusApplyConfiguration { + b.Phase = &value + return b +} + +// WithConditions adds the given value to the Conditions field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Conditions field. +func (b *PodGroupStatusApplyConfiguration) WithConditions(values ...*PodGroupConditionApplyConfiguration) *PodGroupStatusApplyConfiguration { + for i := range values { + if values[i] == nil { + panic("nil value passed to WithConditions") + } + b.Conditions = append(b.Conditions, *values[i]) + } + return b +} + +// WithRunning sets the Running field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Running field is set to the value of the last call. +func (b *PodGroupStatusApplyConfiguration) WithRunning(value int32) *PodGroupStatusApplyConfiguration { + b.Running = &value + return b +} + +// WithSucceeded sets the Succeeded field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Succeeded field is set to the value of the last call. +func (b *PodGroupStatusApplyConfiguration) WithSucceeded(value int32) *PodGroupStatusApplyConfiguration { + b.Succeeded = &value + return b +} + +// WithFailed sets the Failed field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Failed field is set to the value of the last call. +func (b *PodGroupStatusApplyConfiguration) WithFailed(value int32) *PodGroupStatusApplyConfiguration { + b.Failed = &value + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queue.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queue.go new file mode 100644 index 00000000000..5d2b5bc1985 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queue.go @@ -0,0 +1,223 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +// QueueApplyConfiguration represents a declarative configuration of the Queue type for use +// with apply. +type QueueApplyConfiguration struct { + v1.TypeMetaApplyConfiguration `json:",inline"` + *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Spec *QueueSpecApplyConfiguration `json:"spec,omitempty"` + Status *QueueStatusApplyConfiguration `json:"status,omitempty"` +} + +// Queue constructs a declarative configuration of the Queue type for use with +// apply. +func Queue(name string) *QueueApplyConfiguration { + b := &QueueApplyConfiguration{} + b.WithName(name) + b.WithKind("Queue") + b.WithAPIVersion("scheduling.volcano.sh/v1beta1") + return b +} + +// WithKind sets the Kind field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Kind field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithKind(value string) *QueueApplyConfiguration { + b.TypeMetaApplyConfiguration.Kind = &value + return b +} + +// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the APIVersion field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithAPIVersion(value string) *QueueApplyConfiguration { + b.TypeMetaApplyConfiguration.APIVersion = &value + return b +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithName(value string) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Name = &value + return b +} + +// WithGenerateName sets the GenerateName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the GenerateName field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithGenerateName(value string) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.GenerateName = &value + return b +} + +// WithNamespace sets the Namespace field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Namespace field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithNamespace(value string) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Namespace = &value + return b +} + +// WithUID sets the UID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UID field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithUID(value types.UID) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.UID = &value + return b +} + +// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceVersion field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithResourceVersion(value string) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.ResourceVersion = &value + return b +} + +// WithGeneration sets the Generation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Generation field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithGeneration(value int64) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Generation = &value + return b +} + +// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CreationTimestamp field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithCreationTimestamp(value metav1.Time) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.CreationTimestamp = &value + return b +} + +// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionTimestamp field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionTimestamp = &value + return b +} + +// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionGracePeriodSeconds = &value + return b +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *QueueApplyConfiguration) WithLabels(entries map[string]string) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Labels == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *QueueApplyConfiguration) WithAnnotations(entries map[string]string) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Annotations == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Annotations[k] = v + } + return b +} + +// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the OwnerReferences field. +func (b *QueueApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + if values[i] == nil { + panic("nil value passed to WithOwnerReferences") + } + b.ObjectMetaApplyConfiguration.OwnerReferences = append(b.ObjectMetaApplyConfiguration.OwnerReferences, *values[i]) + } + return b +} + +// WithFinalizers adds the given value to the Finalizers field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Finalizers field. +func (b *QueueApplyConfiguration) WithFinalizers(values ...string) *QueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + b.ObjectMetaApplyConfiguration.Finalizers = append(b.ObjectMetaApplyConfiguration.Finalizers, values[i]) + } + return b +} + +func (b *QueueApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { + if b.ObjectMetaApplyConfiguration == nil { + b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} + } +} + +// WithSpec sets the Spec field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Spec field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithSpec(value *QueueSpecApplyConfiguration) *QueueApplyConfiguration { + b.Spec = value + return b +} + +// WithStatus sets the Status field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Status field is set to the value of the last call. +func (b *QueueApplyConfiguration) WithStatus(value *QueueStatusApplyConfiguration) *QueueApplyConfiguration { + b.Status = value + return b +} + +// GetName retrieves the value of the Name field in the declarative configuration. +func (b *QueueApplyConfiguration) GetName() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Name +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queuespec.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queuespec.go new file mode 100644 index 00000000000..2282f05b6fd --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queuespec.go @@ -0,0 +1,128 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" +) + +// QueueSpecApplyConfiguration represents a declarative configuration of the QueueSpec type for use +// with apply. +type QueueSpecApplyConfiguration struct { + Weight *int32 `json:"weight,omitempty"` + Capability *v1.ResourceList `json:"capability,omitempty"` + Reclaimable *bool `json:"reclaimable,omitempty"` + ExtendClusters []ClusterApplyConfiguration `json:"extendClusters,omitempty"` + Guarantee *GuaranteeApplyConfiguration `json:"guarantee,omitempty"` + Affinity *AffinityApplyConfiguration `json:"affinity,omitempty"` + Type *string `json:"type,omitempty"` + Parent *string `json:"parent,omitempty"` + Deserved *v1.ResourceList `json:"deserved,omitempty"` + Priority *int32 `json:"priority,omitempty"` +} + +// QueueSpecApplyConfiguration constructs a declarative configuration of the QueueSpec type for use with +// apply. +func QueueSpec() *QueueSpecApplyConfiguration { + return &QueueSpecApplyConfiguration{} +} + +// WithWeight sets the Weight field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Weight field is set to the value of the last call. +func (b *QueueSpecApplyConfiguration) WithWeight(value int32) *QueueSpecApplyConfiguration { + b.Weight = &value + return b +} + +// WithCapability sets the Capability field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Capability field is set to the value of the last call. +func (b *QueueSpecApplyConfiguration) WithCapability(value v1.ResourceList) *QueueSpecApplyConfiguration { + b.Capability = &value + return b +} + +// WithReclaimable sets the Reclaimable field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Reclaimable field is set to the value of the last call. +func (b *QueueSpecApplyConfiguration) WithReclaimable(value bool) *QueueSpecApplyConfiguration { + b.Reclaimable = &value + return b +} + +// WithExtendClusters adds the given value to the ExtendClusters field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the ExtendClusters field. +func (b *QueueSpecApplyConfiguration) WithExtendClusters(values ...*ClusterApplyConfiguration) *QueueSpecApplyConfiguration { + for i := range values { + if values[i] == nil { + panic("nil value passed to WithExtendClusters") + } + b.ExtendClusters = append(b.ExtendClusters, *values[i]) + } + return b +} + +// WithGuarantee sets the Guarantee field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Guarantee field is set to the value of the last call. +func (b *QueueSpecApplyConfiguration) WithGuarantee(value *GuaranteeApplyConfiguration) *QueueSpecApplyConfiguration { + b.Guarantee = value + return b +} + +// WithAffinity sets the Affinity field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Affinity field is set to the value of the last call. +func (b *QueueSpecApplyConfiguration) WithAffinity(value *AffinityApplyConfiguration) *QueueSpecApplyConfiguration { + b.Affinity = value + return b +} + +// WithType sets the Type field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Type field is set to the value of the last call. +func (b *QueueSpecApplyConfiguration) WithType(value string) *QueueSpecApplyConfiguration { + b.Type = &value + return b +} + +// WithParent sets the Parent field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Parent field is set to the value of the last call. +func (b *QueueSpecApplyConfiguration) WithParent(value string) *QueueSpecApplyConfiguration { + b.Parent = &value + return b +} + +// WithDeserved sets the Deserved field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Deserved field is set to the value of the last call. +func (b *QueueSpecApplyConfiguration) WithDeserved(value v1.ResourceList) *QueueSpecApplyConfiguration { + b.Deserved = &value + return b +} + +// WithPriority sets the Priority field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Priority field is set to the value of the last call. +func (b *QueueSpecApplyConfiguration) WithPriority(value int32) *QueueSpecApplyConfiguration { + b.Priority = &value + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queuestatus.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queuestatus.go new file mode 100644 index 00000000000..615866decc8 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/queuestatus.go @@ -0,0 +1,106 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" +) + +// QueueStatusApplyConfiguration represents a declarative configuration of the QueueStatus type for use +// with apply. +type QueueStatusApplyConfiguration struct { + State *schedulingv1beta1.QueueState `json:"state,omitempty"` + Unknown *int32 `json:"unknown,omitempty"` + Pending *int32 `json:"pending,omitempty"` + Running *int32 `json:"running,omitempty"` + Inqueue *int32 `json:"inqueue,omitempty"` + Completed *int32 `json:"completed,omitempty"` + Reservation *ReservationApplyConfiguration `json:"reservation,omitempty"` + Allocated *v1.ResourceList `json:"allocated,omitempty"` +} + +// QueueStatusApplyConfiguration constructs a declarative configuration of the QueueStatus type for use with +// apply. +func QueueStatus() *QueueStatusApplyConfiguration { + return &QueueStatusApplyConfiguration{} +} + +// WithState sets the State field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the State field is set to the value of the last call. +func (b *QueueStatusApplyConfiguration) WithState(value schedulingv1beta1.QueueState) *QueueStatusApplyConfiguration { + b.State = &value + return b +} + +// WithUnknown sets the Unknown field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Unknown field is set to the value of the last call. +func (b *QueueStatusApplyConfiguration) WithUnknown(value int32) *QueueStatusApplyConfiguration { + b.Unknown = &value + return b +} + +// WithPending sets the Pending field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Pending field is set to the value of the last call. +func (b *QueueStatusApplyConfiguration) WithPending(value int32) *QueueStatusApplyConfiguration { + b.Pending = &value + return b +} + +// WithRunning sets the Running field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Running field is set to the value of the last call. +func (b *QueueStatusApplyConfiguration) WithRunning(value int32) *QueueStatusApplyConfiguration { + b.Running = &value + return b +} + +// WithInqueue sets the Inqueue field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Inqueue field is set to the value of the last call. +func (b *QueueStatusApplyConfiguration) WithInqueue(value int32) *QueueStatusApplyConfiguration { + b.Inqueue = &value + return b +} + +// WithCompleted sets the Completed field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Completed field is set to the value of the last call. +func (b *QueueStatusApplyConfiguration) WithCompleted(value int32) *QueueStatusApplyConfiguration { + b.Completed = &value + return b +} + +// WithReservation sets the Reservation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Reservation field is set to the value of the last call. +func (b *QueueStatusApplyConfiguration) WithReservation(value *ReservationApplyConfiguration) *QueueStatusApplyConfiguration { + b.Reservation = value + return b +} + +// WithAllocated sets the Allocated field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Allocated field is set to the value of the last call. +func (b *QueueStatusApplyConfiguration) WithAllocated(value v1.ResourceList) *QueueStatusApplyConfiguration { + b.Allocated = &value + return b +} diff --git a/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/reservation.go b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/reservation.go new file mode 100644 index 00000000000..cf1fc3f9ec6 --- /dev/null +++ b/vendor/volcano.sh/apis/pkg/client/applyconfiguration/scheduling/v1beta1/reservation.go @@ -0,0 +1,53 @@ +/* +Copyright The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" +) + +// ReservationApplyConfiguration represents a declarative configuration of the Reservation type for use +// with apply. +type ReservationApplyConfiguration struct { + Nodes []string `json:"nodes,omitempty"` + Resource *v1.ResourceList `json:"resource,omitempty"` +} + +// ReservationApplyConfiguration constructs a declarative configuration of the Reservation type for use with +// apply. +func Reservation() *ReservationApplyConfiguration { + return &ReservationApplyConfiguration{} +} + +// WithNodes adds the given value to the Nodes field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Nodes field. +func (b *ReservationApplyConfiguration) WithNodes(values ...string) *ReservationApplyConfiguration { + for i := range values { + b.Nodes = append(b.Nodes, values[i]) + } + return b +} + +// WithResource sets the Resource field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Resource field is set to the value of the last call. +func (b *ReservationApplyConfiguration) WithResource(value v1.ResourceList) *ReservationApplyConfiguration { + b.Resource = &value + return b +} From a416c2e6ce61bc97522cefda296c53f6baf2b1bf Mon Sep 17 00:00:00 2001 From: Harshal Patil <12152047+harche@users.noreply.github.com> Date: Fri, 24 Oct 2025 05:29:37 -0400 Subject: [PATCH 021/119] Add validation for unsupported DRA features (#7226) Signed-off-by: Harshal Patil <12152047+harche@users.noreply.github.com> --- .../core/workload_controller_test.go | 21 +- pkg/dra/claims.go | 72 +++-- pkg/dra/claims_test.go | 159 +++++++++-- .../singlecluster/controller/dra/dra_test.go | 262 ++++++++++++++++++ 4 files changed, 460 insertions(+), 54 deletions(-) diff --git a/pkg/controller/core/workload_controller_test.go b/pkg/controller/core/workload_controller_test.go index b3376d1bf2a..a1878977dda 100644 --- a/pkg/controller/core/workload_controller_test.go +++ b/pkg/controller/core/workload_controller_test.go @@ -20,6 +20,7 @@ import ( "context" stderrors "errors" "fmt" + "strings" "testing" "time" @@ -415,6 +416,7 @@ func TestReconcile(t *testing.T) { wantWorkload *kueue.Workload wantWorkloadUseMergePatch *kueue.Workload // workload version to compensate for the difference between use of Apply and Merge patch in FakeClient wantError error + wantErrorMsg string wantEvents []utiltesting.EventRecord wantResult reconcile.Result reconcilerOpts []Option @@ -574,8 +576,8 @@ func TestReconcile(t *testing.T) { } return wl }(), - wantError: dra.ErrDeviceClassNotMapped, - wantEvents: nil, + wantErrorMsg: "DeviceClass is not mapped in DRA configuration", + wantEvents: nil, }, "reconcile DRA ResourceClaimTemplate not found should return error": { enableDRAFeature: true, @@ -609,8 +611,8 @@ func TestReconcile(t *testing.T) { Message: `failed to get claim spec for ResourceClaimTemplate missing-template in workload wlMissingTemplate podset main: failed to get claim spec: resourceclaimtemplates.resource.k8s.io "missing-template" not found`, }). Obj(), - wantError: dra.ErrClaimSpecNotFound, - wantEvents: nil, + wantErrorMsg: "failed to get claim spec", + wantEvents: nil, }, "assign Admission Checks from ClusterQueue.spec.AdmissionCheckStrategy": { workload: utiltestingapi.MakeWorkload("wl", "ns"). @@ -2485,13 +2487,20 @@ func TestReconcile(t *testing.T) { gotResult, gotError := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(testWl)}) - if tc.wantError != nil { + switch { + case tc.wantError != nil: if gotError == nil { t.Errorf("expected error %v, got nil", tc.wantError) } else if !stderrors.Is(gotError, tc.wantError) { t.Errorf("unexpected error type: want %v, got %v", tc.wantError, gotError) } - } else if gotError != nil { + case tc.wantErrorMsg != "": + if gotError == nil { + t.Errorf("expected error containing %q, got nil", tc.wantErrorMsg) + } else if !strings.Contains(gotError.Error(), tc.wantErrorMsg) { + t.Errorf("expected error containing %q, got %v", tc.wantErrorMsg, gotError) + } + case gotError != nil: t.Errorf("unexpected error: %v", gotError) } diff --git a/pkg/dra/claims.go b/pkg/dra/claims.go index a647ae91240..0781f204347 100644 --- a/pkg/dra/claims.go +++ b/pkg/dra/claims.go @@ -33,38 +33,69 @@ import ( ) var ( - ErrDeviceClassNotMapped = errors.New("DeviceClass is not mapped in DRA configuration") - ErrResourceClaimInUse = errors.New("ResourceClaim is in use") - ErrClaimSpecNotFound = errors.New("failed to get claim spec") + errDeviceClassNotMapped = errors.New("DeviceClass is not mapped in DRA configuration") + errClaimSpecNotFound = errors.New("failed to get claim spec") + errUnsupportedDRAFeature = errors.New("unsupported DRA feature") + + // Specific unsupported DRA feature errors + errUnsupportedDRADeviceConstraints = errors.New("device constraints (MatchAttribute) are not supported") + errUnsupportedDRADeviceConfig = errors.New("device config is not supported") + errUnsupportedDRAFirstAvailable = errors.New("FirstAvailable device selection is not supported") + errUnsupportedDRACELSelectors = errors.New("CEL selectors are not supported") + errUnsupportedDRAAdminAccess = errors.New("AdminAccess is not supported") + errUnsupportedDRAAllocationModeAll = errors.New("AllocationMode 'All' is not supported") ) // countDevicesPerClass returns a resources.Requests representing the // total number of devices requested for each DeviceClass inside the provided -// ResourceClaimSpec. -func countDevicesPerClass(claimSpec *resourcev1.ResourceClaimSpec) resources.Requests { +// ResourceClaimSpec. It validates that only supported DRA features are used +// and returns an error if unsupported features are detected. +func countDevicesPerClass(claimSpec *resourcev1.ResourceClaimSpec) (resources.Requests, error) { out := resources.Requests{} if claimSpec == nil { - return out + return out, nil + } + + // Check for unsupported device constraints + if len(claimSpec.Devices.Constraints) > 0 { + return nil, errUnsupportedDRADeviceConstraints } + + // Check for unsupported device config + if len(claimSpec.Devices.Config) > 0 { + return nil, errUnsupportedDRADeviceConfig + } + for _, req := range claimSpec.Devices.Requests { - // v1beta2 DeviceRequest has Exactly or FirstAvailable. For Step 1, we + // v1 DeviceRequest has Exactly or FirstAvailable. For Step 1, we // preserve existing semantics by only supporting Exactly with Count. var dcName string var q int64 - if req.Exactly != nil { + if req.FirstAvailable != nil { + return nil, errUnsupportedDRAFirstAvailable + } + + switch { + case len(req.Exactly.Selectors) > 0: + return nil, errUnsupportedDRACELSelectors + case req.Exactly.AdminAccess != nil && *req.Exactly.AdminAccess: + return nil, errUnsupportedDRAAdminAccess + case req.Exactly.AllocationMode == resourcev1.DeviceAllocationModeAll: + return nil, errUnsupportedDRAAllocationModeAll + case req.Exactly.AllocationMode == resourcev1.DeviceAllocationModeExactCount: dcName = req.Exactly.DeviceClassName - if req.Exactly.AllocationMode == resourcev1.DeviceAllocationModeExactCount { - q = req.Exactly.Count - } + q = req.Exactly.Count + default: + return nil, fmt.Errorf("%w: unsupported allocation mode: %s", errUnsupportedDRAFeature, req.Exactly.AllocationMode) } + dc := corev1.ResourceName(dcName) if dc == "" { continue } - // TODO: handle other Allocation modes out[dc] += q } - return out + return out, nil } // getClaimSpec resolves the ResourceClaim(Template) referenced by the PodResourceClaim @@ -93,8 +124,8 @@ func getClaimSpec(ctx context.Context, cl client.Client, namespace string, prc c // converts DeviceClass counts into logical resources using the provided lookup function and // returns the aggregated quantities per PodSet. // -// If at least one DeviceClass is not present in the DRA configuration the function -// returns an error. +// If at least one DeviceClass is not present in the DRA configuration or if unsupported DRA +// features are detected, the function returns an error. func GetResourceRequestsForResourceClaimTemplates( ctx context.Context, cl client.Client, @@ -110,16 +141,21 @@ func GetResourceRequestsForResourceClaimTemplates( } spec, err := getClaimSpec(ctx, cl, wl.Namespace, prc) if err != nil { - return nil, fmt.Errorf("failed to get claim spec for ResourceClaimTemplate %s in workload %s podset %s: %w", *prc.ResourceClaimTemplateName, wl.Name, ps.Name, fmt.Errorf("%w: %v", ErrClaimSpecNotFound, err)) + return nil, fmt.Errorf("failed to get claim spec for ResourceClaimTemplate %s in workload %s podset %s: %w", *prc.ResourceClaimTemplateName, wl.Name, ps.Name, fmt.Errorf("%w: %v", errClaimSpecNotFound, err)) } if spec == nil { continue } - for dc, qty := range countDevicesPerClass(spec) { + deviceCounts, err := countDevicesPerClass(spec) + if err != nil { + return nil, fmt.Errorf("unsupported DRA feature in ResourceClaimTemplate %s in workload %s podset %s: %w", *prc.ResourceClaimTemplateName, wl.Name, ps.Name, err) + } + + for dc, qty := range deviceCounts { logical, found := Mapper().lookup(dc) if !found { - return nil, fmt.Errorf("DeviceClass %s is not mapped in DRA configuration for workload %s podset %s: %w", dc, wl.Name, ps.Name, ErrDeviceClassNotMapped) + return nil, fmt.Errorf("DeviceClass %s is not mapped in DRA configuration for workload %s podset %s: %w", dc, wl.Name, ps.Name, errDeviceClassNotMapped) } aggregated = utilresource.MergeResourceListKeepSum(aggregated, corev1.ResourceList{logical: resource.MustParse(strconv.FormatInt(qty, 10))}) } diff --git a/pkg/dra/claims_test.go b/pkg/dra/claims_test.go index 970784b21c5..dc722d049ec 100644 --- a/pkg/dra/claims_test.go +++ b/pkg/dra/claims_test.go @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package dra_test +package dra import ( + "errors" "reflect" "testing" @@ -29,7 +30,6 @@ import ( configapi "sigs.k8s.io/kueue/apis/config/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - "sigs.k8s.io/kueue/pkg/dra" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) @@ -61,13 +61,25 @@ func Test_GetResourceRequests(t *testing.T) { }, } + // Common lookup functions for reuse across test cases + defaultLookup := func(dc corev1.ResourceName) (corev1.ResourceName, bool) { + if dc == "test-deviceclass-1" { + return "res-1", true + } + return "", false + } + + noLookup := func(corev1.ResourceName) (corev1.ResourceName, bool) { + return "", false + } + tests := []struct { name string modifyWL func(w *kueue.Workload) extraObjects []runtime.Object lookup func(corev1.ResourceName) (corev1.ResourceName, bool) want map[kueue.PodSetReference]corev1.ResourceList - wantErr bool + wantErr error }{ { name: "Single claim template with single device", @@ -87,8 +99,8 @@ func Test_GetResourceRequests(t *testing.T) { }, { name: "Unmapped DeviceClass returns error", - lookup: func(corev1.ResourceName) (corev1.ResourceName, bool) { return "", false }, - wantErr: true, + lookup: noLookup, + wantErr: errDeviceClassNotMapped, }, { name: "Two containers each using different claim templates", @@ -149,13 +161,8 @@ func Test_GetResourceRequests(t *testing.T) { {Name: "req-a", ResourceClaimTemplateName: ptr.To("claim-tmpl-1")}, } }, - lookup: func(dc corev1.ResourceName) (corev1.ResourceName, bool) { - if dc == "test-deviceclass-1" { - return "res-1", true - } - return "", false - }, - want: map[kueue.PodSetReference]corev1.ResourceList{"main": {"res-1": resource.MustParse("2")}}, + lookup: defaultLookup, + want: map[kueue.PodSetReference]corev1.ResourceList{"main": {"res-1": resource.MustParse("2")}}, }, { name: "Single template requesting two devices", @@ -178,13 +185,8 @@ func Test_GetResourceRequests(t *testing.T) { {Name: "req-x", ResourceClaimTemplateName: ptr.To("claim-tmpl-3")}, } }, - lookup: func(dc corev1.ResourceName) (corev1.ResourceName, bool) { - if dc == "test-deviceclass-1" { - return "res-1", true - } - return "", false - }, - want: map[kueue.PodSetReference]corev1.ResourceList{"main": {"res-1": resource.MustParse("2")}}, + lookup: defaultLookup, + want: map[kueue.PodSetReference]corev1.ResourceList{"main": {"res-1": resource.MustParse("2")}}, }, { name: "Init and regular container sharing one template", @@ -211,13 +213,104 @@ func Test_GetResourceRequests(t *testing.T) { {Name: "rc", ResourceClaimTemplateName: ptr.To("claim-tmpl-1")}, } }, - lookup: func(dc corev1.ResourceName) (corev1.ResourceName, bool) { - if dc == "test-deviceclass-1" { - return "res-1", true + lookup: defaultLookup, + want: map[kueue.PodSetReference]corev1.ResourceList{"main": {"res-1": resource.MustParse("2")}}, + }, + { + name: "AllocationMode All returns error", + extraObjects: []runtime.Object{ + utiltesting.MakeResourceClaimTemplate("claim-tmpl-all", "ns1"). + DeviceRequest("device-request", "test-deviceclass-1", 0). + AllocationModeAll(). + Obj(), + }, + modifyWL: func(w *kueue.Workload) { + w.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: "req-all", ResourceClaimTemplateName: ptr.To("claim-tmpl-all")}, } - return "", false }, - want: map[kueue.PodSetReference]corev1.ResourceList{"main": {"res-1": resource.MustParse("2")}}, + lookup: defaultLookup, + wantErr: errUnsupportedDRAAllocationModeAll, + }, + { + name: "CEL selectors returns error", + extraObjects: []runtime.Object{ + utiltesting.MakeResourceClaimTemplate("claim-tmpl-cel", "ns1"). + DeviceRequest("req", "test-deviceclass-1", 1). + WithCELSelectors("device.driver == \"test-driver\""). + Obj(), + }, + modifyWL: func(w *kueue.Workload) { + w.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: "req-cel", ResourceClaimTemplateName: ptr.To("claim-tmpl-cel")}, + } + }, + lookup: defaultLookup, + wantErr: errUnsupportedDRACELSelectors, + }, + { + name: "Device constraints returns error", + extraObjects: []runtime.Object{ + utiltesting.MakeResourceClaimTemplate("claim-tmpl-constraints", "ns1"). + DeviceRequest("gpu-1", "test-deviceclass-1", 1). + DeviceRequest("gpu-2", "test-deviceclass-1", 1). + WithDeviceConstraints([]string{"gpu-1", "gpu-2"}, "numa-node"). + Obj(), + }, + modifyWL: func(w *kueue.Workload) { + w.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: "req-constraints", ResourceClaimTemplateName: ptr.To("claim-tmpl-constraints")}, + } + }, + lookup: defaultLookup, + wantErr: errUnsupportedDRADeviceConstraints, + }, + { + name: "FirstAvailable returns error", + extraObjects: []runtime.Object{ + utiltesting.MakeResourceClaimTemplate("claim-tmpl-first", "ns1"). + FirstAvailableRequest("req", "test-deviceclass-1"). + Obj(), + }, + modifyWL: func(w *kueue.Workload) { + w.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: "req-first", ResourceClaimTemplateName: ptr.To("claim-tmpl-first")}, + } + }, + lookup: defaultLookup, + wantErr: errUnsupportedDRAFirstAvailable, + }, + { + name: "AdminAccess returns error", + extraObjects: []runtime.Object{ + utiltesting.MakeResourceClaimTemplate("claim-tmpl-admin", "ns1"). + DeviceRequest("req", "test-deviceclass-1", 1). + WithAdminAccess(true). + Obj(), + }, + modifyWL: func(w *kueue.Workload) { + w.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: "req-admin", ResourceClaimTemplateName: ptr.To("claim-tmpl-admin")}, + } + }, + lookup: defaultLookup, + wantErr: errUnsupportedDRAAdminAccess, + }, + { + name: "Device config returns error", + extraObjects: []runtime.Object{ + utiltesting.MakeResourceClaimTemplate("claim-tmpl-config", "ns1"). + DeviceRequest("req", "test-deviceclass-1", 1). + WithDeviceConfig("req", "", nil). + Obj(), + }, + modifyWL: func(w *kueue.Workload) { + w.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + {Name: "req-config", ResourceClaimTemplateName: ptr.To("claim-tmpl-config")}, + } + }, + lookup: defaultLookup, + wantErr: errUnsupportedDRADeviceConfig, }, } @@ -237,7 +330,7 @@ func Test_GetResourceRequests(t *testing.T) { if tc.name == "Unmapped DeviceClass returns error" { mappings = []configapi.DeviceClassMapping{} } - err := dra.CreateMapperFromConfiguration(mappings) + err := CreateMapperFromConfiguration(mappings) if err != nil { t.Fatalf("Failed to initialize DRA mapper: %v", err) } @@ -257,13 +350,19 @@ func Test_GetResourceRequests(t *testing.T) { } ctx, _ := utiltesting.ContextWithLog(t) - got, err := dra.GetResourceRequestsForResourceClaimTemplates(ctx, baseClient, wlCopy) - if (err != nil) != tc.wantErr { - t.Fatalf("unexpected error status: gotErr=%v wantErr=%v, err=%v", err != nil, tc.wantErr, err) - } - if tc.wantErr { + got, err := GetResourceRequestsForResourceClaimTemplates(ctx, baseClient, wlCopy) + if tc.wantErr != nil { + if err == nil { + t.Fatalf("expected error but got none") + } + if !errors.Is(err, tc.wantErr) { + t.Fatalf("unexpected error: got=%v, want=%v", err, tc.wantErr) + } return } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } if !reflect.DeepEqual(got, tc.want) { t.Fatalf("unexpected result; got=%v want=%v", got, tc.want) } diff --git a/test/integration/singlecluster/controller/dra/dra_test.go b/test/integration/singlecluster/controller/dra/dra_test.go index 64c2bbb48a2..e7e4120c4ad 100644 --- a/test/integration/singlecluster/controller/dra/dra_test.go +++ b/test/integration/singlecluster/controller/dra/dra_test.go @@ -486,5 +486,267 @@ var _ = ginkgo.Describe("DRA Integration", ginkgo.Ordered, ginkgo.ContinueOnFail "DRA resource usage should be a multiple of pod count for webhook validation") }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) + + ginkgo.It("Should reject workload with AllocationMode 'All'", func() { + ginkgo.By("Creating a ResourceClaimTemplate with AllocationMode All") + rct := utiltesting.MakeResourceClaimTemplate("all-mode-template", ns.Name). + DeviceRequest("device-request", "foo.example.com", 0). + AllocationModeAll(). + Obj() + gomega.Expect(k8sClient.Create(ctx, rct)).To(gomega.Succeed()) + + ginkgo.By("Creating a workload with AllocationMode All") + wl := utiltestingapi.MakeWorkload("test-wl-all-mode", ns.Name). + Queue("test-lq"). + Obj() + wl.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + { + Name: "device-template", + ResourceClaimTemplateName: ptr.To("all-mode-template"), + }, + } + wl.Spec.PodSets[0].Template.Spec.Containers[0].Resources.Claims = []corev1.ResourceClaim{ + {Name: "device-template"}, + } + gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) + + ginkgo.By("Verifying workload is marked as inadmissible") + gomega.Eventually(func(g gomega.Gomega) { + var updatedWl kueue.Workload + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedWl)).To(gomega.Succeed()) + g.Expect(workload.HasQuotaReservation(&updatedWl)).To(gomega.BeFalse()) + + g.Expect(updatedWl.Status.Conditions).To(gomega.ContainElement(gomega.And( + gomega.HaveField("Type", kueue.WorkloadQuotaReserved), + gomega.HaveField("Status", metav1.ConditionFalse), + gomega.HaveField("Reason", kueue.WorkloadInadmissible), + gomega.HaveField("Message", gomega.ContainSubstring("AllocationMode 'All' is not supported")), + ))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.It("Should reject workload with CEL selectors", func() { + ginkgo.By("Creating a ResourceClaimTemplate with CEL selectors") + rct := utiltesting.MakeResourceClaimTemplate("cel-selector-template", ns.Name). + DeviceRequest("device-request", "foo.example.com", 2). + WithCELSelectors("device.driver == \"test-driver\""). + Obj() + gomega.Expect(k8sClient.Create(ctx, rct)).To(gomega.Succeed()) + + ginkgo.By("Creating a workload with CEL selectors") + wl := utiltestingapi.MakeWorkload("test-wl-cel-selector", ns.Name). + Queue("test-lq"). + Obj() + wl.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + { + Name: "device-template", + ResourceClaimTemplateName: ptr.To("cel-selector-template"), + }, + } + wl.Spec.PodSets[0].Template.Spec.Containers[0].Resources.Claims = []corev1.ResourceClaim{ + {Name: "device-template"}, + } + gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) + + ginkgo.By("Verifying workload is marked as inadmissible") + gomega.Eventually(func(g gomega.Gomega) { + var updatedWl kueue.Workload + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedWl)).To(gomega.Succeed()) + g.Expect(workload.HasQuotaReservation(&updatedWl)).To(gomega.BeFalse()) + + g.Expect(updatedWl.Status.Conditions).To(gomega.ContainElement(gomega.And( + gomega.HaveField("Type", kueue.WorkloadQuotaReserved), + gomega.HaveField("Status", metav1.ConditionFalse), + gomega.HaveField("Reason", kueue.WorkloadInadmissible), + gomega.HaveField("Message", gomega.ContainSubstring("CEL selectors are not supported")), + ))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.It("Should reject workload with device constraints", func() { + ginkgo.By("Creating a ResourceClaimTemplate with device constraints") + rct := utiltesting.MakeResourceClaimTemplate("constraint-template", ns.Name). + DeviceRequest("gpu-1", "foo.example.com", 1). + DeviceRequest("gpu-2", "foo.example.com", 1). + WithDeviceConstraints([]string{"gpu-1", "gpu-2"}, "example.com/numa_node"). + Obj() + gomega.Expect(k8sClient.Create(ctx, rct)).To(gomega.Succeed()) + + ginkgo.By("Creating a workload with device constraints") + wl := utiltestingapi.MakeWorkload("test-wl-constraint", ns.Name). + Queue("test-lq"). + Obj() + wl.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + { + Name: "device-template", + ResourceClaimTemplateName: ptr.To("constraint-template"), + }, + } + wl.Spec.PodSets[0].Template.Spec.Containers[0].Resources.Claims = []corev1.ResourceClaim{ + {Name: "device-template"}, + } + gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) + + ginkgo.By("Verifying workload is marked as inadmissible") + gomega.Eventually(func(g gomega.Gomega) { + var updatedWl kueue.Workload + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedWl)).To(gomega.Succeed()) + g.Expect(workload.HasQuotaReservation(&updatedWl)).To(gomega.BeFalse()) + + g.Expect(updatedWl.Status.Conditions).To(gomega.ContainElement(gomega.And( + gomega.HaveField("Type", kueue.WorkloadQuotaReserved), + gomega.HaveField("Status", metav1.ConditionFalse), + gomega.HaveField("Reason", kueue.WorkloadInadmissible), + gomega.HaveField("Message", gomega.ContainSubstring("device constraints (MatchAttribute) are not supported")), + ))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.It("Should reject workload with AdminAccess", func() { + ginkgo.By("Adding admin-access label to namespace") + var namespace corev1.Namespace + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: ns.Name}, &namespace)).To(gomega.Succeed()) + if namespace.Labels == nil { + namespace.Labels = make(map[string]string) + } + namespace.Labels["resource.kubernetes.io/admin-access"] = "true" + gomega.Expect(k8sClient.Update(ctx, &namespace)).To(gomega.Succeed()) + + ginkgo.By("Creating a ResourceClaimTemplate with AdminAccess") + rct := utiltesting.MakeResourceClaimTemplate("admin-access-template", ns.Name). + DeviceRequest("device-request", "foo.example.com", 2). + WithAdminAccess(true). + Obj() + gomega.Expect(k8sClient.Create(ctx, rct)).To(gomega.Succeed()) + + ginkgo.By("Creating a workload with AdminAccess") + wl := utiltestingapi.MakeWorkload("test-wl-admin-access", ns.Name). + Queue("test-lq"). + Obj() + wl.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + { + Name: "device-template", + ResourceClaimTemplateName: ptr.To("admin-access-template"), + }, + } + wl.Spec.PodSets[0].Template.Spec.Containers[0].Resources.Claims = []corev1.ResourceClaim{ + {Name: "device-template"}, + } + gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) + + ginkgo.By("Verifying workload is marked as inadmissible") + gomega.Eventually(func(g gomega.Gomega) { + var updatedWl kueue.Workload + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedWl)).To(gomega.Succeed()) + g.Expect(workload.HasQuotaReservation(&updatedWl)).To(gomega.BeFalse()) + + g.Expect(updatedWl.Status.Conditions).To(gomega.ContainElement(gomega.And( + gomega.HaveField("Type", kueue.WorkloadQuotaReserved), + gomega.HaveField("Status", metav1.ConditionFalse), + gomega.HaveField("Reason", kueue.WorkloadInadmissible), + gomega.HaveField("Message", gomega.ContainSubstring("AdminAccess is not supported")), + ))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.It("Should reject workload with device config", func() { + ginkgo.By("Creating a ResourceClaimTemplate with device config") + rct := utiltesting.MakeResourceClaimTemplate("device-config-template", ns.Name). + DeviceRequest("device-request", "foo.example.com", 2). + WithDeviceConfig("device-request", "driver.example.com", []byte(`{"key":"value"}`)). + Obj() + gomega.Expect(k8sClient.Create(ctx, rct)).To(gomega.Succeed()) + + ginkgo.By("Creating a workload with device config") + wl := utiltestingapi.MakeWorkload("test-wl-device-config", ns.Name). + Queue("test-lq"). + Obj() + wl.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + { + Name: "device-template", + ResourceClaimTemplateName: ptr.To("device-config-template"), + }, + } + wl.Spec.PodSets[0].Template.Spec.Containers[0].Resources.Claims = []corev1.ResourceClaim{ + {Name: "device-template"}, + } + gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) + + ginkgo.By("Verifying workload is marked as inadmissible") + gomega.Eventually(func(g gomega.Gomega) { + var updatedWl kueue.Workload + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedWl)).To(gomega.Succeed()) + g.Expect(workload.HasQuotaReservation(&updatedWl)).To(gomega.BeFalse()) + + g.Expect(updatedWl.Status.Conditions).To(gomega.ContainElement(gomega.And( + gomega.HaveField("Type", kueue.WorkloadQuotaReserved), + gomega.HaveField("Status", metav1.ConditionFalse), + gomega.HaveField("Reason", kueue.WorkloadInadmissible), + gomega.HaveField("Message", gomega.ContainSubstring("device config is not supported")), + ))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.It("Should reject workload with FirstAvailable", func() { + ginkgo.By("Creating a ResourceClaimTemplate with FirstAvailable") + rct := utiltesting.MakeResourceClaimTemplate("first-available-template", ns.Name). + FirstAvailableRequest("device-request", "foo.example.com"). + Obj() + gomega.Expect(k8sClient.Create(ctx, rct)).To(gomega.Succeed()) + + ginkgo.By("Creating a workload with FirstAvailable") + wl := utiltestingapi.MakeWorkload("test-wl-first-available", ns.Name). + Queue("test-lq"). + Obj() + wl.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + { + Name: "device-template", + ResourceClaimTemplateName: ptr.To("first-available-template"), + }, + } + wl.Spec.PodSets[0].Template.Spec.Containers[0].Resources.Claims = []corev1.ResourceClaim{ + {Name: "device-template"}, + } + gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) + + ginkgo.By("Verifying workload is marked as inadmissible") + gomega.Eventually(func(g gomega.Gomega) { + var updatedWl kueue.Workload + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedWl)).To(gomega.Succeed()) + g.Expect(workload.HasQuotaReservation(&updatedWl)).To(gomega.BeFalse()) + + g.Expect(updatedWl.Status.Conditions).To(gomega.ContainElement(gomega.And( + gomega.HaveField("Type", kueue.WorkloadQuotaReserved), + gomega.HaveField("Status", metav1.ConditionFalse), + gomega.HaveField("Reason", kueue.WorkloadInadmissible), + gomega.HaveField("Message", gomega.ContainSubstring("FirstAvailable device selection is not supported")), + ))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.It("Should admit workload with empty AllocationMode that defaults to ExactCount", func() { + ginkgo.By("Creating a ResourceClaimTemplate with empty AllocationMode") + rct := utiltesting.MakeResourceClaimTemplate("empty-mode-template", ns.Name). + DeviceRequest("device-request", "foo.example.com", 2). + Obj() + // Set AllocationMode to empty string explicitly + rct.Spec.Spec.Devices.Requests[0].Exactly.AllocationMode = "" + gomega.Expect(k8sClient.Create(ctx, rct)).To(gomega.Succeed()) + + ginkgo.By("Creating a workload that references the template with empty AllocationMode") + wl := utiltestingapi.MakeWorkload("test-wl-empty-mode", ns.Name). + Queue("test-lq"). + Obj() + wl.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{ + { + Name: "device-template", + ResourceClaimTemplateName: ptr.To("empty-mode-template"), + }, + } + wl.Spec.PodSets[0].Template.Spec.Containers[0].Resources.Claims = []corev1.ResourceClaim{ + {Name: "device-template"}, + } + gomega.Expect(k8sClient.Create(ctx, wl)).To(gomega.Succeed()) + }) }) }) From adca57b5d91e34df710a50908cf97ce88a361c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Paj=C4=85k?= Date: Fri, 24 Oct 2025 11:29:43 +0200 Subject: [PATCH 022/119] hotswap reschedule evicted (#7376) --- pkg/controller/jobframework/reconciler.go | 2 +- test/e2e/tas/hotswap_test.go | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index 7394058535f..5ae6435f555 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -545,7 +545,7 @@ func (r *JobReconciler) ReconcileGenericJob(ctx context.Context, req ctrl.Reques log.V(6).Info("The job is no longer active, clear the workloads admission") err := workload.PatchAdmissionStatus(ctx, r.client, wl, r.clock, func() (*kueue.Workload, bool, error) { // The requeued condition status set to true only on EvictedByPreemption - setRequeued := evCond.Reason == kueue.WorkloadEvictedByPreemption + setRequeued := (evCond.Reason == kueue.WorkloadEvictedByPreemption) || (evCond.Reason == kueue.WorkloadEvictedDueToNodeFailures) updated := workload.SetRequeuedCondition(wl, evCond.Reason, evCond.Message, setRequeued) if workload.UnsetQuotaReservationWithCondition(wl, "Pending", evCond.Message, r.clock.Now()) { updated = true diff --git a/test/e2e/tas/hotswap_test.go b/test/e2e/tas/hotswap_test.go index 914e3fae873..90e22290c78 100644 --- a/test/e2e/tas/hotswap_test.go +++ b/test/e2e/tas/hotswap_test.go @@ -25,7 +25,6 @@ import ( "github.com/onsi/gomega" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/utils/ptr" @@ -256,13 +255,10 @@ var _ = ginkgo.Describe("Hotswap for Topology Aware Scheduling", ginkgo.Ordered, nodeToRestore = node.DeepCopy() gomega.Expect(k8sClient.Delete(ctx, node)).To(gomega.Succeed()) }) - wl := &kueue.Workload{} - ginkgo.By("Check that the workload is evicted", func() { - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlKey, wl)).To(gomega.Succeed()) - g.Expect(wl.Status.Admission).To(gomega.BeNil()) - g.Expect(apimeta.IsStatusConditionTrue(wl.Status.Conditions, kueue.WorkloadEvicted)).To(gomega.BeTrue()) - }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + ginkgo.By("Check that workload is rescheduled to a different rack", func() { + expectWorkloadTopologyAssignment(ctx, k8sClient, wlKey, numPods, []string{ + "kind-worker3", "kind-worker4", + }) }) }) }) From ab74ebeee1ab0923120fc582b49a4b6c67e41c08 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 24 Oct 2025 15:47:37 +0530 Subject: [PATCH 023/119] v1beta2: graduate Config API (#7375) * Copy v1beta1 to v1beta2. * v1beta2: graduate Config API * Add conversion logic. * Keep one test with v1beta1 version. --- apis/config/v1beta1/doc.go | 21 + .../config/v1beta1/zz_generated.conversion.go | 855 ++++++++++++++++++ apis/config/v1beta2/configuration_types.go | 590 ++++++++++++ apis/config/v1beta2/defaults.go | 138 +++ apis/config/v1beta2/defaults_test.go | 663 ++++++++++++++ apis/config/v1beta2/doc.go | 20 + apis/config/v1beta2/groupversion_info.go | 48 + apis/config/v1beta2/zz_generated.deepcopy.go | 671 ++++++++++++++ apis/config/v1beta2/zz_generated.defaults.go | 37 + charts/kueue/values.yaml | 2 +- cmd/kueue/main.go | 4 +- .../manager/controller_manager_config.yaml | 2 +- keps/2941-DRA/README.md | 8 +- pkg/cache/queue/cluster_queue.go | 2 +- pkg/cache/queue/cluster_queue_test.go | 2 +- pkg/cache/queue/manager.go | 2 +- pkg/cache/scheduler/cache.go | 2 +- pkg/config/config.go | 2 +- pkg/config/config_test.go | 36 +- pkg/config/validation.go | 2 +- pkg/config/validation_test.go | 2 +- .../admissionchecks/multikueue/controllers.go | 2 +- .../multikueue/externalframeworks/config.go | 2 +- .../externalframeworks/config_test.go | 2 +- .../admissionchecks/multikueue/workload.go | 2 +- .../multikueue/workload_test.go | 2 +- .../core/admissioncheck_controller.go | 2 +- .../core/clusterqueue_controller.go | 2 +- pkg/controller/core/cohort_controller.go | 2 +- pkg/controller/core/core.go | 2 +- .../core/leader_aware_reconciler.go | 2 +- pkg/controller/core/localqueue_controller.go | 2 +- .../core/localqueue_controller_test.go | 2 +- .../core/resourceflavor_controller.go | 2 +- pkg/controller/core/workload_controller.go | 2 +- .../core/workload_controller_test.go | 2 +- pkg/controller/jobframework/reconciler.go | 2 +- .../jobframework/reconciler_test.go | 2 +- .../jobs/job/job_controller_test.go | 2 +- .../jobs/pod/pod_controller_test.go | 2 +- pkg/controller/jobs/pod/pod_webhook.go | 2 +- pkg/controller/jobs/pod/pod_webhook_test.go | 2 +- pkg/controller/tas/controllers.go | 2 +- pkg/controller/tas/node_failure_controller.go | 2 +- pkg/controller/tas/resource_flavor.go | 2 +- pkg/controller/tas/topology_controller.go | 2 +- pkg/controller/tas/topology_ungater.go | 2 +- .../workloaddispatcher/controllers.go | 2 +- .../incrementaldispatcher.go | 2 +- .../incrementaldispatcher_test.go | 2 +- pkg/dra/claims_test.go | 2 +- pkg/dra/mapper.go | 2 +- pkg/dra/mapper_test.go | 2 +- pkg/scheduler/preemption/preemption.go | 2 +- pkg/scheduler/preemption/preemption_test.go | 2 +- pkg/scheduler/scheduler.go | 2 +- pkg/scheduler/scheduler_afs_test.go | 2 +- pkg/scheduler/scheduler_test.go | 2 +- .../admission_fair_sharing.go | 2 +- pkg/util/cert/cert.go | 2 +- pkg/workload/workload.go | 2 +- pkg/workload/workload_test.go | 2 +- site/content/en/docs/concepts/preemption.md | 2 +- site/content/en/docs/installation/_index.md | 2 +- .../en/docs/reference/kueue-config.v1beta1.md | 269 ++---- .../tasks/manage/administer_cluster_quotas.md | 4 +- site/content/en/docs/tasks/run/deployment.md | 2 +- .../en/docs/tasks/run/leaderworkerset.md | 2 +- .../run/multikueue/external-frameworks.md | 2 +- site/content/en/docs/tasks/run/plain_pods.md | 4 +- site/content/en/docs/tasks/run/statefulset.md | 2 +- .../content/zh-CN/docs/concepts/preemption.md | 2 +- .../content/zh-CN/docs/installation/_index.md | 2 +- .../tasks/manage/administer_cluster_quotas.md | 4 +- .../zh-CN/docs/tasks/run/leaderworkerset.md | 2 +- .../zh-CN/docs/tasks/run/plain_pods.md | 4 +- .../zh-CN/docs/tasks/run/statefulset.md | 2 +- .../controller_manager_config.yaml | 2 +- test/e2e/config/certmanager/values.yaml | 2 +- .../managejobswithoutqueuename_test.go | 2 +- .../objectretentionpolicies_test.go | 2 +- .../podintegrationautoenablement_test.go | 2 +- test/e2e/customconfigs/reconcile_test.go | 2 +- test/e2e/customconfigs/suite_test.go | 4 +- .../customconfigs/waitforpodsready_test.go | 2 +- test/e2e/multikueue/e2e_test.go | 2 +- test/integration/framework/framework.go | 2 +- .../integration/multikueue/dispatcher_test.go | 2 +- .../multikueue/enabled_integration_test.go | 2 +- .../multikueue/external_job_test.go | 2 +- test/integration/multikueue/jobs_test.go | 2 +- test/integration/multikueue/no_gc_test.go | 2 +- test/integration/multikueue/setup_test.go | 2 +- test/integration/multikueue/suite_test.go | 2 +- .../provisioning/suite_test.go | 2 +- .../controller/core/suite_test.go | 2 +- .../core/workload_controller_test.go | 2 +- .../singlecluster/controller/dra/dra_test.go | 2 +- .../controller/dra/suite_test.go | 2 +- .../appwrapper/appwrapper_controller_test.go | 2 +- .../controller/jobs/appwrapper/suite_test.go | 2 +- .../jobs/jaxjob/jaxjob_controller_test.go | 2 +- .../controller/jobs/jaxjob/suite_test.go | 2 +- .../jobs/job/job_controller_test.go | 2 +- .../controller/jobs/job/suite_test.go | 2 +- .../jobs/jobset/jobset_controller_test.go | 2 +- .../controller/jobs/jobset/suite_test.go | 2 +- .../jobs/mpijob/mpijob_controller_test.go | 2 +- .../controller/jobs/mpijob/suite_test.go | 2 +- .../paddlejob/paddlejob_controller_test.go | 2 +- .../controller/jobs/paddlejob/suite_test.go | 2 +- .../jobs/pod/pod_controller_test.go | 2 +- .../controller/jobs/pod/suite_test.go | 2 +- .../pytorchjob/pytorchjob_controller_test.go | 2 +- .../controller/jobs/pytorchjob/suite_test.go | 2 +- .../raycluster/raycluster_controller_test.go | 2 +- .../controller/jobs/raycluster/suite_test.go | 2 +- .../jobs/rayjob/rayjob_controller_test.go | 2 +- .../controller/jobs/rayjob/suite_test.go | 2 +- .../controller/jobs/tfjob/suite_test.go | 2 +- .../jobs/tfjob/tfjob_controller_test.go | 2 +- .../controller/jobs/trainjob/suite_test.go | 2 +- .../controller/jobs/xgboostjob/suite_test.go | 2 +- .../xgboostjob/xgboostjob_controller_test.go | 2 +- .../singlecluster/conversion/suite_test.go | 2 +- .../singlecluster/importer/suite_test.go | 2 +- .../singlecluster/kueuectl/suite_test.go | 2 +- .../fairsharing/fair_sharing_test.go | 2 +- .../scheduler/fairsharing/suite_test.go | 2 +- .../scheduler/podsready/scheduler_test.go | 2 +- .../scheduler/podsready/suite_test.go | 2 +- .../singlecluster/scheduler/suite_test.go | 2 +- .../singlecluster/tas/suite_test.go | 2 +- .../singlecluster/webhook/core/suite_test.go | 2 +- .../scheduler/minimalkueue/main.go | 2 +- test/performance/scheduler/runner/main.go | 2 +- test/util/e2e.go | 2 +- 137 files changed, 3254 insertions(+), 364 deletions(-) create mode 100644 apis/config/v1beta1/doc.go create mode 100644 apis/config/v1beta1/zz_generated.conversion.go create mode 100644 apis/config/v1beta2/configuration_types.go create mode 100644 apis/config/v1beta2/defaults.go create mode 100644 apis/config/v1beta2/defaults_test.go create mode 100644 apis/config/v1beta2/doc.go create mode 100644 apis/config/v1beta2/groupversion_info.go create mode 100644 apis/config/v1beta2/zz_generated.deepcopy.go create mode 100644 apis/config/v1beta2/zz_generated.defaults.go diff --git a/apis/config/v1beta1/doc.go b/apis/config/v1beta1/doc.go new file mode 100644 index 00000000000..c419411d85f --- /dev/null +++ b/apis/config/v1beta1/doc.go @@ -0,0 +1,21 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +kubebuilder:object:generate=true +// +groupName=config.kueue.x-k8s.io +// +k8s:conversion-gen=sigs.k8s.io/kueue/apis/config/v1beta2 + +package v1beta1 diff --git a/apis/config/v1beta1/zz_generated.conversion.go b/apis/config/v1beta1/zz_generated.conversion.go new file mode 100644 index 00000000000..3e4c8f2e6bb --- /dev/null +++ b/apis/config/v1beta1/zz_generated.conversion.go @@ -0,0 +1,855 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by conversion-gen. DO NOT EDIT. + +package v1beta1 + +import ( + time "time" + unsafe "unsafe" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + conversion "k8s.io/apimachinery/pkg/conversion" + runtime "k8s.io/apimachinery/pkg/runtime" + v1alpha1 "k8s.io/component-base/config/v1alpha1" + v1beta2 "sigs.k8s.io/kueue/apis/config/v1beta2" +) + +func init() { + localSchemeBuilder.Register(RegisterConversions) +} + +// RegisterConversions adds conversion functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*AdmissionFairSharing)(nil), (*v1beta2.AdmissionFairSharing)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionFairSharing_To_v1beta2_AdmissionFairSharing(a.(*AdmissionFairSharing), b.(*v1beta2.AdmissionFairSharing), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionFairSharing)(nil), (*AdmissionFairSharing)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_AdmissionFairSharing_To_v1beta1_AdmissionFairSharing(a.(*v1beta2.AdmissionFairSharing), b.(*AdmissionFairSharing), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ClientConnection)(nil), (*v1beta2.ClientConnection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClientConnection_To_v1beta2_ClientConnection(a.(*ClientConnection), b.(*v1beta2.ClientConnection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClientConnection)(nil), (*ClientConnection)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClientConnection_To_v1beta1_ClientConnection(a.(*v1beta2.ClientConnection), b.(*ClientConnection), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ClusterQueueVisibility)(nil), (*v1beta2.ClusterQueueVisibility)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility(a.(*ClusterQueueVisibility), b.(*v1beta2.ClusterQueueVisibility), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueueVisibility)(nil), (*ClusterQueueVisibility)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility(a.(*v1beta2.ClusterQueueVisibility), b.(*ClusterQueueVisibility), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Configuration)(nil), (*v1beta2.Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Configuration_To_v1beta2_Configuration(a.(*Configuration), b.(*v1beta2.Configuration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.Configuration)(nil), (*Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_Configuration_To_v1beta1_Configuration(a.(*v1beta2.Configuration), b.(*Configuration), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ControllerConfigurationSpec)(nil), (*v1beta2.ControllerConfigurationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ControllerConfigurationSpec_To_v1beta2_ControllerConfigurationSpec(a.(*ControllerConfigurationSpec), b.(*v1beta2.ControllerConfigurationSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ControllerConfigurationSpec)(nil), (*ControllerConfigurationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ControllerConfigurationSpec_To_v1beta1_ControllerConfigurationSpec(a.(*v1beta2.ControllerConfigurationSpec), b.(*ControllerConfigurationSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ControllerHealth)(nil), (*v1beta2.ControllerHealth)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ControllerHealth_To_v1beta2_ControllerHealth(a.(*ControllerHealth), b.(*v1beta2.ControllerHealth), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ControllerHealth)(nil), (*ControllerHealth)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ControllerHealth_To_v1beta1_ControllerHealth(a.(*v1beta2.ControllerHealth), b.(*ControllerHealth), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ControllerManager)(nil), (*v1beta2.ControllerManager)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ControllerManager_To_v1beta2_ControllerManager(a.(*ControllerManager), b.(*v1beta2.ControllerManager), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ControllerManager)(nil), (*ControllerManager)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ControllerManager_To_v1beta1_ControllerManager(a.(*v1beta2.ControllerManager), b.(*ControllerManager), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ControllerMetrics)(nil), (*v1beta2.ControllerMetrics)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ControllerMetrics_To_v1beta2_ControllerMetrics(a.(*ControllerMetrics), b.(*v1beta2.ControllerMetrics), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ControllerMetrics)(nil), (*ControllerMetrics)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ControllerMetrics_To_v1beta1_ControllerMetrics(a.(*v1beta2.ControllerMetrics), b.(*ControllerMetrics), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ControllerWebhook)(nil), (*v1beta2.ControllerWebhook)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ControllerWebhook_To_v1beta2_ControllerWebhook(a.(*ControllerWebhook), b.(*v1beta2.ControllerWebhook), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ControllerWebhook)(nil), (*ControllerWebhook)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ControllerWebhook_To_v1beta1_ControllerWebhook(a.(*v1beta2.ControllerWebhook), b.(*ControllerWebhook), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*DeviceClassMapping)(nil), (*v1beta2.DeviceClassMapping)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_DeviceClassMapping_To_v1beta2_DeviceClassMapping(a.(*DeviceClassMapping), b.(*v1beta2.DeviceClassMapping), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.DeviceClassMapping)(nil), (*DeviceClassMapping)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_DeviceClassMapping_To_v1beta1_DeviceClassMapping(a.(*v1beta2.DeviceClassMapping), b.(*DeviceClassMapping), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*FairSharing)(nil), (*v1beta2.FairSharing)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_FairSharing_To_v1beta2_FairSharing(a.(*FairSharing), b.(*v1beta2.FairSharing), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.FairSharing)(nil), (*FairSharing)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_FairSharing_To_v1beta1_FairSharing(a.(*v1beta2.FairSharing), b.(*FairSharing), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Integrations)(nil), (*v1beta2.Integrations)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Integrations_To_v1beta2_Integrations(a.(*Integrations), b.(*v1beta2.Integrations), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.Integrations)(nil), (*Integrations)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_Integrations_To_v1beta1_Integrations(a.(*v1beta2.Integrations), b.(*Integrations), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*InternalCertManagement)(nil), (*v1beta2.InternalCertManagement)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_InternalCertManagement_To_v1beta2_InternalCertManagement(a.(*InternalCertManagement), b.(*v1beta2.InternalCertManagement), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.InternalCertManagement)(nil), (*InternalCertManagement)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_InternalCertManagement_To_v1beta1_InternalCertManagement(a.(*v1beta2.InternalCertManagement), b.(*InternalCertManagement), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*MultiKueue)(nil), (*v1beta2.MultiKueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MultiKueue_To_v1beta2_MultiKueue(a.(*MultiKueue), b.(*v1beta2.MultiKueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.MultiKueue)(nil), (*MultiKueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_MultiKueue_To_v1beta1_MultiKueue(a.(*v1beta2.MultiKueue), b.(*MultiKueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*MultiKueueExternalFramework)(nil), (*v1beta2.MultiKueueExternalFramework)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_MultiKueueExternalFramework_To_v1beta2_MultiKueueExternalFramework(a.(*MultiKueueExternalFramework), b.(*v1beta2.MultiKueueExternalFramework), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.MultiKueueExternalFramework)(nil), (*MultiKueueExternalFramework)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_MultiKueueExternalFramework_To_v1beta1_MultiKueueExternalFramework(a.(*v1beta2.MultiKueueExternalFramework), b.(*MultiKueueExternalFramework), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ObjectRetentionPolicies)(nil), (*v1beta2.ObjectRetentionPolicies)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ObjectRetentionPolicies_To_v1beta2_ObjectRetentionPolicies(a.(*ObjectRetentionPolicies), b.(*v1beta2.ObjectRetentionPolicies), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ObjectRetentionPolicies)(nil), (*ObjectRetentionPolicies)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ObjectRetentionPolicies_To_v1beta1_ObjectRetentionPolicies(a.(*v1beta2.ObjectRetentionPolicies), b.(*ObjectRetentionPolicies), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PodIntegrationOptions)(nil), (*v1beta2.PodIntegrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions(a.(*PodIntegrationOptions), b.(*v1beta2.PodIntegrationOptions), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.PodIntegrationOptions)(nil), (*PodIntegrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(a.(*v1beta2.PodIntegrationOptions), b.(*PodIntegrationOptions), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*QueueVisibility)(nil), (*v1beta2.QueueVisibility)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility(a.(*QueueVisibility), b.(*v1beta2.QueueVisibility), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.QueueVisibility)(nil), (*QueueVisibility)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility(a.(*v1beta2.QueueVisibility), b.(*QueueVisibility), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*RequeuingStrategy)(nil), (*v1beta2.RequeuingStrategy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_RequeuingStrategy_To_v1beta2_RequeuingStrategy(a.(*RequeuingStrategy), b.(*v1beta2.RequeuingStrategy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.RequeuingStrategy)(nil), (*RequeuingStrategy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_RequeuingStrategy_To_v1beta1_RequeuingStrategy(a.(*v1beta2.RequeuingStrategy), b.(*RequeuingStrategy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ResourceTransformation)(nil), (*v1beta2.ResourceTransformation)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ResourceTransformation_To_v1beta2_ResourceTransformation(a.(*ResourceTransformation), b.(*v1beta2.ResourceTransformation), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ResourceTransformation)(nil), (*ResourceTransformation)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ResourceTransformation_To_v1beta1_ResourceTransformation(a.(*v1beta2.ResourceTransformation), b.(*ResourceTransformation), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*Resources)(nil), (*v1beta2.Resources)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Resources_To_v1beta2_Resources(a.(*Resources), b.(*v1beta2.Resources), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.Resources)(nil), (*Resources)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_Resources_To_v1beta1_Resources(a.(*v1beta2.Resources), b.(*Resources), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*WaitForPodsReady)(nil), (*v1beta2.WaitForPodsReady)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_WaitForPodsReady_To_v1beta2_WaitForPodsReady(a.(*WaitForPodsReady), b.(*v1beta2.WaitForPodsReady), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.WaitForPodsReady)(nil), (*WaitForPodsReady)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_WaitForPodsReady_To_v1beta1_WaitForPodsReady(a.(*v1beta2.WaitForPodsReady), b.(*WaitForPodsReady), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*WorkloadRetentionPolicy)(nil), (*v1beta2.WorkloadRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_WorkloadRetentionPolicy_To_v1beta2_WorkloadRetentionPolicy(a.(*WorkloadRetentionPolicy), b.(*v1beta2.WorkloadRetentionPolicy), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.WorkloadRetentionPolicy)(nil), (*WorkloadRetentionPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_WorkloadRetentionPolicy_To_v1beta1_WorkloadRetentionPolicy(a.(*v1beta2.WorkloadRetentionPolicy), b.(*WorkloadRetentionPolicy), scope) + }); err != nil { + return err + } + return nil +} + +func autoConvert_v1beta1_AdmissionFairSharing_To_v1beta2_AdmissionFairSharing(in *AdmissionFairSharing, out *v1beta2.AdmissionFairSharing, s conversion.Scope) error { + out.UsageHalfLifeTime = in.UsageHalfLifeTime + out.UsageSamplingInterval = in.UsageSamplingInterval + out.ResourceWeights = *(*map[v1.ResourceName]float64)(unsafe.Pointer(&in.ResourceWeights)) + return nil +} + +// Convert_v1beta1_AdmissionFairSharing_To_v1beta2_AdmissionFairSharing is an autogenerated conversion function. +func Convert_v1beta1_AdmissionFairSharing_To_v1beta2_AdmissionFairSharing(in *AdmissionFairSharing, out *v1beta2.AdmissionFairSharing, s conversion.Scope) error { + return autoConvert_v1beta1_AdmissionFairSharing_To_v1beta2_AdmissionFairSharing(in, out, s) +} + +func autoConvert_v1beta2_AdmissionFairSharing_To_v1beta1_AdmissionFairSharing(in *v1beta2.AdmissionFairSharing, out *AdmissionFairSharing, s conversion.Scope) error { + out.UsageHalfLifeTime = in.UsageHalfLifeTime + out.UsageSamplingInterval = in.UsageSamplingInterval + out.ResourceWeights = *(*map[v1.ResourceName]float64)(unsafe.Pointer(&in.ResourceWeights)) + return nil +} + +// Convert_v1beta2_AdmissionFairSharing_To_v1beta1_AdmissionFairSharing is an autogenerated conversion function. +func Convert_v1beta2_AdmissionFairSharing_To_v1beta1_AdmissionFairSharing(in *v1beta2.AdmissionFairSharing, out *AdmissionFairSharing, s conversion.Scope) error { + return autoConvert_v1beta2_AdmissionFairSharing_To_v1beta1_AdmissionFairSharing(in, out, s) +} + +func autoConvert_v1beta1_ClientConnection_To_v1beta2_ClientConnection(in *ClientConnection, out *v1beta2.ClientConnection, s conversion.Scope) error { + out.QPS = (*float32)(unsafe.Pointer(in.QPS)) + out.Burst = (*int32)(unsafe.Pointer(in.Burst)) + return nil +} + +// Convert_v1beta1_ClientConnection_To_v1beta2_ClientConnection is an autogenerated conversion function. +func Convert_v1beta1_ClientConnection_To_v1beta2_ClientConnection(in *ClientConnection, out *v1beta2.ClientConnection, s conversion.Scope) error { + return autoConvert_v1beta1_ClientConnection_To_v1beta2_ClientConnection(in, out, s) +} + +func autoConvert_v1beta2_ClientConnection_To_v1beta1_ClientConnection(in *v1beta2.ClientConnection, out *ClientConnection, s conversion.Scope) error { + out.QPS = (*float32)(unsafe.Pointer(in.QPS)) + out.Burst = (*int32)(unsafe.Pointer(in.Burst)) + return nil +} + +// Convert_v1beta2_ClientConnection_To_v1beta1_ClientConnection is an autogenerated conversion function. +func Convert_v1beta2_ClientConnection_To_v1beta1_ClientConnection(in *v1beta2.ClientConnection, out *ClientConnection, s conversion.Scope) error { + return autoConvert_v1beta2_ClientConnection_To_v1beta1_ClientConnection(in, out, s) +} + +func autoConvert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility(in *ClusterQueueVisibility, out *v1beta2.ClusterQueueVisibility, s conversion.Scope) error { + out.MaxCount = in.MaxCount + return nil +} + +// Convert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility is an autogenerated conversion function. +func Convert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility(in *ClusterQueueVisibility, out *v1beta2.ClusterQueueVisibility, s conversion.Scope) error { + return autoConvert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility(in, out, s) +} + +func autoConvert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility(in *v1beta2.ClusterQueueVisibility, out *ClusterQueueVisibility, s conversion.Scope) error { + out.MaxCount = in.MaxCount + return nil +} + +// Convert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility is an autogenerated conversion function. +func Convert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility(in *v1beta2.ClusterQueueVisibility, out *ClusterQueueVisibility, s conversion.Scope) error { + return autoConvert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility(in, out, s) +} + +func autoConvert_v1beta1_Configuration_To_v1beta2_Configuration(in *Configuration, out *v1beta2.Configuration, s conversion.Scope) error { + out.Namespace = (*string)(unsafe.Pointer(in.Namespace)) + if err := Convert_v1beta1_ControllerManager_To_v1beta2_ControllerManager(&in.ControllerManager, &out.ControllerManager, s); err != nil { + return err + } + out.ManageJobsWithoutQueueName = in.ManageJobsWithoutQueueName + out.ManagedJobsNamespaceSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.ManagedJobsNamespaceSelector)) + out.InternalCertManagement = (*v1beta2.InternalCertManagement)(unsafe.Pointer(in.InternalCertManagement)) + out.WaitForPodsReady = (*v1beta2.WaitForPodsReady)(unsafe.Pointer(in.WaitForPodsReady)) + out.ClientConnection = (*v1beta2.ClientConnection)(unsafe.Pointer(in.ClientConnection)) + out.Integrations = (*v1beta2.Integrations)(unsafe.Pointer(in.Integrations)) + out.QueueVisibility = (*v1beta2.QueueVisibility)(unsafe.Pointer(in.QueueVisibility)) + out.MultiKueue = (*v1beta2.MultiKueue)(unsafe.Pointer(in.MultiKueue)) + out.FairSharing = (*v1beta2.FairSharing)(unsafe.Pointer(in.FairSharing)) + out.AdmissionFairSharing = (*v1beta2.AdmissionFairSharing)(unsafe.Pointer(in.AdmissionFairSharing)) + out.Resources = (*v1beta2.Resources)(unsafe.Pointer(in.Resources)) + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + out.ObjectRetentionPolicies = (*v1beta2.ObjectRetentionPolicies)(unsafe.Pointer(in.ObjectRetentionPolicies)) + return nil +} + +// Convert_v1beta1_Configuration_To_v1beta2_Configuration is an autogenerated conversion function. +func Convert_v1beta1_Configuration_To_v1beta2_Configuration(in *Configuration, out *v1beta2.Configuration, s conversion.Scope) error { + return autoConvert_v1beta1_Configuration_To_v1beta2_Configuration(in, out, s) +} + +func autoConvert_v1beta2_Configuration_To_v1beta1_Configuration(in *v1beta2.Configuration, out *Configuration, s conversion.Scope) error { + out.Namespace = (*string)(unsafe.Pointer(in.Namespace)) + if err := Convert_v1beta2_ControllerManager_To_v1beta1_ControllerManager(&in.ControllerManager, &out.ControllerManager, s); err != nil { + return err + } + out.ManageJobsWithoutQueueName = in.ManageJobsWithoutQueueName + out.ManagedJobsNamespaceSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.ManagedJobsNamespaceSelector)) + out.InternalCertManagement = (*InternalCertManagement)(unsafe.Pointer(in.InternalCertManagement)) + out.WaitForPodsReady = (*WaitForPodsReady)(unsafe.Pointer(in.WaitForPodsReady)) + out.ClientConnection = (*ClientConnection)(unsafe.Pointer(in.ClientConnection)) + out.Integrations = (*Integrations)(unsafe.Pointer(in.Integrations)) + out.QueueVisibility = (*QueueVisibility)(unsafe.Pointer(in.QueueVisibility)) + out.MultiKueue = (*MultiKueue)(unsafe.Pointer(in.MultiKueue)) + out.FairSharing = (*FairSharing)(unsafe.Pointer(in.FairSharing)) + out.AdmissionFairSharing = (*AdmissionFairSharing)(unsafe.Pointer(in.AdmissionFairSharing)) + out.Resources = (*Resources)(unsafe.Pointer(in.Resources)) + out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates)) + out.ObjectRetentionPolicies = (*ObjectRetentionPolicies)(unsafe.Pointer(in.ObjectRetentionPolicies)) + return nil +} + +// Convert_v1beta2_Configuration_To_v1beta1_Configuration is an autogenerated conversion function. +func Convert_v1beta2_Configuration_To_v1beta1_Configuration(in *v1beta2.Configuration, out *Configuration, s conversion.Scope) error { + return autoConvert_v1beta2_Configuration_To_v1beta1_Configuration(in, out, s) +} + +func autoConvert_v1beta1_ControllerConfigurationSpec_To_v1beta2_ControllerConfigurationSpec(in *ControllerConfigurationSpec, out *v1beta2.ControllerConfigurationSpec, s conversion.Scope) error { + out.GroupKindConcurrency = *(*map[string]int)(unsafe.Pointer(&in.GroupKindConcurrency)) + out.CacheSyncTimeout = (*time.Duration)(unsafe.Pointer(in.CacheSyncTimeout)) + return nil +} + +// Convert_v1beta1_ControllerConfigurationSpec_To_v1beta2_ControllerConfigurationSpec is an autogenerated conversion function. +func Convert_v1beta1_ControllerConfigurationSpec_To_v1beta2_ControllerConfigurationSpec(in *ControllerConfigurationSpec, out *v1beta2.ControllerConfigurationSpec, s conversion.Scope) error { + return autoConvert_v1beta1_ControllerConfigurationSpec_To_v1beta2_ControllerConfigurationSpec(in, out, s) +} + +func autoConvert_v1beta2_ControllerConfigurationSpec_To_v1beta1_ControllerConfigurationSpec(in *v1beta2.ControllerConfigurationSpec, out *ControllerConfigurationSpec, s conversion.Scope) error { + out.GroupKindConcurrency = *(*map[string]int)(unsafe.Pointer(&in.GroupKindConcurrency)) + out.CacheSyncTimeout = (*time.Duration)(unsafe.Pointer(in.CacheSyncTimeout)) + return nil +} + +// Convert_v1beta2_ControllerConfigurationSpec_To_v1beta1_ControllerConfigurationSpec is an autogenerated conversion function. +func Convert_v1beta2_ControllerConfigurationSpec_To_v1beta1_ControllerConfigurationSpec(in *v1beta2.ControllerConfigurationSpec, out *ControllerConfigurationSpec, s conversion.Scope) error { + return autoConvert_v1beta2_ControllerConfigurationSpec_To_v1beta1_ControllerConfigurationSpec(in, out, s) +} + +func autoConvert_v1beta1_ControllerHealth_To_v1beta2_ControllerHealth(in *ControllerHealth, out *v1beta2.ControllerHealth, s conversion.Scope) error { + out.HealthProbeBindAddress = in.HealthProbeBindAddress + out.ReadinessEndpointName = in.ReadinessEndpointName + out.LivenessEndpointName = in.LivenessEndpointName + return nil +} + +// Convert_v1beta1_ControllerHealth_To_v1beta2_ControllerHealth is an autogenerated conversion function. +func Convert_v1beta1_ControllerHealth_To_v1beta2_ControllerHealth(in *ControllerHealth, out *v1beta2.ControllerHealth, s conversion.Scope) error { + return autoConvert_v1beta1_ControllerHealth_To_v1beta2_ControllerHealth(in, out, s) +} + +func autoConvert_v1beta2_ControllerHealth_To_v1beta1_ControllerHealth(in *v1beta2.ControllerHealth, out *ControllerHealth, s conversion.Scope) error { + out.HealthProbeBindAddress = in.HealthProbeBindAddress + out.ReadinessEndpointName = in.ReadinessEndpointName + out.LivenessEndpointName = in.LivenessEndpointName + return nil +} + +// Convert_v1beta2_ControllerHealth_To_v1beta1_ControllerHealth is an autogenerated conversion function. +func Convert_v1beta2_ControllerHealth_To_v1beta1_ControllerHealth(in *v1beta2.ControllerHealth, out *ControllerHealth, s conversion.Scope) error { + return autoConvert_v1beta2_ControllerHealth_To_v1beta1_ControllerHealth(in, out, s) +} + +func autoConvert_v1beta1_ControllerManager_To_v1beta2_ControllerManager(in *ControllerManager, out *v1beta2.ControllerManager, s conversion.Scope) error { + if err := Convert_v1beta1_ControllerWebhook_To_v1beta2_ControllerWebhook(&in.Webhook, &out.Webhook, s); err != nil { + return err + } + out.LeaderElection = (*v1alpha1.LeaderElectionConfiguration)(unsafe.Pointer(in.LeaderElection)) + if err := Convert_v1beta1_ControllerMetrics_To_v1beta2_ControllerMetrics(&in.Metrics, &out.Metrics, s); err != nil { + return err + } + if err := Convert_v1beta1_ControllerHealth_To_v1beta2_ControllerHealth(&in.Health, &out.Health, s); err != nil { + return err + } + out.PprofBindAddress = in.PprofBindAddress + out.Controller = (*v1beta2.ControllerConfigurationSpec)(unsafe.Pointer(in.Controller)) + return nil +} + +// Convert_v1beta1_ControllerManager_To_v1beta2_ControllerManager is an autogenerated conversion function. +func Convert_v1beta1_ControllerManager_To_v1beta2_ControllerManager(in *ControllerManager, out *v1beta2.ControllerManager, s conversion.Scope) error { + return autoConvert_v1beta1_ControllerManager_To_v1beta2_ControllerManager(in, out, s) +} + +func autoConvert_v1beta2_ControllerManager_To_v1beta1_ControllerManager(in *v1beta2.ControllerManager, out *ControllerManager, s conversion.Scope) error { + if err := Convert_v1beta2_ControllerWebhook_To_v1beta1_ControllerWebhook(&in.Webhook, &out.Webhook, s); err != nil { + return err + } + out.LeaderElection = (*v1alpha1.LeaderElectionConfiguration)(unsafe.Pointer(in.LeaderElection)) + if err := Convert_v1beta2_ControllerMetrics_To_v1beta1_ControllerMetrics(&in.Metrics, &out.Metrics, s); err != nil { + return err + } + if err := Convert_v1beta2_ControllerHealth_To_v1beta1_ControllerHealth(&in.Health, &out.Health, s); err != nil { + return err + } + out.PprofBindAddress = in.PprofBindAddress + out.Controller = (*ControllerConfigurationSpec)(unsafe.Pointer(in.Controller)) + return nil +} + +// Convert_v1beta2_ControllerManager_To_v1beta1_ControllerManager is an autogenerated conversion function. +func Convert_v1beta2_ControllerManager_To_v1beta1_ControllerManager(in *v1beta2.ControllerManager, out *ControllerManager, s conversion.Scope) error { + return autoConvert_v1beta2_ControllerManager_To_v1beta1_ControllerManager(in, out, s) +} + +func autoConvert_v1beta1_ControllerMetrics_To_v1beta2_ControllerMetrics(in *ControllerMetrics, out *v1beta2.ControllerMetrics, s conversion.Scope) error { + out.BindAddress = in.BindAddress + out.EnableClusterQueueResources = in.EnableClusterQueueResources + return nil +} + +// Convert_v1beta1_ControllerMetrics_To_v1beta2_ControllerMetrics is an autogenerated conversion function. +func Convert_v1beta1_ControllerMetrics_To_v1beta2_ControllerMetrics(in *ControllerMetrics, out *v1beta2.ControllerMetrics, s conversion.Scope) error { + return autoConvert_v1beta1_ControllerMetrics_To_v1beta2_ControllerMetrics(in, out, s) +} + +func autoConvert_v1beta2_ControllerMetrics_To_v1beta1_ControllerMetrics(in *v1beta2.ControllerMetrics, out *ControllerMetrics, s conversion.Scope) error { + out.BindAddress = in.BindAddress + out.EnableClusterQueueResources = in.EnableClusterQueueResources + return nil +} + +// Convert_v1beta2_ControllerMetrics_To_v1beta1_ControllerMetrics is an autogenerated conversion function. +func Convert_v1beta2_ControllerMetrics_To_v1beta1_ControllerMetrics(in *v1beta2.ControllerMetrics, out *ControllerMetrics, s conversion.Scope) error { + return autoConvert_v1beta2_ControllerMetrics_To_v1beta1_ControllerMetrics(in, out, s) +} + +func autoConvert_v1beta1_ControllerWebhook_To_v1beta2_ControllerWebhook(in *ControllerWebhook, out *v1beta2.ControllerWebhook, s conversion.Scope) error { + out.Port = (*int)(unsafe.Pointer(in.Port)) + out.Host = in.Host + out.CertDir = in.CertDir + return nil +} + +// Convert_v1beta1_ControllerWebhook_To_v1beta2_ControllerWebhook is an autogenerated conversion function. +func Convert_v1beta1_ControllerWebhook_To_v1beta2_ControllerWebhook(in *ControllerWebhook, out *v1beta2.ControllerWebhook, s conversion.Scope) error { + return autoConvert_v1beta1_ControllerWebhook_To_v1beta2_ControllerWebhook(in, out, s) +} + +func autoConvert_v1beta2_ControllerWebhook_To_v1beta1_ControllerWebhook(in *v1beta2.ControllerWebhook, out *ControllerWebhook, s conversion.Scope) error { + out.Port = (*int)(unsafe.Pointer(in.Port)) + out.Host = in.Host + out.CertDir = in.CertDir + return nil +} + +// Convert_v1beta2_ControllerWebhook_To_v1beta1_ControllerWebhook is an autogenerated conversion function. +func Convert_v1beta2_ControllerWebhook_To_v1beta1_ControllerWebhook(in *v1beta2.ControllerWebhook, out *ControllerWebhook, s conversion.Scope) error { + return autoConvert_v1beta2_ControllerWebhook_To_v1beta1_ControllerWebhook(in, out, s) +} + +func autoConvert_v1beta1_DeviceClassMapping_To_v1beta2_DeviceClassMapping(in *DeviceClassMapping, out *v1beta2.DeviceClassMapping, s conversion.Scope) error { + out.Name = v1.ResourceName(in.Name) + out.DeviceClassNames = *(*[]v1.ResourceName)(unsafe.Pointer(&in.DeviceClassNames)) + return nil +} + +// Convert_v1beta1_DeviceClassMapping_To_v1beta2_DeviceClassMapping is an autogenerated conversion function. +func Convert_v1beta1_DeviceClassMapping_To_v1beta2_DeviceClassMapping(in *DeviceClassMapping, out *v1beta2.DeviceClassMapping, s conversion.Scope) error { + return autoConvert_v1beta1_DeviceClassMapping_To_v1beta2_DeviceClassMapping(in, out, s) +} + +func autoConvert_v1beta2_DeviceClassMapping_To_v1beta1_DeviceClassMapping(in *v1beta2.DeviceClassMapping, out *DeviceClassMapping, s conversion.Scope) error { + out.Name = v1.ResourceName(in.Name) + out.DeviceClassNames = *(*[]v1.ResourceName)(unsafe.Pointer(&in.DeviceClassNames)) + return nil +} + +// Convert_v1beta2_DeviceClassMapping_To_v1beta1_DeviceClassMapping is an autogenerated conversion function. +func Convert_v1beta2_DeviceClassMapping_To_v1beta1_DeviceClassMapping(in *v1beta2.DeviceClassMapping, out *DeviceClassMapping, s conversion.Scope) error { + return autoConvert_v1beta2_DeviceClassMapping_To_v1beta1_DeviceClassMapping(in, out, s) +} + +func autoConvert_v1beta1_FairSharing_To_v1beta2_FairSharing(in *FairSharing, out *v1beta2.FairSharing, s conversion.Scope) error { + out.Enable = in.Enable + out.PreemptionStrategies = *(*[]v1beta2.PreemptionStrategy)(unsafe.Pointer(&in.PreemptionStrategies)) + return nil +} + +// Convert_v1beta1_FairSharing_To_v1beta2_FairSharing is an autogenerated conversion function. +func Convert_v1beta1_FairSharing_To_v1beta2_FairSharing(in *FairSharing, out *v1beta2.FairSharing, s conversion.Scope) error { + return autoConvert_v1beta1_FairSharing_To_v1beta2_FairSharing(in, out, s) +} + +func autoConvert_v1beta2_FairSharing_To_v1beta1_FairSharing(in *v1beta2.FairSharing, out *FairSharing, s conversion.Scope) error { + out.Enable = in.Enable + out.PreemptionStrategies = *(*[]PreemptionStrategy)(unsafe.Pointer(&in.PreemptionStrategies)) + return nil +} + +// Convert_v1beta2_FairSharing_To_v1beta1_FairSharing is an autogenerated conversion function. +func Convert_v1beta2_FairSharing_To_v1beta1_FairSharing(in *v1beta2.FairSharing, out *FairSharing, s conversion.Scope) error { + return autoConvert_v1beta2_FairSharing_To_v1beta1_FairSharing(in, out, s) +} + +func autoConvert_v1beta1_Integrations_To_v1beta2_Integrations(in *Integrations, out *v1beta2.Integrations, s conversion.Scope) error { + out.Frameworks = *(*[]string)(unsafe.Pointer(&in.Frameworks)) + out.ExternalFrameworks = *(*[]string)(unsafe.Pointer(&in.ExternalFrameworks)) + out.PodOptions = (*v1beta2.PodIntegrationOptions)(unsafe.Pointer(in.PodOptions)) + out.LabelKeysToCopy = *(*[]string)(unsafe.Pointer(&in.LabelKeysToCopy)) + return nil +} + +// Convert_v1beta1_Integrations_To_v1beta2_Integrations is an autogenerated conversion function. +func Convert_v1beta1_Integrations_To_v1beta2_Integrations(in *Integrations, out *v1beta2.Integrations, s conversion.Scope) error { + return autoConvert_v1beta1_Integrations_To_v1beta2_Integrations(in, out, s) +} + +func autoConvert_v1beta2_Integrations_To_v1beta1_Integrations(in *v1beta2.Integrations, out *Integrations, s conversion.Scope) error { + out.Frameworks = *(*[]string)(unsafe.Pointer(&in.Frameworks)) + out.ExternalFrameworks = *(*[]string)(unsafe.Pointer(&in.ExternalFrameworks)) + out.PodOptions = (*PodIntegrationOptions)(unsafe.Pointer(in.PodOptions)) + out.LabelKeysToCopy = *(*[]string)(unsafe.Pointer(&in.LabelKeysToCopy)) + return nil +} + +// Convert_v1beta2_Integrations_To_v1beta1_Integrations is an autogenerated conversion function. +func Convert_v1beta2_Integrations_To_v1beta1_Integrations(in *v1beta2.Integrations, out *Integrations, s conversion.Scope) error { + return autoConvert_v1beta2_Integrations_To_v1beta1_Integrations(in, out, s) +} + +func autoConvert_v1beta1_InternalCertManagement_To_v1beta2_InternalCertManagement(in *InternalCertManagement, out *v1beta2.InternalCertManagement, s conversion.Scope) error { + out.Enable = (*bool)(unsafe.Pointer(in.Enable)) + out.WebhookServiceName = (*string)(unsafe.Pointer(in.WebhookServiceName)) + out.WebhookSecretName = (*string)(unsafe.Pointer(in.WebhookSecretName)) + return nil +} + +// Convert_v1beta1_InternalCertManagement_To_v1beta2_InternalCertManagement is an autogenerated conversion function. +func Convert_v1beta1_InternalCertManagement_To_v1beta2_InternalCertManagement(in *InternalCertManagement, out *v1beta2.InternalCertManagement, s conversion.Scope) error { + return autoConvert_v1beta1_InternalCertManagement_To_v1beta2_InternalCertManagement(in, out, s) +} + +func autoConvert_v1beta2_InternalCertManagement_To_v1beta1_InternalCertManagement(in *v1beta2.InternalCertManagement, out *InternalCertManagement, s conversion.Scope) error { + out.Enable = (*bool)(unsafe.Pointer(in.Enable)) + out.WebhookServiceName = (*string)(unsafe.Pointer(in.WebhookServiceName)) + out.WebhookSecretName = (*string)(unsafe.Pointer(in.WebhookSecretName)) + return nil +} + +// Convert_v1beta2_InternalCertManagement_To_v1beta1_InternalCertManagement is an autogenerated conversion function. +func Convert_v1beta2_InternalCertManagement_To_v1beta1_InternalCertManagement(in *v1beta2.InternalCertManagement, out *InternalCertManagement, s conversion.Scope) error { + return autoConvert_v1beta2_InternalCertManagement_To_v1beta1_InternalCertManagement(in, out, s) +} + +func autoConvert_v1beta1_MultiKueue_To_v1beta2_MultiKueue(in *MultiKueue, out *v1beta2.MultiKueue, s conversion.Scope) error { + out.GCInterval = (*metav1.Duration)(unsafe.Pointer(in.GCInterval)) + out.Origin = (*string)(unsafe.Pointer(in.Origin)) + out.WorkerLostTimeout = (*metav1.Duration)(unsafe.Pointer(in.WorkerLostTimeout)) + out.DispatcherName = (*string)(unsafe.Pointer(in.DispatcherName)) + out.ExternalFrameworks = *(*[]v1beta2.MultiKueueExternalFramework)(unsafe.Pointer(&in.ExternalFrameworks)) + return nil +} + +// Convert_v1beta1_MultiKueue_To_v1beta2_MultiKueue is an autogenerated conversion function. +func Convert_v1beta1_MultiKueue_To_v1beta2_MultiKueue(in *MultiKueue, out *v1beta2.MultiKueue, s conversion.Scope) error { + return autoConvert_v1beta1_MultiKueue_To_v1beta2_MultiKueue(in, out, s) +} + +func autoConvert_v1beta2_MultiKueue_To_v1beta1_MultiKueue(in *v1beta2.MultiKueue, out *MultiKueue, s conversion.Scope) error { + out.GCInterval = (*metav1.Duration)(unsafe.Pointer(in.GCInterval)) + out.Origin = (*string)(unsafe.Pointer(in.Origin)) + out.WorkerLostTimeout = (*metav1.Duration)(unsafe.Pointer(in.WorkerLostTimeout)) + out.DispatcherName = (*string)(unsafe.Pointer(in.DispatcherName)) + out.ExternalFrameworks = *(*[]MultiKueueExternalFramework)(unsafe.Pointer(&in.ExternalFrameworks)) + return nil +} + +// Convert_v1beta2_MultiKueue_To_v1beta1_MultiKueue is an autogenerated conversion function. +func Convert_v1beta2_MultiKueue_To_v1beta1_MultiKueue(in *v1beta2.MultiKueue, out *MultiKueue, s conversion.Scope) error { + return autoConvert_v1beta2_MultiKueue_To_v1beta1_MultiKueue(in, out, s) +} + +func autoConvert_v1beta1_MultiKueueExternalFramework_To_v1beta2_MultiKueueExternalFramework(in *MultiKueueExternalFramework, out *v1beta2.MultiKueueExternalFramework, s conversion.Scope) error { + out.Name = in.Name + return nil +} + +// Convert_v1beta1_MultiKueueExternalFramework_To_v1beta2_MultiKueueExternalFramework is an autogenerated conversion function. +func Convert_v1beta1_MultiKueueExternalFramework_To_v1beta2_MultiKueueExternalFramework(in *MultiKueueExternalFramework, out *v1beta2.MultiKueueExternalFramework, s conversion.Scope) error { + return autoConvert_v1beta1_MultiKueueExternalFramework_To_v1beta2_MultiKueueExternalFramework(in, out, s) +} + +func autoConvert_v1beta2_MultiKueueExternalFramework_To_v1beta1_MultiKueueExternalFramework(in *v1beta2.MultiKueueExternalFramework, out *MultiKueueExternalFramework, s conversion.Scope) error { + out.Name = in.Name + return nil +} + +// Convert_v1beta2_MultiKueueExternalFramework_To_v1beta1_MultiKueueExternalFramework is an autogenerated conversion function. +func Convert_v1beta2_MultiKueueExternalFramework_To_v1beta1_MultiKueueExternalFramework(in *v1beta2.MultiKueueExternalFramework, out *MultiKueueExternalFramework, s conversion.Scope) error { + return autoConvert_v1beta2_MultiKueueExternalFramework_To_v1beta1_MultiKueueExternalFramework(in, out, s) +} + +func autoConvert_v1beta1_ObjectRetentionPolicies_To_v1beta2_ObjectRetentionPolicies(in *ObjectRetentionPolicies, out *v1beta2.ObjectRetentionPolicies, s conversion.Scope) error { + out.Workloads = (*v1beta2.WorkloadRetentionPolicy)(unsafe.Pointer(in.Workloads)) + return nil +} + +// Convert_v1beta1_ObjectRetentionPolicies_To_v1beta2_ObjectRetentionPolicies is an autogenerated conversion function. +func Convert_v1beta1_ObjectRetentionPolicies_To_v1beta2_ObjectRetentionPolicies(in *ObjectRetentionPolicies, out *v1beta2.ObjectRetentionPolicies, s conversion.Scope) error { + return autoConvert_v1beta1_ObjectRetentionPolicies_To_v1beta2_ObjectRetentionPolicies(in, out, s) +} + +func autoConvert_v1beta2_ObjectRetentionPolicies_To_v1beta1_ObjectRetentionPolicies(in *v1beta2.ObjectRetentionPolicies, out *ObjectRetentionPolicies, s conversion.Scope) error { + out.Workloads = (*WorkloadRetentionPolicy)(unsafe.Pointer(in.Workloads)) + return nil +} + +// Convert_v1beta2_ObjectRetentionPolicies_To_v1beta1_ObjectRetentionPolicies is an autogenerated conversion function. +func Convert_v1beta2_ObjectRetentionPolicies_To_v1beta1_ObjectRetentionPolicies(in *v1beta2.ObjectRetentionPolicies, out *ObjectRetentionPolicies, s conversion.Scope) error { + return autoConvert_v1beta2_ObjectRetentionPolicies_To_v1beta1_ObjectRetentionPolicies(in, out, s) +} + +func autoConvert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions(in *PodIntegrationOptions, out *v1beta2.PodIntegrationOptions, s conversion.Scope) error { + out.NamespaceSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) + out.PodSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.PodSelector)) + return nil +} + +// Convert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions is an autogenerated conversion function. +func Convert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions(in *PodIntegrationOptions, out *v1beta2.PodIntegrationOptions, s conversion.Scope) error { + return autoConvert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions(in, out, s) +} + +func autoConvert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(in *v1beta2.PodIntegrationOptions, out *PodIntegrationOptions, s conversion.Scope) error { + out.NamespaceSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) + out.PodSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.PodSelector)) + return nil +} + +// Convert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions is an autogenerated conversion function. +func Convert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(in *v1beta2.PodIntegrationOptions, out *PodIntegrationOptions, s conversion.Scope) error { + return autoConvert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(in, out, s) +} + +func autoConvert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility(in *QueueVisibility, out *v1beta2.QueueVisibility, s conversion.Scope) error { + out.ClusterQueues = (*v1beta2.ClusterQueueVisibility)(unsafe.Pointer(in.ClusterQueues)) + out.UpdateIntervalSeconds = in.UpdateIntervalSeconds + return nil +} + +// Convert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility is an autogenerated conversion function. +func Convert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility(in *QueueVisibility, out *v1beta2.QueueVisibility, s conversion.Scope) error { + return autoConvert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility(in, out, s) +} + +func autoConvert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility(in *v1beta2.QueueVisibility, out *QueueVisibility, s conversion.Scope) error { + out.ClusterQueues = (*ClusterQueueVisibility)(unsafe.Pointer(in.ClusterQueues)) + out.UpdateIntervalSeconds = in.UpdateIntervalSeconds + return nil +} + +// Convert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility is an autogenerated conversion function. +func Convert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility(in *v1beta2.QueueVisibility, out *QueueVisibility, s conversion.Scope) error { + return autoConvert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility(in, out, s) +} + +func autoConvert_v1beta1_RequeuingStrategy_To_v1beta2_RequeuingStrategy(in *RequeuingStrategy, out *v1beta2.RequeuingStrategy, s conversion.Scope) error { + out.Timestamp = (*v1beta2.RequeuingTimestamp)(unsafe.Pointer(in.Timestamp)) + out.BackoffLimitCount = (*int32)(unsafe.Pointer(in.BackoffLimitCount)) + out.BackoffBaseSeconds = (*int32)(unsafe.Pointer(in.BackoffBaseSeconds)) + out.BackoffMaxSeconds = (*int32)(unsafe.Pointer(in.BackoffMaxSeconds)) + return nil +} + +// Convert_v1beta1_RequeuingStrategy_To_v1beta2_RequeuingStrategy is an autogenerated conversion function. +func Convert_v1beta1_RequeuingStrategy_To_v1beta2_RequeuingStrategy(in *RequeuingStrategy, out *v1beta2.RequeuingStrategy, s conversion.Scope) error { + return autoConvert_v1beta1_RequeuingStrategy_To_v1beta2_RequeuingStrategy(in, out, s) +} + +func autoConvert_v1beta2_RequeuingStrategy_To_v1beta1_RequeuingStrategy(in *v1beta2.RequeuingStrategy, out *RequeuingStrategy, s conversion.Scope) error { + out.Timestamp = (*RequeuingTimestamp)(unsafe.Pointer(in.Timestamp)) + out.BackoffLimitCount = (*int32)(unsafe.Pointer(in.BackoffLimitCount)) + out.BackoffBaseSeconds = (*int32)(unsafe.Pointer(in.BackoffBaseSeconds)) + out.BackoffMaxSeconds = (*int32)(unsafe.Pointer(in.BackoffMaxSeconds)) + return nil +} + +// Convert_v1beta2_RequeuingStrategy_To_v1beta1_RequeuingStrategy is an autogenerated conversion function. +func Convert_v1beta2_RequeuingStrategy_To_v1beta1_RequeuingStrategy(in *v1beta2.RequeuingStrategy, out *RequeuingStrategy, s conversion.Scope) error { + return autoConvert_v1beta2_RequeuingStrategy_To_v1beta1_RequeuingStrategy(in, out, s) +} + +func autoConvert_v1beta1_ResourceTransformation_To_v1beta2_ResourceTransformation(in *ResourceTransformation, out *v1beta2.ResourceTransformation, s conversion.Scope) error { + out.Input = v1.ResourceName(in.Input) + out.Strategy = (*v1beta2.ResourceTransformationStrategy)(unsafe.Pointer(in.Strategy)) + out.Outputs = *(*v1.ResourceList)(unsafe.Pointer(&in.Outputs)) + return nil +} + +// Convert_v1beta1_ResourceTransformation_To_v1beta2_ResourceTransformation is an autogenerated conversion function. +func Convert_v1beta1_ResourceTransformation_To_v1beta2_ResourceTransformation(in *ResourceTransformation, out *v1beta2.ResourceTransformation, s conversion.Scope) error { + return autoConvert_v1beta1_ResourceTransformation_To_v1beta2_ResourceTransformation(in, out, s) +} + +func autoConvert_v1beta2_ResourceTransformation_To_v1beta1_ResourceTransformation(in *v1beta2.ResourceTransformation, out *ResourceTransformation, s conversion.Scope) error { + out.Input = v1.ResourceName(in.Input) + out.Strategy = (*ResourceTransformationStrategy)(unsafe.Pointer(in.Strategy)) + out.Outputs = *(*v1.ResourceList)(unsafe.Pointer(&in.Outputs)) + return nil +} + +// Convert_v1beta2_ResourceTransformation_To_v1beta1_ResourceTransformation is an autogenerated conversion function. +func Convert_v1beta2_ResourceTransformation_To_v1beta1_ResourceTransformation(in *v1beta2.ResourceTransformation, out *ResourceTransformation, s conversion.Scope) error { + return autoConvert_v1beta2_ResourceTransformation_To_v1beta1_ResourceTransformation(in, out, s) +} + +func autoConvert_v1beta1_Resources_To_v1beta2_Resources(in *Resources, out *v1beta2.Resources, s conversion.Scope) error { + out.ExcludeResourcePrefixes = *(*[]string)(unsafe.Pointer(&in.ExcludeResourcePrefixes)) + out.Transformations = *(*[]v1beta2.ResourceTransformation)(unsafe.Pointer(&in.Transformations)) + out.DeviceClassMappings = *(*[]v1beta2.DeviceClassMapping)(unsafe.Pointer(&in.DeviceClassMappings)) + return nil +} + +// Convert_v1beta1_Resources_To_v1beta2_Resources is an autogenerated conversion function. +func Convert_v1beta1_Resources_To_v1beta2_Resources(in *Resources, out *v1beta2.Resources, s conversion.Scope) error { + return autoConvert_v1beta1_Resources_To_v1beta2_Resources(in, out, s) +} + +func autoConvert_v1beta2_Resources_To_v1beta1_Resources(in *v1beta2.Resources, out *Resources, s conversion.Scope) error { + out.ExcludeResourcePrefixes = *(*[]string)(unsafe.Pointer(&in.ExcludeResourcePrefixes)) + out.Transformations = *(*[]ResourceTransformation)(unsafe.Pointer(&in.Transformations)) + out.DeviceClassMappings = *(*[]DeviceClassMapping)(unsafe.Pointer(&in.DeviceClassMappings)) + return nil +} + +// Convert_v1beta2_Resources_To_v1beta1_Resources is an autogenerated conversion function. +func Convert_v1beta2_Resources_To_v1beta1_Resources(in *v1beta2.Resources, out *Resources, s conversion.Scope) error { + return autoConvert_v1beta2_Resources_To_v1beta1_Resources(in, out, s) +} + +func autoConvert_v1beta1_WaitForPodsReady_To_v1beta2_WaitForPodsReady(in *WaitForPodsReady, out *v1beta2.WaitForPodsReady, s conversion.Scope) error { + out.Enable = in.Enable + out.Timeout = (*metav1.Duration)(unsafe.Pointer(in.Timeout)) + out.BlockAdmission = (*bool)(unsafe.Pointer(in.BlockAdmission)) + out.RequeuingStrategy = (*v1beta2.RequeuingStrategy)(unsafe.Pointer(in.RequeuingStrategy)) + out.RecoveryTimeout = (*metav1.Duration)(unsafe.Pointer(in.RecoveryTimeout)) + return nil +} + +// Convert_v1beta1_WaitForPodsReady_To_v1beta2_WaitForPodsReady is an autogenerated conversion function. +func Convert_v1beta1_WaitForPodsReady_To_v1beta2_WaitForPodsReady(in *WaitForPodsReady, out *v1beta2.WaitForPodsReady, s conversion.Scope) error { + return autoConvert_v1beta1_WaitForPodsReady_To_v1beta2_WaitForPodsReady(in, out, s) +} + +func autoConvert_v1beta2_WaitForPodsReady_To_v1beta1_WaitForPodsReady(in *v1beta2.WaitForPodsReady, out *WaitForPodsReady, s conversion.Scope) error { + out.Enable = in.Enable + out.Timeout = (*metav1.Duration)(unsafe.Pointer(in.Timeout)) + out.BlockAdmission = (*bool)(unsafe.Pointer(in.BlockAdmission)) + out.RequeuingStrategy = (*RequeuingStrategy)(unsafe.Pointer(in.RequeuingStrategy)) + out.RecoveryTimeout = (*metav1.Duration)(unsafe.Pointer(in.RecoveryTimeout)) + return nil +} + +// Convert_v1beta2_WaitForPodsReady_To_v1beta1_WaitForPodsReady is an autogenerated conversion function. +func Convert_v1beta2_WaitForPodsReady_To_v1beta1_WaitForPodsReady(in *v1beta2.WaitForPodsReady, out *WaitForPodsReady, s conversion.Scope) error { + return autoConvert_v1beta2_WaitForPodsReady_To_v1beta1_WaitForPodsReady(in, out, s) +} + +func autoConvert_v1beta1_WorkloadRetentionPolicy_To_v1beta2_WorkloadRetentionPolicy(in *WorkloadRetentionPolicy, out *v1beta2.WorkloadRetentionPolicy, s conversion.Scope) error { + out.AfterFinished = (*metav1.Duration)(unsafe.Pointer(in.AfterFinished)) + out.AfterDeactivatedByKueue = (*metav1.Duration)(unsafe.Pointer(in.AfterDeactivatedByKueue)) + return nil +} + +// Convert_v1beta1_WorkloadRetentionPolicy_To_v1beta2_WorkloadRetentionPolicy is an autogenerated conversion function. +func Convert_v1beta1_WorkloadRetentionPolicy_To_v1beta2_WorkloadRetentionPolicy(in *WorkloadRetentionPolicy, out *v1beta2.WorkloadRetentionPolicy, s conversion.Scope) error { + return autoConvert_v1beta1_WorkloadRetentionPolicy_To_v1beta2_WorkloadRetentionPolicy(in, out, s) +} + +func autoConvert_v1beta2_WorkloadRetentionPolicy_To_v1beta1_WorkloadRetentionPolicy(in *v1beta2.WorkloadRetentionPolicy, out *WorkloadRetentionPolicy, s conversion.Scope) error { + out.AfterFinished = (*metav1.Duration)(unsafe.Pointer(in.AfterFinished)) + out.AfterDeactivatedByKueue = (*metav1.Duration)(unsafe.Pointer(in.AfterDeactivatedByKueue)) + return nil +} + +// Convert_v1beta2_WorkloadRetentionPolicy_To_v1beta1_WorkloadRetentionPolicy is an autogenerated conversion function. +func Convert_v1beta2_WorkloadRetentionPolicy_To_v1beta1_WorkloadRetentionPolicy(in *v1beta2.WorkloadRetentionPolicy, out *WorkloadRetentionPolicy, s conversion.Scope) error { + return autoConvert_v1beta2_WorkloadRetentionPolicy_To_v1beta1_WorkloadRetentionPolicy(in, out, s) +} diff --git a/apis/config/v1beta2/configuration_types.go b/apis/config/v1beta2/configuration_types.go new file mode 100644 index 00000000000..bad26216229 --- /dev/null +++ b/apis/config/v1beta2/configuration_types.go @@ -0,0 +1,590 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + configv1alpha1 "k8s.io/component-base/config/v1alpha1" +) + +// +k8s:defaulter-gen=true +// +kubebuilder:object:root=true + +// Configuration is the Schema for the kueueconfigurations API +type Configuration struct { + metav1.TypeMeta `json:",inline"` + + // Namespace is the namespace in which kueue is deployed. It is used as part of DNSName of the webhook Service. + // If not set, the value is set from the file /var/run/secrets/kubernetes.io/serviceaccount/namespace + // If the file doesn't exist, default value is kueue-system. + Namespace *string `json:"namespace,omitempty"` + + // ControllerManager returns the configurations for controllers + ControllerManager `json:",inline"` + + // ManageJobsWithoutQueueName controls whether or not Kueue reconciles + // jobs that don't set the label kueue.x-k8s.io/queue-name. + // If set to true, then those jobs will be suspended and never started unless + // they are assigned a queue and eventually admitted. This also applies to + // jobs created before starting the kueue controller. + // Defaults to false; therefore, those jobs are not managed and if they are created + // unsuspended, they will start immediately. + ManageJobsWithoutQueueName bool `json:"manageJobsWithoutQueueName"` + + // ManagedJobsNamespaceSelector provides a namespace-based mechanism to exempt jobs + // from management by Kueue. + // + // It provides a strong exemption for the Pod-based integrations (pod, deployment, statefulset, etc.), + // For Pod-based integrations, only jobs whose namespaces match ManagedJobsNamespaceSelector are + // eligible to be managed by Kueue. Pods, deployments, etc. in non-matching namespaces will + // never be managed by Kueue, even if they have a kueue.x-k8s.io/queue-name label. + // This strong exemption ensures that Kueue will not interfere with the basic operation + // of system namespace. + // + // For all other integrations, ManagedJobsNamespaceSelector provides a weaker exemption + // by only modulating the effects of ManageJobsWithoutQueueName. For these integrations, + // a job that has a kueue.x-k8s.io/queue-name label will always be managed by Kueue. Jobs without + // a kueue.x-k8s.io/queue-name label will be managed by Kueue only when ManageJobsWithoutQueueName is + // true and the job's namespace matches ManagedJobsNamespaceSelector. + ManagedJobsNamespaceSelector *metav1.LabelSelector `json:"managedJobsNamespaceSelector,omitempty"` + + // InternalCertManagement is configuration for internalCertManagement + InternalCertManagement *InternalCertManagement `json:"internalCertManagement,omitempty"` + + // WaitForPodsReady is configuration to provide a time-based all-or-nothing + // scheduling semantics for Jobs, by ensuring all pods are ready (running + // and passing the readiness probe) within the specified time. If the timeout + // is exceeded, then the workload is evicted. + WaitForPodsReady *WaitForPodsReady `json:"waitForPodsReady,omitempty"` + + // ClientConnection provides additional configuration options for Kubernetes + // API server client. + ClientConnection *ClientConnection `json:"clientConnection,omitempty"` + + // Integrations provide configuration options for AI/ML/Batch frameworks + // integrations (including K8S job). + Integrations *Integrations `json:"integrations,omitempty"` + + // QueueVisibility is configuration to expose the information about the top + // pending workloads. + // Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand + // (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) + // instead. + QueueVisibility *QueueVisibility `json:"queueVisibility,omitempty"` + + // MultiKueue controls the behaviour of the MultiKueue AdmissionCheck Controller. + MultiKueue *MultiKueue `json:"multiKueue,omitempty"` + + // FairSharing controls the Fair Sharing semantics across the cluster. + FairSharing *FairSharing `json:"fairSharing,omitempty"` + + // admissionFairSharing indicates configuration of FairSharing with the `AdmissionTime` mode on + AdmissionFairSharing *AdmissionFairSharing `json:"admissionFairSharing,omitempty"` + + // Resources provides additional configuration options for handling the resources. + Resources *Resources `json:"resources,omitempty"` + + // FeatureGates is a map of feature names to bools that allows to override the + // default enablement status of a feature. The map cannot be used in conjunction + // with passing the list of features via the command line argument "--feature-gates" + // for the Kueue Deployment. + FeatureGates map[string]bool `json:"featureGates,omitempty"` + + // ObjectRetentionPolicies provides configuration options for automatic deletion + // of Kueue-managed objects. A nil value disables all automatic deletions. + // +optional + ObjectRetentionPolicies *ObjectRetentionPolicies `json:"objectRetentionPolicies,omitempty"` +} + +type ControllerManager struct { + // Webhook contains the controllers webhook configuration + // +optional + Webhook ControllerWebhook `json:"webhook,omitempty"` + + // LeaderElection is the LeaderElection config to be used when configuring + // the manager.Manager leader election + // +optional + LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"` + + // Metrics contains the controller metrics configuration + // +optional + Metrics ControllerMetrics `json:"metrics,omitempty"` + + // Health contains the controller health configuration + // +optional + Health ControllerHealth `json:"health,omitempty"` + + // PprofBindAddress is the TCP address that the controller should bind to + // for serving pprof. + // It can be set to "" or "0" to disable the pprof serving. + // Since pprof may contain sensitive information, make sure to protect it + // before exposing it to public. + // +optional + PprofBindAddress string `json:"pprofBindAddress,omitempty"` + + // Controller contains global configuration options for controllers + // registered within this manager. + // +optional + Controller *ControllerConfigurationSpec `json:"controller,omitempty"` +} + +// ControllerWebhook defines the webhook server for the controller. +type ControllerWebhook struct { + // Port is the port that the webhook server serves at. + // It is used to set webhook.Server.Port. + // +optional + Port *int `json:"port,omitempty"` + + // Host is the hostname that the webhook server binds to. + // It is used to set webhook.Server.Host. + // +optional + Host string `json:"host,omitempty"` + + // CertDir is the directory that contains the server key and certificate. + // if not set, webhook server would look up the server key and certificate in + // {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate + // must be named tls.key and tls.crt, respectively. + // +optional + CertDir string `json:"certDir,omitempty"` +} + +// ControllerMetrics defines the metrics configs. +type ControllerMetrics struct { + // BindAddress is the TCP address that the controller should bind to + // for serving prometheus metrics. + // It can be set to "0" to disable the metrics serving. + // +optional + BindAddress string `json:"bindAddress,omitempty"` + + // EnableClusterQueueResources, if true the cluster queue resource usage and quotas + // metrics will be reported. + // +optional + EnableClusterQueueResources bool `json:"enableClusterQueueResources,omitempty"` +} + +// ControllerHealth defines the health configs. +type ControllerHealth struct { + // HealthProbeBindAddress is the TCP address that the controller should bind to + // for serving health probes + // It can be set to "0" or "" to disable serving the health probe. + // +optional + HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"` + + // ReadinessEndpointName, defaults to "readyz" + // +optional + ReadinessEndpointName string `json:"readinessEndpointName,omitempty"` + + // LivenessEndpointName, defaults to "healthz" + // +optional + LivenessEndpointName string `json:"livenessEndpointName,omitempty"` +} + +// ControllerConfigurationSpec defines the global configuration for +// controllers registered with the manager. +type ControllerConfigurationSpec struct { + // GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation + // allowed for that controller. + // + // When a controller is registered within this manager using the builder utilities, + // users have to specify the type the controller reconciles in the For(...) call. + // If the object's kind passed matches one of the keys in this map, the concurrency + // for that controller is set to the number specified. + // + // The key is expected to be consistent in form with GroupKind.String(), + // e.g. ReplicaSet in apps group (regardless of version) would be `ReplicaSet.apps`. + // + // +optional + GroupKindConcurrency map[string]int `json:"groupKindConcurrency,omitempty"` + + // CacheSyncTimeout refers to the time limit set to wait for syncing caches. + // Defaults to 2 minutes if not set. + // +optional + CacheSyncTimeout *time.Duration `json:"cacheSyncTimeout,omitempty"` +} + +// WaitForPodsReady defines configuration for the Wait For Pods Ready feature, +// which is used to ensure that all Pods are ready within the specified time. +type WaitForPodsReady struct { + // Enable indicates whether to enable wait for pods ready feature. + // Defaults to false. + Enable bool `json:"enable,omitempty"` + + // Timeout defines the time for an admitted workload to reach the + // PodsReady=true condition. When the timeout is exceeded, the workload + // evicted and requeued in the same cluster queue. + // Defaults to 5min. + // +optional + Timeout *metav1.Duration `json:"timeout,omitempty"` + + // BlockAdmission when true, cluster queue will block admissions for all + // subsequent jobs until the jobs reach the PodsReady=true condition. + // This setting is only honored when `Enable` is set to true. + BlockAdmission *bool `json:"blockAdmission,omitempty"` + + // RequeuingStrategy defines the strategy for requeuing a Workload. + // +optional + RequeuingStrategy *RequeuingStrategy `json:"requeuingStrategy,omitempty"` + + // RecoveryTimeout defines an opt-in timeout, measured since the + // last transition to the PodsReady=false condition after a Workload is Admitted and running. + // Such a transition may happen when a Pod failed and the replacement Pod + // is awaited to be scheduled. + // After exceeding the timeout the corresponding job gets suspended again + // and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. + // If not set, there is no timeout. + // +optional + RecoveryTimeout *metav1.Duration `json:"recoveryTimeout,omitempty"` +} + +type MultiKueue struct { + // GCInterval defines the time interval between two consecutive garbage collection runs. + // Defaults to 1min. If 0, the garbage collection is disabled. + // +optional + GCInterval *metav1.Duration `json:"gcInterval"` + + // Origin defines a label value used to track the creator of workloads in the worker + // clusters. + // This is used by multikueue in components like its garbage collector to identify + // remote objects that ware created by this multikueue manager cluster and delete + // them if their local counterpart no longer exists. + // +optional + Origin *string `json:"origin,omitempty"` + + // WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready + // if the connection with its reserving worker cluster is lost. + // + // Defaults to 15 minutes. + // +optional + WorkerLostTimeout *metav1.Duration `json:"workerLostTimeout,omitempty"` + + // DispatcherName defines the dispatcher responsible for selecting worker clusters to handle the workload. + // - If specified, the workload will be handled by the named dispatcher. + // - If not specified, the workload will be handled by the default ("kueue.x-k8s.io/multikueue-dispatcher-all-at-once") dispatcher. + // +optional + DispatcherName *string `json:"dispatcherName,omitempty"` + + // ExternalFrameworks defines a list of external frameworks that should be supported + // by the generic MultiKueue adapter. Each entry defines how to handle a specific + // GroupVersionKind (GVK) for MultiKueue operations. + // +optional + ExternalFrameworks []MultiKueueExternalFramework `json:"externalFrameworks,omitempty"` +} + +// MultiKueueExternalFramework defines a framework that is not built-in. +type MultiKueueExternalFramework struct { + // Name is the GVK of the resource that are + // managed by external controllers + // the expected format is `kind.version.group`. + Name string `json:"name"` +} + +const ( + // MultiKueueDispatcherModeAllAtOnce is the name of dispatcher mode where all worker clusters are considered at once + // and the first one accepting the workload is selected. + MultiKueueDispatcherModeAllAtOnce = "kueue.x-k8s.io/multikueue-dispatcher-all-at-once" + + // MultiKueueDispatcherModeIncremental is the name of dispatcher mode where worker clusters are incrementally added to the pool of nominated clusters. + // The process begins with up to 3 initial clusters and expands the pool by up to 3 clusters at a time (if fewer remain, all are added). + MultiKueueDispatcherModeIncremental = "kueue.x-k8s.io/multikueue-dispatcher-incremental" +) + +type RequeuingStrategy struct { + // Timestamp defines the timestamp used for re-queuing a Workload + // that was evicted due to Pod readiness. The possible values are: + // + // - `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. + // - `Creation` indicates from Workload .metadata.creationTimestamp. + // + // +optional + Timestamp *RequeuingTimestamp `json:"timestamp,omitempty"` + + // BackoffLimitCount defines the maximum number of re-queuing retries. + // Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). + // When it is null, the workloads will repeatedly and endless re-queueing. + // + // Every backoff duration is about "b*2^(n-1)+Rand" where: + // - "b" represents the base set by "BackoffBaseSeconds" parameter, + // - "n" represents the "workloadStatus.requeueState.count", + // - "Rand" represents the random jitter. + // During this time, the workload is taken as an inadmissible and + // other workloads will have a chance to be admitted. + // By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). + // + // Defaults to null. + // +optional + BackoffLimitCount *int32 `json:"backoffLimitCount,omitempty"` + + // BackoffBaseSeconds defines the base for the exponential backoff for + // re-queuing an evicted workload. + // + // Defaults to 60. + // +optional + BackoffBaseSeconds *int32 `json:"backoffBaseSeconds,omitempty"` + + // BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. + // + // Defaults to 3600. + // +optional + BackoffMaxSeconds *int32 `json:"backoffMaxSeconds,omitempty"` +} + +type RequeuingTimestamp string + +const ( + // CreationTimestamp timestamp (from Workload .metadata.creationTimestamp). + CreationTimestamp RequeuingTimestamp = "Creation" + + // EvictionTimestamp timestamp (from Workload .status.conditions). + EvictionTimestamp RequeuingTimestamp = "Eviction" +) + +type InternalCertManagement struct { + // Enable controls the use of internal cert management for the webhook, + // metrics and visibility endpoints. + // When enabled Kueue is using libraries to generate and + // self-sign the certificates. + // When disabled, you need to provide the certificates for + // the webhooks, metrics and visibility through a third party certificate + // This secret is mounted to the kueue controller manager pod. The mount + // path for webhooks is /tmp/k8s-webhook-server/serving-certs, for + // metrics endpoint the expected path is `/etc/kueue/metrics/certs` and for + // visibility endpoint the expected path is `/visibility`. + // The keys and certs are named tls.key and tls.crt. + Enable *bool `json:"enable,omitempty"` + + // WebhookServiceName is the name of the Service used as part of the DNSName. + // Defaults to kueue-webhook-service. + WebhookServiceName *string `json:"webhookServiceName,omitempty"` + + // WebhookSecretName is the name of the Secret used to store CA and server certs. + // Defaults to kueue-webhook-server-cert. + WebhookSecretName *string `json:"webhookSecretName,omitempty"` +} + +type ClientConnection struct { + // QPS controls the number of queries per second allowed for K8S api server + // connection. + // + // Setting this to a negative value will disable client-side ratelimiting. + QPS *float32 `json:"qps,omitempty"` + + // Burst allows extra queries to accumulate when a client is exceeding its rate. + Burst *int32 `json:"burst,omitempty"` +} + +type Integrations struct { + // List of framework names to be enabled. + // Possible options: + // - "batch/job" + // - "kubeflow.org/mpijob" + // - "ray.io/rayjob" + // - "ray.io/raycluster" + // - "jobset.x-k8s.io/jobset" + // - "kubeflow.org/paddlejob" + // - "kubeflow.org/pytorchjob" + // - "kubeflow.org/tfjob" + // - "kubeflow.org/xgboostjob" + // - "kubeflow.org/jaxjob" + // - "trainer.kubeflow.org/trainjob" + // - "workload.codeflare.dev/appwrapper" + // - "pod" + // - "deployment" (requires enabling pod integration) + // - "statefulset" (requires enabling pod integration) + // - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) + Frameworks []string `json:"frameworks,omitempty"` + // List of GroupVersionKinds that are managed for Kueue by external controllers; + // the expected format is `Kind.version.group.com`. + ExternalFrameworks []string `json:"externalFrameworks,omitempty"` + // PodOptions defines kueue controller behaviour for pod objects + // Deprecated: This field will be removed on v1beta2, use ManagedJobsNamespaceSelector + // (https://kueue.sigs.k8s.io/docs/tasks/run/plain_pods/) + // instead. + PodOptions *PodIntegrationOptions `json:"podOptions,omitempty"` + + // labelKeysToCopy is a list of label keys that should be copied from the job into the + // workload object. It is not required for the job to have all the labels from this + // list. If a job does not have some label with the given key from this list, the + // constructed workload object will be created without this label. In the case + // of creating a workload from a composable job (pod group), if multiple objects + // have labels with some key from the list, the values of these labels must + // match or otherwise the workload creation would fail. The labels are copied only + // during the workload creation and are not updated even if the labels of the + // underlying job are changed. + LabelKeysToCopy []string `json:"labelKeysToCopy,omitempty"` +} + +type PodIntegrationOptions struct { + // NamespaceSelector can be used to omit some namespaces from pod reconciliation + NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` + // PodSelector can be used to choose what pods to reconcile + PodSelector *metav1.LabelSelector `json:"podSelector,omitempty"` +} + +type QueueVisibility struct { + // ClusterQueues is configuration to expose the information + // about the top pending workloads in the cluster queue. + ClusterQueues *ClusterQueueVisibility `json:"clusterQueues,omitempty"` + + // UpdateIntervalSeconds specifies the time interval for updates to the structure + // of the top pending workloads in the queues. + // The minimum value is 1. + // Defaults to 5. + UpdateIntervalSeconds int32 `json:"updateIntervalSeconds,omitempty"` +} + +type ClusterQueueVisibility struct { + // MaxCount indicates the maximal number of pending workloads exposed in the + // cluster queue status. When the value is set to 0, then ClusterQueue + // visibility updates are disabled. + // The maximal value is 4000. + // Defaults to 10. + MaxCount int32 `json:"maxCount,omitempty"` +} + +type Resources struct { + // ExcludedResourcePrefixes defines which resources should be ignored by Kueue + ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"` + + // Transformations defines how to transform PodSpec resources into Workload resource requests. + // This is intended to be a map with Input as the key (enforced by validation code) + Transformations []ResourceTransformation `json:"transformations,omitempty"` + + // DeviceClassMappings defines mappings from device classes to logical resources + // for Dynamic Resource Allocation support. + // +optional + DeviceClassMappings []DeviceClassMapping `json:"deviceClassMappings,omitempty"` +} + +type ResourceTransformationStrategy string + +const Retain ResourceTransformationStrategy = "Retain" +const Replace ResourceTransformationStrategy = "Replace" + +type ResourceTransformation struct { + // Input is the name of the input resource. + Input corev1.ResourceName `json:"input"` + + // Strategy specifies if the input resource should be replaced or retained. + // Defaults to Retain + Strategy *ResourceTransformationStrategy `json:"strategy,omitempty"` + + // Outputs specifies the output resources and quantities per unit of input resource. + // An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue. + Outputs corev1.ResourceList `json:"outputs,omitempty"` +} + +// DeviceClassMapping holds device class to logical resource mappings +// for Dynamic Resource Allocation support. +type DeviceClassMapping struct { + // Name is referenced in ClusterQueue.nominalQuota and Workload status. + // Must be a valid fully qualified name consisting of an optional DNS subdomain prefix + // followed by a slash and a DNS label, or just a DNS label. + // DNS labels consist of lower-case alphanumeric characters or hyphens, + // and must start and end with an alphanumeric character. + // DNS subdomain prefixes follow the same rules as DNS labels but can contain periods. + // The total length must not exceed 253 characters. + Name corev1.ResourceName `json:"name"` + + // DeviceClassNames enumerates the DeviceClasses represented by this resource name. + // Each device class name must be a valid qualified name consisting of an optional DNS subdomain prefix + // followed by a slash and a DNS label, or just a DNS label. + // DNS labels consist of lower-case alphanumeric characters or hyphens, + // and must start and end with an alphanumeric character. + // DNS subdomain prefixes follow the same rules as DNS labels but can contain periods. + // The total length of each name must not exceed 253 characters. + DeviceClassNames []corev1.ResourceName `json:"deviceClassNames"` +} + +type PreemptionStrategy string + +const ( + LessThanOrEqualToFinalShare PreemptionStrategy = "LessThanOrEqualToFinalShare" + LessThanInitialShare PreemptionStrategy = "LessThanInitialShare" +) + +type FairSharing struct { + // enable indicates whether to enable Fair Sharing for all cohorts. + // Defaults to false. + Enable bool `json:"enable"` + + // preemptionStrategies indicates which constraints should a preemption satisfy. + // The preemption algorithm will only use the next strategy in the list if the + // incoming workload (preemptor) doesn't fit after using the previous strategies. + // Possible values are: + // - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ + // with the preemptor workload is less than or equal to the share of the preemptee CQ + // without the workload to be preempted. + // This strategy might favor preemption of smaller workloads in the preemptee CQ, + // regardless of priority or start time, in an effort to keep the share of the CQ + // as high as possible. + // - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ + // with the incoming workload is strictly less than the share of the preemptee CQ. + // This strategy doesn't depend on the share usage of the workload being preempted. + // As a result, the strategy chooses to preempt workloads with the lowest priority and + // newest start time first. + // The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. + PreemptionStrategies []PreemptionStrategy `json:"preemptionStrategies,omitempty"` +} + +type AdmissionFairSharing struct { + // usageHalfLifeTime indicates the time after which the current usage will decay by a half + // If set to 0, usage will be reset to 0 immediately. + UsageHalfLifeTime metav1.Duration `json:"usageHalfLifeTime"` + + // usageSamplingInterval indicates how often Kueue updates consumedResources in FairSharingStatus + // Defaults to 5min. + UsageSamplingInterval metav1.Duration `json:"usageSamplingInterval"` + + // resourceWeights assigns weights to resources which then are used to calculate LocalQueue's + // resource usage and order Workloads. + // Defaults to 1. + ResourceWeights map[corev1.ResourceName]float64 `json:"resourceWeights,omitempty"` +} + +// ObjectRetentionPolicies holds retention settings for different object types. +type ObjectRetentionPolicies struct { + // Workloads configures retention for Workloads. + // A nil value disables automatic deletion of Workloads. + // +optional + Workloads *WorkloadRetentionPolicy `json:"workloads,omitempty"` +} + +// WorkloadRetentionPolicy defines the policies for when Workloads should be deleted. +type WorkloadRetentionPolicy struct { + // AfterFinished is the duration to wait after a Workload finishes + // before deleting it. + // A duration of 0 will delete immediately. + // A nil value disables automatic deletion. + // Represented using metav1.Duration (e.g. "10m", "1h30m"). + // +optional + AfterFinished *metav1.Duration `json:"afterFinished,omitempty"` + + // AfterDeactivatedByKueue is the duration to wait after *any* Kueue-managed Workload + // (such as a Job, JobSet, or other custom workload types) has been marked + // as deactivated by Kueue before automatically deleting it. + // Deletion of deactivated workloads may cascade to objects not created by + // Kueue, since deleting the parent Workload owner (e.g. JobSet) can trigger + // garbage-collection of dependent resources. + // A duration of 0 will delete immediately. + // A nil value disables automatic deletion. + // Represented using metav1.Duration (e.g. "10m", "1h30m"). + // +optional + AfterDeactivatedByKueue *metav1.Duration `json:"afterDeactivatedByKueue,omitempty"` +} diff --git a/apis/config/v1beta2/defaults.go b/apis/config/v1beta2/defaults.go new file mode 100644 index 00000000000..cbbcbd82174 --- /dev/null +++ b/apis/config/v1beta2/defaults.go @@ -0,0 +1,138 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "cmp" + "os" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/leaderelection/resourcelock" + configv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/utils/ptr" +) + +const ( + DefaultNamespace = "kueue-system" + DefaultWebhookServiceName = "kueue-webhook-service" + DefaultWebhookSecretName = "kueue-webhook-server-cert" + DefaultWebhookPort = 9443 + DefaultWebhookCertDir = "/tmp/k8s-webhook-server/serving-certs" + DefaultHealthProbeBindAddress = ":8081" + DefaultMetricsBindAddress = ":8443" + DefaultLeaderElectionID = "c1f6bfd2.kueue.x-k8s.io" + DefaultLeaderElectionLeaseDuration = 15 * time.Second + DefaultLeaderElectionRenewDeadline = 10 * time.Second + DefaultLeaderElectionRetryPeriod = 2 * time.Second + DefaultClientConnectionQPS float32 = 20.0 + DefaultClientConnectionBurst int32 = 30 + defaultPodsReadyTimeout = 5 * time.Minute + defaultJobFrameworkName = "batch/job" + DefaultMultiKueueGCInterval = time.Minute + DefaultMultiKueueOrigin = "multikueue" + DefaultMultiKueueWorkerLostTimeout = 15 * time.Minute + DefaultRequeuingBackoffBaseSeconds = 60 + DefaultRequeuingBackoffMaxSeconds = 3600 + DefaultResourceTransformationStrategy = Retain +) + +func getOperatorNamespace() string { + if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { + if ns := strings.TrimSpace(string(data)); len(ns) > 0 { + return ns + } + } + return DefaultNamespace +} + +// SetDefaults_Configuration sets default values for ComponentConfig. +// +//nolint:revive // format required by generated code for defaulting +func SetDefaults_Configuration(cfg *Configuration) { + cfg.Namespace = cmp.Or(cfg.Namespace, ptr.To(getOperatorNamespace())) + cfg.Webhook.Port = cmp.Or(cfg.Webhook.Port, ptr.To(DefaultWebhookPort)) + cfg.Webhook.CertDir = cmp.Or(cfg.Webhook.CertDir, DefaultWebhookCertDir) + cfg.Metrics.BindAddress = cmp.Or(cfg.Metrics.BindAddress, DefaultMetricsBindAddress) + cfg.Health.HealthProbeBindAddress = cmp.Or(cfg.Health.HealthProbeBindAddress, DefaultHealthProbeBindAddress) + cfg.LeaderElection = cmp.Or(cfg.LeaderElection, &configv1alpha1.LeaderElectionConfiguration{}) + cfg.LeaderElection.ResourceName = cmp.Or(cfg.LeaderElection.ResourceName, DefaultLeaderElectionID) + + // Default to Lease as component-base still defaults to endpoint resources + // until core components migrate to using Leases. See k/k #80289 for more details. + cfg.LeaderElection.ResourceLock = cmp.Or(cfg.LeaderElection.ResourceLock, resourcelock.LeasesResourceLock) + + // Use the default LeaderElectionConfiguration options + configv1alpha1.RecommendedDefaultLeaderElectionConfiguration(cfg.LeaderElection) + + cfg.InternalCertManagement = cmp.Or(cfg.InternalCertManagement, &InternalCertManagement{}) + cfg.InternalCertManagement.Enable = cmp.Or(cfg.InternalCertManagement.Enable, ptr.To(true)) + if *cfg.InternalCertManagement.Enable { + cfg.InternalCertManagement.WebhookServiceName = cmp.Or(cfg.InternalCertManagement.WebhookServiceName, ptr.To(DefaultWebhookServiceName)) + cfg.InternalCertManagement.WebhookSecretName = cmp.Or(cfg.InternalCertManagement.WebhookSecretName, ptr.To(DefaultWebhookSecretName)) + } + + cfg.ClientConnection = cmp.Or(cfg.ClientConnection, &ClientConnection{}) + cfg.ClientConnection.QPS = cmp.Or(cfg.ClientConnection.QPS, ptr.To(DefaultClientConnectionQPS)) + cfg.ClientConnection.Burst = cmp.Or(cfg.ClientConnection.Burst, ptr.To(DefaultClientConnectionBurst)) + + cfg.WaitForPodsReady = cmp.Or(cfg.WaitForPodsReady, &WaitForPodsReady{Enable: false}) + if cfg.WaitForPodsReady.Enable { + cfg.WaitForPodsReady.Timeout = cmp.Or(cfg.WaitForPodsReady.Timeout, &metav1.Duration{Duration: defaultPodsReadyTimeout}) + cfg.WaitForPodsReady.BlockAdmission = cmp.Or(cfg.WaitForPodsReady.BlockAdmission, &cfg.WaitForPodsReady.Enable) + cfg.WaitForPodsReady.RequeuingStrategy = cmp.Or(cfg.WaitForPodsReady.RequeuingStrategy, &RequeuingStrategy{}) + cfg.WaitForPodsReady.RequeuingStrategy.Timestamp = cmp.Or(cfg.WaitForPodsReady.RequeuingStrategy.Timestamp, ptr.To(EvictionTimestamp)) + cfg.WaitForPodsReady.RequeuingStrategy.BackoffBaseSeconds = cmp.Or(cfg.WaitForPodsReady.RequeuingStrategy.BackoffBaseSeconds, ptr.To[int32](DefaultRequeuingBackoffBaseSeconds)) + cfg.WaitForPodsReady.RequeuingStrategy.BackoffMaxSeconds = cmp.Or(cfg.WaitForPodsReady.RequeuingStrategy.BackoffMaxSeconds, ptr.To[int32](DefaultRequeuingBackoffMaxSeconds)) + } + + cfg.Integrations = cmp.Or(cfg.Integrations, &Integrations{}) + if len(cfg.Integrations.Frameworks) == 0 { + cfg.Integrations.Frameworks = []string{defaultJobFrameworkName} + } + + cfg.ManagedJobsNamespaceSelector = cmp.Or(cfg.ManagedJobsNamespaceSelector, &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1.LabelMetadataName, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"kube-system", *cfg.Namespace}, + }, + }, + }) + + cfg.MultiKueue = cmp.Or(cfg.MultiKueue, &MultiKueue{}) + cfg.MultiKueue.GCInterval = cmp.Or(cfg.MultiKueue.GCInterval, &metav1.Duration{Duration: DefaultMultiKueueGCInterval}) + cfg.MultiKueue.Origin = ptr.To(cmp.Or(ptr.Deref(cfg.MultiKueue.Origin, ""), DefaultMultiKueueOrigin)) + cfg.MultiKueue.WorkerLostTimeout = cmp.Or(cfg.MultiKueue.WorkerLostTimeout, &metav1.Duration{Duration: DefaultMultiKueueWorkerLostTimeout}) + cfg.MultiKueue.DispatcherName = cmp.Or(cfg.MultiKueue.DispatcherName, ptr.To(MultiKueueDispatcherModeAllAtOnce)) + + if fs := cfg.FairSharing; fs != nil && fs.Enable && len(fs.PreemptionStrategies) == 0 { + fs.PreemptionStrategies = []PreemptionStrategy{LessThanOrEqualToFinalShare, LessThanInitialShare} + } + if afs := cfg.AdmissionFairSharing; afs != nil { + afs.UsageSamplingInterval.Duration = cmp.Or(afs.UsageSamplingInterval.Duration, 5*time.Minute) + } + + if cfg.Resources != nil { + for idx := range cfg.Resources.Transformations { + cfg.Resources.Transformations[idx].Strategy = ptr.To(cmp.Or(ptr.Deref(cfg.Resources.Transformations[idx].Strategy, ""), DefaultResourceTransformationStrategy)) + } + } +} diff --git a/apis/config/v1beta2/defaults_test.go b/apis/config/v1beta2/defaults_test.go new file mode 100644 index 00000000000..db28943f0c4 --- /dev/null +++ b/apis/config/v1beta2/defaults_test.go @@ -0,0 +1,663 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + componentconfigv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/utils/ptr" +) + +const ( + overwriteNamespace = "kueue-tenant-a" + overwriteWebhookPort = 9444 + overwriteWebhookCertDir = "/tmp/test" + overwriteMetricBindAddress = ":38081" + overwriteHealthProbeBindAddress = ":38080" + overwriteLeaderElectionID = "foo.kueue.x-k8s.io" +) + +func TestSetDefaults_Configuration(t *testing.T) { + defaultCtrlManagerConfigurationSpec := ControllerManager{ + LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: "leases", + ResourceName: "c1f6bfd2.kueue.x-k8s.io", + }, + Webhook: ControllerWebhook{ + Port: ptr.To(DefaultWebhookPort), + CertDir: DefaultWebhookCertDir, + }, + Metrics: ControllerMetrics{ + BindAddress: DefaultMetricsBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: DefaultHealthProbeBindAddress, + }, + } + defaultClientConnection := &ClientConnection{ + QPS: ptr.To(DefaultClientConnectionQPS), + Burst: ptr.To(DefaultClientConnectionBurst), + } + defaultIntegrations := &Integrations{ + Frameworks: []string{defaultJobFrameworkName}, + } + defaultManagedJobsNamespaceSelector := &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1.LabelMetadataName, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"kube-system", "kueue-system"}, + }, + }, + } + + overwriteNamespaceIntegrations := &Integrations{ + Frameworks: []string{defaultJobFrameworkName}, + } + + overwriteNamespaceSelector := &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: corev1.LabelMetadataName, + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{"kube-system", overwriteNamespace}, + }, + }, + } + + defaultMultiKueue := &MultiKueue{ + GCInterval: &metav1.Duration{Duration: DefaultMultiKueueGCInterval}, + Origin: ptr.To(DefaultMultiKueueOrigin), + WorkerLostTimeout: &metav1.Duration{Duration: DefaultMultiKueueWorkerLostTimeout}, + DispatcherName: ptr.To(MultiKueueDispatcherModeAllAtOnce), + } + + podsReadyTimeout := metav1.Duration{Duration: defaultPodsReadyTimeout} + podsReadyTimeoutOverwrite := metav1.Duration{Duration: time.Minute} + + testCases := map[string]struct { + original *Configuration + want *Configuration + }{ + "defaulting namespace": { + original: &Configuration{ + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "defaulting ControllerManager": { + original: &Configuration{ + ControllerManager: ControllerManager{ + LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ + Port: ptr.To(DefaultWebhookPort), + CertDir: DefaultWebhookCertDir, + }, + Metrics: ControllerMetrics{ + BindAddress: DefaultMetricsBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: DefaultHealthProbeBindAddress, + }, + LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: "leases", + ResourceName: DefaultLeaderElectionID, + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "should not default ControllerManager": { + original: &Configuration{ + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ + Port: ptr.To(overwriteWebhookPort), + CertDir: overwriteWebhookCertDir, + }, + Metrics: ControllerMetrics{ + BindAddress: overwriteMetricBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: overwriteHealthProbeBindAddress, + }, + LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: "leases", + ResourceName: overwriteLeaderElectionID, + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + Integrations: defaultIntegrations, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ + Port: ptr.To(overwriteWebhookPort), + CertDir: overwriteWebhookCertDir, + }, + Metrics: ControllerMetrics{ + BindAddress: overwriteMetricBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: overwriteHealthProbeBindAddress, + }, + LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(true), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: "leases", + ResourceName: overwriteLeaderElectionID, + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "should not set LeaderElectionID": { + original: &Configuration{ + ControllerManager: ControllerManager{ + LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(false), + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: ControllerManager{ + Webhook: ControllerWebhook{ + Port: ptr.To(DefaultWebhookPort), + CertDir: DefaultWebhookCertDir, + }, + Metrics: ControllerMetrics{ + BindAddress: DefaultMetricsBindAddress, + }, + Health: ControllerHealth{ + HealthProbeBindAddress: DefaultHealthProbeBindAddress, + }, + LeaderElection: &componentconfigv1alpha1.LeaderElectionConfiguration{ + LeaderElect: ptr.To(false), + LeaseDuration: metav1.Duration{Duration: DefaultLeaderElectionLeaseDuration}, + RenewDeadline: metav1.Duration{Duration: DefaultLeaderElectionRenewDeadline}, + RetryPeriod: metav1.Duration{Duration: DefaultLeaderElectionRetryPeriod}, + ResourceLock: "leases", + ResourceName: "c1f6bfd2.kueue.x-k8s.io", + }, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "defaulting InternalCertManagement": { + original: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + }, + want: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(true), + WebhookServiceName: ptr.To(DefaultWebhookServiceName), + WebhookSecretName: ptr.To(DefaultWebhookSecretName), + }, + ClientConnection: defaultClientConnection, + Integrations: overwriteNamespaceIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: overwriteNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "should not default InternalCertManagement": { + original: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: overwriteNamespaceIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: overwriteNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "should not default values in custom ClientConnection": { + original: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: &ClientConnection{ + QPS: ptr.To[float32](123.0), + Burst: ptr.To[int32](456), + }, + }, + want: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: &ClientConnection{ + QPS: ptr.To[float32](123.0), + Burst: ptr.To[int32](456), + }, + Integrations: overwriteNamespaceIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: overwriteNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "should default empty custom ClientConnection": { + original: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: &ClientConnection{}, + }, + want: &Configuration{ + Namespace: ptr.To(overwriteNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: overwriteNamespaceIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: overwriteNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "defaulting waitForPodsReady values": { + original: &Configuration{ + WaitForPodsReady: &WaitForPodsReady{ + Enable: true, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + WaitForPodsReady: &WaitForPodsReady{ + Enable: true, + BlockAdmission: ptr.To(true), + Timeout: &podsReadyTimeout, + RecoveryTimeout: nil, + RequeuingStrategy: &RequeuingStrategy{ + Timestamp: ptr.To(EvictionTimestamp), + BackoffBaseSeconds: ptr.To[int32](DefaultRequeuingBackoffBaseSeconds), + BackoffMaxSeconds: ptr.To[int32](DefaultRequeuingBackoffMaxSeconds), + }, + }, + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + }, + }, + "set waitForPodsReady.blockAdmission to false, and waitForPodsReady.recoveryTimeout to nil when enable is false": { + original: &Configuration{ + WaitForPodsReady: &WaitForPodsReady{ + Enable: false, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + WaitForPodsReady: &WaitForPodsReady{ + Enable: false, + }, + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + }, + }, + "respecting provided waitForPodsReady values": { + original: &Configuration{ + WaitForPodsReady: &WaitForPodsReady{ + Enable: true, + Timeout: &podsReadyTimeoutOverwrite, + RequeuingStrategy: &RequeuingStrategy{ + Timestamp: ptr.To(CreationTimestamp), + BackoffBaseSeconds: ptr.To[int32](63), + BackoffMaxSeconds: ptr.To[int32](1800), + }, + RecoveryTimeout: &metav1.Duration{Duration: time.Minute}, + }, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + }, + want: &Configuration{ + WaitForPodsReady: &WaitForPodsReady{ + Enable: true, + BlockAdmission: ptr.To(true), + Timeout: &podsReadyTimeoutOverwrite, + RecoveryTimeout: &metav1.Duration{Duration: time.Minute}, + RequeuingStrategy: &RequeuingStrategy{ + Timestamp: ptr.To(CreationTimestamp), + BackoffBaseSeconds: ptr.To[int32](63), + BackoffMaxSeconds: ptr.To[int32](1800), + }, + }, + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + }, + }, + "integrations": { + original: &Configuration{ + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + Integrations: &Integrations{ + Frameworks: []string{"a", "b"}, + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: &Integrations{ + Frameworks: []string{"a", "b"}, + }, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "multiKueue": { + original: &Configuration{ + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + MultiKueue: &MultiKueue{ + GCInterval: &metav1.Duration{Duration: time.Second}, + Origin: ptr.To("multikueue-manager1"), + WorkerLostTimeout: &metav1.Duration{Duration: time.Minute}, + DispatcherName: ptr.To(MultiKueueDispatcherModeIncremental), + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: &MultiKueue{ + GCInterval: &metav1.Duration{Duration: time.Second}, + Origin: ptr.To("multikueue-manager1"), + WorkerLostTimeout: &metav1.Duration{Duration: time.Minute}, + DispatcherName: ptr.To(MultiKueueDispatcherModeIncremental), + }, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "multiKueue origin is an empty value": { + original: &Configuration{ + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + MultiKueue: &MultiKueue{ + GCInterval: &metav1.Duration{Duration: time.Second}, + Origin: ptr.To(""), + WorkerLostTimeout: &metav1.Duration{Duration: time.Minute}, + DispatcherName: defaultMultiKueue.DispatcherName, + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: &MultiKueue{ + GCInterval: &metav1.Duration{Duration: time.Second}, + Origin: ptr.To(DefaultMultiKueueOrigin), + WorkerLostTimeout: &metav1.Duration{Duration: time.Minute}, + DispatcherName: defaultMultiKueue.DispatcherName, + }, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "multiKueue GCInterval 0": { + original: &Configuration{ + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + MultiKueue: &MultiKueue{ + GCInterval: &metav1.Duration{}, + Origin: ptr.To("multikueue-manager1"), + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: &MultiKueue{ + GCInterval: &metav1.Duration{}, + Origin: ptr.To("multikueue-manager1"), + WorkerLostTimeout: &metav1.Duration{Duration: 15 * time.Minute}, + DispatcherName: defaultMultiKueue.DispatcherName, + }, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "add default fair sharing configuration when enabled": { + original: &Configuration{ + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + FairSharing: &FairSharing{ + Enable: true, + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + FairSharing: &FairSharing{ + Enable: true, + PreemptionStrategies: []PreemptionStrategy{LessThanOrEqualToFinalShare, LessThanInitialShare}, + }, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "set object retention policy for workloads": { + original: &Configuration{ + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ObjectRetentionPolicies: &ObjectRetentionPolicies{ + Workloads: &WorkloadRetentionPolicy{ + AfterFinished: &metav1.Duration{Duration: 30 * time.Minute}, + AfterDeactivatedByKueue: &metav1.Duration{Duration: 30 * time.Minute}, + }, + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + ObjectRetentionPolicies: &ObjectRetentionPolicies{ + Workloads: &WorkloadRetentionPolicy{ + AfterFinished: &metav1.Duration{Duration: 30 * time.Minute}, + AfterDeactivatedByKueue: &metav1.Duration{Duration: 30 * time.Minute}, + }, + }, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + "resources.transformations strategy": { + original: &Configuration{ + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + Resources: &Resources{ + Transformations: []ResourceTransformation{ + {Input: corev1.ResourceCPU}, + {Input: corev1.ResourceMemory, Strategy: ptr.To(Replace)}, + {Input: corev1.ResourceEphemeralStorage, Strategy: ptr.To[ResourceTransformationStrategy]("")}, + }, + }, + }, + want: &Configuration{ + Namespace: ptr.To(DefaultNamespace), + ControllerManager: defaultCtrlManagerConfigurationSpec, + InternalCertManagement: &InternalCertManagement{ + Enable: ptr.To(false), + }, + ClientConnection: defaultClientConnection, + Integrations: defaultIntegrations, + MultiKueue: defaultMultiKueue, + ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, + Resources: &Resources{ + Transformations: []ResourceTransformation{ + {Input: corev1.ResourceCPU, Strategy: ptr.To(DefaultResourceTransformationStrategy)}, + {Input: corev1.ResourceMemory, Strategy: ptr.To(Replace)}, + {Input: corev1.ResourceEphemeralStorage, Strategy: ptr.To(DefaultResourceTransformationStrategy)}, + }, + }, + WaitForPodsReady: &WaitForPodsReady{}, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + SetDefaults_Configuration(tc.original) + if diff := cmp.Diff(tc.want, tc.original); diff != "" { + t.Errorf("unexpected error (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/apis/config/v1beta2/doc.go b/apis/config/v1beta2/doc.go new file mode 100644 index 00000000000..1cb1ffb869b --- /dev/null +++ b/apis/config/v1beta2/doc.go @@ -0,0 +1,20 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +kubebuilder:object:generate=true +// +groupName=config.kueue.x-k8s.io + +package v1beta2 diff --git a/apis/config/v1beta2/groupversion_info.go b/apis/config/v1beta2/groupversion_info.go new file mode 100644 index 00000000000..12d8c6e9c15 --- /dev/null +++ b/apis/config/v1beta2/groupversion_info.go @@ -0,0 +1,48 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta2 contains API Schema definitions for the config v1beta2 API group +// +kubebuilder:object:generate=true +// +groupName=config.kueue.x-k8s.io +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "config.kueue.x-k8s.io", Version: "v1beta2"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // localSchemeBuilder is used to register autogenerated conversion and defaults functions + // It is required by ./zz_generated.conversion.go and ./zz_generated.defaults.go + localSchemeBuilder = &SchemeBuilder.SchemeBuilder + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +func init() { + SchemeBuilder.Register(&Configuration{}) + // We only register manually written functions here. The registration of the + // generated functions takes place in the generated files. The separation + // makes the code compile even when the generated files are missing. + localSchemeBuilder.Register(RegisterDefaults) +} diff --git a/apis/config/v1beta2/zz_generated.deepcopy.go b/apis/config/v1beta2/zz_generated.deepcopy.go new file mode 100644 index 00000000000..859b944e305 --- /dev/null +++ b/apis/config/v1beta2/zz_generated.deepcopy.go @@ -0,0 +1,671 @@ +//go:build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta2 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/component-base/config/v1alpha1" + timex "time" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AdmissionFairSharing) DeepCopyInto(out *AdmissionFairSharing) { + *out = *in + out.UsageHalfLifeTime = in.UsageHalfLifeTime + out.UsageSamplingInterval = in.UsageSamplingInterval + if in.ResourceWeights != nil { + in, out := &in.ResourceWeights, &out.ResourceWeights + *out = make(map[corev1.ResourceName]float64, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdmissionFairSharing. +func (in *AdmissionFairSharing) DeepCopy() *AdmissionFairSharing { + if in == nil { + return nil + } + out := new(AdmissionFairSharing) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClientConnection) DeepCopyInto(out *ClientConnection) { + *out = *in + if in.QPS != nil { + in, out := &in.QPS, &out.QPS + *out = new(float32) + **out = **in + } + if in.Burst != nil { + in, out := &in.Burst, &out.Burst + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClientConnection. +func (in *ClientConnection) DeepCopy() *ClientConnection { + if in == nil { + return nil + } + out := new(ClientConnection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterQueueVisibility) DeepCopyInto(out *ClusterQueueVisibility) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterQueueVisibility. +func (in *ClusterQueueVisibility) DeepCopy() *ClusterQueueVisibility { + if in == nil { + return nil + } + out := new(ClusterQueueVisibility) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Namespace != nil { + in, out := &in.Namespace, &out.Namespace + *out = new(string) + **out = **in + } + in.ControllerManager.DeepCopyInto(&out.ControllerManager) + if in.ManagedJobsNamespaceSelector != nil { + in, out := &in.ManagedJobsNamespaceSelector, &out.ManagedJobsNamespaceSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.InternalCertManagement != nil { + in, out := &in.InternalCertManagement, &out.InternalCertManagement + *out = new(InternalCertManagement) + (*in).DeepCopyInto(*out) + } + if in.WaitForPodsReady != nil { + in, out := &in.WaitForPodsReady, &out.WaitForPodsReady + *out = new(WaitForPodsReady) + (*in).DeepCopyInto(*out) + } + if in.ClientConnection != nil { + in, out := &in.ClientConnection, &out.ClientConnection + *out = new(ClientConnection) + (*in).DeepCopyInto(*out) + } + if in.Integrations != nil { + in, out := &in.Integrations, &out.Integrations + *out = new(Integrations) + (*in).DeepCopyInto(*out) + } + if in.QueueVisibility != nil { + in, out := &in.QueueVisibility, &out.QueueVisibility + *out = new(QueueVisibility) + (*in).DeepCopyInto(*out) + } + if in.MultiKueue != nil { + in, out := &in.MultiKueue, &out.MultiKueue + *out = new(MultiKueue) + (*in).DeepCopyInto(*out) + } + if in.FairSharing != nil { + in, out := &in.FairSharing, &out.FairSharing + *out = new(FairSharing) + (*in).DeepCopyInto(*out) + } + if in.AdmissionFairSharing != nil { + in, out := &in.AdmissionFairSharing, &out.AdmissionFairSharing + *out = new(AdmissionFairSharing) + (*in).DeepCopyInto(*out) + } + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(Resources) + (*in).DeepCopyInto(*out) + } + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ObjectRetentionPolicies != nil { + in, out := &in.ObjectRetentionPolicies, &out.ObjectRetentionPolicies + *out = new(ObjectRetentionPolicies) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Configuration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerConfigurationSpec) DeepCopyInto(out *ControllerConfigurationSpec) { + *out = *in + if in.GroupKindConcurrency != nil { + in, out := &in.GroupKindConcurrency, &out.GroupKindConcurrency + *out = make(map[string]int, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.CacheSyncTimeout != nil { + in, out := &in.CacheSyncTimeout, &out.CacheSyncTimeout + *out = new(timex.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerConfigurationSpec. +func (in *ControllerConfigurationSpec) DeepCopy() *ControllerConfigurationSpec { + if in == nil { + return nil + } + out := new(ControllerConfigurationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerHealth) DeepCopyInto(out *ControllerHealth) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerHealth. +func (in *ControllerHealth) DeepCopy() *ControllerHealth { + if in == nil { + return nil + } + out := new(ControllerHealth) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerManager) DeepCopyInto(out *ControllerManager) { + *out = *in + in.Webhook.DeepCopyInto(&out.Webhook) + if in.LeaderElection != nil { + in, out := &in.LeaderElection, &out.LeaderElection + *out = new(v1alpha1.LeaderElectionConfiguration) + (*in).DeepCopyInto(*out) + } + out.Metrics = in.Metrics + out.Health = in.Health + if in.Controller != nil { + in, out := &in.Controller, &out.Controller + *out = new(ControllerConfigurationSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManager. +func (in *ControllerManager) DeepCopy() *ControllerManager { + if in == nil { + return nil + } + out := new(ControllerManager) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerMetrics) DeepCopyInto(out *ControllerMetrics) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerMetrics. +func (in *ControllerMetrics) DeepCopy() *ControllerMetrics { + if in == nil { + return nil + } + out := new(ControllerMetrics) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ControllerWebhook) DeepCopyInto(out *ControllerWebhook) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerWebhook. +func (in *ControllerWebhook) DeepCopy() *ControllerWebhook { + if in == nil { + return nil + } + out := new(ControllerWebhook) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceClassMapping) DeepCopyInto(out *DeviceClassMapping) { + *out = *in + if in.DeviceClassNames != nil { + in, out := &in.DeviceClassNames, &out.DeviceClassNames + *out = make([]corev1.ResourceName, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceClassMapping. +func (in *DeviceClassMapping) DeepCopy() *DeviceClassMapping { + if in == nil { + return nil + } + out := new(DeviceClassMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FairSharing) DeepCopyInto(out *FairSharing) { + *out = *in + if in.PreemptionStrategies != nil { + in, out := &in.PreemptionStrategies, &out.PreemptionStrategies + *out = make([]PreemptionStrategy, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FairSharing. +func (in *FairSharing) DeepCopy() *FairSharing { + if in == nil { + return nil + } + out := new(FairSharing) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Integrations) DeepCopyInto(out *Integrations) { + *out = *in + if in.Frameworks != nil { + in, out := &in.Frameworks, &out.Frameworks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalFrameworks != nil { + in, out := &in.ExternalFrameworks, &out.ExternalFrameworks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PodOptions != nil { + in, out := &in.PodOptions, &out.PodOptions + *out = new(PodIntegrationOptions) + (*in).DeepCopyInto(*out) + } + if in.LabelKeysToCopy != nil { + in, out := &in.LabelKeysToCopy, &out.LabelKeysToCopy + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Integrations. +func (in *Integrations) DeepCopy() *Integrations { + if in == nil { + return nil + } + out := new(Integrations) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternalCertManagement) DeepCopyInto(out *InternalCertManagement) { + *out = *in + if in.Enable != nil { + in, out := &in.Enable, &out.Enable + *out = new(bool) + **out = **in + } + if in.WebhookServiceName != nil { + in, out := &in.WebhookServiceName, &out.WebhookServiceName + *out = new(string) + **out = **in + } + if in.WebhookSecretName != nil { + in, out := &in.WebhookSecretName, &out.WebhookSecretName + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternalCertManagement. +func (in *InternalCertManagement) DeepCopy() *InternalCertManagement { + if in == nil { + return nil + } + out := new(InternalCertManagement) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultiKueue) DeepCopyInto(out *MultiKueue) { + *out = *in + if in.GCInterval != nil { + in, out := &in.GCInterval, &out.GCInterval + *out = new(v1.Duration) + **out = **in + } + if in.Origin != nil { + in, out := &in.Origin, &out.Origin + *out = new(string) + **out = **in + } + if in.WorkerLostTimeout != nil { + in, out := &in.WorkerLostTimeout, &out.WorkerLostTimeout + *out = new(v1.Duration) + **out = **in + } + if in.DispatcherName != nil { + in, out := &in.DispatcherName, &out.DispatcherName + *out = new(string) + **out = **in + } + if in.ExternalFrameworks != nil { + in, out := &in.ExternalFrameworks, &out.ExternalFrameworks + *out = make([]MultiKueueExternalFramework, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultiKueue. +func (in *MultiKueue) DeepCopy() *MultiKueue { + if in == nil { + return nil + } + out := new(MultiKueue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultiKueueExternalFramework) DeepCopyInto(out *MultiKueueExternalFramework) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultiKueueExternalFramework. +func (in *MultiKueueExternalFramework) DeepCopy() *MultiKueueExternalFramework { + if in == nil { + return nil + } + out := new(MultiKueueExternalFramework) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectRetentionPolicies) DeepCopyInto(out *ObjectRetentionPolicies) { + *out = *in + if in.Workloads != nil { + in, out := &in.Workloads, &out.Workloads + *out = new(WorkloadRetentionPolicy) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectRetentionPolicies. +func (in *ObjectRetentionPolicies) DeepCopy() *ObjectRetentionPolicies { + if in == nil { + return nil + } + out := new(ObjectRetentionPolicies) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodIntegrationOptions) DeepCopyInto(out *PodIntegrationOptions) { + *out = *in + if in.NamespaceSelector != nil { + in, out := &in.NamespaceSelector, &out.NamespaceSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.PodSelector != nil { + in, out := &in.PodSelector, &out.PodSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodIntegrationOptions. +func (in *PodIntegrationOptions) DeepCopy() *PodIntegrationOptions { + if in == nil { + return nil + } + out := new(PodIntegrationOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueVisibility) DeepCopyInto(out *QueueVisibility) { + *out = *in + if in.ClusterQueues != nil { + in, out := &in.ClusterQueues, &out.ClusterQueues + *out = new(ClusterQueueVisibility) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueVisibility. +func (in *QueueVisibility) DeepCopy() *QueueVisibility { + if in == nil { + return nil + } + out := new(QueueVisibility) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RequeuingStrategy) DeepCopyInto(out *RequeuingStrategy) { + *out = *in + if in.Timestamp != nil { + in, out := &in.Timestamp, &out.Timestamp + *out = new(RequeuingTimestamp) + **out = **in + } + if in.BackoffLimitCount != nil { + in, out := &in.BackoffLimitCount, &out.BackoffLimitCount + *out = new(int32) + **out = **in + } + if in.BackoffBaseSeconds != nil { + in, out := &in.BackoffBaseSeconds, &out.BackoffBaseSeconds + *out = new(int32) + **out = **in + } + if in.BackoffMaxSeconds != nil { + in, out := &in.BackoffMaxSeconds, &out.BackoffMaxSeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RequeuingStrategy. +func (in *RequeuingStrategy) DeepCopy() *RequeuingStrategy { + if in == nil { + return nil + } + out := new(RequeuingStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTransformation) DeepCopyInto(out *ResourceTransformation) { + *out = *in + if in.Strategy != nil { + in, out := &in.Strategy, &out.Strategy + *out = new(ResourceTransformationStrategy) + **out = **in + } + if in.Outputs != nil { + in, out := &in.Outputs, &out.Outputs + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTransformation. +func (in *ResourceTransformation) DeepCopy() *ResourceTransformation { + if in == nil { + return nil + } + out := new(ResourceTransformation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Resources) DeepCopyInto(out *Resources) { + *out = *in + if in.ExcludeResourcePrefixes != nil { + in, out := &in.ExcludeResourcePrefixes, &out.ExcludeResourcePrefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Transformations != nil { + in, out := &in.Transformations, &out.Transformations + *out = make([]ResourceTransformation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.DeviceClassMappings != nil { + in, out := &in.DeviceClassMappings, &out.DeviceClassMappings + *out = make([]DeviceClassMapping, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Resources. +func (in *Resources) DeepCopy() *Resources { + if in == nil { + return nil + } + out := new(Resources) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WaitForPodsReady) DeepCopyInto(out *WaitForPodsReady) { + *out = *in + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(v1.Duration) + **out = **in + } + if in.BlockAdmission != nil { + in, out := &in.BlockAdmission, &out.BlockAdmission + *out = new(bool) + **out = **in + } + if in.RequeuingStrategy != nil { + in, out := &in.RequeuingStrategy, &out.RequeuingStrategy + *out = new(RequeuingStrategy) + (*in).DeepCopyInto(*out) + } + if in.RecoveryTimeout != nil { + in, out := &in.RecoveryTimeout, &out.RecoveryTimeout + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WaitForPodsReady. +func (in *WaitForPodsReady) DeepCopy() *WaitForPodsReady { + if in == nil { + return nil + } + out := new(WaitForPodsReady) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadRetentionPolicy) DeepCopyInto(out *WorkloadRetentionPolicy) { + *out = *in + if in.AfterFinished != nil { + in, out := &in.AfterFinished, &out.AfterFinished + *out = new(v1.Duration) + **out = **in + } + if in.AfterDeactivatedByKueue != nil { + in, out := &in.AfterDeactivatedByKueue, &out.AfterDeactivatedByKueue + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadRetentionPolicy. +func (in *WorkloadRetentionPolicy) DeepCopy() *WorkloadRetentionPolicy { + if in == nil { + return nil + } + out := new(WorkloadRetentionPolicy) + in.DeepCopyInto(out) + return out +} diff --git a/apis/config/v1beta2/zz_generated.defaults.go b/apis/config/v1beta2/zz_generated.defaults.go new file mode 100644 index 00000000000..304886432b1 --- /dev/null +++ b/apis/config/v1beta2/zz_generated.defaults.go @@ -0,0 +1,37 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1beta2 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&Configuration{}, func(obj interface{}) { SetObjectDefaults_Configuration(obj.(*Configuration)) }) + return nil +} + +func SetObjectDefaults_Configuration(in *Configuration) { + SetDefaults_Configuration(in) +} diff --git a/charts/kueue/values.yaml b/charts/kueue/values.yaml index e2aa043e2e5..dac543d6a9c 100644 --- a/charts/kueue/values.yaml +++ b/charts/kueue/values.yaml @@ -95,7 +95,7 @@ managerConfig: # ControllerManager utilizes this yaml via manager-config Configmap. # @default -- controllerManagerConfigYaml controllerManagerConfigYaml: |- - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration health: healthProbeBindAddress: :8081 diff --git a/cmd/kueue/main.go b/cmd/kueue/main.go index 1f8cff3dd37..5f8221c1768 100644 --- a/cmd/kueue/main.go +++ b/cmd/kueue/main.go @@ -48,7 +48,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapiv1beta1 "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueuealpha "sigs.k8s.io/kueue/apis/kueue/v1alpha1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" @@ -96,6 +97,7 @@ func init() { utilruntime.Must(kueue.AddToScheme(scheme)) utilruntime.Must(kueuev1beta2.AddToScheme(scheme)) utilruntime.Must(kueuealpha.AddToScheme(scheme)) + utilruntime.Must(configapiv1beta1.AddToScheme(scheme)) utilruntime.Must(configapi.AddToScheme(scheme)) utilruntime.Must(autoscaling.AddToScheme(scheme)) // Add any additional framework integration types. diff --git a/config/components/manager/controller_manager_config.yaml b/config/components/manager/controller_manager_config.yaml index bed137f8440..5a1614fc8f8 100644 --- a/config/components/manager/controller_manager_config.yaml +++ b/config/components/manager/controller_manager_config.yaml @@ -1,4 +1,4 @@ -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration health: healthProbeBindAddress: :8081 diff --git a/keps/2941-DRA/README.md b/keps/2941-DRA/README.md index b380b0c455b..a12cbfab5af 100644 --- a/keps/2941-DRA/README.md +++ b/keps/2941-DRA/README.md @@ -285,7 +285,7 @@ metadata: namespace: kueue-system data: config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system manageJobsWithoutQueueName: false @@ -350,7 +350,7 @@ metadata: namespace: kueue-system data: config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration resources: deviceClassMappings: @@ -371,7 +371,7 @@ metadata: namespace: kueue-system data: config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration resources: deviceClassMappings: @@ -467,7 +467,7 @@ metadata: namespace: kueue-system data: config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system resources: diff --git a/pkg/cache/queue/cluster_queue.go b/pkg/cache/queue/cluster_queue.go index 3d151fe1bc9..714b6783e05 100644 --- a/pkg/cache/queue/cluster_queue.go +++ b/pkg/cache/queue/cluster_queue.go @@ -31,7 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/cache/hierarchy" afs "sigs.k8s.io/kueue/pkg/util/admissionfairsharing" diff --git a/pkg/cache/queue/cluster_queue_test.go b/pkg/cache/queue/cluster_queue_test.go index bc93ef9bf7d..115b0e1061e 100644 --- a/pkg/cache/queue/cluster_queue_test.go +++ b/pkg/cache/queue/cluster_queue_test.go @@ -30,7 +30,7 @@ import ( testingclock "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/cache/queue/manager.go b/pkg/cache/queue/manager.go index aa7701f363c..878c67a74e6 100644 --- a/pkg/cache/queue/manager.go +++ b/pkg/cache/queue/manager.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/cache/hierarchy" utilindexer "sigs.k8s.io/kueue/pkg/controller/core/indexer" diff --git a/pkg/cache/scheduler/cache.go b/pkg/cache/scheduler/cache.go index 233689f5069..d2871c23073 100644 --- a/pkg/cache/scheduler/cache.go +++ b/pkg/cache/scheduler/cache.go @@ -34,7 +34,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/cache/hierarchy" utilindexer "sigs.k8s.io/kueue/pkg/controller/core/indexer" diff --git a/pkg/config/config.go b/pkg/config/config.go index b84741dc476..5bf2ca91e91 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -31,7 +31,7 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" ) var ( diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index bd62c1a5165..1fc909c6c41 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -42,7 +42,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/job" _ "sigs.k8s.io/kueue/pkg/controller/jobs" @@ -94,7 +94,7 @@ func TestLoad(t *testing.T) { namespaceOverWriteConfig := filepath.Join(tmpDir, "namespace-overwrite.yaml") if err := os.WriteFile(namespaceOverWriteConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-tenant-a health: @@ -112,7 +112,7 @@ webhook: emptyConfig := filepath.Join(tmpDir, "empty-config.yaml") if err := os.WriteFile(emptyConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration `), os.FileMode(0600)); err != nil { t.Fatal(err) @@ -120,7 +120,7 @@ kind: Configuration ctrlManagerConfigSpecOverWriteConfig := filepath.Join(tmpDir, "ctrl-manager-config-spec-overwrite.yaml") if err := os.WriteFile(ctrlManagerConfigSpecOverWriteConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system health: @@ -138,7 +138,7 @@ webhook: certOverWriteConfig := filepath.Join(tmpDir, "cert-overwrite.yaml") if err := os.WriteFile(certOverWriteConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system health: @@ -160,7 +160,7 @@ internalCertManagement: disableCertOverWriteConfig := filepath.Join(tmpDir, "disable-cert-overwrite.yaml") if err := os.WriteFile(disableCertOverWriteConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system health: @@ -180,7 +180,7 @@ internalCertManagement: leaderElectionDisabledConfig := filepath.Join(tmpDir, "leaderElection-disabled.yaml") if err := os.WriteFile(leaderElectionDisabledConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system health: @@ -197,7 +197,7 @@ webhook: waitForPodsReadyEnabledConfig := filepath.Join(tmpDir, "waitForPodsReady-enabled.yaml") if err := os.WriteFile(waitForPodsReadyEnabledConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration waitForPodsReady: enable: true @@ -215,7 +215,7 @@ waitForPodsReady: clientConnectionConfig := filepath.Join(tmpDir, "clientConnection.yaml") if err := os.WriteFile(clientConnectionConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system health: @@ -236,7 +236,7 @@ clientConnection: fullControllerConfig := filepath.Join(tmpDir, "fullControllerConfig.yaml") if err := os.WriteFile(fullControllerConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system health: @@ -271,7 +271,7 @@ clientConnection: integrationsConfig := filepath.Join(tmpDir, "integrations.yaml") if err := os.WriteFile(integrationsConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration integrations: frameworks: @@ -284,7 +284,7 @@ integrations: podIntegrationOptionsConfig := filepath.Join(tmpDir, "podIntegrationOptions.yaml") if err := os.WriteFile(podIntegrationOptionsConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration integrations: frameworks: @@ -306,7 +306,7 @@ integrations: multiKueueConfig := filepath.Join(tmpDir, "multiKueue.yaml") if err := os.WriteFile(multiKueueConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system multiKueue: @@ -320,7 +320,7 @@ multiKueue: resourceTransformConfig := filepath.Join(tmpDir, "resourceXForm.yaml") if err := os.WriteFile(resourceTransformConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system resources: @@ -345,7 +345,7 @@ resources: invalidConfig := filepath.Join(tmpDir, "invalid-config.yaml") if err := os.WriteFile(invalidConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespaces: kueue-system invalidField: invalidValue @@ -363,7 +363,7 @@ webhook: } objectRetentionPoliciesConfig := filepath.Join(tmpDir, "objectRetentionPolicies.yaml") - if err := os.WriteFile(objectRetentionPoliciesConfig, []byte(`apiVersion: config.kueue.x-k8s.io/v1beta1 + if err := os.WriteFile(objectRetentionPoliciesConfig, []byte(`apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system objectRetentionPolicies: @@ -1015,7 +1015,7 @@ func TestEncode(t *testing.T) { scheme: testScheme, cfg: &configapi.Configuration{}, wantResult: map[string]any{ - "apiVersion": "config.kueue.x-k8s.io/v1beta1", + "apiVersion": "config.kueue.x-k8s.io/v1beta2", "kind": "Configuration", "manageJobsWithoutQueueName": false, "health": map[string]any{}, @@ -1028,7 +1028,7 @@ func TestEncode(t *testing.T) { scheme: testScheme, cfg: defaultConfig, wantResult: map[string]any{ - "apiVersion": "config.kueue.x-k8s.io/v1beta1", + "apiVersion": "config.kueue.x-k8s.io/v1beta2", "kind": "Configuration", "namespace": configapi.DefaultNamespace, "webhook": map[string]any{ diff --git a/pkg/config/validation.go b/pkg/config/validation.go index c207103ed4e..7ef3c56513b 100644 --- a/pkg/config/validation.go +++ b/pkg/config/validation.go @@ -36,7 +36,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" podworkload "sigs.k8s.io/kueue/pkg/controller/jobs/pod" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/config/validation_test.go b/pkg/config/validation_test.go index 88dfad3eb40..ee1c87a6364 100644 --- a/pkg/config/validation_test.go +++ b/pkg/config/validation_test.go @@ -31,7 +31,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" ) func TestValidate(t *testing.T) { diff --git a/pkg/controller/admissionchecks/multikueue/controllers.go b/pkg/controller/admissionchecks/multikueue/controllers.go index 888b11f85da..8cdd382b123 100644 --- a/pkg/controller/admissionchecks/multikueue/controllers.go +++ b/pkg/controller/admissionchecks/multikueue/controllers.go @@ -21,7 +21,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/admissioncheck" diff --git a/pkg/controller/admissionchecks/multikueue/externalframeworks/config.go b/pkg/controller/admissionchecks/multikueue/externalframeworks/config.go index 2eb0f4b3a96..874e6b8a539 100644 --- a/pkg/controller/admissionchecks/multikueue/externalframeworks/config.go +++ b/pkg/controller/admissionchecks/multikueue/externalframeworks/config.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" k8serrors "k8s.io/apimachinery/pkg/util/errors" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" ) // NewAdapters creates and returns adapters from the given configurations. diff --git a/pkg/controller/admissionchecks/multikueue/externalframeworks/config_test.go b/pkg/controller/admissionchecks/multikueue/externalframeworks/config_test.go index 2c505fd9673..5ce45046488 100644 --- a/pkg/controller/admissionchecks/multikueue/externalframeworks/config_test.go +++ b/pkg/controller/admissionchecks/multikueue/externalframeworks/config_test.go @@ -22,7 +22,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" ) func TestNewAdapters(t *testing.T) { diff --git a/pkg/controller/admissionchecks/multikueue/workload.go b/pkg/controller/admissionchecks/multikueue/workload.go index 6ad58de7158..4f87ab85f87 100644 --- a/pkg/controller/admissionchecks/multikueue/workload.go +++ b/pkg/controller/admissionchecks/multikueue/workload.go @@ -42,7 +42,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/controller/admissionchecks/multikueue/workload_test.go b/pkg/controller/admissionchecks/multikueue/workload_test.go index 6453997bfc1..6064f184644 100644 --- a/pkg/controller/admissionchecks/multikueue/workload_test.go +++ b/pkg/controller/admissionchecks/multikueue/workload_test.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/controller/core/admissioncheck_controller.go b/pkg/controller/core/admissioncheck_controller.go index 6c8fe09c226..326bc6b1da3 100644 --- a/pkg/controller/core/admissioncheck_controller.go +++ b/pkg/controller/core/admissioncheck_controller.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/core/clusterqueue_controller.go b/pkg/controller/core/clusterqueue_controller.go index dbe1cb60979..0eb8644ea1a 100644 --- a/pkg/controller/core/clusterqueue_controller.go +++ b/pkg/controller/core/clusterqueue_controller.go @@ -43,7 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/core/cohort_controller.go b/pkg/controller/core/cohort_controller.go index 895e6acc427..98b2c1d86dc 100644 --- a/pkg/controller/core/cohort_controller.go +++ b/pkg/controller/core/cohort_controller.go @@ -34,7 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/core/core.go b/pkg/controller/core/core.go index f5cc6dd3961..b30c4e582c2 100644 --- a/pkg/controller/core/core.go +++ b/pkg/controller/core/core.go @@ -21,7 +21,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/pkg/controller/core/leader_aware_reconciler.go b/pkg/controller/core/leader_aware_reconciler.go index 0d99df92b16..faac231982b 100644 --- a/pkg/controller/core/leader_aware_reconciler.go +++ b/pkg/controller/core/leader_aware_reconciler.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" ) // WithLeadingManager returns a decorating reconcile.Reconciler that discards reconciliation requests diff --git a/pkg/controller/core/localqueue_controller.go b/pkg/controller/core/localqueue_controller.go index 1381b553fbe..9c0f886360e 100644 --- a/pkg/controller/core/localqueue_controller.go +++ b/pkg/controller/core/localqueue_controller.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/core/localqueue_controller_test.go b/pkg/controller/core/localqueue_controller_test.go index f323bc8ca4d..ecf9f2230a5 100644 --- a/pkg/controller/core/localqueue_controller_test.go +++ b/pkg/controller/core/localqueue_controller_test.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/core/resourceflavor_controller.go b/pkg/controller/core/resourceflavor_controller.go index f0b4af52581..9c14cc51e22 100644 --- a/pkg/controller/core/resourceflavor_controller.go +++ b/pkg/controller/core/resourceflavor_controller.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index 382130ae8a0..0f1c3e1cf52 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -50,7 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/core/workload_controller_test.go b/pkg/controller/core/workload_controller_test.go index a1878977dda..3e8c79fd9f6 100644 --- a/pkg/controller/core/workload_controller_test.go +++ b/pkg/controller/core/workload_controller_test.go @@ -39,7 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index 5ae6435f555..66d6f7ff5d2 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -43,7 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/jobframework/reconciler_test.go b/pkg/controller/jobframework/reconciler_test.go index 7d4e3ae3298..50e31eeef4e 100644 --- a/pkg/controller/jobframework/reconciler_test.go +++ b/pkg/controller/jobframework/reconciler_test.go @@ -37,7 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/jobset/api/jobset/v1alpha2" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" mocks "sigs.k8s.io/kueue/internal/mocks/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/pkg/controller/jobs/job/job_controller_test.go b/pkg/controller/jobs/job/job_controller_test.go index 29574b599a9..76546cd6cb2 100644 --- a/pkg/controller/jobs/job/job_controller_test.go +++ b/pkg/controller/jobs/job/job_controller_test.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/pkg/controller/jobs/pod/pod_controller_test.go b/pkg/controller/jobs/pod/pod_controller_test.go index bf98dec0cd3..685610ee5b3 100644 --- a/pkg/controller/jobs/pod/pod_controller_test.go +++ b/pkg/controller/jobs/pod/pod_controller_test.go @@ -39,7 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/controller/jobs/pod/pod_webhook.go b/pkg/controller/jobs/pod/pod_webhook.go index b15d43c8d95..4f8ee2e4a41 100644 --- a/pkg/controller/jobs/pod/pod_webhook.go +++ b/pkg/controller/jobs/pod/pod_webhook.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" diff --git a/pkg/controller/jobs/pod/pod_webhook_test.go b/pkg/controller/jobs/pod/pod_webhook_test.go index 2642f99e859..cfe3280dff6 100644 --- a/pkg/controller/jobs/pod/pod_webhook_test.go +++ b/pkg/controller/jobs/pod/pod_webhook_test.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/tas/controllers.go b/pkg/controller/tas/controllers.go index 6e32fee1e55..941ebced5a9 100644 --- a/pkg/controller/tas/controllers.go +++ b/pkg/controller/tas/controllers.go @@ -19,7 +19,7 @@ package tas import ( ctrl "sigs.k8s.io/controller-runtime" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/controller/tas/node_failure_controller.go b/pkg/controller/tas/node_failure_controller.go index 56ef9b3d229..0f172be2d7e 100644 --- a/pkg/controller/tas/node_failure_controller.go +++ b/pkg/controller/tas/node_failure_controller.go @@ -43,7 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/controller/tas/indexer" diff --git a/pkg/controller/tas/resource_flavor.go b/pkg/controller/tas/resource_flavor.go index 39d15ad4fd6..e7165615dd0 100644 --- a/pkg/controller/tas/resource_flavor.go +++ b/pkg/controller/tas/resource_flavor.go @@ -39,7 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/tas/topology_controller.go b/pkg/controller/tas/topology_controller.go index 931c5ad4072..cea24de59b5 100644 --- a/pkg/controller/tas/topology_controller.go +++ b/pkg/controller/tas/topology_controller.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/controller/tas/topology_ungater.go b/pkg/controller/tas/topology_ungater.go index 70ef8a1a050..7042cffa440 100644 --- a/pkg/controller/tas/topology_ungater.go +++ b/pkg/controller/tas/topology_ungater.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/pkg/controller/workloaddispatcher/controllers.go b/pkg/controller/workloaddispatcher/controllers.go index 59bb0da5ed4..12d2bb86e7b 100644 --- a/pkg/controller/workloaddispatcher/controllers.go +++ b/pkg/controller/workloaddispatcher/controllers.go @@ -19,7 +19,7 @@ package dispatcher import ( ctrl "sigs.k8s.io/controller-runtime" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" "sigs.k8s.io/kueue/pkg/util/admissioncheck" ) diff --git a/pkg/controller/workloaddispatcher/incrementaldispatcher.go b/pkg/controller/workloaddispatcher/incrementaldispatcher.go index 115c506c469..76228f2fe52 100644 --- a/pkg/controller/workloaddispatcher/incrementaldispatcher.go +++ b/pkg/controller/workloaddispatcher/incrementaldispatcher.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueueconfig "sigs.k8s.io/kueue/apis/config/v1beta1" + kueueconfig "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/util/admissioncheck" diff --git a/pkg/controller/workloaddispatcher/incrementaldispatcher_test.go b/pkg/controller/workloaddispatcher/incrementaldispatcher_test.go index c987269d524..73c22243719 100644 --- a/pkg/controller/workloaddispatcher/incrementaldispatcher_test.go +++ b/pkg/controller/workloaddispatcher/incrementaldispatcher_test.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueueconfig "sigs.k8s.io/kueue/apis/config/v1beta1" + kueueconfig "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utilmaps "sigs.k8s.io/kueue/pkg/util/maps" diff --git a/pkg/dra/claims_test.go b/pkg/dra/claims_test.go index dc722d049ec..7e7cce60004 100644 --- a/pkg/dra/claims_test.go +++ b/pkg/dra/claims_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) diff --git a/pkg/dra/mapper.go b/pkg/dra/mapper.go index 64977e0c3c7..0167aa680ab 100644 --- a/pkg/dra/mapper.go +++ b/pkg/dra/mapper.go @@ -19,7 +19,7 @@ package dra import ( corev1 "k8s.io/api/core/v1" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" ) var ( diff --git a/pkg/dra/mapper_test.go b/pkg/dra/mapper_test.go index 183decd2b24..9c4ad67362c 100644 --- a/pkg/dra/mapper_test.go +++ b/pkg/dra/mapper_test.go @@ -21,7 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" ) func TestNewDRAResourceMapper(t *testing.T) { diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index f20387fe168..e6a25c997ea 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -33,7 +33,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/pkg/scheduler/preemption/preemption_test.go b/pkg/scheduler/preemption/preemption_test.go index dfe925d08f8..67b6871dab3 100644 --- a/pkg/scheduler/preemption/preemption_test.go +++ b/pkg/scheduler/preemption/preemption_test.go @@ -36,7 +36,7 @@ import ( clocktesting "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/cache/hierarchy" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 38e14e47975..c5008773df1 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -37,7 +37,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/scheduler/scheduler_afs_test.go b/pkg/scheduler/scheduler_afs_test.go index eb4fe32b3ad..05047398659 100644 --- a/pkg/scheduler/scheduler_afs_test.go +++ b/pkg/scheduler/scheduler_afs_test.go @@ -32,7 +32,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index cf70be31a37..cb114f2230c 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/pkg/util/admissionfairsharing/admission_fair_sharing.go b/pkg/util/admissionfairsharing/admission_fair_sharing.go index b12298b681a..b867af8a4c9 100644 --- a/pkg/util/admissionfairsharing/admission_fair_sharing.go +++ b/pkg/util/admissionfairsharing/admission_fair_sharing.go @@ -21,7 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/resource" diff --git a/pkg/util/cert/cert.go b/pkg/util/cert/cert.go index 7fe7e236ed2..18c6d4e5ec5 100644 --- a/pkg/util/cert/cert.go +++ b/pkg/util/cert/cert.go @@ -25,7 +25,7 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" ) const ( diff --git a/pkg/workload/workload.go b/pkg/workload/workload.go index bf38b943840..3fa7312825c 100644 --- a/pkg/workload/workload.go +++ b/pkg/workload/workload.go @@ -39,7 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/workload/workload_test.go b/pkg/workload/workload_test.go index 3797617a15d..c05b11900b5 100644 --- a/pkg/workload/workload_test.go +++ b/pkg/workload/workload_test.go @@ -32,7 +32,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" diff --git a/site/content/en/docs/concepts/preemption.md b/site/content/en/docs/concepts/preemption.md index 75f545d49ac..29fe0874d97 100644 --- a/site/content/en/docs/concepts/preemption.md +++ b/site/content/en/docs/concepts/preemption.md @@ -135,7 +135,7 @@ unsupported, and results in undefined behavior. To enable Fair Sharing, [use a Kueue Configuration](/docs/installation#install-a-custom-configured-release-version) similar to the following: ```yaml -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration fairSharing: enable: true diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index e83ad0014c0..03c2da36dd8 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -140,7 +140,7 @@ metadata: namespace: kueue-system data: controller_manager_config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system health: diff --git a/site/content/en/docs/reference/kueue-config.v1beta1.md b/site/content/en/docs/reference/kueue-config.v1beta1.md index 75f61714c80..3f0e2025fc6 100644 --- a/site/content/en/docs/reference/kueue-config.v1beta1.md +++ b/site/content/en/docs/reference/kueue-config.v1beta1.md @@ -1,21 +1,18 @@ --- title: Kueue Configuration API content_type: tool-reference -package: /v1beta1 +package: config.kueue.x-k8s.io/v1beta1 auto_generated: true -description: Generated API reference documentation for Kueue Configuration. +description: Generated API reference documentation for config.kueue.x-k8s.io/v1beta1. --- ## Resource Types -- [Configuration](#Configuration) - - -## `AdmissionFairSharing` {#AdmissionFairSharing} +## `AdmissionFairSharing` {#config-kueue-x-k8s-io-v1beta1-AdmissionFairSharing} **Appears in:** @@ -56,7 +53,7 @@ Defaults to 1.

-## `ClientConnection` {#ClientConnection} +## `ClientConnection` {#config-kueue-x-k8s-io-v1beta1-ClientConnection} **Appears in:** @@ -88,12 +85,12 @@ connection.

-## `ClusterQueueVisibility` {#ClusterQueueVisibility} +## `ClusterQueueVisibility` {#config-kueue-x-k8s-io-v1beta1-ClusterQueueVisibility} **Appears in:** -- [QueueVisibility](#QueueVisibility) +- [QueueVisibility](#config-kueue-x-k8s-io-v1beta1-QueueVisibility) @@ -116,165 +113,12 @@ Defaults to 10.

-## `Configuration` {#Configuration} - - - -

Configuration is the Schema for the kueueconfigurations API

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldDescription
namespace [Required]
-string -
-

Namespace is the namespace in which kueue is deployed. It is used as part of DNSName of the webhook Service. -If not set, the value is set from the file /var/run/secrets/kubernetes.io/serviceaccount/namespace -If the file doesn't exist, default value is kueue-system.

-
ControllerManager [Required]
-ControllerManager -
(Members of ControllerManager are embedded into this type.) -

ControllerManager returns the configurations for controllers

-
manageJobsWithoutQueueName [Required]
-bool -
-

ManageJobsWithoutQueueName controls whether or not Kueue reconciles -jobs that don't set the label kueue.x-k8s.io/queue-name. -If set to true, then those jobs will be suspended and never started unless -they are assigned a queue and eventually admitted. This also applies to -jobs created before starting the kueue controller. -Defaults to false; therefore, those jobs are not managed and if they are created -unsuspended, they will start immediately.

-
managedJobsNamespaceSelector [Required]
-k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector -
-

ManagedJobsNamespaceSelector provides a namespace-based mechanism to exempt jobs -from management by Kueue.

-

It provides a strong exemption for the Pod-based integrations (pod, deployment, statefulset, etc.), -For Pod-based integrations, only jobs whose namespaces match ManagedJobsNamespaceSelector are -eligible to be managed by Kueue. Pods, deployments, etc. in non-matching namespaces will -never be managed by Kueue, even if they have a kueue.x-k8s.io/queue-name label. -This strong exemption ensures that Kueue will not interfere with the basic operation -of system namespace.

-

For all other integrations, ManagedJobsNamespaceSelector provides a weaker exemption -by only modulating the effects of ManageJobsWithoutQueueName. For these integrations, -a job that has a kueue.x-k8s.io/queue-name label will always be managed by Kueue. Jobs without -a kueue.x-k8s.io/queue-name label will be managed by Kueue only when ManageJobsWithoutQueueName is -true and the job's namespace matches ManagedJobsNamespaceSelector.

-
internalCertManagement [Required]
-InternalCertManagement -
-

InternalCertManagement is configuration for internalCertManagement

-
waitForPodsReady [Required]
-WaitForPodsReady -
-

WaitForPodsReady is configuration to provide a time-based all-or-nothing -scheduling semantics for Jobs, by ensuring all pods are ready (running -and passing the readiness probe) within the specified time. If the timeout -is exceeded, then the workload is evicted.

-
clientConnection [Required]
-ClientConnection -
-

ClientConnection provides additional configuration options for Kubernetes -API server client.

-
integrations [Required]
-Integrations -
-

Integrations provide configuration options for AI/ML/Batch frameworks -integrations (including K8S job).

-
queueVisibility [Required]
-QueueVisibility -
-

QueueVisibility is configuration to expose the information about the top -pending workloads. -Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand -(https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) -instead.

-
multiKueue [Required]
-MultiKueue -
-

MultiKueue controls the behaviour of the MultiKueue AdmissionCheck Controller.

-
fairSharing [Required]
-FairSharing -
-

FairSharing controls the Fair Sharing semantics across the cluster.

-
admissionFairSharing [Required]
-AdmissionFairSharing -
-

admissionFairSharing indicates configuration of FairSharing with the AdmissionTime mode on

-
resources [Required]
-Resources -
-

Resources provides additional configuration options for handling the resources.

-
featureGates [Required]
-map[string]bool -
-

FeatureGates is a map of feature names to bools that allows to override the -default enablement status of a feature. The map cannot be used in conjunction -with passing the list of features via the command line argument "--feature-gates" -for the Kueue Deployment.

-
objectRetentionPolicies
-ObjectRetentionPolicies -
-

ObjectRetentionPolicies provides configuration options for automatic deletion -of Kueue-managed objects. A nil value disables all automatic deletions.

-
- -## `ControllerConfigurationSpec` {#ControllerConfigurationSpec} +## `ControllerConfigurationSpec` {#config-kueue-x-k8s-io-v1beta1-ControllerConfigurationSpec} **Appears in:** -- [ControllerManager](#ControllerManager) +- [ControllerManager](#config-kueue-x-k8s-io-v1beta1-ControllerManager)

ControllerConfigurationSpec defines the global configuration for @@ -311,12 +155,12 @@ Defaults to 2 minutes if not set.

-## `ControllerHealth` {#ControllerHealth} +## `ControllerHealth` {#config-kueue-x-k8s-io-v1beta1-ControllerHealth} **Appears in:** -- [ControllerManager](#ControllerManager) +- [ControllerManager](#config-kueue-x-k8s-io-v1beta1-ControllerManager)

ControllerHealth defines the health configs.

@@ -353,7 +197,7 @@ It can be set to "0" or "" to disable serving the health pro -## `ControllerManager` {#ControllerManager} +## `ControllerManager` {#config-kueue-x-k8s-io-v1beta1-ControllerManager} **Appears in:** @@ -367,7 +211,7 @@ It can be set to "0" or "" to disable serving the health pro webhook
-ControllerWebhook +ControllerWebhook

Webhook contains the controllers webhook configuration

@@ -382,14 +226,14 @@ the manager.Manager leader election

metrics
-ControllerMetrics +ControllerMetrics

Metrics contains the controller metrics configuration

health
-ControllerHealth +ControllerHealth

Health contains the controller health configuration

@@ -407,7 +251,7 @@ before exposing it to public.

controller
-ControllerConfigurationSpec +ControllerConfigurationSpec

Controller contains global configuration options for controllers @@ -417,12 +261,12 @@ registered within this manager.

-## `ControllerMetrics` {#ControllerMetrics} +## `ControllerMetrics` {#config-kueue-x-k8s-io-v1beta1-ControllerMetrics} **Appears in:** -- [ControllerManager](#ControllerManager) +- [ControllerManager](#config-kueue-x-k8s-io-v1beta1-ControllerManager)

ControllerMetrics defines the metrics configs.

@@ -453,12 +297,12 @@ metrics will be reported.

-## `ControllerWebhook` {#ControllerWebhook} +## `ControllerWebhook` {#config-kueue-x-k8s-io-v1beta1-ControllerWebhook} **Appears in:** -- [ControllerManager](#ControllerManager) +- [ControllerManager](#config-kueue-x-k8s-io-v1beta1-ControllerManager)

ControllerWebhook defines the webhook server for the controller.

@@ -498,12 +342,12 @@ must be named tls.key and tls.crt, respectively.

-## `DeviceClassMapping` {#DeviceClassMapping} +## `DeviceClassMapping` {#config-kueue-x-k8s-io-v1beta1-DeviceClassMapping} **Appears in:** -- [Resources](#Resources) +- [Resources](#config-kueue-x-k8s-io-v1beta1-Resources)

DeviceClassMapping holds device class to logical resource mappings @@ -544,7 +388,7 @@ The total length of each name must not exceed 253 characters.

-## `FairSharing` {#FairSharing} +## `FairSharing` {#config-kueue-x-k8s-io-v1beta1-FairSharing} **Appears in:** @@ -566,7 +410,7 @@ Defaults to false.

preemptionStrategies [Required]
-[]PreemptionStrategy +[]PreemptionStrategy

preemptionStrategies indicates which constraints should a preemption satisfy. @@ -592,7 +436,7 @@ The default strategy is ["LessThanOrEqualToFinalShare", "LessThan -## `Integrations` {#Integrations} +## `Integrations` {#config-kueue-x-k8s-io-v1beta1-Integrations} **Appears in:** @@ -640,7 +484,7 @@ the expected format is Kind.version.group.com.

podOptions [Required]
-PodIntegrationOptions +PodIntegrationOptions

PodOptions defines kueue controller behaviour for pod objects @@ -667,7 +511,7 @@ underlying job are changed.

-## `InternalCertManagement` {#InternalCertManagement} +## `InternalCertManagement` {#config-kueue-x-k8s-io-v1beta1-InternalCertManagement} **Appears in:** @@ -716,7 +560,7 @@ Defaults to kueue-webhook-server-cert.

-## `MultiKueue` {#MultiKueue} +## `MultiKueue` {#config-kueue-x-k8s-io-v1beta1-MultiKueue} **Appears in:** @@ -769,7 +613,7 @@ if the connection with its reserving worker cluster is lost.

externalFrameworks
-[]MultiKueueExternalFramework +[]MultiKueueExternalFramework

ExternalFrameworks defines a list of external frameworks that should be supported @@ -780,12 +624,12 @@ GroupVersionKind (GVK) for MultiKueue operations.

-## `MultiKueueExternalFramework` {#MultiKueueExternalFramework} +## `MultiKueueExternalFramework` {#config-kueue-x-k8s-io-v1beta1-MultiKueueExternalFramework} **Appears in:** -- [MultiKueue](#MultiKueue) +- [MultiKueue](#config-kueue-x-k8s-io-v1beta1-MultiKueue)

MultiKueueExternalFramework defines a framework that is not built-in.

@@ -808,7 +652,7 @@ the expected format is kind.version.group.

-## `ObjectRetentionPolicies` {#ObjectRetentionPolicies} +## `ObjectRetentionPolicies` {#config-kueue-x-k8s-io-v1beta1-ObjectRetentionPolicies} **Appears in:** @@ -824,7 +668,7 @@ the expected format is kind.version.group.

workloads
-WorkloadRetentionPolicy +WorkloadRetentionPolicy

Workloads configures retention for Workloads. @@ -834,12 +678,12 @@ A nil value disables automatic deletion of Workloads.

-## `PodIntegrationOptions` {#PodIntegrationOptions} +## `PodIntegrationOptions` {#config-kueue-x-k8s-io-v1beta1-PodIntegrationOptions} **Appears in:** -- [Integrations](#Integrations) +- [Integrations](#config-kueue-x-k8s-io-v1beta1-Integrations) @@ -865,19 +709,19 @@ A nil value disables automatic deletion of Workloads.

-## `PreemptionStrategy` {#PreemptionStrategy} +## `PreemptionStrategy` {#config-kueue-x-k8s-io-v1beta1-PreemptionStrategy} (Alias of `string`) **Appears in:** -- [FairSharing](#FairSharing) +- [FairSharing](#config-kueue-x-k8s-io-v1beta1-FairSharing) -## `QueueVisibility` {#QueueVisibility} +## `QueueVisibility` {#config-kueue-x-k8s-io-v1beta1-QueueVisibility} **Appears in:** @@ -891,7 +735,7 @@ A nil value disables automatic deletion of Workloads.

clusterQueues [Required]
-ClusterQueueVisibility +ClusterQueueVisibility

ClusterQueues is configuration to expose the information @@ -911,12 +755,12 @@ Defaults to 5.

-## `RequeuingStrategy` {#RequeuingStrategy} +## `RequeuingStrategy` {#config-kueue-x-k8s-io-v1beta1-RequeuingStrategy} **Appears in:** -- [WaitForPodsReady](#WaitForPodsReady) +- [WaitForPodsReady](#config-kueue-x-k8s-io-v1beta1-WaitForPodsReady) @@ -926,7 +770,7 @@ Defaults to 5.

timestamp
-RequeuingTimestamp +RequeuingTimestamp

Timestamp defines the timestamp used for re-queuing a Workload @@ -976,24 +820,24 @@ re-queuing an evicted workload.

-## `RequeuingTimestamp` {#RequeuingTimestamp} +## `RequeuingTimestamp` {#config-kueue-x-k8s-io-v1beta1-RequeuingTimestamp} (Alias of `string`) **Appears in:** -- [RequeuingStrategy](#RequeuingStrategy) +- [RequeuingStrategy](#config-kueue-x-k8s-io-v1beta1-RequeuingStrategy) -## `ResourceTransformation` {#ResourceTransformation} +## `ResourceTransformation` {#config-kueue-x-k8s-io-v1beta1-ResourceTransformation} **Appears in:** -- [Resources](#Resources) +- [Resources](#config-kueue-x-k8s-io-v1beta1-Resources) @@ -1010,7 +854,7 @@ re-queuing an evicted workload.

strategy [Required]
-ResourceTransformationStrategy +ResourceTransformationStrategy

Strategy specifies if the input resource should be replaced or retained. @@ -1028,19 +872,19 @@ An empty Outputs combined with a Replace Strategy causes the Input -## `ResourceTransformationStrategy` {#ResourceTransformationStrategy} +## `ResourceTransformationStrategy` {#config-kueue-x-k8s-io-v1beta1-ResourceTransformationStrategy} (Alias of `string`) **Appears in:** -- [ResourceTransformation](#ResourceTransformation) +- [ResourceTransformation](#config-kueue-x-k8s-io-v1beta1-ResourceTransformation) -## `Resources` {#Resources} +## `Resources` {#config-kueue-x-k8s-io-v1beta1-Resources} **Appears in:** @@ -1061,7 +905,7 @@ An empty Outputs combined with a Replace Strategy causes the Input transformations [Required]
-[]ResourceTransformation +[]ResourceTransformation

Transformations defines how to transform PodSpec resources into Workload resource requests. @@ -1069,7 +913,7 @@ This is intended to be a map with Input as the key (enforced by validation code) deviceClassMappings
-[]DeviceClassMapping +[]DeviceClassMapping

DeviceClassMappings defines mappings from device classes to logical resources @@ -1079,7 +923,7 @@ for Dynamic Resource Allocation support.

-## `WaitForPodsReady` {#WaitForPodsReady} +## `WaitForPodsReady` {#config-kueue-x-k8s-io-v1beta1-WaitForPodsReady} **Appears in:** @@ -1123,7 +967,7 @@ This setting is only honored when Enable is set to true.

requeuingStrategy
-RequeuingStrategy +RequeuingStrategy

RequeuingStrategy defines the strategy for requeuing a Workload.

@@ -1145,12 +989,12 @@ If not set, there is no timeout.

-## `WorkloadRetentionPolicy` {#WorkloadRetentionPolicy} +## `WorkloadRetentionPolicy` {#config-kueue-x-k8s-io-v1beta1-WorkloadRetentionPolicy} **Appears in:** -- [ObjectRetentionPolicies](#ObjectRetentionPolicies) +- [ObjectRetentionPolicies](#config-kueue-x-k8s-io-v1beta1-ObjectRetentionPolicies)

WorkloadRetentionPolicy defines the policies for when Workloads should be deleted.

@@ -1188,4 +1032,5 @@ Represented using metav1.Duration (e.g. "10m", "1h30m").

- \ No newline at end of file + + \ No newline at end of file diff --git a/site/content/en/docs/tasks/manage/administer_cluster_quotas.md b/site/content/en/docs/tasks/manage/administer_cluster_quotas.md index fa2366c8a90..d8074d21a94 100644 --- a/site/content/en/docs/tasks/manage/administer_cluster_quotas.md +++ b/site/content/en/docs/tasks/manage/administer_cluster_quotas.md @@ -394,7 +394,7 @@ Follow the [installation instructions for using a custom configuration](/docs/in and extend the configuration with fields similar to the following: ```yaml -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration resources: excludeResourcePrefixes: @@ -422,7 +422,7 @@ Follow the [installation instructions for using a custom configuration](/docs/in and extend the Kueue configuration with fields similar to the following: ```yaml -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration resources: transformations: diff --git a/site/content/en/docs/tasks/run/deployment.md b/site/content/en/docs/tasks/run/deployment.md index 61cd8edcd9b..5b5310fb276 100644 --- a/site/content/en/docs/tasks/run/deployment.md +++ b/site/content/en/docs/tasks/run/deployment.md @@ -25,7 +25,7 @@ For more information, see [Kueue's overview](/docs/overview). 2. Ensure that you have the `deployment` integration enabled, for example: ```yaml - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration integrations: frameworks: diff --git a/site/content/en/docs/tasks/run/leaderworkerset.md b/site/content/en/docs/tasks/run/leaderworkerset.md index 193e2d9996d..a41e00ff563 100644 --- a/site/content/en/docs/tasks/run/leaderworkerset.md +++ b/site/content/en/docs/tasks/run/leaderworkerset.md @@ -25,7 +25,7 @@ For more information, see [Kueue's overview](/docs/overview). 2. Ensure that you have the `leaderworkerset.x-k8s.io/leaderworkerset` integration enabled, for example: ```yaml - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration integrations: frameworks: diff --git a/site/content/en/docs/tasks/run/multikueue/external-frameworks.md b/site/content/en/docs/tasks/run/multikueue/external-frameworks.md index a603ee44b10..85f1fa7a90b 100644 --- a/site/content/en/docs/tasks/run/multikueue/external-frameworks.md +++ b/site/content/en/docs/tasks/run/multikueue/external-frameworks.md @@ -60,7 +60,7 @@ To demonstrate how to configure the adapter, let's use Tekton `PipelineRun` as a First, update your Kueue configuration to include `PipelineRun` in the `externalFrameworks` list: ```yaml -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration data: multikueue: diff --git a/site/content/en/docs/tasks/run/plain_pods.md b/site/content/en/docs/tasks/run/plain_pods.md index 99e4087d148..d748ec5b333 100644 --- a/site/content/en/docs/tasks/run/plain_pods.md +++ b/site/content/en/docs/tasks/run/plain_pods.md @@ -25,7 +25,7 @@ This guide is for [batch users](/docs/tasks#batch-user) that have a basic unders One approach is to only enable management only for specific namespaces: ```yaml - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration managedJobsNamespaceSelector: matchLabels: @@ -36,7 +36,7 @@ This guide is for [batch users](/docs/tasks#batch-user) that have a basic unders ``` An alternate approach is to exempt system namespaces from management: ```yaml - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration managedJobsNamespaceSelector: matchExpressions: diff --git a/site/content/en/docs/tasks/run/statefulset.md b/site/content/en/docs/tasks/run/statefulset.md index 48cc78e5e93..f6219690be6 100644 --- a/site/content/en/docs/tasks/run/statefulset.md +++ b/site/content/en/docs/tasks/run/statefulset.md @@ -23,7 +23,7 @@ For more information, see [Kueue's overview](/docs/overview). 2. Ensure that you have the v1/statefulset integration enabled, for example: ```yaml - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration integrations: frameworks: diff --git a/site/content/zh-CN/docs/concepts/preemption.md b/site/content/zh-CN/docs/concepts/preemption.md index ed03293c66b..3b6490fe4e0 100644 --- a/site/content/zh-CN/docs/concepts/preemption.md +++ b/site/content/zh-CN/docs/concepts/preemption.md @@ -101,7 +101,7 @@ Kueue 提供了两种抢占算法。它们的主要区别在于:当抢占 Clus 要启用公平共享,[请使用如下 Kueue 配置](/docs/installation#install-a-custom-configured-release-version): ```yaml -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration fairSharing: enable: true diff --git a/site/content/zh-CN/docs/installation/_index.md b/site/content/zh-CN/docs/installation/_index.md index 08f9a92483f..8ad673b0312 100644 --- a/site/content/zh-CN/docs/installation/_index.md +++ b/site/content/zh-CN/docs/installation/_index.md @@ -139,7 +139,7 @@ metadata: namespace: kueue-system data: controller_manager_config.yaml: | - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration namespace: kueue-system health: diff --git a/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md b/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md index f696b3d4afc..5ac6202af4c 100644 --- a/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md +++ b/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md @@ -383,7 +383,7 @@ kubectl apply -f team-a-cq.yaml -f team-b-cq.yaml -f shared-cq.yaml 操作,并在配置中添加如下字段: ```yaml -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration resources: excludeResourcePrefixes: @@ -410,7 +410,7 @@ resources: 操作,并在 Kueue 配置中添加如下字段: ```yaml -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration resources: transformations: diff --git a/site/content/zh-CN/docs/tasks/run/leaderworkerset.md b/site/content/zh-CN/docs/tasks/run/leaderworkerset.md index fdc25bcd054..78a5f7115d8 100644 --- a/site/content/zh-CN/docs/tasks/run/leaderworkerset.md +++ b/site/content/zh-CN/docs/tasks/run/leaderworkerset.md @@ -24,7 +24,7 @@ description: 将 LeaderWorkerSet 作为由 Kueue 管理的工作负载运行 2. 确保你已启用 `leaderworkerset.x-k8s.io/leaderworkerset` 集成,例如: ```yaml - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration integrations: frameworks: diff --git a/site/content/zh-CN/docs/tasks/run/plain_pods.md b/site/content/zh-CN/docs/tasks/run/plain_pods.md index 3523f14c15f..f6861a469b9 100644 --- a/site/content/zh-CN/docs/tasks/run/plain_pods.md +++ b/site/content/zh-CN/docs/tasks/run/plain_pods.md @@ -24,7 +24,7 @@ Kueue 支持管理[单个 Pod](#running-a-single-pod-admitted-by-kueue) 一种方法是仅对特定命名空间启用管理: ```yaml - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration managedJobsNamespaceSelector: matchLabels: @@ -35,7 +35,7 @@ Kueue 支持管理[单个 Pod](#running-a-single-pod-admitted-by-kueue) ``` 另一种方法是免除系统命名空间的管理: ```yaml - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration managedJobsNamespaceSelector: matchExpressions: diff --git a/site/content/zh-CN/docs/tasks/run/statefulset.md b/site/content/zh-CN/docs/tasks/run/statefulset.md index a008ec65486..d4f6666f853 100644 --- a/site/content/zh-CN/docs/tasks/run/statefulset.md +++ b/site/content/zh-CN/docs/tasks/run/statefulset.md @@ -20,7 +20,7 @@ description: 将 StatefulSet 作为 Kueue 管理的工作负载运行。 2. 确保你启用了 v1/statefulset 集成,例如: ```yaml - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration integrations: frameworks: diff --git a/test/e2e/config/certmanager/controller_manager_config.yaml b/test/e2e/config/certmanager/controller_manager_config.yaml index e2415fc3bda..7bbcc75c568 100644 --- a/test/e2e/config/certmanager/controller_manager_config.yaml +++ b/test/e2e/config/certmanager/controller_manager_config.yaml @@ -1,4 +1,4 @@ -apiVersion: config.kueue.x-k8s.io/v1beta1 +apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration metrics: enableClusterQueueResources: true diff --git a/test/e2e/config/certmanager/values.yaml b/test/e2e/config/certmanager/values.yaml index 2f5a22b5859..da13f7b9164 100644 --- a/test/e2e/config/certmanager/values.yaml +++ b/test/e2e/config/certmanager/values.yaml @@ -30,7 +30,7 @@ controllerManager: kubernetesClusterDomain: cluster.local managerConfig: controllerManagerConfigYaml: |- - apiVersion: config.kueue.x-k8s.io/v1beta1 + apiVersion: config.kueue.x-k8s.io/v1beta2 kind: Configuration metrics: enableClusterQueueResources: true diff --git a/test/e2e/customconfigs/managejobswithoutqueuename_test.go b/test/e2e/customconfigs/managejobswithoutqueuename_test.go index ae4b205f889..27aea660325 100644 --- a/test/e2e/customconfigs/managejobswithoutqueuename_test.go +++ b/test/e2e/customconfigs/managejobswithoutqueuename_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/jobset/api/jobset/v1alpha2" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/constants" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/test/e2e/customconfigs/objectretentionpolicies_test.go b/test/e2e/customconfigs/objectretentionpolicies_test.go index bbc015cb2da..dcb42dc79d9 100644 --- a/test/e2e/customconfigs/objectretentionpolicies_test.go +++ b/test/e2e/customconfigs/objectretentionpolicies_test.go @@ -26,7 +26,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" diff --git a/test/e2e/customconfigs/podintegrationautoenablement_test.go b/test/e2e/customconfigs/podintegrationautoenablement_test.go index a1cf25b3415..0ed80e32421 100644 --- a/test/e2e/customconfigs/podintegrationautoenablement_test.go +++ b/test/e2e/customconfigs/podintegrationautoenablement_test.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/statefulset" diff --git a/test/e2e/customconfigs/reconcile_test.go b/test/e2e/customconfigs/reconcile_test.go index 1790ef549e5..b72d16ad801 100644 --- a/test/e2e/customconfigs/reconcile_test.go +++ b/test/e2e/customconfigs/reconcile_test.go @@ -24,7 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/features" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" diff --git a/test/e2e/customconfigs/suite_test.go b/test/e2e/customconfigs/suite_test.go index 9c2f95c6d38..d9672eee538 100644 --- a/test/e2e/customconfigs/suite_test.go +++ b/test/e2e/customconfigs/suite_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -37,7 +37,7 @@ var ( cfg *rest.Config restClient *rest.RESTClient ctx context.Context - defaultKueueCfg *v1beta1.Configuration + defaultKueueCfg *config.Configuration kueueNS = util.GetKueueNamespace() kindClusterName = os.Getenv("KIND_CLUSTER_NAME") ) diff --git a/test/e2e/customconfigs/waitforpodsready_test.go b/test/e2e/customconfigs/waitforpodsready_test.go index e98a0282027..42c14262fc7 100644 --- a/test/e2e/customconfigs/waitforpodsready_test.go +++ b/test/e2e/customconfigs/waitforpodsready_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/test/e2e/multikueue/e2e_test.go b/test/e2e/multikueue/e2e_test.go index 6419b00313b..0eda05067bf 100644 --- a/test/e2e/multikueue/e2e_test.go +++ b/test/e2e/multikueue/e2e_test.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueueconfig "sigs.k8s.io/kueue/apis/config/v1beta1" + kueueconfig "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" workloadaw "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" diff --git a/test/integration/framework/framework.go b/test/integration/framework/framework.go index 3af92c5bb75..2d98e0f0759 100644 --- a/test/integration/framework/framework.go +++ b/test/integration/framework/framework.go @@ -49,7 +49,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" diff --git a/test/integration/multikueue/dispatcher_test.go b/test/integration/multikueue/dispatcher_test.go index 2d47498643f..9ddee6b0ec8 100644 --- a/test/integration/multikueue/dispatcher_test.go +++ b/test/integration/multikueue/dispatcher_test.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" diff --git a/test/integration/multikueue/enabled_integration_test.go b/test/integration/multikueue/enabled_integration_test.go index a493efd264c..200edaff591 100644 --- a/test/integration/multikueue/enabled_integration_test.go +++ b/test/integration/multikueue/enabled_integration_test.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" workloadmpijob "sigs.k8s.io/kueue/pkg/controller/jobs/mpijob" diff --git a/test/integration/multikueue/external_job_test.go b/test/integration/multikueue/external_job_test.go index ac2cd90d7ff..b90c0785ac2 100644 --- a/test/integration/multikueue/external_job_test.go +++ b/test/integration/multikueue/external_job_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" diff --git a/test/integration/multikueue/jobs_test.go b/test/integration/multikueue/jobs_test.go index 80bd38fe845..83e1514e319 100644 --- a/test/integration/multikueue/jobs_test.go +++ b/test/integration/multikueue/jobs_test.go @@ -41,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadappwrapper "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" diff --git a/test/integration/multikueue/no_gc_test.go b/test/integration/multikueue/no_gc_test.go index ee7828fd9dc..93213cae937 100644 --- a/test/integration/multikueue/no_gc_test.go +++ b/test/integration/multikueue/no_gc_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" diff --git a/test/integration/multikueue/setup_test.go b/test/integration/multikueue/setup_test.go index a69b9839577..f790d1d66af 100644 --- a/test/integration/multikueue/setup_test.go +++ b/test/integration/multikueue/setup_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" diff --git a/test/integration/multikueue/suite_test.go b/test/integration/multikueue/suite_test.go index 9559d9157f5..f67267f413d 100644 --- a/test/integration/multikueue/suite_test.go +++ b/test/integration/multikueue/suite_test.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/admissionchecks/provisioning/suite_test.go b/test/integration/singlecluster/controller/admissionchecks/provisioning/suite_test.go index ed341020b12..bc2f60a011c 100644 --- a/test/integration/singlecluster/controller/admissionchecks/provisioning/suite_test.go +++ b/test/integration/singlecluster/controller/admissionchecks/provisioning/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/core/suite_test.go b/test/integration/singlecluster/controller/core/suite_test.go index 3c5cb9f8c98..c15d63d030f 100644 --- a/test/integration/singlecluster/controller/core/suite_test.go +++ b/test/integration/singlecluster/controller/core/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/core" diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go index 153d6afa85b..67cec5acaab 100644 --- a/test/integration/singlecluster/controller/core/workload_controller_test.go +++ b/test/integration/singlecluster/controller/core/workload_controller_test.go @@ -29,7 +29,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" diff --git a/test/integration/singlecluster/controller/dra/dra_test.go b/test/integration/singlecluster/controller/dra/dra_test.go index e7e4120c4ad..f06511d5dbd 100644 --- a/test/integration/singlecluster/controller/dra/dra_test.go +++ b/test/integration/singlecluster/controller/dra/dra_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" diff --git a/test/integration/singlecluster/controller/dra/suite_test.go b/test/integration/singlecluster/controller/dra/suite_test.go index d337546dc1b..e743e82a87d 100644 --- a/test/integration/singlecluster/controller/dra/suite_test.go +++ b/test/integration/singlecluster/controller/dra/suite_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/core" diff --git a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go index cb975e82a82..377d256c73c 100644 --- a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go @@ -35,7 +35,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/test/integration/singlecluster/controller/jobs/appwrapper/suite_test.go b/test/integration/singlecluster/controller/jobs/appwrapper/suite_test.go index 72acf3af554..730f0ffe624 100644 --- a/test/integration/singlecluster/controller/jobs/appwrapper/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/appwrapper/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go b/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go index 17d8f61de78..24a4c5f394d 100644 --- a/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go @@ -29,7 +29,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/test/integration/singlecluster/controller/jobs/jaxjob/suite_test.go b/test/integration/singlecluster/controller/jobs/jaxjob/suite_test.go index d5a8b4e4eb6..d879ce5859a 100644 --- a/test/integration/singlecluster/controller/jobs/jaxjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/jaxjob/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go index b7a45f1e2ff..ea1d39e9740 100644 --- a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go @@ -40,7 +40,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/test/integration/singlecluster/controller/jobs/job/suite_test.go b/test/integration/singlecluster/controller/jobs/job/suite_test.go index 2ed7a785d3d..f61be92ecb3 100644 --- a/test/integration/singlecluster/controller/jobs/job/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/job/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go index bfdfbcbbb3c..a3c02f888e8 100644 --- a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go @@ -34,7 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/test/integration/singlecluster/controller/jobs/jobset/suite_test.go b/test/integration/singlecluster/controller/jobs/jobset/suite_test.go index 2901938cc5b..33683d8352a 100644 --- a/test/integration/singlecluster/controller/jobs/jobset/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/jobset/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go index d9edce41cfb..c2da406a23d 100644 --- a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go @@ -33,7 +33,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/test/integration/singlecluster/controller/jobs/mpijob/suite_test.go b/test/integration/singlecluster/controller/jobs/mpijob/suite_test.go index 5e1e6fbcefd..8eaeb86145c 100644 --- a/test/integration/singlecluster/controller/jobs/mpijob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/mpijob/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go b/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go index dd917c52086..7675e25faf0 100644 --- a/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadpaddlejob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/paddlejob" diff --git a/test/integration/singlecluster/controller/jobs/paddlejob/suite_test.go b/test/integration/singlecluster/controller/jobs/paddlejob/suite_test.go index d20cecfdb86..2c61fa214e7 100644 --- a/test/integration/singlecluster/controller/jobs/paddlejob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/paddlejob/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go index 741ad5bf2e0..d8e7c367f5b 100644 --- a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/test/integration/singlecluster/controller/jobs/pod/suite_test.go b/test/integration/singlecluster/controller/jobs/pod/suite_test.go index 9ace5306ede..d7c310d5351 100644 --- a/test/integration/singlecluster/controller/jobs/pod/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/pod/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go b/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go index 0228ebd2754..31aa0e64e26 100644 --- a/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go @@ -29,7 +29,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/test/integration/singlecluster/controller/jobs/pytorchjob/suite_test.go b/test/integration/singlecluster/controller/jobs/pytorchjob/suite_test.go index 3d0520fbc0c..6a09357b788 100644 --- a/test/integration/singlecluster/controller/jobs/pytorchjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/pytorchjob/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go index ec37b656dca..6f876cefb39 100644 --- a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go @@ -34,7 +34,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/test/integration/singlecluster/controller/jobs/raycluster/suite_test.go b/test/integration/singlecluster/controller/jobs/raycluster/suite_test.go index 080ad9fe962..260d47c17b0 100644 --- a/test/integration/singlecluster/controller/jobs/raycluster/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/raycluster/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go b/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go index ffd488711df..2a49df48248 100644 --- a/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go @@ -31,7 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/test/integration/singlecluster/controller/jobs/rayjob/suite_test.go b/test/integration/singlecluster/controller/jobs/rayjob/suite_test.go index 27fa38ecff7..b62b1e02517 100644 --- a/test/integration/singlecluster/controller/jobs/rayjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/rayjob/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/tfjob/suite_test.go b/test/integration/singlecluster/controller/jobs/tfjob/suite_test.go index 644ed3d93cd..ed90ef2b868 100644 --- a/test/integration/singlecluster/controller/jobs/tfjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/tfjob/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go b/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go index ecc4681590f..9505179327a 100644 --- a/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadtfjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/tfjob" diff --git a/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go b/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go index 386a7362399..74c130e70c4 100644 --- a/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/xgboostjob/suite_test.go b/test/integration/singlecluster/controller/jobs/xgboostjob/suite_test.go index ef1219213c4..54b1646269a 100644 --- a/test/integration/singlecluster/controller/jobs/xgboostjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/xgboostjob/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go b/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go index efec6877ab4..a54790d7b44 100644 --- a/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadxgboostjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/xgboostjob" diff --git a/test/integration/singlecluster/conversion/suite_test.go b/test/integration/singlecluster/conversion/suite_test.go index 6d902a5895d..7f6c86dda9c 100644 --- a/test/integration/singlecluster/conversion/suite_test.go +++ b/test/integration/singlecluster/conversion/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/importer/suite_test.go b/test/integration/singlecluster/importer/suite_test.go index 547c5859211..5700f93b4a3 100644 --- a/test/integration/singlecluster/importer/suite_test.go +++ b/test/integration/singlecluster/importer/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/kueuectl/suite_test.go b/test/integration/singlecluster/kueuectl/suite_test.go index c2c13274d3e..fb32ef6f9cf 100644 --- a/test/integration/singlecluster/kueuectl/suite_test.go +++ b/test/integration/singlecluster/kueuectl/suite_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/core" diff --git a/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go b/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go index 4cef95808e6..7df837b9efa 100644 --- a/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go +++ b/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" diff --git a/test/integration/singlecluster/scheduler/fairsharing/suite_test.go b/test/integration/singlecluster/scheduler/fairsharing/suite_test.go index e1730bc1d80..c02423a3259 100644 --- a/test/integration/singlecluster/scheduler/fairsharing/suite_test.go +++ b/test/integration/singlecluster/scheduler/fairsharing/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go index 7b9f5c933f7..1082d86e146 100644 --- a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go +++ b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go @@ -30,7 +30,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" "sigs.k8s.io/kueue/pkg/workload" diff --git a/test/integration/singlecluster/scheduler/podsready/suite_test.go b/test/integration/singlecluster/scheduler/podsready/suite_test.go index c0a868e82a7..b8bdbc14d59 100644 --- a/test/integration/singlecluster/scheduler/podsready/suite_test.go +++ b/test/integration/singlecluster/scheduler/podsready/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/scheduler/suite_test.go b/test/integration/singlecluster/scheduler/suite_test.go index 3ad1ea3d359..ea8eaa9c961 100644 --- a/test/integration/singlecluster/scheduler/suite_test.go +++ b/test/integration/singlecluster/scheduler/suite_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/tas/suite_test.go b/test/integration/singlecluster/tas/suite_test.go index be1d3c55ec4..e5b007a9218 100644 --- a/test/integration/singlecluster/tas/suite_test.go +++ b/test/integration/singlecluster/tas/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/integration/singlecluster/webhook/core/suite_test.go b/test/integration/singlecluster/webhook/core/suite_test.go index 66b66d61bea..e140608b7fb 100644 --- a/test/integration/singlecluster/webhook/core/suite_test.go +++ b/test/integration/singlecluster/webhook/core/suite_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - config "sigs.k8s.io/kueue/apis/config/v1beta1" + config "sigs.k8s.io/kueue/apis/config/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/core" diff --git a/test/performance/scheduler/minimalkueue/main.go b/test/performance/scheduler/minimalkueue/main.go index 51307d13057..df95d6bbc01 100644 --- a/test/performance/scheduler/minimalkueue/main.go +++ b/test/performance/scheduler/minimalkueue/main.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueuealpha "sigs.k8s.io/kueue/apis/kueue/v1alpha1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" diff --git a/test/performance/scheduler/runner/main.go b/test/performance/scheduler/runner/main.go index 59863ec2fd6..1c67c8a3930 100644 --- a/test/performance/scheduler/runner/main.go +++ b/test/performance/scheduler/runner/main.go @@ -45,7 +45,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/yaml" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueuealpha "sigs.k8s.io/kueue/apis/kueue/v1alpha1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/test/util/e2e.go b/test/util/e2e.go index 4c423df7797..1132070d03f 100644 --- a/test/util/e2e.go +++ b/test/util/e2e.go @@ -49,7 +49,7 @@ import ( leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" "sigs.k8s.io/yaml" - configapi "sigs.k8s.io/kueue/apis/config/v1beta1" + configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" From 3e019537677eb5f5392bb21be75d0740f986fd77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Fri, 24 Oct 2025 12:17:44 +0200 Subject: [PATCH 024/119] Align imports for Kueue (#7378) --- pkg/controller/jobframework/tas_validation.go | 72 +++++++++---------- .../leaderworkerset_webhook.go | 6 +- .../jobs/raycluster/raycluster_webhook.go | 16 ++--- pkg/controller/jobs/rayjob/rayjob_webhook.go | 10 +-- 4 files changed, 52 insertions(+), 52 deletions(-) diff --git a/pkg/controller/jobframework/tas_validation.go b/pkg/controller/jobframework/tas_validation.go index 051a125405d..0a1e41b2310 100644 --- a/pkg/controller/jobframework/tas_validation.go +++ b/pkg/controller/jobframework/tas_validation.go @@ -25,17 +25,17 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" - kueuebeta "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/util/orderedgroups" ) func ValidateTASPodSetRequest(replicaPath *field.Path, replicaMetadata *metav1.ObjectMeta) field.ErrorList { var allErrs field.ErrorList - requiredValue, requiredFound := replicaMetadata.Annotations[kueuebeta.PodSetRequiredTopologyAnnotation] - preferredValue, preferredFound := replicaMetadata.Annotations[kueuebeta.PodSetPreferredTopologyAnnotation] - _, unconstrainedFound := replicaMetadata.Annotations[kueuebeta.PodSetUnconstrainedTopologyAnnotation] - sliceRequiredValue, sliceRequiredFound := replicaMetadata.Annotations[kueuebeta.PodSetSliceRequiredTopologyAnnotation] - _, sliceSizeFound := replicaMetadata.Annotations[kueuebeta.PodSetSliceSizeAnnotation] + requiredValue, requiredFound := replicaMetadata.Annotations[kueue.PodSetRequiredTopologyAnnotation] + preferredValue, preferredFound := replicaMetadata.Annotations[kueue.PodSetPreferredTopologyAnnotation] + _, unconstrainedFound := replicaMetadata.Annotations[kueue.PodSetUnconstrainedTopologyAnnotation] + sliceRequiredValue, sliceRequiredFound := replicaMetadata.Annotations[kueue.PodSetSliceRequiredTopologyAnnotation] + _, sliceSizeFound := replicaMetadata.Annotations[kueue.PodSetSliceSizeAnnotation] // validate no more than 1 annotation asInt := func(b bool) int { @@ -49,38 +49,38 @@ func ValidateTASPodSetRequest(replicaPath *field.Path, replicaMetadata *metav1.O if annotationFoundCount > 1 { allErrs = append(allErrs, field.Invalid(annotationsPath, field.OmitValueType{}, fmt.Sprintf("must not contain more than one topology annotation: [%q, %q, %q]", - kueuebeta.PodSetRequiredTopologyAnnotation, - kueuebeta.PodSetPreferredTopologyAnnotation, - kueuebeta.PodSetUnconstrainedTopologyAnnotation), + kueue.PodSetRequiredTopologyAnnotation, + kueue.PodSetPreferredTopologyAnnotation, + kueue.PodSetUnconstrainedTopologyAnnotation), )) } // validate labels if requiredFound { - allErrs = append(allErrs, metavalidation.ValidateLabelName(requiredValue, annotationsPath.Key(kueuebeta.PodSetRequiredTopologyAnnotation))...) + allErrs = append(allErrs, metavalidation.ValidateLabelName(requiredValue, annotationsPath.Key(kueue.PodSetRequiredTopologyAnnotation))...) } if preferredFound { - allErrs = append(allErrs, metavalidation.ValidateLabelName(preferredValue, annotationsPath.Key(kueuebeta.PodSetPreferredTopologyAnnotation))...) + allErrs = append(allErrs, metavalidation.ValidateLabelName(preferredValue, annotationsPath.Key(kueue.PodSetPreferredTopologyAnnotation))...) } if sliceRequiredFound { - allErrs = append(allErrs, metavalidation.ValidateLabelName(sliceRequiredValue, annotationsPath.Key(kueuebeta.PodSetSliceRequiredTopologyAnnotation))...) + allErrs = append(allErrs, metavalidation.ValidateLabelName(sliceRequiredValue, annotationsPath.Key(kueue.PodSetSliceRequiredTopologyAnnotation))...) } // validate PodSetGroupName annotation - podSetGroupNameValue, podSetGroupNameFound := replicaMetadata.Annotations[kueuebeta.PodSetGroupName] + podSetGroupNameValue, podSetGroupNameFound := replicaMetadata.Annotations[kueue.PodSetGroupName] if podSetGroupNameFound { - allErrs = append(allErrs, validatePodSetGroupNameAnnotation(podSetGroupNameValue, annotationsPath.Key(kueuebeta.PodSetGroupName))...) + allErrs = append(allErrs, validatePodSetGroupNameAnnotation(podSetGroupNameValue, annotationsPath.Key(kueue.PodSetGroupName))...) if sliceSizeFound { - allErrs = append(allErrs, field.Forbidden(annotationsPath.Key(kueuebeta.PodSetGroupName), fmt.Sprintf("may not be set when '%s' is specified", kueuebeta.PodSetSliceSizeAnnotation))) + allErrs = append(allErrs, field.Forbidden(annotationsPath.Key(kueue.PodSetGroupName), fmt.Sprintf("may not be set when '%s' is specified", kueue.PodSetSliceSizeAnnotation))) } if sliceRequiredFound { - allErrs = append(allErrs, field.Forbidden(annotationsPath.Key(kueuebeta.PodSetGroupName), fmt.Sprintf("may not be set when '%s' is specified", kueuebeta.PodSetSliceRequiredTopologyAnnotation))) + allErrs = append(allErrs, field.Forbidden(annotationsPath.Key(kueue.PodSetGroupName), fmt.Sprintf("may not be set when '%s' is specified", kueue.PodSetSliceRequiredTopologyAnnotation))) } if !preferredFound && !requiredFound { - allErrs = append(allErrs, field.Forbidden(annotationsPath.Key(kueuebeta.PodSetGroupName), fmt.Sprintf("may not be set when neither '%s' nor '%s' is specified", kueuebeta.PodSetPreferredTopologyAnnotation, kueuebeta.PodSetRequiredTopologyAnnotation))) + allErrs = append(allErrs, field.Forbidden(annotationsPath.Key(kueue.PodSetGroupName), fmt.Sprintf("may not be set when neither '%s' nor '%s' is specified", kueue.PodSetPreferredTopologyAnnotation, kueue.PodSetRequiredTopologyAnnotation))) } } @@ -92,21 +92,21 @@ func ValidateTASPodSetRequest(replicaPath *field.Path, replicaMetadata *metav1.O // validate slice annotations if sliceRequiredFound && !sliceSizeFound { - allErrs = append(allErrs, field.Required(annotationsPath.Key(kueuebeta.PodSetSliceSizeAnnotation), fmt.Sprintf("must be set when '%s' is specified", kueuebeta.PodSetSliceRequiredTopologyAnnotation))) + allErrs = append(allErrs, field.Required(annotationsPath.Key(kueue.PodSetSliceSizeAnnotation), fmt.Sprintf("must be set when '%s' is specified", kueue.PodSetSliceRequiredTopologyAnnotation))) } if !sliceRequiredFound && sliceSizeFound { - allErrs = append(allErrs, field.Forbidden(annotationsPath.Key(kueuebeta.PodSetSliceSizeAnnotation), fmt.Sprintf("may not be set when '%s' is not specified", kueuebeta.PodSetSliceRequiredTopologyAnnotation))) + allErrs = append(allErrs, field.Forbidden(annotationsPath.Key(kueue.PodSetSliceSizeAnnotation), fmt.Sprintf("may not be set when '%s' is not specified", kueue.PodSetSliceRequiredTopologyAnnotation))) } return allErrs } func validateTASUnconstrained(annotationsPath *field.Path, replicaMetadata *metav1.ObjectMeta) field.ErrorList { - if val, ok := replicaMetadata.Annotations[kueuebeta.PodSetUnconstrainedTopologyAnnotation]; ok { + if val, ok := replicaMetadata.Annotations[kueue.PodSetUnconstrainedTopologyAnnotation]; ok { if _, err := strconv.ParseBool(val); err != nil { return field.ErrorList{ field.Invalid( - annotationsPath.Key(kueuebeta.PodSetUnconstrainedTopologyAnnotation), val, "must be a boolean value", + annotationsPath.Key(kueue.PodSetUnconstrainedTopologyAnnotation), val, "must be a boolean value", ), } } @@ -115,7 +115,7 @@ func validateTASUnconstrained(annotationsPath *field.Path, replicaMetadata *meta } func validateSliceSizeAnnotation(annotationsPath *field.Path, replicaMetadata *metav1.ObjectMeta) field.ErrorList { - sliceSizeValue, sliceSizeFound := replicaMetadata.Annotations[kueuebeta.PodSetSliceSizeAnnotation] + sliceSizeValue, sliceSizeFound := replicaMetadata.Annotations[kueue.PodSetSliceSizeAnnotation] if !sliceSizeFound { return nil } @@ -124,7 +124,7 @@ func validateSliceSizeAnnotation(annotationsPath *field.Path, replicaMetadata *m if err != nil { return field.ErrorList{ field.Invalid( - annotationsPath.Key(kueuebeta.PodSetSliceSizeAnnotation), sliceSizeValue, "must be a numeric value", + annotationsPath.Key(kueue.PodSetSliceSizeAnnotation), sliceSizeValue, "must be a numeric value", ), } } @@ -132,7 +132,7 @@ func validateSliceSizeAnnotation(annotationsPath *field.Path, replicaMetadata *m if int32(val) < 1 { return field.ErrorList{ field.Invalid( - annotationsPath.Key(kueuebeta.PodSetSliceSizeAnnotation), sliceSizeValue, + annotationsPath.Key(kueue.PodSetSliceSizeAnnotation), sliceSizeValue, "must be greater than or equal to 1", ), } @@ -153,8 +153,8 @@ func validatePodSetGroupNameAnnotation(groupName string, annotationPath *field.P return nil } -func ValidateSliceSizeAnnotationUpperBound(replicaPath *field.Path, replicaMetadata *metav1.ObjectMeta, podSet *kueuebeta.PodSet) field.ErrorList { - sliceSizeValue, sliceSizeFound := replicaMetadata.Annotations[kueuebeta.PodSetSliceSizeAnnotation] +func ValidateSliceSizeAnnotationUpperBound(replicaPath *field.Path, replicaMetadata *metav1.ObjectMeta, podSet *kueue.PodSet) field.ErrorList { + sliceSizeValue, sliceSizeFound := replicaMetadata.Annotations[kueue.PodSetSliceSizeAnnotation] if !sliceSizeFound || podSet == nil { return nil } @@ -165,7 +165,7 @@ func ValidateSliceSizeAnnotationUpperBound(replicaPath *field.Path, replicaMetad if err != nil { return field.ErrorList{ field.Invalid( - annotationsPath.Key(kueuebeta.PodSetSliceSizeAnnotation), sliceSizeValue, "must be a numeric value", + annotationsPath.Key(kueue.PodSetSliceSizeAnnotation), sliceSizeValue, "must be a numeric value", ), } } @@ -173,7 +173,7 @@ func ValidateSliceSizeAnnotationUpperBound(replicaPath *field.Path, replicaMetad if int32(val) > podSet.Count { return field.ErrorList{ field.Invalid( - annotationsPath.Key(kueuebeta.PodSetSliceSizeAnnotation), sliceSizeValue, + annotationsPath.Key(kueue.PodSetSliceSizeAnnotation), sliceSizeValue, fmt.Sprintf("must not be greater than pod set count %d", podSet.Count), ), } @@ -182,8 +182,8 @@ func ValidateSliceSizeAnnotationUpperBound(replicaPath *field.Path, replicaMetad return nil } -func ValidatePodSetGroupingTopology(podSets []kueuebeta.PodSet, podSetAnnotationsByName map[kueuebeta.PodSetReference]*field.Path) field.ErrorList { - podSetGroups := orderedgroups.NewOrderedGroups[string, kueuebeta.PodSet]() +func ValidatePodSetGroupingTopology(podSets []kueue.PodSet, podSetAnnotationsByName map[kueue.PodSetReference]*field.Path) field.ErrorList { + podSetGroups := orderedgroups.NewOrderedGroups[string, kueue.PodSet]() for _, podSet := range podSets { if podSet.TopologyRequest == nil || podSet.TopologyRequest.PodSetGroupName == nil { continue @@ -200,7 +200,7 @@ func ValidatePodSetGroupingTopology(podSets []kueuebeta.PodSet, podSetAnnotation allErrs = append( allErrs, field.Invalid( - podSetAnnotationsByName[podSet.Name].Key(kueuebeta.PodSetGroupName), + podSetAnnotationsByName[podSet.Name].Key(kueue.PodSetGroupName), groupName, fmt.Sprintf("can only define groups of exactly 2 pod sets, got: %d pod set(s)", groupSize), ), @@ -222,12 +222,12 @@ func ValidatePodSetGroupingTopology(podSets []kueuebeta.PodSet, podSetAnnotation ) allErrs = append(allErrs, field.Invalid( - annotationsPath1.Key(kueuebeta.PodSetGroupName), + annotationsPath1.Key(kueue.PodSetGroupName), groupName, sizeErrorMessage, ), field.Invalid( - annotationsPath2.Key(kueuebeta.PodSetGroupName), + annotationsPath2.Key(kueue.PodSetGroupName), groupName, sizeErrorMessage, ), @@ -237,8 +237,8 @@ func ValidatePodSetGroupingTopology(podSets []kueuebeta.PodSet, podSetAnnotation if !topologyRequestsValid(podSet1.TopologyRequest, podSet2.TopologyRequest) { errorMessageTemplate := fmt.Sprintf( "must specify '%s' or '%s' topology consistent with '%%s' in group '%s'", - kueuebeta.PodSetRequiredTopologyAnnotation, - kueuebeta.PodSetPreferredTopologyAnnotation, + kueue.PodSetRequiredTopologyAnnotation, + kueue.PodSetPreferredTopologyAnnotation, groupName, ) allErrs = append( @@ -260,7 +260,7 @@ func ValidatePodSetGroupingTopology(podSets []kueuebeta.PodSet, podSetAnnotation return allErrs } -func topologyRequestsValid(r1, r2 *kueuebeta.PodSetTopologyRequest) bool { +func topologyRequestsValid(r1, r2 *kueue.PodSetTopologyRequest) bool { // Check that the requests have exactly one of `Required` and `Preferred`. if r1.Required == nil && r1.Preferred == nil { return false diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go index af985e1e243..edbe8686c5b 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueuebeta "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" @@ -116,7 +116,7 @@ var ( workerTemplatePath = leaderWorkerTemplatePath.Child("workerTemplate") workerTemplateMetaPath = workerTemplatePath.Child("metadata") workerTemplateAnnotationsPath = workerTemplateMetaPath.Child("annotations") - podSetAnnotationsPathByName = map[kueuebeta.PodSetReference]*field.Path{ + podSetAnnotationsPathByName = map[kueue.PodSetReference]*field.Path{ "leader": leaderTemplateMetaPath.Child("annotations"), "worker": workerTemplateAnnotationsPath, "main": workerTemplateAnnotationsPath, @@ -211,7 +211,7 @@ func validateTopologyRequest(lws *LeaderWorkerSet) (field.ErrorList, error) { lwsv1 := leaderworkersetv1.LeaderWorkerSet(*lws) podSets, podSetsErr := podSets(&lwsv1) - defaultPodSetName := kueuebeta.DefaultPodSetName + defaultPodSetName := kueue.DefaultPodSetName if lws.Spec.LeaderWorkerTemplate.LeaderTemplate != nil { defaultPodSetName = workerPodSetName diff --git a/pkg/controller/jobs/raycluster/raycluster_webhook.go b/pkg/controller/jobs/raycluster/raycluster_webhook.go index a2b5a82ed79..613dd24b056 100644 --- a/pkg/controller/jobs/raycluster/raycluster_webhook.go +++ b/pkg/controller/jobs/raycluster/raycluster_webhook.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueuebeta "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/jobframework" @@ -96,12 +96,12 @@ func (w *RayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err if isAnElasticJob(rjob) { // Ensure that the PodSchedulingGate is present in the RayCluster's pod Templates for its Head and all its Workers - utilpod.GateTemplate(&job.Spec.HeadGroupSpec.Template, kueuebeta.ElasticJobSchedulingGate) + utilpod.GateTemplate(&job.Spec.HeadGroupSpec.Template, kueue.ElasticJobSchedulingGate) for index := range job.Spec.WorkerGroupSpecs { wgs := &job.Spec.WorkerGroupSpecs[index] - utilpod.GateTemplate(&wgs.Template, kueuebeta.ElasticJobSchedulingGate) + utilpod.GateTemplate(&wgs.Template, kueue.ElasticJobSchedulingGate) } } @@ -175,7 +175,7 @@ func validateElasticJob(job *rayv1.RayCluster) field.ErrorList { specPath := field.NewPath("spec") workloadSliceSchedulingGate := corev1.PodSchedulingGate{ - Name: kueuebeta.ElasticJobSchedulingGate, + Name: kueue.ElasticJobSchedulingGate, } for index := range job.Spec.WorkerGroupSpecs { @@ -214,7 +214,7 @@ func (w *RayClusterWebhook) validateTopologyRequest(ctx context.Context, rayJob continue } - podSet := podset.FindPodSetByName(podSets, kueuebeta.NewPodSetReference(wgs.GroupName)) + podSet := podset.FindPodSetByName(podSets, kueue.NewPodSetReference(wgs.GroupName)) allErrs = append(allErrs, jobframework.ValidateSliceSizeAnnotationUpperBound(workerGroupMetaPath, &rayJob.Spec.WorkerGroupSpecs[i].Template.ObjectMeta, podSet)...) } @@ -225,11 +225,11 @@ func (w *RayClusterWebhook) validateTopologyRequest(ctx context.Context, rayJob return nil, podSetsErr } -func buildPodSetAnnotationsPathByNameMap(rayJob *RayCluster) map[kueuebeta.PodSetReference]*field.Path { - podSetAnnotationsPathByName := make(map[kueuebeta.PodSetReference]*field.Path) +func buildPodSetAnnotationsPathByNameMap(rayJob *RayCluster) map[kueue.PodSetReference]*field.Path { + podSetAnnotationsPathByName := make(map[kueue.PodSetReference]*field.Path) podSetAnnotationsPathByName[headGroupPodSetName] = headGroupMetaPath.Child("annotations") for i, wgs := range rayJob.Spec.WorkerGroupSpecs { - podSetAnnotationsPathByName[kueuebeta.PodSetReference(wgs.GroupName)] = workerGroupSpecsPath.Index(i).Child("template", "metadata", "annotations") + podSetAnnotationsPathByName[kueue.PodSetReference(wgs.GroupName)] = workerGroupSpecsPath.Index(i).Child("template", "metadata", "annotations") } return podSetAnnotationsPathByName } diff --git a/pkg/controller/jobs/rayjob/rayjob_webhook.go b/pkg/controller/jobs/rayjob/rayjob_webhook.go index 96571c129ed..4e1a222a6fa 100644 --- a/pkg/controller/jobs/rayjob/rayjob_webhook.go +++ b/pkg/controller/jobs/rayjob/rayjob_webhook.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueuebeta "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/jobframework" @@ -195,7 +195,7 @@ func (w *RayJobWebhook) validateTopologyRequest(ctx context.Context, rayJob *ray continue } - workerPodSetName := podset.FindPodSetByName(podSets, kueuebeta.NewPodSetReference(wgs.GroupName)) + workerPodSetName := podset.FindPodSetByName(podSets, kueue.NewPodSetReference(wgs.GroupName)) allErrs = append(allErrs, jobframework.ValidateSliceSizeAnnotationUpperBound(workerGroupMetaPath, &rayJob.Spec.RayClusterSpec.WorkerGroupSpecs[i].Template.ObjectMeta, workerPodSetName)...) } @@ -206,12 +206,12 @@ func (w *RayJobWebhook) validateTopologyRequest(ctx context.Context, rayJob *ray return nil, podSetsErr } -func buildPodSetAnnotationsPathByNameMap(rayJob *rayv1.RayJob) map[kueuebeta.PodSetReference]*field.Path { - podSetAnnotationsPathByName := make(map[kueuebeta.PodSetReference]*field.Path) +func buildPodSetAnnotationsPathByNameMap(rayJob *rayv1.RayJob) map[kueue.PodSetReference]*field.Path { + podSetAnnotationsPathByName := make(map[kueue.PodSetReference]*field.Path) podSetAnnotationsPathByName[headGroupPodSetName] = headGroupMetaPath.Child("annotations") podSetAnnotationsPathByName[submitterJobPodSetName] = field.NewPath("spec", "submitterPodTemplate", "metadata", "annotations") for i, wgs := range rayJob.Spec.RayClusterSpec.WorkerGroupSpecs { - podSetAnnotationsPathByName[kueuebeta.PodSetReference(wgs.GroupName)] = workerGroupSpecsPath.Index(i).Child("template", "metadata", "annotations") + podSetAnnotationsPathByName[kueue.PodSetReference(wgs.GroupName)] = workerGroupSpecsPath.Index(i).Child("template", "metadata", "annotations") } return podSetAnnotationsPathByName } From e0a733ccfc08e04c90aaa44963ed4e9d2613ba4d Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 24 Oct 2025 16:17:35 +0530 Subject: [PATCH 025/119] Remove workers from Pytorch e2e test. (#7381) --- .../testingjobs/pytorchjob/wrappers_pytorchjob.go | 8 +++++++- test/e2e/multikueue/e2e_test.go | 8 -------- test/e2e/singlecluster/pytorchjob_test.go | 11 ++--------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/pkg/util/testingjobs/pytorchjob/wrappers_pytorchjob.go b/pkg/util/testingjobs/pytorchjob/wrappers_pytorchjob.go index e28bd115100..853989daa1f 100644 --- a/pkg/util/testingjobs/pytorchjob/wrappers_pytorchjob.go +++ b/pkg/util/testingjobs/pytorchjob/wrappers_pytorchjob.go @@ -75,7 +75,7 @@ func (j *PyTorchJobWrapper) PyTorchReplicaSpecs(replicaSpecs ...PyTorchReplicaSp return j } -func (j *PyTorchJobWrapper) PyTorchReplicaSpecsDefault() *PyTorchJobWrapper { +func (j *PyTorchJobWrapper) PyTorchReplicaSpecsOnlyMasterDefault() *PyTorchJobWrapper { j.Spec.PyTorchReplicaSpecs[kftraining.PyTorchJobReplicaTypeMaster] = &kftraining.ReplicaSpec{ Replicas: ptr.To[int32](1), Template: corev1.PodTemplateSpec{ @@ -97,6 +97,12 @@ func (j *PyTorchJobWrapper) PyTorchReplicaSpecsDefault() *PyTorchJobWrapper { }, } + return j +} + +func (j *PyTorchJobWrapper) PyTorchReplicaSpecsDefault() *PyTorchJobWrapper { + j.PyTorchReplicaSpecsOnlyMasterDefault() + j.Spec.PyTorchReplicaSpecs[kftraining.PyTorchJobReplicaTypeWorker] = &kftraining.ReplicaSpec{ Replicas: ptr.To[int32](1), Template: corev1.PodTemplateSpec{ diff --git a/test/e2e/multikueue/e2e_test.go b/test/e2e/multikueue/e2e_test.go index 0eda05067bf..fab0844a8b1 100644 --- a/test/e2e/multikueue/e2e_test.go +++ b/test/e2e/multikueue/e2e_test.go @@ -674,18 +674,10 @@ var _ = ginkgo.Describe("MultiKueue", func() { ReplicaCount: 1, RestartPolicy: "Never", }, - testingpytorchjob.PyTorchReplicaSpecRequirement{ - ReplicaType: kftraining.PyTorchJobReplicaTypeWorker, - ReplicaCount: 1, - RestartPolicy: "OnFailure", - }, ). RequestAndLimit(kftraining.PyTorchJobReplicaTypeMaster, corev1.ResourceCPU, "0.2"). RequestAndLimit(kftraining.PyTorchJobReplicaTypeMaster, corev1.ResourceMemory, "800M"). - RequestAndLimit(kftraining.PyTorchJobReplicaTypeWorker, corev1.ResourceCPU, "0.5"). - RequestAndLimit(kftraining.PyTorchJobReplicaTypeWorker, corev1.ResourceMemory, "800M"). Image(kftraining.PyTorchJobReplicaTypeMaster, util.GetAgnHostImage(), util.BehaviorExitFast). - Image(kftraining.PyTorchJobReplicaTypeWorker, util.GetAgnHostImage(), util.BehaviorExitFast). Obj() ginkgo.By("Creating the PyTorchJob", func() { diff --git a/test/e2e/singlecluster/pytorchjob_test.go b/test/e2e/singlecluster/pytorchjob_test.go index 00dfb025329..103969ad557 100644 --- a/test/e2e/singlecluster/pytorchjob_test.go +++ b/test/e2e/singlecluster/pytorchjob_test.go @@ -82,14 +82,10 @@ var _ = ginkgo.Describe("PyTorch integration", func() { Queue(localQueueName). Suspend(false). SetTypeMeta(). - PyTorchReplicaSpecsDefault(). - Parallelism(2). + PyTorchReplicaSpecsOnlyMasterDefault(). Image(kftraining.PyTorchJobReplicaTypeMaster, util.GetAgnHostImage(), util.BehaviorWaitForDeletion). Request(kftraining.PyTorchJobReplicaTypeMaster, corev1.ResourceCPU, "1"). Request(kftraining.PyTorchJobReplicaTypeMaster, corev1.ResourceMemory, "200Mi"). - Image(kftraining.PyTorchJobReplicaTypeWorker, util.GetAgnHostImage(), util.BehaviorWaitForDeletion). - Request(kftraining.PyTorchJobReplicaTypeWorker, corev1.ResourceCPU, "1"). - Request(kftraining.PyTorchJobReplicaTypeWorker, corev1.ResourceMemory, "200Mi"). Obj() ginkgo.By("Create a PyTorch", func() { @@ -107,12 +103,9 @@ var _ = ginkgo.Describe("PyTorch integration", func() { // Check the worker replica status exists masterReplicaStatus, ok := createdPyTorch.Status.ReplicaStatuses[kftraining.PyTorchJobReplicaTypeMaster] g.Expect(ok).To(gomega.BeTrue(), "Master replica status not found in PyTorch status") - workerReplicaStatus, ok := createdPyTorch.Status.ReplicaStatuses[kftraining.PyTorchJobReplicaTypeWorker] - g.Expect(ok).To(gomega.BeTrue(), "Worker replica status not found in PyTorch status") // Check the number of active replicas g.Expect(masterReplicaStatus.Active).To(gomega.Equal(int32(1)), "Unexpected number of active %s replicas", kftraining.PyTorchJobReplicaTypeMaster) - g.Expect(workerReplicaStatus.Active).To(gomega.Equal(int32(2)), "Unexpected number of active %s replicas", kftraining.PyTorchJobReplicaTypeWorker) // Ensure PyTorch job has "Running" condition with status "True" g.Expect(createdPyTorch.Status.Conditions).To(gomega.ContainElements( @@ -139,7 +132,7 @@ var _ = ginkgo.Describe("PyTorch integration", func() { ginkgo.By("Check workload is finished", func() { // Wait for active pods and terminate them - util.WaitForActivePodsAndTerminate(ctx, k8sClient, restClient, cfg, ns.Name, 3, 0, client.InNamespace(ns.Name)) + util.WaitForActivePodsAndTerminate(ctx, k8sClient, restClient, cfg, ns.Name, 1, 0, client.InNamespace(ns.Name)) util.ExpectWorkloadToFinish(ctx, k8sClient, wlLookupKey) }) From 02af80c87b7e731e0d35272ebce96d13445e215b Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 24 Oct 2025 19:13:36 +0530 Subject: [PATCH 026/119] Fix Should run a kubeflow PyTorchJob on worker if admitted e2e test. (#7386) --- test/e2e/multikueue/e2e_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/multikueue/e2e_test.go b/test/e2e/multikueue/e2e_test.go index fab0844a8b1..ebc3de37bfe 100644 --- a/test/e2e/multikueue/e2e_test.go +++ b/test/e2e/multikueue/e2e_test.go @@ -675,8 +675,8 @@ var _ = ginkgo.Describe("MultiKueue", func() { RestartPolicy: "Never", }, ). - RequestAndLimit(kftraining.PyTorchJobReplicaTypeMaster, corev1.ResourceCPU, "0.2"). - RequestAndLimit(kftraining.PyTorchJobReplicaTypeMaster, corev1.ResourceMemory, "800M"). + RequestAndLimit(kftraining.PyTorchJobReplicaTypeMaster, corev1.ResourceCPU, "1"). + RequestAndLimit(kftraining.PyTorchJobReplicaTypeMaster, corev1.ResourceMemory, "1600M"). Image(kftraining.PyTorchJobReplicaTypeMaster, util.GetAgnHostImage(), util.BehaviorExitFast). Obj() From aa299d3f85986e88982eb0021e7a27b6ddb293ea Mon Sep 17 00:00:00 2001 From: Tomas Tormo Date: Fri, 24 Oct 2025 19:27:35 +0200 Subject: [PATCH 027/119] [Trainer] Use podset label to identify Kueue injected config (#7389) --- .../jobs/trainjob/trainjob_controller.go | 27 +++---- .../jobs/trainjob/trainjob_controller_test.go | 71 +++++++++++-------- test/e2e/singlecluster/tas_test.go | 9 +-- 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/pkg/controller/jobs/trainjob/trainjob_controller.go b/pkg/controller/jobs/trainjob/trainjob_controller.go index 5386fac2f48..1af7476d664 100644 --- a/pkg/controller/jobs/trainjob/trainjob_controller.go +++ b/pkg/controller/jobs/trainjob/trainjob_controller.go @@ -21,7 +21,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "sync" kftrainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" @@ -43,6 +42,7 @@ import ( jobsetapplyapi "sigs.k8s.io/jobset/client-go/applyconfiguration/jobset/v1alpha2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjobset "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" "sigs.k8s.io/kueue/pkg/features" @@ -57,11 +57,6 @@ var ( TrainJobControllerName = "trainer.kubeflow.org/trainjob-controller" ) -const ( - // This is alpha level annotation - firstOverrideIdx = "kueue.x-k8s.io/trainjob-override-idx" -) - func init() { utilruntime.Must(jobframework.RegisterIntegration(FrameworkName, jobframework.IntegrationCallbacks{ SetupIndexes: SetupIndexes, @@ -272,7 +267,6 @@ func (t *TrainJob) RunWithPodSetsInfo(ctx context.Context, podSetsInfo []podset. if t.Annotations == nil { t.Annotations = map[string]string{} } - t.Annotations[firstOverrideIdx] = strconv.Itoa(len(t.Spec.PodTemplateOverrides)) for _, info := range podSetsInfo { // The trainjob controller merges each podSpecOverride sequentially, so any existing user provided override will be processed first t.Spec.PodTemplateOverrides = append(t.Spec.PodTemplateOverrides, kftrainer.PodTemplateOverride{ @@ -316,7 +310,6 @@ func (t *TrainJob) Stop(ctx context.Context, c client.Client, podSetsInfo []pods if !t.RestorePodSetsInfo(podSetsInfo) { return t.Object(), false, errors.New("error restoring info to the trainjob") } - delete(t.Annotations, firstOverrideIdx) return t.Object(), true, nil }); err != nil { return false, err @@ -325,16 +318,16 @@ func (t *TrainJob) Stop(ctx context.Context, c client.Client, podSetsInfo []pods } func (t *TrainJob) RestorePodSetsInfo(_ []podset.PodSetInfo) bool { - idx, ok := t.Annotations[firstOverrideIdx] - if !ok { - // Kueue didn't inject any config yet - return true - } - idxInt, err := strconv.Atoi(idx) - if err != nil { - return false + for i, o := range t.Spec.PodTemplateOverrides { + if o.Metadata != nil { + _, exists := o.Metadata.Labels[controllerconsts.PodSetLabel] + if exists { + t.Spec.PodTemplateOverrides = t.Spec.PodTemplateOverrides[:i] + break + } + } } - t.Spec.PodTemplateOverrides = t.Spec.PodTemplateOverrides[:idxInt] + return true } diff --git a/pkg/controller/jobs/trainjob/trainjob_controller_test.go b/pkg/controller/jobs/trainjob/trainjob_controller_test.go index a5a4757711c..3753635b78e 100644 --- a/pkg/controller/jobs/trainjob/trainjob_controller_test.go +++ b/pkg/controller/jobs/trainjob/trainjob_controller_test.go @@ -32,6 +32,7 @@ import ( jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/podset" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" @@ -91,24 +92,33 @@ func TestRunWithPodsetsInfo(t *testing.T) { trainJob: testTrainJob.Clone().Obj(), podsetsInfo: []podset.PodSetInfo{ { - Name: "node", - Annotations: map[string]string{"test-annotation": "test"}, - Labels: map[string]string{"test-label": "label"}, + Name: "node", + Annotations: map[string]string{ + "test-annotation": "test", + }, + Labels: map[string]string{ + controllerconsts.PodSetLabel: "node", + "test-label": "label", + }, NodeSelector: map[string]string{"disktype": "ssd"}, Tolerations: []corev1.Toleration{*toleration1.DeepCopy()}, SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-1"}}, }, }, wantTrainJob: testTrainJob.Clone(). - Annotation(firstOverrideIdx, "0"). PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ { TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "node"}, }, Metadata: &metav1.ObjectMeta{ - Annotations: map[string]string{"test-annotation": "test"}, - Labels: map[string]string{"test-label": "label"}, + Annotations: map[string]string{ + "test-annotation": "test", + }, + Labels: map[string]string{ + controllerconsts.PodSetLabel: "node", + "test-label": "label", + }, }, Spec: &kftrainerapi.PodTemplateSpecOverride{ NodeSelector: map[string]string{"disktype": "ssd"}, @@ -137,16 +147,20 @@ func TestRunWithPodsetsInfo(t *testing.T) { }).Obj(), podsetsInfo: []podset.PodSetInfo{ { - Name: "node", - Annotations: map[string]string{"test-annotation": "test"}, - Labels: map[string]string{"test-label": "label"}, + Name: "node", + Annotations: map[string]string{ + "test-annotation": "test", + }, + Labels: map[string]string{ + controllerconsts.PodSetLabel: "node", + "test-label": "label", + }, NodeSelector: map[string]string{"gpu": "nvidia"}, Tolerations: []corev1.Toleration{*toleration2.DeepCopy()}, SchedulingGates: []corev1.PodSchedulingGate{{Name: "test-scheduling-gate-2"}}, }, }, wantTrainJob: testTrainJob.Clone(). - Annotation(firstOverrideIdx, "1"). PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ { TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ @@ -163,8 +177,13 @@ func TestRunWithPodsetsInfo(t *testing.T) { {Name: "node"}, }, Metadata: &metav1.ObjectMeta{ - Annotations: map[string]string{"test-annotation": "test"}, - Labels: map[string]string{"test-label": "label"}, + Annotations: map[string]string{ + "test-annotation": "test", + }, + Labels: map[string]string{ + controllerconsts.PodSetLabel: "node", + "test-label": "label", + }, }, Spec: &kftrainerapi.PodTemplateSpecOverride{ NodeSelector: map[string]string{"gpu": "nvidia"}, @@ -244,23 +263,8 @@ func TestRestorePodSetsInfo(t *testing.T) { wantTrainJob *kftrainerapi.TrainJob wantReturn bool }{ - "should not modify the trainjob if it doesn't have the first override index annotation": { - trainJob: testTrainJob.Clone().Obj(), - wantTrainJob: testTrainJob.Clone().Obj(), - wantReturn: true, - }, - "should not modify the trainjob if it fails parsing the annotation": { + "should remove all the podTemplateOverrides added by kueue": { trainJob: testTrainJob.Clone(). - Annotation(firstOverrideIdx, "+"). - Obj(), - wantTrainJob: testTrainJob.Clone(). - Annotation(firstOverrideIdx, "+"). - Obj(), - wantReturn: false, - }, - "should remove all the podTemplateOverrides starting from the index specified in the annotation": { - trainJob: testTrainJob.Clone(). - Annotation(firstOverrideIdx, "2"). PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ { TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ @@ -282,6 +286,11 @@ func TestRestorePodSetsInfo(t *testing.T) { TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "kueue-provided-1"}, }, + Metadata: &metav1.ObjectMeta{ + Labels: map[string]string{ + controllerconsts.PodSetLabel: "kueue-provided-1", + }, + }, Spec: &kftrainerapi.PodTemplateSpecOverride{ NodeSelector: map[string]string{"disktype": "sdd"}, }, @@ -290,6 +299,11 @@ func TestRestorePodSetsInfo(t *testing.T) { TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ {Name: "kueue-provided-2"}, }, + Metadata: &metav1.ObjectMeta{ + Labels: map[string]string{ + controllerconsts.PodSetLabel: "kueue-provided-2", + }, + }, Spec: &kftrainerapi.PodTemplateSpecOverride{ NodeSelector: map[string]string{"disktype": "sdd"}, }, @@ -297,7 +311,6 @@ func TestRestorePodSetsInfo(t *testing.T) { }). Obj(), wantTrainJob: testTrainJob.Clone(). - Annotation(firstOverrideIdx, "2"). PodTemplateOverrides([]kftrainerapi.PodTemplateOverride{ { TargetJobs: []kftrainerapi.PodTemplateOverrideTargetJob{ diff --git a/test/e2e/singlecluster/tas_test.go b/test/e2e/singlecluster/tas_test.go index bcc94ed0594..bb933118cd4 100644 --- a/test/e2e/singlecluster/tas_test.go +++ b/test/e2e/singlecluster/tas_test.go @@ -18,7 +18,6 @@ package e2e import ( "fmt" - "strconv" kftrainerapi "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1" "github.com/onsi/ginkgo/v2" @@ -523,12 +522,8 @@ var _ = ginkgo.Describe("TopologyAwareScheduling", func() { }) ginkgo.By("verify the TrainJob has nodeSelector set", func() { - firstKueueOverride, exists := trainjob.Annotations["kueue.x-k8s.io/trainjob-override-idx"] - gomega.Expect(exists).Should(gomega.BeTrue()) - firstKueueOverrideIdx, err := strconv.Atoi(firstKueueOverride) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Expect(trainjob.Spec.PodTemplateOverrides[firstKueueOverrideIdx].Spec.NodeSelector).To(gomega.Equal( + gomega.Expect(trainjob.Spec.PodTemplateOverrides).To(gomega.HaveLen(2)) + gomega.Expect(trainjob.Spec.PodTemplateOverrides[1].Spec.NodeSelector).To(gomega.Equal( map[string]string{ "instance-type": "on-demand", }, From ab990ab49de6a951db8105aa33a55252040f6e34 Mon Sep 17 00:00:00 2001 From: j-skiba Date: Mon, 27 Oct 2025 10:41:36 +0100 Subject: [PATCH 028/119] Expose contextualized fair sharing weights for cluster queues as metrics (#7338) * use float instead of int in cluster_queue_weighted_share metric and add cohort label * don't use two fields for weighted share * adjust metric test util to the changes * make ExpectClusterQueueWeightedShareMetric accept float64 as value * adjust integration test * report NaN instead of max_int when weight is 0 * remove unused imports in e2e tests * use float instead of int in cohort_weighted_share metric * fix format and naming cleanup --- pkg/cache/scheduler/cache.go | 10 +- pkg/cache/scheduler/fair_sharing.go | 2 - .../core/clusterqueue_controller.go | 9 +- pkg/controller/core/cohort_controller.go | 2 +- pkg/controller/core/helpers.go | 28 +++ pkg/metrics/metrics.go | 16 +- .../fairsharing/fair_sharing_test.go | 211 ++++++++++-------- test/util/util.go | 6 +- 8 files changed, 170 insertions(+), 114 deletions(-) create mode 100644 pkg/controller/core/helpers.go diff --git a/pkg/cache/scheduler/cache.go b/pkg/cache/scheduler/cache.go index d2871c23073..46d7a995bc6 100644 --- a/pkg/cache/scheduler/cache.go +++ b/pkg/cache/scheduler/cache.go @@ -686,7 +686,7 @@ type ClusterQueueUsageStats struct { ReservingWorkloads int AdmittedResources []kueue.FlavorUsage AdmittedWorkloads int - WeightedShare int64 + WeightedShare float64 } // Usage reports the reserved and admitted resources and number of workloads holding them in the ClusterQueue. @@ -708,14 +708,13 @@ func (c *Cache) Usage(cqObj *kueue.ClusterQueue) (*ClusterQueueUsageStats, error if c.fairSharingEnabled { drs := dominantResourceShare(cq, nil) - weightedShare, _ := drs.roundedWeightedShare() - stats.WeightedShare = weightedShare + stats.WeightedShare = drs.PreciseWeightedShare() } return stats, nil } type CohortUsageStats struct { - WeightedShare int64 + WeightedShare float64 } func (c *Cache) CohortStats(cohortObj *kueue.Cohort) (*CohortUsageStats, error) { @@ -730,8 +729,7 @@ func (c *Cache) CohortStats(cohortObj *kueue.Cohort) (*CohortUsageStats, error) stats := &CohortUsageStats{} if c.fairSharingEnabled { drs := dominantResourceShare(cohort, nil) - weightedShare, _ := drs.roundedWeightedShare() - stats.WeightedShare = weightedShare + stats.WeightedShare = drs.PreciseWeightedShare() } return stats, nil diff --git a/pkg/cache/scheduler/fair_sharing.go b/pkg/cache/scheduler/fair_sharing.go index 3db3df730b6..ceeb9db67d2 100644 --- a/pkg/cache/scheduler/fair_sharing.go +++ b/pkg/cache/scheduler/fair_sharing.go @@ -67,8 +67,6 @@ func (d DRS) PreciseWeightedShare() float64 { return 0.0 } if d.isWeightZero() { - // This branch is used only for logging; functional - // branches never reach here. return math.Inf(1) } return d.unweightedRatio / d.fairWeight diff --git a/pkg/controller/core/clusterqueue_controller.go b/pkg/controller/core/clusterqueue_controller.go index 0eb8644ea1a..7ff8acf76da 100644 --- a/pkg/controller/core/clusterqueue_controller.go +++ b/pkg/controller/core/clusterqueue_controller.go @@ -19,6 +19,7 @@ package core import ( "context" "iter" + "math" "slices" "github.com/go-logr/logr" @@ -563,12 +564,16 @@ func (r *ClusterQueueReconciler) updateCqStatusIfChanged( }) if r.fairSharingEnabled { if r.reportResourceMetrics { - metrics.ReportClusterQueueWeightedShare(cq.Name, stats.WeightedShare) + weightedShare := stats.WeightedShare + if weightedShare == math.Inf(1) { + weightedShare = math.NaN() + } + metrics.ReportClusterQueueWeightedShare(cq.Name, string(cq.Spec.Cohort), weightedShare) } if cq.Status.FairSharing == nil { cq.Status.FairSharing = &kueue.FairSharingStatus{} } - cq.Status.FairSharing.WeightedShare = stats.WeightedShare + cq.Status.FairSharing.WeightedShare = WeightedShare(stats.WeightedShare) } else { cq.Status.FairSharing = nil } diff --git a/pkg/controller/core/cohort_controller.go b/pkg/controller/core/cohort_controller.go index 98b2c1d86dc..77f03a9e91d 100644 --- a/pkg/controller/core/cohort_controller.go +++ b/pkg/controller/core/cohort_controller.go @@ -171,7 +171,7 @@ func (r *CohortReconciler) updateCohortStatusIfChanged(ctx context.Context, coho if cohort.Status.FairSharing == nil { cohort.Status.FairSharing = &kueue.FairSharingStatus{} } - cohort.Status.FairSharing.WeightedShare = stats.WeightedShare + cohort.Status.FairSharing.WeightedShare = WeightedShare(stats.WeightedShare) } else { cohort.Status.FairSharing = nil } diff --git a/pkg/controller/core/helpers.go b/pkg/controller/core/helpers.go new file mode 100644 index 00000000000..c7a7e338136 --- /dev/null +++ b/pkg/controller/core/helpers.go @@ -0,0 +1,28 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "math" +) + +func WeightedShare(f float64) int64 { + if f == math.Inf(1) { + return math.MaxInt64 + } + return int64(math.Ceil(f)) +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 8b5bc2ffefe..b7de2ef408f 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -477,9 +477,8 @@ For a LocalQueue, the metric only reports a value of 1 for one of the statuses.` quota to the lendable resources in the cohort, among all the resources provided by the ClusterQueue, and divided by the weight. If zero, it means that the usage of the ClusterQueue is below the nominal quota. -If the ClusterQueue has a weight of zero and is borrowing, this will return 9223372036854775807, -the maximum possible share value.`, - }, []string{"cluster_queue"}, +If the ClusterQueue has a weight of zero and is borrowing, this will return NaN.`, + }, []string{"cluster_queue", "cohort"}, ) CohortWeightedShare = prometheus.NewGaugeVec( @@ -490,8 +489,7 @@ the maximum possible share value.`, quota to the lendable resources in the Cohort, among all the resources provided by the Cohort, and divided by the weight. If zero, it means that the usage of the Cohort is below the nominal quota. -If the Cohort has a weight of zero and is borrowing, this will return 9223372036854775807, -the maximum possible share value.`, +If the Cohort has a weight of zero and is borrowing, this will return NaN.`, }, []string{"cohort"}, ) ) @@ -685,12 +683,12 @@ func ReportLocalQueueResourceUsage(lq LocalQueueReference, flavor, resource stri LocalQueueResourceUsage.WithLabelValues(string(lq.Name), lq.Namespace, flavor, resource).Set(usage) } -func ReportClusterQueueWeightedShare(cq string, weightedShare int64) { - ClusterQueueWeightedShare.WithLabelValues(cq).Set(float64(weightedShare)) +func ReportClusterQueueWeightedShare(cq, cohort string, weightedShare float64) { + ClusterQueueWeightedShare.WithLabelValues(cq, cohort).Set(weightedShare) } -func ReportCohortWeightedShare(cohort string, weightedShare int64) { - CohortWeightedShare.WithLabelValues(cohort).Set(float64(weightedShare)) +func ReportCohortWeightedShare(cohort string, weightedShare float64) { + CohortWeightedShare.WithLabelValues(cohort).Set(weightedShare) } func ClearClusterQueueResourceMetrics(cqName string) { diff --git a/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go b/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go index 7df837b9efa..f963bcd9476 100644 --- a/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go +++ b/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go @@ -17,6 +17,7 @@ limitations under the License. package fairsharing import ( + "math" "time" "github.com/onsi/ginkgo/v2" @@ -30,6 +31,7 @@ import ( config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" @@ -161,9 +163,9 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f } util.ExpectReservingActiveWorkloadsMetric(cqA, 8) util.ExpectPendingWorkloadsMetric(cqA, 0, 2) - util.ExpectClusterQueueWeightedShareMetric(cqA, 625) - util.ExpectClusterQueueWeightedShareMetric(cqB, 0) - util.ExpectClusterQueueWeightedShareMetric(cqShared, 0) + util.ExpectClusterQueueWeightedShareMetric(cqA, 625.0) + util.ExpectClusterQueueWeightedShareMetric(cqB, 0.0) + util.ExpectClusterQueueWeightedShareMetric(cqShared, 0.0) ginkgo.By("Creating newer workloads in cq-b") util.WaitForNextSecondAfterCreation(wls[len(wls)-1]) @@ -171,9 +173,9 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f createWorkload("b", "1") } util.ExpectPendingWorkloadsMetric(cqB, 0, 5) - util.ExpectClusterQueueWeightedShareMetric(cqA, 625) - util.ExpectClusterQueueWeightedShareMetric(cqB, 0) - util.ExpectClusterQueueWeightedShareMetric(cqShared, 0) + util.ExpectClusterQueueWeightedShareMetric(cqA, 625.0) + util.ExpectClusterQueueWeightedShareMetric(cqB, 0.0) + util.ExpectClusterQueueWeightedShareMetric(cqShared, 0.0) ginkgo.By("Terminating 4 running workloads in cqA: shared quota is fair-shared") util.FinishRunningWorkloadsInCQ(ctx, k8sClient, cqA, 4) @@ -183,9 +185,9 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f util.ExpectPendingWorkloadsMetric(cqA, 0, 1) util.ExpectReservingActiveWorkloadsMetric(cqB, 3) util.ExpectPendingWorkloadsMetric(cqB, 0, 2) - util.ExpectClusterQueueWeightedShareMetric(cqA, 250) - util.ExpectClusterQueueWeightedShareMetric(cqB, 250) - util.ExpectClusterQueueWeightedShareMetric(cqShared, 0) + util.ExpectClusterQueueWeightedShareMetric(cqA, 250.0) + util.ExpectClusterQueueWeightedShareMetric(cqB, 250.0) + util.ExpectClusterQueueWeightedShareMetric(cqShared, 0.0) ginkgo.By("Terminating 2 more running workloads in cqA: cqB starts to take over shared quota") util.FinishRunningWorkloadsInCQ(ctx, k8sClient, cqA, 2) @@ -195,9 +197,9 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f util.ExpectPendingWorkloadsMetric(cqA, 0, 0) util.ExpectReservingActiveWorkloadsMetric(cqB, 4) util.ExpectPendingWorkloadsMetric(cqB, 0, 1) - util.ExpectClusterQueueWeightedShareMetric(cqA, 125) - util.ExpectClusterQueueWeightedShareMetric(cqB, 375) - util.ExpectClusterQueueWeightedShareMetric(cqShared, 0) + util.ExpectClusterQueueWeightedShareMetric(cqA, 125.0) + util.ExpectClusterQueueWeightedShareMetric(cqB, 375.0) + util.ExpectClusterQueueWeightedShareMetric(cqShared, 0.0) ginkgo.By("Checking that weight share status changed") cqAKey := client.ObjectKeyFromObject(cqA) @@ -235,28 +237,34 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f Cohort("all"). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "3").Obj(), - ).Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - }).Obj()) + ). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Obj()) cqB = createQueue(utiltestingapi.MakeClusterQueue("b"). Cohort("all"). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "3").Obj(), - ).Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - }).Obj()) + ). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Obj()) cqC = createQueue(utiltestingapi.MakeClusterQueue("c"). Cohort("all"). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "3").Obj(), - ).Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - }).Obj()) + ). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Obj()) }) ginkgo.It("Admits workloads respecting fair share", framework.SlowSpec, func() { @@ -266,9 +274,9 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f } util.ExpectReservingActiveWorkloadsMetric(cqA, 9) util.ExpectPendingWorkloadsMetric(cqA, 0, 1) - util.ExpectClusterQueueWeightedShareMetric(cqA, 667) - util.ExpectClusterQueueWeightedShareMetric(cqB, 0) - util.ExpectClusterQueueWeightedShareMetric(cqC, 0) + util.ExpectClusterQueueWeightedShareMetric(cqA, 6.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqB, 0.0) + util.ExpectClusterQueueWeightedShareMetric(cqC, 0.0) ginkgo.By("Creating newer workloads in cq-b") for range 5 { @@ -279,24 +287,24 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Finishing eviction of 4 running workloads in cqA: shared quota is fair-shared") util.FinishEvictionOfWorkloadsInCQ(ctx, k8sClient, cqA, 4) util.ExpectReservingActiveWorkloadsMetric(cqB, 4) - util.ExpectClusterQueueWeightedShareMetric(cqA, 223) - util.ExpectClusterQueueWeightedShareMetric(cqB, 112) - util.ExpectClusterQueueWeightedShareMetric(cqC, 0) + util.ExpectClusterQueueWeightedShareMetric(cqA, 2.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqB, 1.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqC, 0.0) ginkgo.By("cq-c reclaims one unit, preemption happens in cq-a") cWorkload := utiltestingapi.MakeWorkload("c0", ns.Name).Queue("c").Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, cWorkload) util.ExpectPendingWorkloadsMetric(cqC, 1, 0) - util.ExpectClusterQueueWeightedShareMetric(cqA, 223) - util.ExpectClusterQueueWeightedShareMetric(cqB, 112) - util.ExpectClusterQueueWeightedShareMetric(cqC, 0) + util.ExpectClusterQueueWeightedShareMetric(cqA, 2.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqB, 1.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqC, 0.0) ginkgo.By("Finishing eviction of 1 running workloads in the CQ with highest usage: cqA") util.FinishEvictionOfWorkloadsInCQ(ctx, k8sClient, cqA, 1) util.ExpectReservingActiveWorkloadsMetric(cqC, 1) - util.ExpectClusterQueueWeightedShareMetric(cqA, 112) - util.ExpectClusterQueueWeightedShareMetric(cqB, 112) - util.ExpectClusterQueueWeightedShareMetric(cqC, 0) + util.ExpectClusterQueueWeightedShareMetric(cqA, 1.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqB, 1.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqC, 0.0) ginkgo.By("Checking that weight share status changed") cqAKey := client.ObjectKeyFromObject(cqA) @@ -378,6 +386,22 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f util.ExpectEvictedWorkloadsTotalMetric(cqA.Name, kueue.WorkloadEvictedByPreemption, "", "", 1) util.ExpectEvictedWorkloadsTotalMetric(cqB.Name, kueue.WorkloadEvictedByPreemption, "", "", 0) }) + + ginkgo.It("should have NaN weighted share metric", func() { + ginkgo.By("Creating a workload in cqA") + wlA1 := createWorkloadWithPriority("best-effort-cq-a", "4", 100) + util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, wlA1) + util.ExpectAdmittedWorkloadsTotalMetric(cqA, "", 1) + util.ExpectReservingActiveWorkloadsMetric(cqA, 1) + + ginkgo.By("checking the weighted share metric") + gomega.Eventually(func(g gomega.Gomega) { + metric := metrics.ClusterQueueWeightedShare.WithLabelValues(cqA.Name, string(cqA.Spec.Cohort)) + v, err := testutil.GetGaugeMetricValue(metric) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(math.IsNaN(v)).Should(gomega.BeTrue()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) }) ginkgo.When("using hierarchical cohorts", func() { @@ -409,9 +433,9 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ResourceGroup( *utiltestingapi.MakeFlavorQuotas(defaultFlavor.Name).Resource(corev1.ResourceCPU, "2").Obj(), ).Obj()) - expectCohortWeightedShare(cohortFirstLeft.Name, 0) - expectCohortWeightedShare(cohortFirstRight.Name, 0) - expectCohortWeightedShare(cohortBank.Name, 0) + expectCohortWeightedShare(cohortFirstLeft.Name, 0.0) + expectCohortWeightedShare(cohortFirstRight.Name, 0.0) + expectCohortWeightedShare(cohortBank.Name, 0.0) ginkgo.By("Adding workloads to cqSecondLeft and cqSecondRight in round-robin fashion") for range 5 { @@ -423,11 +447,11 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f util.ExpectReservingActiveWorkloadsMetric(cqSecondLeft, 5) util.ExpectAdmittedWorkloadsTotalMetric(cqSecondRight, "", 5) util.ExpectReservingActiveWorkloadsMetric(cqSecondRight, 5) - expectCohortWeightedShare(cohortFirstLeft.Name, 429) - expectCohortWeightedShare(cohortFirstRight.Name, 0) - expectCohortWeightedShare(cohortSecondLeft.Name, 215) - expectCohortWeightedShare(cohortSecondRight.Name, 215) - expectCohortWeightedShare(cohortBank.Name, 0) + expectCohortWeightedShare(cohortFirstLeft.Name, 6.0*1000.0/14.0) + expectCohortWeightedShare(cohortFirstRight.Name, 0.0) + expectCohortWeightedShare(cohortSecondLeft.Name, 3.0*1000.0/14.0) + expectCohortWeightedShare(cohortSecondRight.Name, 3.0*1000.0/14.0) + expectCohortWeightedShare(cohortBank.Name, 0.0) }) ginkgo.It("preempts workloads to enforce fair share", func() { // below are Cohorts and their fair @@ -462,8 +486,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f createWorkloadWithPriority(kueue.LocalQueueName(bestEffortQueue.GetName()), "1", -1) createWorkloadWithPriority(kueue.LocalQueueName(physicsQueue.GetName()), "1", -1) } - expectCohortWeightedShare("best-effort", 1000) - expectCohortWeightedShare("physics", 500) + expectCohortWeightedShare("best-effort", 1000.0) + expectCohortWeightedShare("physics", 500.0) util.ExpectAdmittedWorkloadsTotalMetric(bestEffortQueue, "", 6) util.ExpectReservingActiveWorkloadsMetric(bestEffortQueue, 6) util.ExpectAdmittedWorkloadsTotalMetric(physicsQueue, "", 6) @@ -482,12 +506,12 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("share is fair with respect to each parent") // parent root - expectCohortWeightedShare("best-effort", 667) - expectCohortWeightedShare("research", 667) + expectCohortWeightedShare("best-effort", 4.0*1000.0/12.0/0.5) + expectCohortWeightedShare("research", 8.0*1000.0/12.0) // parent research - expectCohortWeightedShare("chemistry", 167) - expectCohortWeightedShare("physics", 167) - expectCohortWeightedShare("llm", 167) + expectCohortWeightedShare("chemistry", 2.0*1000.0/12.0) + expectCohortWeightedShare("physics", 2.0*1000.0/12.0) + expectCohortWeightedShare("llm", 4.0*1000.0/12.0/2.0) ginkgo.By("number workloads admitted proportional to share at each level") util.ExpectReservingActiveWorkloadsMetric(bestEffortQueue, 4) @@ -608,8 +632,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f util.ExpectReservingActiveWorkloadsMetric(cqp1, 18) ginkgo.By("Expected Weighted Shares") - util.ExpectClusterQueueWeightedShareMetric(cqp1, 600) - expectCohortWeightedShare("cohort-a", 0) + util.ExpectClusterQueueWeightedShareMetric(cqp1, 600.0) + expectCohortWeightedShare("cohort-a", 0.0) }) ginkgo.It("Prefers flavor with remaining guarantees at Cohort level (FlavorFungibilityImplicitPreferenceDefault=true)", func() { _ = features.SetEnable(features.FlavorFungibilityImplicitPreferenceDefault, true) @@ -621,8 +645,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f util.ExpectReservingActiveWorkloadsMetric(cqp1, 18) ginkgo.By("Expected Weighted Shares") - util.ExpectClusterQueueWeightedShareMetric(cqp1, 600) - expectCohortWeightedShare("cohort-a", 0) + util.ExpectClusterQueueWeightedShareMetric(cqp1, 600.0) + expectCohortWeightedShare("cohort-a", 0.0) }) // scenario from Kueue#7015 @@ -638,7 +662,7 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f } util.ExpectAdmittedWorkloadsTotalMetric(cqp1, "", 20) util.ExpectReservingActiveWorkloadsMetric(cqp1, 20) - expectCohortWeightedShare("cohort-a", 100) + expectCohortWeightedShare("cohort-a", 100.0) ginkgo.By("Create workloads in CohortB which will preempt CohortA") createWorkload("cq-p5", "1") @@ -652,10 +676,10 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f util.ExpectReservingActiveWorkloadsMetric(cqp5, 2) ginkgo.By("Expected Weighted Shares") - util.ExpectClusterQueueWeightedShareMetric(cqp1, 600) - util.ExpectClusterQueueWeightedShareMetric(cqp5, 100) - expectCohortWeightedShare("cohort-a", 0) - expectCohortWeightedShare("cohort-b", 0) + util.ExpectClusterQueueWeightedShareMetric(cqp1, 600.0) + util.ExpectClusterQueueWeightedShareMetric(cqp5, 100.0) + expectCohortWeightedShare("cohort-a", 0.0) + expectCohortWeightedShare("cohort-b", 0.0) }) }) @@ -728,8 +752,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Expected Total Admitted Workloads and Weighted Share") util.ExpectAdmittedWorkloadsTotalMetric(cqp1, "", 4) util.ExpectAdmittedWorkloadsTotalMetric(cqp2, "", 1) - util.ExpectClusterQueueWeightedShareMetric(cqp1, 445) - util.ExpectClusterQueueWeightedShareMetric(cqp2, 556) + util.ExpectClusterQueueWeightedShareMetric(cqp1, 4.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqp2, 5.0*1000.0/9.0) }) // The larger workload, size 6, satisfies @@ -761,8 +785,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Expected Total Admitted Workloads and Weighted Share") util.ExpectAdmittedWorkloadsTotalMetric(cqp1, "", 4) util.ExpectAdmittedWorkloadsTotalMetric(cqp2, "", 1) - util.ExpectClusterQueueWeightedShareMetric(cqp1, 445) - util.ExpectClusterQueueWeightedShareMetric(cqp2, 556) + util.ExpectClusterQueueWeightedShareMetric(cqp1, 4.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqp2, 5.0*1000.0/9.0) }) ginkgo.It("workload of size 4 admits with inadmissible higher priority workload at ClusterQueue head", func() { @@ -787,8 +811,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Expected Total Admitted Workloads and Weighted Share") util.ExpectAdmittedWorkloadsTotalMetric(cqp1, "", 4) util.ExpectAdmittedWorkloadsTotalMetric(cqp2, "", 1) - util.ExpectClusterQueueWeightedShareMetric(cqp1, 445) - util.ExpectClusterQueueWeightedShareMetric(cqp2, 445) + util.ExpectClusterQueueWeightedShareMetric(cqp1, 4.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqp2, 4.0*1000.0/9.0) }) ginkgo.It("workload admits when several higher priority blocking workloads in front", func() { @@ -814,8 +838,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Expected Total Admitted Workloads and Weighted Share") util.ExpectAdmittedWorkloadsTotalMetric(cqp1, "", 4) util.ExpectAdmittedWorkloadsTotalMetric(cqp2, "", 1) - util.ExpectClusterQueueWeightedShareMetric(cqp1, 445) - util.ExpectClusterQueueWeightedShareMetric(cqp2, 556) + util.ExpectClusterQueueWeightedShareMetric(cqp1, 4.0*1000.0/9.0) + util.ExpectClusterQueueWeightedShareMetric(cqp2, 5.0*1000.0/9.0) }) }) @@ -885,8 +909,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Expected Total Admitted Workloads and Weighted Share") util.ExpectAdmittedWorkloadsTotalMetric(cq1, "", 1) util.ExpectAdmittedWorkloadsTotalMetric(cq2, "", 2) - util.ExpectClusterQueueWeightedShareMetric(cq1, 0) - util.ExpectClusterQueueWeightedShareMetric(cq2, 0) + util.ExpectClusterQueueWeightedShareMetric(cq1, 0.0) + util.ExpectClusterQueueWeightedShareMetric(cq2, 0.0) }) ginkgo.It("sticky workload becomes inadmissible. next workload admits", func() { @@ -923,8 +947,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Expected Total Admitted Workloads and Weighted Share") util.ExpectAdmittedWorkloadsTotalMetric(cq1, "", 1) - util.ExpectClusterQueueWeightedShareMetric(cq1, 0) - util.ExpectClusterQueueWeightedShareMetric(cq2, 0) + util.ExpectClusterQueueWeightedShareMetric(cq1, 0.0) + util.ExpectClusterQueueWeightedShareMetric(cq2, 0.0) }) ginkgo.It("sticky workload deleted, next workload can admit", func() { @@ -956,8 +980,8 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("Expected Total Admitted Workloads and Weighted Share") util.ExpectAdmittedWorkloadsTotalMetric(cq1, "", 1) - util.ExpectClusterQueueWeightedShareMetric(cq1, 0) - util.ExpectClusterQueueWeightedShareMetric(cq2, 0) + util.ExpectClusterQueueWeightedShareMetric(cq1, 0.0) + util.ExpectClusterQueueWeightedShareMetric(cq2, 0.0) }) }) @@ -1135,19 +1159,21 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f Cohort("all").FairWeight(resource.MustParse("300")). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "600").Obj(), - ).Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - }).Obj()) + ). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }).Obj()) cqB = createQueue(utiltestingapi.MakeClusterQueue("b"). Cohort("all").FairWeight(resource.MustParse("300")). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "600").Obj(), - ).Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - }).Obj()) + ). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }).Obj()) }) ginkgo.It("Queue can reclaim its nominal quota", framework.SlowSpec, func() { @@ -1187,8 +1213,9 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f Cohort("all"). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "8").Obj(), - ).Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny}). + ). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny}). Obj()) features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.AdmissionFairSharing, false) @@ -1218,13 +1245,13 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f }) }) -func expectCohortWeightedShare(cohortName string, weightedShare int64) { +func expectCohortWeightedShare(cohortName string, weightedShare float64) { // check Status gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { cohort := &kueue.Cohort{} g.ExpectWithOffset(1, k8sClient.Get(ctx, client.ObjectKey{Name: cohortName}, cohort)).Should(gomega.Succeed()) g.ExpectWithOffset(1, cohort.Status.FairSharing).ShouldNot(gomega.BeNil()) - g.ExpectWithOffset(1, cohort.Status.FairSharing.WeightedShare).Should(gomega.Equal(weightedShare)) + g.ExpectWithOffset(1, cohort.Status.FairSharing.WeightedShare).Should(gomega.Equal(core.WeightedShare(weightedShare))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) // check Metric @@ -1232,7 +1259,7 @@ func expectCohortWeightedShare(cohortName string, weightedShare int64) { gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { v, err := testutil.GetGaugeMetricValue(metric) g.ExpectWithOffset(1, err).ToNot(gomega.HaveOccurred()) - g.ExpectWithOffset(1, int64(v)).Should(gomega.Equal(weightedShare)) + g.ExpectWithOffset(1, v).Should(gomega.Equal(weightedShare)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) } @@ -1328,10 +1355,12 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f Cohort("all"). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "16").Obj(), - ).Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - }).Obj() + ). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Obj() util.MustCreate(ctx, k8sClient, cq2) cqs = append(cqs, cq2) diff --git a/test/util/util.go b/test/util/util.go index 25e7495c825..e84704e5ba4 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -728,12 +728,12 @@ func ExpectClusterQueueStatusMetric(cq *kueue.ClusterQueue, status metrics.Clust } } -func ExpectClusterQueueWeightedShareMetric(cq *kueue.ClusterQueue, value int64) { - metric := metrics.ClusterQueueWeightedShare.WithLabelValues(cq.Name) +func ExpectClusterQueueWeightedShareMetric(cq *kueue.ClusterQueue, value float64) { + metric := metrics.ClusterQueueWeightedShare.WithLabelValues(cq.Name, string(cq.Spec.Cohort)) gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { v, err := testutil.GetGaugeMetricValue(metric) g.Expect(err).ToNot(gomega.HaveOccurred()) - g.Expect(int64(v)).Should(gomega.Equal(value)) + g.Expect(v).Should(gomega.Equal(value)) }, Timeout, Interval).Should(gomega.Succeed()) } From 9a4452acac5aaaef3bcc11faadbc00e190dab268 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 03:41:37 -0700 Subject: [PATCH 029/119] Bump e2e-test-images/agnhost from 2.57 to 2.59 in /hack/agnhost (#7399) Bumps e2e-test-images/agnhost from 2.57 to 2.59. --- updated-dependencies: - dependency-name: e2e-test-images/agnhost dependency-version: '2.59' dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- hack/agnhost/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/agnhost/Dockerfile b/hack/agnhost/Dockerfile index d4e719830ca..11ece0494c7 100644 --- a/hack/agnhost/Dockerfile +++ b/hack/agnhost/Dockerfile @@ -1 +1 @@ -FROM registry.k8s.io/e2e-test-images/agnhost:2.57@sha256:80658cfc42bff7977efaa188e661631202fdbe810c4e17b2fcc84e2a3ac79298 +FROM registry.k8s.io/e2e-test-images/agnhost:2.59@sha256:10ed98804c877e505e17bced991682b6fd50ab1da9cddcd19935b48d645da40a From 33ceb56e617eee2c7e5dda2538e3dd1394c8978e Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Mon, 27 Oct 2025 16:39:36 +0530 Subject: [PATCH 030/119] Use clock on preemption. (#7395) --- pkg/scheduler/preemption/preemption.go | 2 +- pkg/workload/workload.go | 16 +++++++++------- .../controller/core/workload_controller_test.go | 2 +- .../controller/jobs/pod/pod_controller_test.go | 4 ++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index e6a25c997ea..9c721fb4132 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -197,7 +197,7 @@ func (p *Preemptor) IssuePreemptions(ctx context.Context, preemptor *workload.In func (p *Preemptor) patchPreemption(ctx context.Context, w *kueue.Workload, reason, message string) error { w = w.DeepCopy() return workload.Evict(ctx, p.client, p.recorder, w, kueue.WorkloadEvictedByPreemption, message, "", p.clock, workload.WithCustomPrepare(func() (*kueue.Workload, error) { - workload.SetPreemptedCondition(w, reason, message) + workload.SetPreemptedCondition(w, p.clock.Now(), reason, message) return w, nil })) } diff --git a/pkg/workload/workload.go b/pkg/workload/workload.go index 3fa7312825c..10013ceb0cd 100644 --- a/pkg/workload/workload.go +++ b/pkg/workload/workload.go @@ -782,12 +782,13 @@ func HasTopologyAssignmentsPending(w *kueue.Workload) bool { return false } -func SetPreemptedCondition(w *kueue.Workload, reason string, message string) { +func SetPreemptedCondition(w *kueue.Workload, now time.Time, reason string, message string) { condition := metav1.Condition{ - Type: kueue.WorkloadPreempted, - Status: metav1.ConditionTrue, - Reason: reason, - Message: api.TruncateConditionMessage(message), + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(now), + Reason: reason, + Message: api.TruncateConditionMessage(message), } apimeta.SetStatusCondition(&w.Status.Conditions, condition) } @@ -803,10 +804,11 @@ func SetDeactivationTarget(w *kueue.Workload, reason string, message string) boo return apimeta.SetStatusCondition(&w.Status.Conditions, condition) } -func SetEvictedCondition(w *kueue.Workload, reason string, message string) bool { +func SetEvictedCondition(w *kueue.Workload, now time.Time, reason string, message string) bool { condition := metav1.Condition{ Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(now), Reason: reason, Message: api.TruncateConditionMessage(message), ObservedGeneration: w.Generation, @@ -1297,7 +1299,7 @@ func Evict(ctx context.Context, c client.Client, recorder record.EventRecorder, } func prepareForEviction(w *kueue.Workload, now time.Time, reason, message string) { - SetEvictedCondition(w, reason, message) + SetEvictedCondition(w, now, reason, message) resetClusterNomination(w) resetChecksOnEviction(w, now) resetUnhealthyNodes(w) diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go index 67cec5acaab..a39a491c0db 100644 --- a/test/integration/singlecluster/controller/core/workload_controller_test.go +++ b/test/integration/singlecluster/controller/core/workload_controller_test.go @@ -502,7 +502,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, key, wl)).To(gomega.Succeed()) g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, wl, realClock, func() (*kueue.Workload, bool, error) { - return wl, workload.SetEvictedCondition(wl, "ByTest", "by test"), nil + return wl, workload.SetEvictedCondition(wl, realClock.Now(), "ByTest", "by test"), nil })).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.FinishEvictionForWorkloads(ctx, k8sClient, wl) diff --git a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go index d8e7c367f5b..7ab3449e5bf 100644 --- a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go @@ -835,7 +835,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, createdWorkload, realClock, func() (*kueue.Workload, bool, error) { - return createdWorkload, workload.SetEvictedCondition(createdWorkload, "ByTest", "by test"), nil + return createdWorkload, workload.SetEvictedCondition(createdWorkload, realClock.Now(), "ByTest", "by test"), nil })).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -1428,7 +1428,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, wl)).Should(gomega.Succeed()) g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, wl, realClock, func() (*kueue.Workload, bool, error) { - return wl, workload.SetEvictedCondition(wl, kueue.WorkloadEvictedByPreemption, "By test"), nil + return wl, workload.SetEvictedCondition(wl, realClock.Now(), kueue.WorkloadEvictedByPreemption, "By test"), nil })).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) From c7ef18f9f779606ed569fceab311d50b81661758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Mon, 27 Oct 2025 12:51:34 +0100 Subject: [PATCH 031/119] v1beta2-convert-logic-and-tests (#7369) --- apis/kueue/v1beta1/localqueue_conversion.go | 20 +++-- charts/kueue/templates/webhook/manifests.yaml | 28 +++---- cmd/importer/pod/import.go | 74 +++++++++++++++---- cmd/kueue/main.go | 6 +- cmd/kueuectl/app/completion/completion.go | 8 +- .../app/completion/completion_test.go | 4 +- .../app/create/create_clusterqueue.go | 10 +-- .../app/create/create_clusterqueue_test.go | 8 +- cmd/kueuectl/app/create/create_localqueue.go | 8 +- .../app/create/create_localqueue_test.go | 4 +- .../app/create/create_resourceflavor.go | 8 +- .../app/create/create_resourceflavor_test.go | 6 +- cmd/kueuectl/app/delete/delete_workload.go | 8 +- .../app/delete/delete_workload_test.go | 6 +- cmd/kueuectl/app/list/list_clusterqueue.go | 8 +- .../app/list/list_clusterqueue_printer.go | 4 +- .../app/list/list_clusterqueue_test.go | 4 +- cmd/kueuectl/app/list/list_localqueue.go | 8 +- .../app/list/list_localqueue_printer.go | 2 +- .../app/list/list_localqueue_printer_test.go | 2 +- cmd/kueuectl/app/list/list_localqueue_test.go | 4 +- cmd/kueuectl/app/list/list_resourceflavor.go | 6 +- .../app/list/list_resourceflavor_printer.go | 2 +- .../list/list_resourceflavor_printer_test.go | 4 +- .../app/list/list_resourceflavor_test.go | 2 +- cmd/kueuectl/app/list/list_test.go | 4 +- cmd/kueuectl/app/list/list_workload.go | 6 +- .../app/list/list_workload_printer.go | 2 +- cmd/kueuectl/app/list/list_workload_test.go | 4 +- .../update_workload_activation_options.go | 6 +- .../app/resume/resume_clusterqueue.go | 8 +- cmd/kueuectl/app/resume/resume_localqueue.go | 8 +- cmd/kueuectl/app/stop/stop_clusterqueue.go | 8 +- cmd/kueuectl/app/stop/stop_localqueue.go | 8 +- config/components/webhook/manifests.yaml | 28 +++---- .../controller/jobframework/interface.go | 6 +- keps/79-hierarchical-cohorts/README.md | 8 +- pkg/cache/hierarchy/clusterqueue.go | 2 +- pkg/cache/hierarchy/cohort.go | 2 +- pkg/cache/hierarchy/cycle.go | 2 +- pkg/cache/hierarchy/manager.go | 2 +- pkg/cache/hierarchy/manager_test.go | 2 +- pkg/cache/queue/cluster_queue.go | 2 +- pkg/cache/queue/cluster_queue_test.go | 4 +- pkg/cache/queue/cohort.go | 2 +- pkg/cache/queue/dumper.go | 2 +- pkg/cache/queue/local_queue.go | 2 +- pkg/cache/queue/manager.go | 6 +- pkg/cache/queue/manager_test.go | 6 +- pkg/cache/queue/second_pass_queue.go | 2 +- pkg/cache/queue/status_checker.go | 2 +- pkg/cache/scheduler/cache.go | 10 +-- pkg/cache/scheduler/cache_test.go | 10 +-- pkg/cache/scheduler/clusterqueue.go | 2 +- pkg/cache/scheduler/clusterqueue_snapshot.go | 2 +- pkg/cache/scheduler/clusterqueue_test.go | 4 +- pkg/cache/scheduler/cohort.go | 2 +- pkg/cache/scheduler/cohort_snapshot.go | 2 +- pkg/cache/scheduler/fair_sharing.go | 2 +- pkg/cache/scheduler/fair_sharing_test.go | 4 +- pkg/cache/scheduler/resource.go | 2 +- pkg/cache/scheduler/resource_node_test.go | 2 +- pkg/cache/scheduler/resource_test.go | 4 +- pkg/cache/scheduler/snapshot.go | 2 +- pkg/cache/scheduler/snapshot_test.go | 4 +- pkg/cache/scheduler/tas_cache.go | 2 +- pkg/cache/scheduler/tas_cache_test.go | 2 +- pkg/cache/scheduler/tas_flavor.go | 2 +- pkg/cache/scheduler/tas_flavor_snapshot.go | 2 +- .../scheduler/tas_flavor_snapshot_test.go | 2 +- .../multikueue/admissioncheck.go | 2 +- .../multikueue/admissioncheck_test.go | 4 +- .../multikueue/externalframeworks/adapter.go | 2 +- .../externalframeworks/adapter_test.go | 2 +- .../admissionchecks/multikueue/fswatch.go | 2 +- .../admissionchecks/multikueue/indexer.go | 2 +- .../multikueue/indexer_test.go | 4 +- .../multikueue/multikueuecluster.go | 2 +- .../multikueue/multikueuecluster_test.go | 4 +- .../admissionchecks/multikueue/workload.go | 2 +- .../multikueue/workload_test.go | 4 +- .../provisioning/admissioncheck_reconciler.go | 2 +- .../admissioncheck_reconciler_test.go | 4 +- .../provisioning/controller.go | 2 +- .../provisioning/controller_test.go | 10 +-- .../admissionchecks/provisioning/indexer.go | 2 +- .../provisioning/indexer_test.go | 4 +- .../provisioning/provisioning.go | 2 +- pkg/controller/constants/constants.go | 2 +- .../core/admissioncheck_controller.go | 2 +- .../core/clusterqueue_controller.go | 12 +-- .../core/clusterqueue_controller_test.go | 8 +- pkg/controller/core/cohort_controller.go | 4 +- pkg/controller/core/cohort_controller_test.go | 4 +- pkg/controller/core/indexer/indexer.go | 2 +- pkg/controller/core/localqueue_controller.go | 6 +- .../core/localqueue_controller_test.go | 4 +- .../core/resourceflavor_controller.go | 2 +- pkg/controller/core/workload_controller.go | 8 +- .../core/workload_controller_test.go | 4 +- .../jobframework/base_webhook_test.go | 4 +- pkg/controller/jobframework/defaults.go | 2 +- pkg/controller/jobframework/interface.go | 2 +- pkg/controller/jobframework/reconciler.go | 2 +- .../jobframework/reconciler_test.go | 4 +- pkg/controller/jobframework/setup_test.go | 4 +- pkg/controller/jobframework/tas.go | 2 +- pkg/controller/jobframework/tas_test.go | 2 +- pkg/controller/jobframework/tas_validation.go | 2 +- .../jobs/appwrapper/appwrapper_controller.go | 6 +- .../appwrapper/appwrapper_controller_test.go | 4 +- .../appwrapper_multikueue_adapter.go | 2 +- .../appwrapper_multikueue_adapter_test.go | 2 +- .../deployment/deployment_webhook_test.go | 2 +- pkg/controller/jobs/job/job_controller.go | 2 +- .../jobs/job/job_controller_test.go | 4 +- .../jobs/job/job_multikueue_adapter.go | 2 +- .../jobs/job/job_multikueue_adapter_test.go | 2 +- pkg/controller/jobs/job/job_webhook.go | 2 +- pkg/controller/jobs/job/job_webhook_test.go | 4 +- .../jobs/jobset/jobset_controller.go | 2 +- .../jobs/jobset/jobset_controller_test.go | 4 +- .../jobs/jobset/jobset_multikueue_adapter.go | 2 +- .../jobset/jobset_multikueue_adapter_test.go | 2 +- pkg/controller/jobs/jobset/jobset_webhook.go | 2 +- .../jobs/jobset/jobset_webhook_test.go | 4 +- .../jobs/jaxjob/jaxjob_controller_test.go | 4 +- .../jaxjob/jaxjob_multikueue_adapter_test.go | 2 +- .../paddlejob/paddlejob_controller_test.go | 4 +- .../paddlejob_multikueue_adapter_test.go | 2 +- .../pytorch_multikueue_adapter_test.go | 2 +- .../pytorchjob/pytorchjob_controller_test.go | 4 +- .../jobs/tfjob/tfjob_controller_test.go | 4 +- .../tfjob/tfjob_multikueue_adapter_test.go | 2 +- .../xgboostjob/xgboostjob_controller_test.go | 4 +- .../xgboostjob_multikueue_adapter_test.go | 2 +- .../kubeflowjob/kubeflowjob_controller.go | 2 +- .../kubeflowjob_multikueue_adapter.go | 2 +- .../leaderworkerset_pod_reconciler.go | 2 +- .../leaderworkerset_pod_reconciler_test.go | 2 +- .../leaderworkerset_reconciler.go | 2 +- .../leaderworkerset_reconciler_test.go | 4 +- .../leaderworkerset_webhook.go | 2 +- .../leaderworkerset_webhook_test.go | 4 +- .../jobs/mpijob/mpijob_controller.go | 2 +- .../jobs/mpijob/mpijob_controller_test.go | 4 +- .../jobs/mpijob/mpijob_multikueue_adapter.go | 2 +- .../mpijob/mpijob_multikueue_adapter_test.go | 2 +- pkg/controller/jobs/mpijob/mpijob_webhook.go | 2 +- .../jobs/mpijob/mpijob_webhook_test.go | 4 +- pkg/controller/jobs/pod/event_handlers.go | 2 +- pkg/controller/jobs/pod/pod_controller.go | 2 +- .../jobs/pod/pod_controller_test.go | 4 +- .../jobs/pod/pod_multikueue_adapter.go | 2 +- .../jobs/pod/pod_multikueue_adapter_test.go | 2 +- pkg/controller/jobs/pod/pod_webhook.go | 2 +- pkg/controller/jobs/pod/pod_webhook_test.go | 4 +- .../jobs/raycluster/raycluster_controller.go | 2 +- .../raycluster/raycluster_controller_test.go | 4 +- .../raycluster_multikueue_adapter.go | 2 +- .../raycluster_multikueue_adapter_test.go | 2 +- .../jobs/raycluster/raycluster_webhook.go | 2 +- .../raycluster/raycluster_webhook_test.go | 4 +- .../jobs/rayjob/rayjob_controller.go | 2 +- .../jobs/rayjob/rayjob_controller_test.go | 4 +- .../jobs/rayjob/rayjob_multikueue_adapter.go | 2 +- .../rayjob/rayjob_multikueue_adapter_test.go | 2 +- pkg/controller/jobs/rayjob/rayjob_webhook.go | 2 +- .../jobs/rayjob/rayjob_webhook_test.go | 4 +- .../jobs/statefulset/statefulset_webhook.go | 2 +- .../statefulset/statefulset_webhook_test.go | 2 +- .../jobs/trainjob/trainjob_controller.go | 2 +- .../jobs/trainjob/trainjob_controller_test.go | 4 +- .../trainjob/trainjob_multikueue_adapter.go | 2 +- .../trainjob_multikueue_adapter_test.go | 2 +- .../jobs/trainjob/trainjob_webhook_test.go | 4 +- pkg/controller/tas/indexer/indexer.go | 2 +- pkg/controller/tas/node_failure_controller.go | 2 +- .../tas/node_failure_controller_test.go | 4 +- pkg/controller/tas/resource_flavor.go | 2 +- pkg/controller/tas/topology_controller.go | 2 +- pkg/controller/tas/topology_ungater.go | 2 +- pkg/controller/tas/topology_ungater_test.go | 4 +- .../incrementaldispatcher.go | 2 +- .../incrementaldispatcher_test.go | 4 +- pkg/dra/claims.go | 2 +- pkg/dra/claims_test.go | 2 +- pkg/metrics/metrics.go | 2 +- pkg/metrics/metrics_test.go | 2 +- pkg/podset/podset.go | 2 +- pkg/podset/podset_test.go | 4 +- pkg/resources/resource.go | 2 +- pkg/scheduler/fair_sharing_iterator.go | 2 +- .../flavorassigner/flavorassigner.go | 2 +- .../flavorassigner/flavorassigner_test.go | 6 +- .../flavorassigner/podset_reducer.go | 2 +- .../flavorassigner/podset_reducer_test.go | 4 +- .../flavorassigner/tas_flavorassigner.go | 2 +- .../classical/candidate_generator.go | 2 +- .../classical/hierarchical_preemption.go | 2 +- pkg/scheduler/preemption/common/ordering.go | 2 +- .../preemption/common/preemption_policy.go | 2 +- .../preemption/fairsharing/ordering.go | 2 +- pkg/scheduler/preemption/policy.go | 2 +- pkg/scheduler/preemption/preemption.go | 2 +- pkg/scheduler/preemption/preemption_test.go | 4 +- pkg/scheduler/scheduler.go | 2 +- pkg/scheduler/scheduler_afs_test.go | 4 +- pkg/scheduler/scheduler_tas_test.go | 4 +- pkg/scheduler/scheduler_test.go | 4 +- pkg/util/admissioncheck/admissioncheck.go | 2 +- .../admissioncheck/admissioncheck_test.go | 4 +- .../admission_fair_sharing.go | 2 +- pkg/util/equality/podset.go | 2 +- pkg/util/equality/podset_test.go | 4 +- pkg/util/podset/podset.go | 2 +- pkg/util/priority/priority.go | 2 +- pkg/util/priority/priority_test.go | 4 +- pkg/util/queue/cluster_queue.go | 2 +- pkg/util/queue/local_queue.go | 2 +- pkg/util/tas/tas.go | 2 +- pkg/util/testing/client.go | 4 +- pkg/util/testingjobs/job/wrappers.go | 2 +- pkg/util/testingjobs/pod/wrappers.go | 2 +- pkg/util/testingjobs/statefulset/wrappers.go | 2 +- .../api/v1beta1/pending_workloads_cq.go | 2 +- .../api/v1beta1/pending_workloads_cq_test.go | 4 +- .../api/v1beta1/pending_workloads_lq.go | 2 +- .../api/v1beta1/pending_workloads_lq_test.go | 4 +- pkg/visibility/api/v1beta1/utils.go | 3 +- pkg/webhooks/clusterqueue_webhook.go | 8 +- pkg/webhooks/clusterqueue_webhook_test.go | 4 +- pkg/webhooks/cohort_webhook.go | 4 +- pkg/webhooks/cohort_webhook_test.go | 4 +- pkg/webhooks/common.go | 2 +- pkg/webhooks/resourceflavor_webhook.go | 6 +- pkg/webhooks/resourceflavor_webhook_test.go | 4 +- pkg/webhooks/webhooks.go | 2 +- pkg/webhooks/workload_webhook.go | 6 +- pkg/webhooks/workload_webhook_test.go | 4 +- pkg/workload/admissionchecks.go | 8 +- pkg/workload/admissionchecks_test.go | 12 +-- pkg/workload/podsetscounts.go | 2 +- pkg/workload/podsetscounts_test.go | 2 +- pkg/workload/resources.go | 2 +- pkg/workload/resources_test.go | 4 +- pkg/workload/usage.go | 2 +- pkg/workload/workload.go | 4 +- pkg/workload/workload_test.go | 4 +- pkg/workloadslicing/workloadslicing.go | 2 +- pkg/workloadslicing/workloadslicing_test.go | 4 +- test/e2e/certmanager/certmanager_test.go | 4 +- test/e2e/certmanager/metrics_test.go | 4 +- test/e2e/certmanager/visibility_test.go | 4 +- .../managejobswithoutqueuename_test.go | 4 +- .../objectretentionpolicies_test.go | 4 +- .../podintegrationautoenablement_test.go | 4 +- test/e2e/customconfigs/reconcile_test.go | 4 +- .../customconfigs/waitforpodsready_test.go | 4 +- test/e2e/multikueue/e2e_test.go | 4 +- test/e2e/multikueue/suite_test.go | 2 +- test/e2e/singlecluster/appwrapper_test.go | 4 +- test/e2e/singlecluster/deployment_test.go | 4 +- test/e2e/singlecluster/e2e_test.go | 4 +- ...2e_v1beta2_test.go => e2e_v1beta1_test.go} | 4 +- test/e2e/singlecluster/fair_sharing_test.go | 4 +- test/e2e/singlecluster/jaxjob_test.go | 4 +- test/e2e/singlecluster/jobset_test.go | 4 +- test/e2e/singlecluster/kuberay_test.go | 4 +- test/e2e/singlecluster/kueuectl_test.go | 4 +- .../e2e/singlecluster/leaderworkerset_test.go | 4 +- test/e2e/singlecluster/metrics_test.go | 4 +- test/e2e/singlecluster/pod_test.go | 4 +- test/e2e/singlecluster/pytorchjob_test.go | 4 +- test/e2e/singlecluster/statefulset_test.go | 4 +- test/e2e/singlecluster/tas_test.go | 4 +- test/e2e/singlecluster/trainjob_test.go | 4 +- test/e2e/singlecluster/visibility_test.go | 23 +++--- test/e2e/tas/appwrapper_test.go | 4 +- test/e2e/tas/hotswap_test.go | 4 +- test/e2e/tas/job_test.go | 4 +- test/e2e/tas/jobset_test.go | 4 +- test/e2e/tas/leaderworkerset_test.go | 4 +- test/e2e/tas/mpijob_test.go | 4 +- test/e2e/tas/pod_group_test.go | 4 +- test/e2e/tas/pytorch_test.go | 4 +- test/e2e/tas/rayjob_test.go | 4 +- test/e2e/tas/statefulset_test.go | 4 +- test/e2e/tas/trainjob_test.go | 4 +- test/integration/framework/framework.go | 8 +- .../integration/multikueue/dispatcher_test.go | 4 +- .../multikueue/enabled_integration_test.go | 4 +- .../multikueue/external_job_test.go | 4 +- test/integration/multikueue/jobs_test.go | 4 +- test/integration/multikueue/no_gc_test.go | 4 +- test/integration/multikueue/setup_test.go | 4 +- .../provisioning/provisioning_test.go | 4 +- .../core/admissioncheck_controller_test.go | 6 +- .../core/clusterqueue_controller_test.go | 4 +- .../core/localqueue_controller_test.go | 22 +++--- .../core/resourceflavor_controller_test.go | 6 +- .../core/workload_controller_test.go | 8 +- .../singlecluster/controller/dra/dra_test.go | 4 +- .../setup/setup_controllers_test.go | 4 +- .../jobframework/setup/suite_test.go | 4 + .../appwrapper/appwrapper_controller_test.go | 4 +- .../controller/jobs/appwrapper/suite_test.go | 3 + .../jobs/jaxjob/jaxjob_controller_test.go | 4 +- .../controller/jobs/jaxjob/suite_test.go | 3 + .../jobs/job/job_controller_test.go | 4 +- .../controller/jobs/job/suite_test.go | 3 + .../jobs/jobset/jobset_controller_test.go | 4 +- .../controller/jobs/jobset/suite_test.go | 3 + .../controller/jobs/kubeflow/kubeflowjob.go | 4 +- .../jobs/mpijob/mpijob_controller_test.go | 4 +- .../controller/jobs/mpijob/suite_test.go | 3 + .../paddlejob/paddlejob_controller_test.go | 4 +- .../controller/jobs/paddlejob/suite_test.go | 3 + .../jobs/pod/pod_controller_test.go | 4 +- .../pytorchjob/pytorchjob_controller_test.go | 4 +- .../controller/jobs/pytorchjob/suite_test.go | 3 + .../raycluster/raycluster_controller_test.go | 4 +- .../controller/jobs/raycluster/suite_test.go | 6 ++ .../jobs/rayjob/rayjob_controller_test.go | 4 +- .../controller/jobs/rayjob/suite_test.go | 6 ++ .../controller/jobs/tfjob/suite_test.go | 3 + .../jobs/tfjob/tfjob_controller_test.go | 4 +- .../controller/jobs/trainjob/suite_test.go | 3 + .../jobs/trainjob/trainjob_controller_test.go | 4 +- .../controller/jobs/xgboostjob/suite_test.go | 3 + .../xgboostjob/xgboostjob_controller_test.go | 4 +- ...a2_test.go => conversions_v1beta1_test.go} | 18 ++--- .../singlecluster/importer/importer_test.go | 63 ++++++++++++++-- .../singlecluster/kueuectl/create_test.go | 10 +-- .../singlecluster/kueuectl/list_test.go | 4 +- .../kueuectl/passthrough_test.go | 2 +- .../singlecluster/kueuectl/resume_test.go | 4 +- .../singlecluster/kueuectl/stop_test.go | 4 +- .../fairsharing/fair_sharing_test.go | 6 +- .../scheduler/podsready/scheduler_test.go | 4 +- .../scheduler/preemption_test.go | 4 +- .../singlecluster/scheduler/scheduler_test.go | 14 ++-- .../scheduler/workload_controller_test.go | 4 +- .../integration/singlecluster/tas/tas_test.go | 4 +- .../webhook/core/admissioncheck_test.go | 4 +- .../webhook/core/clusterqueue_test.go | 4 +- .../singlecluster/webhook/core/cohort_test.go | 4 +- .../webhook/core/localqueue_test.go | 10 ++- .../webhook/core/resourceflavor_test.go | 4 +- .../singlecluster/webhook/core/suite_test.go | 6 +- .../webhook/core/workload_test.go | 38 +++++++--- .../webhook/jobs/jobset_webhook_test.go | 2 +- .../webhook/jobs/paddlejob_webhook_test.go | 2 +- .../webhook/jobs/pytorchjob_webhook_test.go | 2 +- .../webhook/jobs/raycluster_webhook_test.go | 4 +- .../webhook/jobs/tfjob_webhook_test.go | 2 +- .../webhook/jobs/xgboostjob_webhook_test.go | 2 +- .../scheduler/minimalkueue/main.go | 2 +- .../scheduler/runner/controller/controller.go | 2 +- .../scheduler/runner/generator/generator.go | 4 +- .../runner/generator/generator_test.go | 2 +- test/performance/scheduler/runner/main.go | 2 +- .../scheduler/runner/recorder/recorder.go | 4 +- test/util/e2e.go | 6 +- test/util/util.go | 10 +-- test/util/util_scheduling.go | 2 +- 366 files changed, 895 insertions(+), 718 deletions(-) rename test/e2e/singlecluster/{e2e_v1beta2_test.go => e2e_v1beta1_test.go} (98%) rename test/integration/singlecluster/conversion/{conversions_v1beta2_test.go => conversions_v1beta1_test.go} (96%) diff --git a/apis/kueue/v1beta1/localqueue_conversion.go b/apis/kueue/v1beta1/localqueue_conversion.go index c50d2809d3a..4a22fc85525 100644 --- a/apis/kueue/v1beta1/localqueue_conversion.go +++ b/apis/kueue/v1beta1/localqueue_conversion.go @@ -37,20 +37,24 @@ func (dst *LocalQueue) ConvertFrom(srcRaw conversion.Hub) error { } func Convert_v1beta2_LocalQueueStatus_To_v1beta1_LocalQueueStatus(in *v1beta2.LocalQueueStatus, out *LocalQueueStatus, s conversionapi.Scope) error { - out.FlavorUsage = make([]LocalQueueFlavorUsage, len(in.FlavorsUsage)) - for i := range in.FlavorsUsage { - if err := autoConvert_v1beta2_LocalQueueFlavorUsage_To_v1beta1_LocalQueueFlavorUsage(&in.FlavorsUsage[i], &out.FlavorUsage[i], s); err != nil { - return err + if in.FlavorsUsage != nil { + out.FlavorUsage = make([]LocalQueueFlavorUsage, len(in.FlavorsUsage)) + for i := range in.FlavorsUsage { + if err := autoConvert_v1beta2_LocalQueueFlavorUsage_To_v1beta1_LocalQueueFlavorUsage(&in.FlavorsUsage[i], &out.FlavorUsage[i], s); err != nil { + return err + } } } return autoConvert_v1beta2_LocalQueueStatus_To_v1beta1_LocalQueueStatus(in, out, s) } func Convert_v1beta1_LocalQueueStatus_To_v1beta2_LocalQueueStatus(in *LocalQueueStatus, out *v1beta2.LocalQueueStatus, s conversionapi.Scope) error { - out.FlavorsUsage = make([]v1beta2.LocalQueueFlavorUsage, len(in.FlavorUsage)) - for i := range in.FlavorUsage { - if err := autoConvert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage(&in.FlavorUsage[i], &out.FlavorsUsage[i], s); err != nil { - return err + if in.FlavorUsage != nil { + out.FlavorsUsage = make([]v1beta2.LocalQueueFlavorUsage, len(in.FlavorUsage)) + for i := range in.FlavorUsage { + if err := autoConvert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage(&in.FlavorUsage[i], &out.FlavorsUsage[i], s); err != nil { + return err + } } } return autoConvert_v1beta1_LocalQueueStatus_To_v1beta2_LocalQueueStatus(in, out, s) diff --git a/charts/kueue/templates/webhook/manifests.yaml b/charts/kueue/templates/webhook/manifests.yaml index 784e03f67d9..4cfb6b87812 100644 --- a/charts/kueue/templates/webhook/manifests.yaml +++ b/charts/kueue/templates/webhook/manifests.yaml @@ -436,14 +436,14 @@ webhooks: service: name: '{{ include "kueue.fullname" . }}-webhook-service' namespace: '{{ .Release.Namespace }}' - path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue + path: /mutate-kueue-x-k8s-io-v1beta2-clusterqueue failurePolicy: Fail name: mclusterqueue.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE resources: @@ -456,14 +456,14 @@ webhooks: service: name: '{{ include "kueue.fullname" . }}-webhook-service' namespace: '{{ .Release.Namespace }}' - path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor + path: /mutate-kueue-x-k8s-io-v1beta2-resourceflavor failurePolicy: Fail name: mresourceflavor.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE resources: @@ -476,14 +476,14 @@ webhooks: service: name: '{{ include "kueue.fullname" . }}-webhook-service' namespace: '{{ .Release.Namespace }}' - path: /mutate-kueue-x-k8s-io-v1beta1-workload + path: /mutate-kueue-x-k8s-io-v1beta2-workload failurePolicy: Fail name: mworkload.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE resources: @@ -922,14 +922,14 @@ webhooks: service: name: '{{ include "kueue.fullname" . }}-webhook-service' namespace: '{{ .Release.Namespace }}' - path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue + path: /validate-kueue-x-k8s-io-v1beta2-clusterqueue failurePolicy: Fail name: vclusterqueue.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE - UPDATE @@ -942,14 +942,14 @@ webhooks: service: name: '{{ include "kueue.fullname" . }}-webhook-service' namespace: '{{ .Release.Namespace }}' - path: /validate-kueue-x-k8s-io-v1beta1-cohort + path: /validate-kueue-x-k8s-io-v1beta2-cohort failurePolicy: Fail name: vcohort.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE - UPDATE @@ -962,14 +962,14 @@ webhooks: service: name: '{{ include "kueue.fullname" . }}-webhook-service' namespace: '{{ .Release.Namespace }}' - path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor + path: /validate-kueue-x-k8s-io-v1beta2-resourceflavor failurePolicy: Fail name: vresourceflavor.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE - UPDATE @@ -982,14 +982,14 @@ webhooks: service: name: '{{ include "kueue.fullname" . }}-webhook-service' namespace: '{{ .Release.Namespace }}' - path: /validate-kueue-x-k8s-io-v1beta1-workload + path: /validate-kueue-x-k8s-io-v1beta2-workload failurePolicy: Fail name: vworkload.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE - UPDATE diff --git a/cmd/importer/pod/import.go b/cmd/importer/pod/import.go index 45857d75b98..6cadf7a7d37 100644 --- a/cmd/importer/pod/import.go +++ b/cmd/importer/pod/import.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/importer/util" "sigs.k8s.io/kueue/pkg/constants" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" @@ -84,11 +85,21 @@ func Import(ctx context.Context, c client.Client, cache *util.ImportCache, jobs wl.Spec.PriorityClassSource = constants.PodPriorityClassSource } - if err := createWorkload(ctx, c, wl); err != nil { + wlv1beta1 := &kueue.Workload{ + TypeMeta: metav1.TypeMeta{ + Kind: "Workload", + APIVersion: kueue.SchemeGroupVersion.String(), + }, + } + if err := kueue.Convert_v1beta2_Workload_To_v1beta1_Workload(wl, wlv1beta1, nil); err != nil { + return false, fmt.Errorf("failed to convert workload: %w", err) + } + + if err := createWorkload(ctx, c, wlv1beta1); err != nil { return false, fmt.Errorf("creating workload: %w", err) } - if err := admitWorkload(ctx, c, wl, cache.ClusterQueues[string(lq.Spec.ClusterQueue)]); err != nil { + if err := admitWorkload(ctx, c, wlv1beta1, cache.ClusterQueues[string(lq.Spec.ClusterQueue)]); err != nil { return false, err } log.V(2).Info("Successfully imported", "pod", klog.KObj(p), "workload", klog.KObj(wl)) @@ -168,45 +179,58 @@ func createWorkload(ctx context.Context, c client.Client, wl *kueue.Workload) er } func admitWorkload(ctx context.Context, c client.Client, wl *kueue.Workload, cq *kueue.ClusterQueue) error { - update := func() (*kueue.Workload, bool, error) { + // make its admission and update its status + // Here we convert temporarily to v1beta2 just so that we can use the helper functions for admission + // from the new Kueue. + newCQ := &kueuev1beta2.ClusterQueue{} + if err := kueue.Convert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(cq, newCQ, nil); err != nil { + return fmt.Errorf("failed to convert ClusterQueue: %w", err) + } + + update := func() (*kueuev1beta2.Workload, bool, error) { // make its admission and update its status - info := workload.NewInfo(wl) + newWl := &kueuev1beta2.Workload{} + if err := kueue.Convert_v1beta1_Workload_To_v1beta2_Workload(wl, newWl, nil); err != nil { + return nil, false, fmt.Errorf("failed to convert workload: %w", err) + } - admission := kueue.Admission{ - ClusterQueue: kueue.ClusterQueueReference(cq.Name), - PodSetAssignments: []kueue.PodSetAssignment{ + info := workload.NewInfo(newWl) + + admission := kueuev1beta2.Admission{ + ClusterQueue: kueuev1beta2.ClusterQueueReference(cq.Name), + PodSetAssignments: []kueuev1beta2.PodSetAssignment{ { Name: info.TotalRequests[0].Name, - Flavors: make(map[corev1.ResourceName]kueue.ResourceFlavorReference), + Flavors: make(map[corev1.ResourceName]kueuev1beta2.ResourceFlavorReference), ResourceUsage: info.TotalRequests[0].Requests.ToResourceList(), Count: ptr.To[int32](1), }, }, } - flv := cq.Spec.ResourceGroups[0].Flavors[0].Name + flv := newCQ.Spec.ResourceGroups[0].Flavors[0].Name for r := range info.TotalRequests[0].Requests { admission.PodSetAssignments[0].Flavors[r] = flv } - wl.Status.Admission = &admission + newWl.Status.Admission = &admission reservedCond := metav1.Condition{ Type: kueue.WorkloadQuotaReserved, Status: metav1.ConditionTrue, Reason: "Imported", Message: fmt.Sprintf("Imported into ClusterQueue %s", cq.Name), } - apimeta.SetStatusCondition(&wl.Status.Conditions, reservedCond) + apimeta.SetStatusCondition(&newWl.Status.Conditions, reservedCond) admittedCond := metav1.Condition{ Type: kueue.WorkloadAdmitted, Status: metav1.ConditionTrue, Reason: "Imported", Message: fmt.Sprintf("Imported into ClusterQueue %s", cq.Name), } - apimeta.SetStatusCondition(&wl.Status.Conditions, admittedCond) - return wl, true, nil + apimeta.SetStatusCondition(&newWl.Status.Conditions, admittedCond) + return newWl, true, nil } - err := workload.PatchAdmissionStatus(ctx, c, wl, realClock, update) + err := patchAdmissionStatus(ctx, c, update) retry, _, timeout := checkError(err) for retry { if timeout >= 0 { @@ -216,8 +240,28 @@ func admitWorkload(ctx context.Context, c client.Client, wl *kueue.Workload, cq case <-time.After(timeout): } } - err = workload.PatchAdmissionStatus(ctx, c, wl, realClock, update) + err = patchAdmissionStatus(ctx, c, update) retry, _, timeout = checkError(err) } return err } + +func patchAdmissionStatus(ctx context.Context, c client.Client, update func() (*kueuev1beta2.Workload, bool, error)) error { + wPatched, updated, err := update() + if err != nil || !updated { + return err + } + wlCopy := workload.PrepareWorkloadPatch(wPatched, true, realClock) + // Downgrade the version of the workload API used for Patching to v1beta1 so that it matches the storage version + // We use the storage version as importer is running before Kueue is enabled. + oldWl := &kueue.Workload{ + TypeMeta: metav1.TypeMeta{ + Kind: "Workload", + APIVersion: kueue.SchemeGroupVersion.String(), + }, + } + if err := kueue.Convert_v1beta2_Workload_To_v1beta1_Workload(wlCopy, oldWl, nil); err != nil { + return fmt.Errorf("failed to convert workload to v1beta1: %w", err) + } + return c.Status().Patch(ctx, oldWl, client.Apply, client.FieldOwner(constants.AdmissionName), client.ForceOwnership) +} diff --git a/cmd/kueue/main.go b/cmd/kueue/main.go index 5f8221c1768..94a63080c17 100644 --- a/cmd/kueue/main.go +++ b/cmd/kueue/main.go @@ -51,8 +51,8 @@ import ( configapiv1beta1 "sigs.k8s.io/kueue/apis/config/v1beta1" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueuealpha "sigs.k8s.io/kueue/apis/kueue/v1alpha1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" + kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/config" @@ -95,7 +95,7 @@ func init() { utilruntime.Must(schedulingv1.AddToScheme(scheme)) utilruntime.Must(kueue.AddToScheme(scheme)) - utilruntime.Must(kueuev1beta2.AddToScheme(scheme)) + utilruntime.Must(kueuev1beta1.AddToScheme(scheme)) utilruntime.Must(kueuealpha.AddToScheme(scheme)) utilruntime.Must(configapiv1beta1.AddToScheme(scheme)) utilruntime.Must(configapi.AddToScheme(scheme)) diff --git a/cmd/kueuectl/app/completion/completion.go b/cmd/kueuectl/app/completion/completion.go index 8dfa22eaf86..9adb6c6b8eb 100644 --- a/cmd/kueuectl/app/completion/completion.go +++ b/cmd/kueuectl/app/completion/completion.go @@ -24,7 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -111,7 +111,7 @@ func WorkloadNameFunc(clientGetter util.ClientGetter, activeStatus *bool) func(* return []string{}, cobra.ShellCompDirectiveError } - list, err := clientSet.KueueV1beta1().Workloads(namespace).List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) + list, err := clientSet.KueueV1beta2().Workloads(namespace).List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) if err != nil { return []string{}, cobra.ShellCompDirectiveError } @@ -148,7 +148,7 @@ func ClusterQueueNameFunc(clientGetter util.ClientGetter, activeStatus *bool) fu return []string{}, cobra.ShellCompDirectiveError } - list, err := clientSet.KueueV1beta1().ClusterQueues().List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) + list, err := clientSet.KueueV1beta2().ClusterQueues().List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) if err != nil { return []string{}, cobra.ShellCompDirectiveError } @@ -191,7 +191,7 @@ func LocalQueueNameFunc(clientGetter util.ClientGetter, activeStatus *bool) func return []string{}, cobra.ShellCompDirectiveError } - list, err := clientSet.KueueV1beta1().LocalQueues(namespace).List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) + list, err := clientSet.KueueV1beta2().LocalQueues(namespace).List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) if err != nil { return []string{}, cobra.ShellCompDirectiveError } diff --git a/cmd/kueuectl/app/completion/completion_test.go b/cmd/kueuectl/app/completion/completion_test.go index bba87c85180..817877d323a 100644 --- a/cmd/kueuectl/app/completion/completion_test.go +++ b/cmd/kueuectl/app/completion/completion_test.go @@ -25,10 +25,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" cmdtesting "sigs.k8s.io/kueue/cmd/kueuectl/app/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestWorkloadNameCompletionFunc(t *testing.T) { diff --git a/cmd/kueuectl/app/create/create_clusterqueue.go b/cmd/kueuectl/app/create/create_clusterqueue.go index 5cb12485627..0a91e0e3ec5 100644 --- a/cmd/kueuectl/app/create/create_clusterqueue.go +++ b/cmd/kueuectl/app/create/create_clusterqueue.go @@ -33,9 +33,9 @@ import ( "k8s.io/kubectl/pkg/util/templates" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" utilslices "sigs.k8s.io/kueue/pkg/util/slices" ) @@ -105,7 +105,7 @@ type ClusterQueueOptions struct { UserSpecifiedBorrowingLimit []string UserSpecifiedLendingLimit []string - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface PrintObj printers.ResourcePrinterFunc @@ -214,7 +214,7 @@ func (o *ClusterQueueOptions) Complete(clientGetter util.ClientGetter, cmd *cobr return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) if err != nil { @@ -269,7 +269,7 @@ func (o *ClusterQueueOptions) createClusterQueue() *kueue.ClusterQueue { TypeMeta: metav1.TypeMeta{APIVersion: kueue.SchemeGroupVersion.String(), Kind: "ClusterQueue"}, ObjectMeta: metav1.ObjectMeta{Name: o.Name}, Spec: kueue.ClusterQueueSpec{ - Cohort: kueue.CohortReference(o.Cohort), + CohortName: kueue.CohortReference(o.Cohort), QueueingStrategy: o.QueueingStrategy, NamespaceSelector: &o.NamespaceSelector, Preemption: &kueue.ClusterQueuePreemption{ diff --git a/cmd/kueuectl/app/create/create_clusterqueue_test.go b/cmd/kueuectl/app/create/create_clusterqueue_test.go index 31260ab03dc..5cfc435166c 100644 --- a/cmd/kueuectl/app/create/create_clusterqueue_test.go +++ b/cmd/kueuectl/app/create/create_clusterqueue_test.go @@ -24,8 +24,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestCreateClusterQueue(t *testing.T) { @@ -56,10 +56,10 @@ func TestCreateClusterQueue(t *testing.T) { }, }, expected: &kueue.ClusterQueue{ - TypeMeta: metav1.TypeMeta{APIVersion: "kueue.x-k8s.io/v1beta1", Kind: "ClusterQueue"}, + TypeMeta: metav1.TypeMeta{APIVersion: "kueue.x-k8s.io/v1beta2", Kind: "ClusterQueue"}, ObjectMeta: metav1.ObjectMeta{Name: "cq1"}, Spec: kueue.ClusterQueueSpec{ - Cohort: "cohort", + CohortName: "cohort", QueueingStrategy: kueue.StrictFIFO, NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{"foo": "bar"}, diff --git a/cmd/kueuectl/app/create/create_localqueue.go b/cmd/kueuectl/app/create/create_localqueue.go index df6c7c37c22..818531a988b 100644 --- a/cmd/kueuectl/app/create/create_localqueue.go +++ b/cmd/kueuectl/app/create/create_localqueue.go @@ -27,9 +27,9 @@ import ( "k8s.io/cli-runtime/pkg/printers" "k8s.io/kubectl/pkg/util/templates" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/completion" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -57,7 +57,7 @@ type LocalQueueOptions struct { UserSpecifiedClusterQueue string - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface PrintObj printers.ResourcePrinterFunc @@ -129,7 +129,7 @@ func (o *LocalQueueOptions) Complete(clientGetter util.ClientGetter, cmd *cobra. return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) if err != nil { diff --git a/cmd/kueuectl/app/create/create_localqueue_test.go b/cmd/kueuectl/app/create/create_localqueue_test.go index f459d8790c5..f9faa826d05 100644 --- a/cmd/kueuectl/app/create/create_localqueue_test.go +++ b/cmd/kueuectl/app/create/create_localqueue_test.go @@ -22,7 +22,7 @@ import ( "github.com/google/go-cmp/cmp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) func TestCreateLocalQueue(t *testing.T) { @@ -37,7 +37,7 @@ func TestCreateLocalQueue(t *testing.T) { ClusterQueue: "cq1", }, expected: &kueue.LocalQueue{ - TypeMeta: metav1.TypeMeta{APIVersion: "kueue.x-k8s.io/v1beta1", Kind: "LocalQueue"}, + TypeMeta: metav1.TypeMeta{APIVersion: "kueue.x-k8s.io/v1beta2", Kind: "LocalQueue"}, ObjectMeta: metav1.ObjectMeta{Name: "lq1", Namespace: "ns1"}, Spec: kueue.LocalQueueSpec{ClusterQueue: "cq1"}, }, diff --git a/cmd/kueuectl/app/create/create_resourceflavor.go b/cmd/kueuectl/app/create/create_resourceflavor.go index d8048afdb96..30f3594bf64 100644 --- a/cmd/kueuectl/app/create/create_resourceflavor.go +++ b/cmd/kueuectl/app/create/create_resourceflavor.go @@ -32,9 +32,9 @@ import ( "k8s.io/cli-runtime/pkg/printers" "k8s.io/kubectl/pkg/util/templates" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -73,7 +73,7 @@ type ResourceFlavorOptions struct { UserSpecifiedNodeTaints []string UserSpecifiedTolerations []string - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface PrintObj printers.ResourcePrinterFunc @@ -145,7 +145,7 @@ func (o *ResourceFlavorOptions) Complete(clientGetter util.ClientGetter, cmd *co return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) if err != nil { diff --git a/cmd/kueuectl/app/create/create_resourceflavor_test.go b/cmd/kueuectl/app/create/create_resourceflavor_test.go index 8f1fd0b7104..d6b4124a73f 100644 --- a/cmd/kueuectl/app/create/create_resourceflavor_test.go +++ b/cmd/kueuectl/app/create/create_resourceflavor_test.go @@ -26,7 +26,7 @@ import ( "k8s.io/cli-runtime/pkg/genericiooptions" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" cmdtesting "sigs.k8s.io/kueue/cmd/kueuectl/app/testing" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" @@ -214,7 +214,7 @@ func TestResourceFlavorOptions_createResourceFlavor(t *testing.T) { NodeTaints: []corev1.Taint{{Key: "key1", Value: "value", Effect: corev1.TaintEffectNoSchedule}}, }, expected: &kueue.ResourceFlavor{ - TypeMeta: metav1.TypeMeta{APIVersion: "kueue.x-k8s.io/v1beta1", Kind: "ResourceFlavor"}, + TypeMeta: metav1.TypeMeta{APIVersion: "kueue.x-k8s.io/v1beta2", Kind: "ResourceFlavor"}, ObjectMeta: metav1.ObjectMeta{Name: "rf"}, Spec: kueue.ResourceFlavorSpec{ NodeLabels: map[string]string{"key1": "value"}, @@ -359,7 +359,7 @@ func TestResourceFlavorCmd(t *testing.T) { } ctx, _ := utiltesting.ContextWithLog(t) - gotRf, err := clientset.KueueV1beta1().ResourceFlavors().Get(ctx, tc.rfName, metav1.GetOptions{}) + gotRf, err := clientset.KueueV1beta2().ResourceFlavors().Get(ctx, tc.rfName, metav1.GetOptions{}) if client.IgnoreNotFound(err) != nil { t.Error(err) return diff --git a/cmd/kueuectl/app/delete/delete_workload.go b/cmd/kueuectl/app/delete/delete_workload.go index 3285371fd4a..dd4c5fa430e 100644 --- a/cmd/kueuectl/app/delete/delete_workload.go +++ b/cmd/kueuectl/app/delete/delete_workload.go @@ -34,9 +34,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/completion" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -65,7 +65,7 @@ type WorkloadOptions struct { DryRunStrategy util.DryRunStrategy - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface DynamicClient dynamic.Interface RestMapper meta.RESTMapper @@ -158,7 +158,7 @@ func (o *WorkloadOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Co return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() o.DynamicClient, err = clientGetter.DynamicClient() if err != nil { diff --git a/cmd/kueuectl/app/delete/delete_workload_test.go b/cmd/kueuectl/app/delete/delete_workload_test.go index 69e0cbd904a..be3d10b2ff9 100644 --- a/cmd/kueuectl/app/delete/delete_workload_test.go +++ b/cmd/kueuectl/app/delete/delete_workload_test.go @@ -31,11 +31,11 @@ import ( k8sscheme "k8s.io/client-go/kubernetes/scheme" kubetesting "k8s.io/client-go/testing" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" cmdtesting "sigs.k8s.io/kueue/cmd/kueuectl/app/testing" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestWorkloadCmd(t *testing.T) { @@ -354,7 +354,7 @@ Do you want to proceed (y/n)? jobs.batch/j1 deleted } ctx, _ := utiltesting.ContextWithLog(t) - gotWorkloadList, err := clientset.KueueV1beta1().Workloads(tc.ns).List(ctx, metav1.ListOptions{}) + gotWorkloadList, err := clientset.KueueV1beta2().Workloads(tc.ns).List(ctx, metav1.ListOptions{}) if err != nil { t.Fatal(err) } diff --git a/cmd/kueuectl/app/list/list_clusterqueue.go b/cmd/kueuectl/app/list/list_clusterqueue.go index 1d4629a38ab..71ea4f5b468 100644 --- a/cmd/kueuectl/app/list/list_clusterqueue.go +++ b/cmd/kueuectl/app/list/list_clusterqueue.go @@ -30,9 +30,9 @@ import ( "k8s.io/kubectl/pkg/util/templates" "k8s.io/utils/clock" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -61,7 +61,7 @@ type ClusterQueueOptions struct { // Active means the cluster queue has kueue.ClusterQueueActive condition with status=metav1.ConditionTrue Active []bool - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface genericiooptions.IOStreams } @@ -116,7 +116,7 @@ func (o *ClusterQueueOptions) Complete(clientGetter util.ClientGetter, cmd *cobr return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() if len(args) > 0 { activeFlag, err := cmd.Flags().GetBoolSlice("active") diff --git a/cmd/kueuectl/app/list/list_clusterqueue_printer.go b/cmd/kueuectl/app/list/list_clusterqueue_printer.go index 7730760df9d..bac034d47d5 100644 --- a/cmd/kueuectl/app/list/list_clusterqueue_printer.go +++ b/cmd/kueuectl/app/list/list_clusterqueue_printer.go @@ -26,7 +26,7 @@ import ( "k8s.io/cli-runtime/pkg/printers" "k8s.io/utils/clock" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type listClusterQueuePrinter struct { @@ -89,7 +89,7 @@ func (p *listClusterQueuePrinter) printClusterQueue(clusterQueue *kueue.ClusterQ } row.Cells = []any{ clusterQueue.Name, - clusterQueue.Spec.Cohort, + clusterQueue.Spec.CohortName, clusterQueue.Status.PendingWorkloads, clusterQueue.Status.AdmittedWorkloads, isActiveStatus(clusterQueue), diff --git a/cmd/kueuectl/app/list/list_clusterqueue_test.go b/cmd/kueuectl/app/list/list_clusterqueue_test.go index 20503f3f873..7847fce8d4f 100644 --- a/cmd/kueuectl/app/list/list_clusterqueue_test.go +++ b/cmd/kueuectl/app/list/list_clusterqueue_test.go @@ -27,10 +27,10 @@ import ( "k8s.io/cli-runtime/pkg/genericiooptions" testingclock "k8s.io/utils/clock/testing" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" cmdtesting "sigs.k8s.io/kueue/cmd/kueuectl/app/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestClusterQueueRun(t *testing.T) { diff --git a/cmd/kueuectl/app/list/list_localqueue.go b/cmd/kueuectl/app/list/list_localqueue.go index d9d55cc6bfe..2a2a8e65a36 100644 --- a/cmd/kueuectl/app/list/list_localqueue.go +++ b/cmd/kueuectl/app/list/list_localqueue.go @@ -28,9 +28,9 @@ import ( "k8s.io/kubectl/pkg/util/templates" "k8s.io/utils/clock" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/completion" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -58,7 +58,7 @@ type LocalQueueOptions struct { LabelSelector string ClusterQueueFilter string - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface genericiooptions.IOStreams } @@ -123,7 +123,7 @@ func (o *LocalQueueOptions) Complete(clientGetter util.ClientGetter) error { return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() return nil } diff --git a/cmd/kueuectl/app/list/list_localqueue_printer.go b/cmd/kueuectl/app/list/list_localqueue_printer.go index f5bc3bd156a..b550dde3c59 100644 --- a/cmd/kueuectl/app/list/list_localqueue_printer.go +++ b/cmd/kueuectl/app/list/list_localqueue_printer.go @@ -26,7 +26,7 @@ import ( "k8s.io/cli-runtime/pkg/printers" "k8s.io/utils/clock" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type listLocalQueuePrinter struct { diff --git a/cmd/kueuectl/app/list/list_localqueue_printer_test.go b/cmd/kueuectl/app/list/list_localqueue_printer_test.go index b11dfe0802e..c4a879707d6 100644 --- a/cmd/kueuectl/app/list/list_localqueue_printer_test.go +++ b/cmd/kueuectl/app/list/list_localqueue_printer_test.go @@ -25,7 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" testingclock "k8s.io/utils/clock/testing" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) func TestLocalQueuePrint(t *testing.T) { diff --git a/cmd/kueuectl/app/list/list_localqueue_test.go b/cmd/kueuectl/app/list/list_localqueue_test.go index 50df7836224..06797b82b52 100644 --- a/cmd/kueuectl/app/list/list_localqueue_test.go +++ b/cmd/kueuectl/app/list/list_localqueue_test.go @@ -28,10 +28,10 @@ import ( "k8s.io/cli-runtime/pkg/genericiooptions" testingclock "k8s.io/utils/clock/testing" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" cmdtesting "sigs.k8s.io/kueue/cmd/kueuectl/app/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestLocalQueueFilter(t *testing.T) { diff --git a/cmd/kueuectl/app/list/list_resourceflavor.go b/cmd/kueuectl/app/list/list_resourceflavor.go index 69c1c3d4e96..6442607e3c0 100644 --- a/cmd/kueuectl/app/list/list_resourceflavor.go +++ b/cmd/kueuectl/app/list/list_resourceflavor.go @@ -29,7 +29,7 @@ import ( "k8s.io/utils/clock" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -48,7 +48,7 @@ type ResourceFlavorOptions struct { FieldSelector string LabelSelector string - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface genericiooptions.IOStreams } @@ -102,7 +102,7 @@ func (o *ResourceFlavorOptions) Complete(clientGetter util.ClientGetter) error { return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() return nil } diff --git a/cmd/kueuectl/app/list/list_resourceflavor_printer.go b/cmd/kueuectl/app/list/list_resourceflavor_printer.go index c70cc1d328e..57135dc22ac 100644 --- a/cmd/kueuectl/app/list/list_resourceflavor_printer.go +++ b/cmd/kueuectl/app/list/list_resourceflavor_printer.go @@ -29,7 +29,7 @@ import ( "k8s.io/cli-runtime/pkg/printers" "k8s.io/utils/clock" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type listResourceFlavorPrinter struct { diff --git a/cmd/kueuectl/app/list/list_resourceflavor_printer_test.go b/cmd/kueuectl/app/list/list_resourceflavor_printer_test.go index 54ff1a2e72a..b8fe7daa255 100644 --- a/cmd/kueuectl/app/list/list_resourceflavor_printer_test.go +++ b/cmd/kueuectl/app/list/list_resourceflavor_printer_test.go @@ -25,8 +25,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" testingclock "k8s.io/utils/clock/testing" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestResourceFlavorPrint(t *testing.T) { diff --git a/cmd/kueuectl/app/list/list_resourceflavor_test.go b/cmd/kueuectl/app/list/list_resourceflavor_test.go index 0eb8b8d3896..51559486bcc 100644 --- a/cmd/kueuectl/app/list/list_resourceflavor_test.go +++ b/cmd/kueuectl/app/list/list_resourceflavor_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" cmdtesting "sigs.k8s.io/kueue/cmd/kueuectl/app/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestResourceFlavorCmd(t *testing.T) { diff --git a/cmd/kueuectl/app/list/list_test.go b/cmd/kueuectl/app/list/list_test.go index 5832395b182..b3cc3ba52e2 100644 --- a/cmd/kueuectl/app/list/list_test.go +++ b/cmd/kueuectl/app/list/list_test.go @@ -29,10 +29,10 @@ import ( "k8s.io/cli-runtime/pkg/genericiooptions" testingclock "k8s.io/utils/clock/testing" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" cmdtesting "sigs.k8s.io/kueue/cmd/kueuectl/app/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestListCmd(t *testing.T) { diff --git a/cmd/kueuectl/app/list/list_workload.go b/cmd/kueuectl/app/list/list_workload.go index db67f35f0b7..6d7933319de 100644 --- a/cmd/kueuectl/app/list/list_workload.go +++ b/cmd/kueuectl/app/list/list_workload.go @@ -35,7 +35,7 @@ import ( "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" clientset "sigs.k8s.io/kueue/client-go/clientset/versioned" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" @@ -283,7 +283,7 @@ func (o *WorkloadOptions) Run(ctx context.Context) error { for { headers := totalCount == 0 - list, err := o.ClientSet.KueueV1beta1().Workloads(namespace).List(ctx, opts) + list, err := o.ClientSet.KueueV1beta2().Workloads(namespace).List(ctx, opts) if err != nil { return err } @@ -417,7 +417,7 @@ func (o *WorkloadOptions) localQueues(ctx context.Context, list *kueue.WorkloadL continue } if _, ok := localQueues[localQueueKeyForWorkload(&wl)]; !ok { - lq, err := o.ClientSet.KueueV1beta1().LocalQueues(wl.Namespace).Get(ctx, string(wl.Spec.QueueName), metav1.GetOptions{}) + lq, err := o.ClientSet.KueueV1beta2().LocalQueues(wl.Namespace).Get(ctx, string(wl.Spec.QueueName), metav1.GetOptions{}) if client.IgnoreNotFound(err) != nil { return nil, err } diff --git a/cmd/kueuectl/app/list/list_workload_printer.go b/cmd/kueuectl/app/list/list_workload_printer.go index 09346a5eb32..0bec93d1c18 100644 --- a/cmd/kueuectl/app/list/list_workload_printer.go +++ b/cmd/kueuectl/app/list/list_workload_printer.go @@ -30,7 +30,7 @@ import ( "k8s.io/cli-runtime/pkg/printers" "k8s.io/utils/clock" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/cmd/kueuectl/app/list/list_workload_test.go b/cmd/kueuectl/app/list/list_workload_test.go index 332ee656d9d..066fed37982 100644 --- a/cmd/kueuectl/app/list/list_workload_test.go +++ b/cmd/kueuectl/app/list/list_workload_test.go @@ -43,12 +43,12 @@ import ( kubetesting "k8s.io/client-go/testing" testingclock "k8s.io/utils/clock/testing" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" cmdtesting "sigs.k8s.io/kueue/cmd/kueuectl/app/testing" "sigs.k8s.io/kueue/pkg/controller/constants" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" ) diff --git a/cmd/kueuectl/app/options/update_workload_activation_options.go b/cmd/kueuectl/app/options/update_workload_activation_options.go index 8962d844e53..94b794630f6 100644 --- a/cmd/kueuectl/app/options/update_workload_activation_options.go +++ b/cmd/kueuectl/app/options/update_workload_activation_options.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -42,7 +42,7 @@ type UpdateWorkloadActivationOptions struct { Namespace string EnforceNamespace bool - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface PrintObj printers.ResourcePrinterFunc @@ -72,7 +72,7 @@ func (o *UpdateWorkloadActivationOptions) Complete(clientGetter util.ClientGette return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) if err != nil { diff --git a/cmd/kueuectl/app/resume/resume_clusterqueue.go b/cmd/kueuectl/app/resume/resume_clusterqueue.go index 02806f2a8b7..f5a81e1bd43 100644 --- a/cmd/kueuectl/app/resume/resume_clusterqueue.go +++ b/cmd/kueuectl/app/resume/resume_clusterqueue.go @@ -29,9 +29,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/completion" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -46,7 +46,7 @@ var ( type ClusterQueueOptions struct { ClusterQueueName string - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface PrintFlags *genericclioptions.PrintFlags PrintObj printers.ResourcePrinterFunc genericiooptions.IOStreams @@ -95,7 +95,7 @@ func (o *ClusterQueueOptions) Complete(clientGetter util.ClientGetter, args []st return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() printer, err := o.PrintFlags.ToPrinter() if err != nil { diff --git a/cmd/kueuectl/app/resume/resume_localqueue.go b/cmd/kueuectl/app/resume/resume_localqueue.go index 55dce566628..565d95e11c9 100644 --- a/cmd/kueuectl/app/resume/resume_localqueue.go +++ b/cmd/kueuectl/app/resume/resume_localqueue.go @@ -29,9 +29,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/completion" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -51,7 +51,7 @@ type LocalQueueOptions struct { LocalQueueName string Namespace string - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface genericiooptions.IOStreams } @@ -106,7 +106,7 @@ func (o *LocalQueueOptions) Complete(clientGetter util.ClientGetter, args []stri return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() printer, err := o.PrintFlags.ToPrinter() if err != nil { diff --git a/cmd/kueuectl/app/stop/stop_clusterqueue.go b/cmd/kueuectl/app/stop/stop_clusterqueue.go index d7794ae59fe..7dfc39ceb91 100644 --- a/cmd/kueuectl/app/stop/stop_clusterqueue.go +++ b/cmd/kueuectl/app/stop/stop_clusterqueue.go @@ -29,9 +29,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/completion" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -47,7 +47,7 @@ var ( type ClusterQueueOptions struct { ClusterQueueName string KeepAlreadyRunning bool - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface PrintFlags *genericclioptions.PrintFlags PrintObj printers.ResourcePrinterFunc genericiooptions.IOStreams @@ -98,7 +98,7 @@ func (o *ClusterQueueOptions) Complete(clientGetter util.ClientGetter, args []st return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() printer, err := o.PrintFlags.ToPrinter() if err != nil { diff --git a/cmd/kueuectl/app/stop/stop_localqueue.go b/cmd/kueuectl/app/stop/stop_localqueue.go index 6532a8c17ef..2e022dee8dd 100644 --- a/cmd/kueuectl/app/stop/stop_localqueue.go +++ b/cmd/kueuectl/app/stop/stop_localqueue.go @@ -29,9 +29,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" - kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app/completion" "sigs.k8s.io/kueue/cmd/kueuectl/app/util" ) @@ -52,7 +52,7 @@ type LocalQueueOptions struct { Namespace string KeepAlreadyRunning bool - Client kueuev1beta1.KueueV1beta1Interface + Client kueuev1beta2.KueueV1beta2Interface genericiooptions.IOStreams } @@ -109,7 +109,7 @@ func (o *LocalQueueOptions) Complete(clientGetter util.ClientGetter, args []stri return err } - o.Client = clientset.KueueV1beta1() + o.Client = clientset.KueueV1beta2() printer, err := o.PrintFlags.ToPrinter() if err != nil { diff --git a/config/components/webhook/manifests.yaml b/config/components/webhook/manifests.yaml index 26caf441f4b..068540b5c31 100644 --- a/config/components/webhook/manifests.yaml +++ b/config/components/webhook/manifests.yaml @@ -317,14 +317,14 @@ webhooks: service: name: webhook-service namespace: system - path: /mutate-kueue-x-k8s-io-v1beta1-clusterqueue + path: /mutate-kueue-x-k8s-io-v1beta2-clusterqueue failurePolicy: Fail name: mclusterqueue.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE resources: @@ -336,14 +336,14 @@ webhooks: service: name: webhook-service namespace: system - path: /mutate-kueue-x-k8s-io-v1beta1-resourceflavor + path: /mutate-kueue-x-k8s-io-v1beta2-resourceflavor failurePolicy: Fail name: mresourceflavor.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE resources: @@ -355,14 +355,14 @@ webhooks: service: name: webhook-service namespace: system - path: /mutate-kueue-x-k8s-io-v1beta1-workload + path: /mutate-kueue-x-k8s-io-v1beta2-workload failurePolicy: Fail name: mworkload.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE resources: @@ -700,14 +700,14 @@ webhooks: service: name: webhook-service namespace: system - path: /validate-kueue-x-k8s-io-v1beta1-clusterqueue + path: /validate-kueue-x-k8s-io-v1beta2-clusterqueue failurePolicy: Fail name: vclusterqueue.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE - UPDATE @@ -720,14 +720,14 @@ webhooks: service: name: webhook-service namespace: system - path: /validate-kueue-x-k8s-io-v1beta1-cohort + path: /validate-kueue-x-k8s-io-v1beta2-cohort failurePolicy: Fail name: vcohort.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE - UPDATE @@ -740,14 +740,14 @@ webhooks: service: name: webhook-service namespace: system - path: /validate-kueue-x-k8s-io-v1beta1-resourceflavor + path: /validate-kueue-x-k8s-io-v1beta2-resourceflavor failurePolicy: Fail name: vresourceflavor.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE - UPDATE @@ -760,14 +760,14 @@ webhooks: service: name: webhook-service namespace: system - path: /validate-kueue-x-k8s-io-v1beta1-workload + path: /validate-kueue-x-k8s-io-v1beta2-workload failurePolicy: Fail name: vworkload.kb.io rules: - apiGroups: - kueue.x-k8s.io apiVersions: - - v1beta1 + - v1beta2 operations: - CREATE - UPDATE diff --git a/internal/mocks/controller/jobframework/interface.go b/internal/mocks/controller/jobframework/interface.go index b12fdcd5fcb..8b3588ce779 100644 --- a/internal/mocks/controller/jobframework/interface.go +++ b/internal/mocks/controller/jobframework/interface.go @@ -17,7 +17,7 @@ import ( schema "k8s.io/apimachinery/pkg/runtime/schema" field "k8s.io/apimachinery/pkg/util/validation/field" client "sigs.k8s.io/controller-runtime/pkg/client" - v1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" + v1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" jobframework "sigs.k8s.io/kueue/pkg/controller/jobframework" podset "sigs.k8s.io/kueue/pkg/podset" ) @@ -119,10 +119,10 @@ func (mr *MockGenericJobMockRecorder) Object() *gomock.Call { } // PodSets mocks base method. -func (m *MockGenericJob) PodSets(ctx context.Context) ([]v1beta1.PodSet, error) { +func (m *MockGenericJob) PodSets(ctx context.Context) ([]v1beta2.PodSet, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "PodSets", ctx) - ret0, _ := ret[0].([]v1beta1.PodSet) + ret0, _ := ret[0].([]v1beta2.PodSet) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/keps/79-hierarchical-cohorts/README.md b/keps/79-hierarchical-cohorts/README.md index faeac348e80..1117411c11d 100644 --- a/keps/79-hierarchical-cohorts/README.md +++ b/keps/79-hierarchical-cohorts/README.md @@ -142,7 +142,7 @@ type CohortSpec struct { // Cohort, including ClusterQueues, until the cycle is // removed. We prevent further admission while the cycle // exists. - Parent kueuebeta.CohortReference `json:"parent,omitempty"` + Parent kueue.CohortReference `json:"parent,omitempty"` // ResourceGroups describes groupings of Resources and // Flavors. Each ResourceGroup defines a list of Resources @@ -163,18 +163,18 @@ type CohortSpec struct { // //+listType=atomic //+kubebuilder:validation:MaxItems=16 - ResourceGroups []kueuebeta.ResourceGroup `json:"resourceGroups,omitempty"` + ResourceGroups []kueue.ResourceGroup `json:"resourceGroups,omitempty"` // fairSharing defines the properties of the Cohort when // participating in FairSharing. The values are only relevant // if FairSharing is enabled in the Kueue configuration. // +optional - FairSharing *kueuebeta.FairSharing `json:"fairSharing,omitempty"` + FairSharing *kueue.FairSharing `json:"fairSharing,omitempty"` } type CohortStatus struct { // +optional - FairSharing *kueuebeta.FairSharingStatus `json:"fairSharing,omitempty"` + FairSharing *kueue.FairSharingStatus `json:"fairSharing,omitempty"` } ``` diff --git a/pkg/cache/hierarchy/clusterqueue.go b/pkg/cache/hierarchy/clusterqueue.go index bd5a51364cf..2e544f61d7c 100644 --- a/pkg/cache/hierarchy/clusterqueue.go +++ b/pkg/cache/hierarchy/clusterqueue.go @@ -16,7 +16,7 @@ limitations under the License. package hierarchy -import kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" +import kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" type ClusterQueue[C nodeBase[kueue.CohortReference]] struct { cohort C diff --git a/pkg/cache/hierarchy/cohort.go b/pkg/cache/hierarchy/cohort.go index 73030b104a8..9c23780649d 100644 --- a/pkg/cache/hierarchy/cohort.go +++ b/pkg/cache/hierarchy/cohort.go @@ -19,7 +19,7 @@ package hierarchy import ( "k8s.io/apimachinery/pkg/util/sets" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) //lint:ignore U1000 due to https://github.com/dominikh/go-tools/issues/1602. diff --git a/pkg/cache/hierarchy/cycle.go b/pkg/cache/hierarchy/cycle.go index 3a4c31def23..62226b5bbfd 100644 --- a/pkg/cache/hierarchy/cycle.go +++ b/pkg/cache/hierarchy/cycle.go @@ -19,7 +19,7 @@ package hierarchy import ( "k8s.io/apimachinery/pkg/util/sets" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type CycleCheckable interface { diff --git a/pkg/cache/hierarchy/manager.go b/pkg/cache/hierarchy/manager.go index 67a08f54935..3d71b69de8d 100644 --- a/pkg/cache/hierarchy/manager.go +++ b/pkg/cache/hierarchy/manager.go @@ -19,7 +19,7 @@ package hierarchy import ( "maps" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) // Manager stores Cohorts and ClusterQueues, and maintains the edges diff --git a/pkg/cache/hierarchy/manager_test.go b/pkg/cache/hierarchy/manager_test.go index 79d22360cff..f63180fe874 100644 --- a/pkg/cache/hierarchy/manager_test.go +++ b/pkg/cache/hierarchy/manager_test.go @@ -23,7 +23,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "k8s.io/apimachinery/pkg/util/sets" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) func TestManager(t *testing.T) { diff --git a/pkg/cache/queue/cluster_queue.go b/pkg/cache/queue/cluster_queue.go index 714b6783e05..48ce5e128a7 100644 --- a/pkg/cache/queue/cluster_queue.go +++ b/pkg/cache/queue/cluster_queue.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" afs "sigs.k8s.io/kueue/pkg/util/admissionfairsharing" "sigs.k8s.io/kueue/pkg/util/heap" diff --git a/pkg/cache/queue/cluster_queue_test.go b/pkg/cache/queue/cluster_queue_test.go index 115b0e1061e..1c55073024b 100644 --- a/pkg/cache/queue/cluster_queue_test.go +++ b/pkg/cache/queue/cluster_queue_test.go @@ -31,10 +31,10 @@ import ( "k8s.io/utils/ptr" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/cache/queue/cohort.go b/pkg/cache/queue/cohort.go index f9e2fa476a6..db04fe56900 100644 --- a/pkg/cache/queue/cohort.go +++ b/pkg/cache/queue/cohort.go @@ -17,7 +17,7 @@ limitations under the License. package queue import ( - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" ) diff --git a/pkg/cache/queue/dumper.go b/pkg/cache/queue/dumper.go index bda4bf4e6ff..84dc59bcfb9 100644 --- a/pkg/cache/queue/dumper.go +++ b/pkg/cache/queue/dumper.go @@ -20,7 +20,7 @@ import ( "github.com/go-logr/logr" "k8s.io/klog/v2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/cache/queue/local_queue.go b/pkg/cache/queue/local_queue.go index 5906d335332..9ce4841336a 100644 --- a/pkg/cache/queue/local_queue.go +++ b/pkg/cache/queue/local_queue.go @@ -17,7 +17,7 @@ limitations under the License. package queue import ( - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/queue" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/cache/queue/manager.go b/pkg/cache/queue/manager.go index 878c67a74e6..d853d7455a0 100644 --- a/pkg/cache/queue/manager.go +++ b/pkg/cache/queue/manager.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" utilindexer "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/features" @@ -199,7 +199,7 @@ func (m *Manager) AddClusterQueue(ctx context.Context, cq *kueue.ClusterQueue) e return err } m.hm.AddClusterQueue(cqImpl) - m.hm.UpdateClusterQueueEdge(kueue.ClusterQueueReference(cq.Name), cq.Spec.Cohort) + m.hm.UpdateClusterQueueEdge(kueue.ClusterQueueReference(cq.Name), cq.Spec.CohortName) // Iterate through existing queues, as queues corresponding to this cluster // queue might have been added earlier. @@ -251,7 +251,7 @@ func (m *Manager) UpdateClusterQueue(ctx context.Context, cq *kueue.ClusterQueue if err := cqImpl.Update(cq); err != nil { return err } - m.hm.UpdateClusterQueueEdge(cqName, cq.Spec.Cohort) + m.hm.UpdateClusterQueueEdge(cqName, cq.Spec.CohortName) // TODO(#8): Selectively move workloads based on the exact event. // If any workload becomes admissible or the queue becomes active. diff --git a/pkg/cache/queue/manager_test.go b/pkg/cache/queue/manager_test.go index 3cec975c922..7674d5d69f5 100644 --- a/pkg/cache/queue/manager_test.go +++ b/pkg/cache/queue/manager_test.go @@ -32,10 +32,10 @@ import ( testingclock "k8s.io/utils/clock/testing" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/queue" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) @@ -178,7 +178,7 @@ func TestUpdateClusterQueue(t *testing.T) { } // Put cq2 in the same cohort as cq1. - clusterQueues[1].Spec.Cohort = clusterQueues[0].Spec.Cohort + clusterQueues[1].Spec.CohortName = clusterQueues[0].Spec.CohortName if err := manager.UpdateClusterQueue(ctx, clusterQueues[1], true); err != nil { t.Fatalf("Failed to update ClusterQueue: %v", err) } diff --git a/pkg/cache/queue/second_pass_queue.go b/pkg/cache/queue/second_pass_queue.go index 9297531526f..d17839926f2 100644 --- a/pkg/cache/queue/second_pass_queue.go +++ b/pkg/cache/queue/second_pass_queue.go @@ -22,7 +22,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/wait" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/cache/queue/status_checker.go b/pkg/cache/queue/status_checker.go index a71b67cf996..4ba9e79840a 100644 --- a/pkg/cache/queue/status_checker.go +++ b/pkg/cache/queue/status_checker.go @@ -16,7 +16,7 @@ limitations under the License. package queue -import kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" +import kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" // StatusChecker checks status of clusterQueue. type StatusChecker interface { diff --git a/pkg/cache/scheduler/cache.go b/pkg/cache/scheduler/cache.go index 46d7a995bc6..ac9eabcd495 100644 --- a/pkg/cache/scheduler/cache.go +++ b/pkg/cache/scheduler/cache.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" utilindexer "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/features" @@ -147,7 +147,7 @@ func (c *Cache) newClusterQueue(log logr.Logger, cq *kueue.ClusterQueue) (*clust workloadsNotAccountedForTAS: sets.New[workload.Reference](), } c.hm.AddClusterQueue(cqImpl) - c.hm.UpdateClusterQueueEdge(kueue.ClusterQueueReference(cq.Name), cq.Spec.Cohort) + c.hm.UpdateClusterQueueEdge(kueue.ClusterQueueReference(cq.Name), cq.Spec.CohortName) if err := cqImpl.updateClusterQueue(log, cq, c.resourceFlavors, c.admissionChecks, nil); err != nil { return nil, err } @@ -432,7 +432,7 @@ func (c *Cache) UpdateClusterQueue(log logr.Logger, cq *kueue.ClusterQueue) erro return ErrCqNotFound } oldParent := cqImpl.Parent() - c.hm.UpdateClusterQueueEdge(kueue.ClusterQueueReference(cq.Name), cq.Spec.Cohort) + c.hm.UpdateClusterQueueEdge(kueue.ClusterQueueReference(cq.Name), cq.Spec.CohortName) if err := cqImpl.updateClusterQueue(log, cq, c.resourceFlavors, c.admissionChecks, oldParent); err != nil { return err } @@ -742,11 +742,11 @@ func (c *Cache) ClusterQueueAncestors(cqObj *kueue.ClusterQueue) ([]kueue.Cohort c.RLock() defer c.RUnlock() - if cqObj.Spec.Cohort == "" { + if cqObj.Spec.CohortName == "" { return nil, nil } - cohort := c.hm.Cohort(cqObj.Spec.Cohort) + cohort := c.hm.Cohort(cqObj.Spec.CohortName) if cohort == nil { return nil, nil } diff --git a/pkg/cache/scheduler/cache_test.go b/pkg/cache/scheduler/cache_test.go index 54e3802e1ef..776c7dba4ad 100644 --- a/pkg/cache/scheduler/cache_test.go +++ b/pkg/cache/scheduler/cache_test.go @@ -37,13 +37,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" "sigs.k8s.io/kueue/pkg/util/queue" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) @@ -1750,7 +1750,7 @@ func TestClusterQueueUsage(t *testing.T) { ). Cohort("one").Obj() cqWithOutCohort := cq.DeepCopy() - cqWithOutCohort.Spec.Cohort = "" + cqWithOutCohort.Spec.CohortName = "" workloads := []kueue.Workload{ *utiltestingapi.MakeWorkload("one", ""). Request(corev1.ResourceCPU, "8"). @@ -3656,7 +3656,7 @@ func TestCohortCycles(t *testing.T) { } // Successfully updated to cohort without cycle - cq.Spec.Cohort = "cohort" + cq.Spec.CohortName = "cohort" if err := cache.UpdateClusterQueue(log, cq); err != nil { t.Fatal("Expected success") } @@ -3713,7 +3713,7 @@ func TestCohortCycles(t *testing.T) { } // Updated to cycle - cq.Spec.Cohort = "cycle" + cq.Spec.CohortName = "cycle" if err := cache.UpdateClusterQueue(log, cq); err == nil { t.Fatal("Expected failure") } diff --git a/pkg/cache/scheduler/clusterqueue.go b/pkg/cache/scheduler/clusterqueue.go index 5add8c53a3b..53bb2ed0edf 100644 --- a/pkg/cache/scheduler/clusterqueue.go +++ b/pkg/cache/scheduler/clusterqueue.go @@ -31,7 +31,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" diff --git a/pkg/cache/scheduler/clusterqueue_snapshot.go b/pkg/cache/scheduler/clusterqueue_snapshot.go index 8c132c6bb2f..6ba1320980a 100644 --- a/pkg/cache/scheduler/clusterqueue_snapshot.go +++ b/pkg/cache/scheduler/clusterqueue_snapshot.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" diff --git a/pkg/cache/scheduler/clusterqueue_test.go b/pkg/cache/scheduler/clusterqueue_test.go index c32c8a96c19..31cdf2c89fa 100644 --- a/pkg/cache/scheduler/clusterqueue_test.go +++ b/pkg/cache/scheduler/clusterqueue_test.go @@ -23,12 +23,12 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestClusterQueueUpdateWithFlavors(t *testing.T) { diff --git a/pkg/cache/scheduler/cohort.go b/pkg/cache/scheduler/cohort.go index fc89fe5fdc5..5d22241bbd3 100644 --- a/pkg/cache/scheduler/cohort.go +++ b/pkg/cache/scheduler/cohort.go @@ -19,7 +19,7 @@ package scheduler import ( "iter" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" ) diff --git a/pkg/cache/scheduler/cohort_snapshot.go b/pkg/cache/scheduler/cohort_snapshot.go index 556ad5ff108..c355c916151 100644 --- a/pkg/cache/scheduler/cohort_snapshot.go +++ b/pkg/cache/scheduler/cohort_snapshot.go @@ -17,7 +17,7 @@ limitations under the License. package scheduler import ( - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" "sigs.k8s.io/kueue/pkg/resources" ) diff --git a/pkg/cache/scheduler/fair_sharing.go b/pkg/cache/scheduler/fair_sharing.go index ceeb9db67d2..144cde44db3 100644 --- a/pkg/cache/scheduler/fair_sharing.go +++ b/pkg/cache/scheduler/fair_sharing.go @@ -22,7 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/resources" ) diff --git a/pkg/cache/scheduler/fair_sharing_test.go b/pkg/cache/scheduler/fair_sharing_test.go index 772a58da18d..2f7b7e05391 100644 --- a/pkg/cache/scheduler/fair_sharing_test.go +++ b/pkg/cache/scheduler/fair_sharing_test.go @@ -26,10 +26,10 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/resources" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/cache/scheduler/resource.go b/pkg/cache/scheduler/resource.go index 208ff9438c2..f995a9bde63 100644 --- a/pkg/cache/scheduler/resource.go +++ b/pkg/cache/scheduler/resource.go @@ -21,7 +21,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" utilslices "sigs.k8s.io/kueue/pkg/util/slices" diff --git a/pkg/cache/scheduler/resource_node_test.go b/pkg/cache/scheduler/resource_node_test.go index 98f4612206f..e1f416ff1a6 100644 --- a/pkg/cache/scheduler/resource_node_test.go +++ b/pkg/cache/scheduler/resource_node_test.go @@ -23,7 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestCohortLendable(t *testing.T) { diff --git a/pkg/cache/scheduler/resource_test.go b/pkg/cache/scheduler/resource_test.go index 739eba38a33..ac331c81d4e 100644 --- a/pkg/cache/scheduler/resource_test.go +++ b/pkg/cache/scheduler/resource_test.go @@ -21,10 +21,10 @@ import ( "github.com/google/go-cmp/cmp" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/resources" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/cache/scheduler/snapshot.go b/pkg/cache/scheduler/snapshot.go index 8aa0af873c2..da410c69fa1 100644 --- a/pkg/cache/scheduler/snapshot.go +++ b/pkg/cache/scheduler/snapshot.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" "sigs.k8s.io/kueue/pkg/features" afs "sigs.k8s.io/kueue/pkg/util/admissionfairsharing" diff --git a/pkg/cache/scheduler/snapshot_test.go b/pkg/cache/scheduler/snapshot_test.go index b2174d3ebef..a7fabfd731a 100644 --- a/pkg/cache/scheduler/snapshot_test.go +++ b/pkg/cache/scheduler/snapshot_test.go @@ -28,12 +28,12 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/cache/scheduler/tas_cache.go b/pkg/cache/scheduler/tas_cache.go index 3573e22b532..db861a30bbf 100644 --- a/pkg/cache/scheduler/tas_cache.go +++ b/pkg/cache/scheduler/tas_cache.go @@ -23,7 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltas "sigs.k8s.io/kueue/pkg/util/tas" ) diff --git a/pkg/cache/scheduler/tas_cache_test.go b/pkg/cache/scheduler/tas_cache_test.go index 8f063afe84d..a4044c8d59d 100644 --- a/pkg/cache/scheduler/tas_cache_test.go +++ b/pkg/cache/scheduler/tas_cache_test.go @@ -26,7 +26,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" diff --git a/pkg/cache/scheduler/tas_flavor.go b/pkg/cache/scheduler/tas_flavor.go index 425800fa86f..492987cb64a 100644 --- a/pkg/cache/scheduler/tas_flavor.go +++ b/pkg/cache/scheduler/tas_flavor.go @@ -30,7 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/resources" utilpod "sigs.k8s.io/kueue/pkg/util/pod" diff --git a/pkg/cache/scheduler/tas_flavor_snapshot.go b/pkg/cache/scheduler/tas_flavor_snapshot.go index 347338f4b5a..c0e86ba3b3b 100644 --- a/pkg/cache/scheduler/tas_flavor_snapshot.go +++ b/pkg/cache/scheduler/tas_flavor_snapshot.go @@ -33,7 +33,7 @@ import ( corev1helpers "k8s.io/component-helpers/scheduling/corev1" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" "sigs.k8s.io/kueue/pkg/resources" diff --git a/pkg/cache/scheduler/tas_flavor_snapshot_test.go b/pkg/cache/scheduler/tas_flavor_snapshot_test.go index f5d6ba72a41..48f08817b2c 100644 --- a/pkg/cache/scheduler/tas_flavor_snapshot_test.go +++ b/pkg/cache/scheduler/tas_flavor_snapshot_test.go @@ -24,7 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/resources" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" "sigs.k8s.io/kueue/pkg/util/testingjobs/node" diff --git a/pkg/controller/admissionchecks/multikueue/admissioncheck.go b/pkg/controller/admissionchecks/multikueue/admissioncheck.go index 6438579676e..07f8225e148 100644 --- a/pkg/controller/admissionchecks/multikueue/admissioncheck.go +++ b/pkg/controller/admissionchecks/multikueue/admissioncheck.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/admissioncheck" ) diff --git a/pkg/controller/admissionchecks/multikueue/admissioncheck_test.go b/pkg/controller/admissionchecks/multikueue/admissioncheck_test.go index 1d4c9a7b9b2..cc634843d40 100644 --- a/pkg/controller/admissionchecks/multikueue/admissioncheck_test.go +++ b/pkg/controller/admissionchecks/multikueue/admissioncheck_test.go @@ -25,10 +25,10 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestReconcile(t *testing.T) { diff --git a/pkg/controller/admissionchecks/multikueue/externalframeworks/adapter.go b/pkg/controller/admissionchecks/multikueue/externalframeworks/adapter.go index dd27681bc05..871a63dd62a 100644 --- a/pkg/controller/admissionchecks/multikueue/externalframeworks/adapter.go +++ b/pkg/controller/admissionchecks/multikueue/externalframeworks/adapter.go @@ -31,7 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/controller/admissionchecks/multikueue/externalframeworks/adapter_test.go b/pkg/controller/admissionchecks/multikueue/externalframeworks/adapter_test.go index 3fca2844525..ee608d2a9b1 100644 --- a/pkg/controller/admissionchecks/multikueue/externalframeworks/adapter_test.go +++ b/pkg/controller/admissionchecks/multikueue/externalframeworks/adapter_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client/fake" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/features" ) diff --git a/pkg/controller/admissionchecks/multikueue/fswatch.go b/pkg/controller/admissionchecks/multikueue/fswatch.go index 1eadd2c794e..c2a7152076a 100644 --- a/pkg/controller/admissionchecks/multikueue/fswatch.go +++ b/pkg/controller/admissionchecks/multikueue/fswatch.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/manager" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) var ( diff --git a/pkg/controller/admissionchecks/multikueue/indexer.go b/pkg/controller/admissionchecks/multikueue/indexer.go index 7be067f7ed1..4a41079e7e8 100644 --- a/pkg/controller/admissionchecks/multikueue/indexer.go +++ b/pkg/controller/admissionchecks/multikueue/indexer.go @@ -23,7 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/admissioncheck" "sigs.k8s.io/kueue/pkg/util/slices" ) diff --git a/pkg/controller/admissionchecks/multikueue/indexer_test.go b/pkg/controller/admissionchecks/multikueue/indexer_test.go index 3c2f70a9c1f..3f746c59318 100644 --- a/pkg/controller/admissionchecks/multikueue/indexer_test.go +++ b/pkg/controller/admissionchecks/multikueue/indexer_test.go @@ -28,11 +28,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) const ( diff --git a/pkg/controller/admissionchecks/multikueue/multikueuecluster.go b/pkg/controller/admissionchecks/multikueue/multikueuecluster.go index 1c145b39504..278a585f0e7 100644 --- a/pkg/controller/admissionchecks/multikueue/multikueuecluster.go +++ b/pkg/controller/admissionchecks/multikueue/multikueuecluster.go @@ -50,7 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" ) diff --git a/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go b/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go index 2fd7884f6a5..594cdbb476e 100644 --- a/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go +++ b/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go @@ -34,11 +34,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" _ "sigs.k8s.io/kueue/pkg/controller/jobs" diff --git a/pkg/controller/admissionchecks/multikueue/workload.go b/pkg/controller/admissionchecks/multikueue/workload.go index 4f87ab85f87..7c3f692d96d 100644 --- a/pkg/controller/admissionchecks/multikueue/workload.go +++ b/pkg/controller/admissionchecks/multikueue/workload.go @@ -43,7 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" diff --git a/pkg/controller/admissionchecks/multikueue/workload_test.go b/pkg/controller/admissionchecks/multikueue/workload_test.go index 6064f184644..e389aaaf1ae 100644 --- a/pkg/controller/admissionchecks/multikueue/workload_test.go +++ b/pkg/controller/admissionchecks/multikueue/workload_test.go @@ -41,14 +41,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workloadslicing" diff --git a/pkg/controller/admissionchecks/provisioning/admissioncheck_reconciler.go b/pkg/controller/admissionchecks/provisioning/admissioncheck_reconciler.go index dd792e2f658..f9aa7f3b5cb 100644 --- a/pkg/controller/admissionchecks/provisioning/admissioncheck_reconciler.go +++ b/pkg/controller/admissionchecks/provisioning/admissioncheck_reconciler.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type acReconciler struct { diff --git a/pkg/controller/admissionchecks/provisioning/admissioncheck_reconciler_test.go b/pkg/controller/admissionchecks/provisioning/admissioncheck_reconciler_test.go index 9d316b1aaf1..7929c418274 100644 --- a/pkg/controller/admissionchecks/provisioning/admissioncheck_reconciler_test.go +++ b/pkg/controller/admissionchecks/provisioning/admissioncheck_reconciler_test.go @@ -25,9 +25,9 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestReconcileAdmissionCheck(t *testing.T) { diff --git a/pkg/controller/admissionchecks/provisioning/controller.go b/pkg/controller/admissionchecks/provisioning/controller.go index 8d4ca818dc4..06064c4ed65 100644 --- a/pkg/controller/admissionchecks/provisioning/controller.go +++ b/pkg/controller/admissionchecks/provisioning/controller.go @@ -44,7 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/podset" "sigs.k8s.io/kueue/pkg/util/admissioncheck" diff --git a/pkg/controller/admissionchecks/provisioning/controller_test.go b/pkg/controller/admissionchecks/provisioning/controller_test.go index c97c3d7b291..0ada52b1d74 100644 --- a/pkg/controller/admissionchecks/provisioning/controller_test.go +++ b/pkg/controller/admissionchecks/provisioning/controller_test.go @@ -39,11 +39,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) @@ -457,7 +457,7 @@ func TestReconcile(t *testing.T) { *baseTemplate1.Clone(). ControllerReference(schema.GroupVersionKind{ Group: "kueue.x-k8s.io", - Version: "v1beta1", + Version: "v1beta2", Kind: "Workload", }, "wl", ""). Obj(), @@ -1133,14 +1133,14 @@ func TestReconcile(t *testing.T) { baseTemplate1.Name: baseTemplate1.Clone(). ControllerReference(schema.GroupVersionKind{ Group: "kueue.x-k8s.io", - Version: "v1beta1", + Version: "v1beta2", Kind: "Workload", }, "wl", ""). Obj(), baseTemplate2.Name: baseTemplate2.Clone(). ControllerReference(schema.GroupVersionKind{ Group: "kueue.x-k8s.io", - Version: "v1beta1", + Version: "v1beta2", Kind: "Workload", }, "wl", ""). Obj(), diff --git a/pkg/controller/admissionchecks/provisioning/indexer.go b/pkg/controller/admissionchecks/provisioning/indexer.go index 58b7307b2d5..060b7fe787e 100644 --- a/pkg/controller/admissionchecks/provisioning/indexer.go +++ b/pkg/controller/admissionchecks/provisioning/indexer.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/manager" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/admissioncheck" "sigs.k8s.io/kueue/pkg/util/slices" ) diff --git a/pkg/controller/admissionchecks/provisioning/indexer_test.go b/pkg/controller/admissionchecks/provisioning/indexer_test.go index ed5ccd59c52..02107bf3d2d 100644 --- a/pkg/controller/admissionchecks/provisioning/indexer_test.go +++ b/pkg/controller/admissionchecks/provisioning/indexer_test.go @@ -30,10 +30,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) const ( diff --git a/pkg/controller/admissionchecks/provisioning/provisioning.go b/pkg/controller/admissionchecks/provisioning/provisioning.go index 5426ceb8577..edd8b3c92f8 100644 --- a/pkg/controller/admissionchecks/provisioning/provisioning.go +++ b/pkg/controller/admissionchecks/provisioning/provisioning.go @@ -27,7 +27,7 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" autoscaling "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/util/admissioncheck" ) diff --git a/pkg/controller/constants/constants.go b/pkg/controller/constants/constants.go index c1b258e7b3d..af13467b11d 100644 --- a/pkg/controller/constants/constants.go +++ b/pkg/controller/constants/constants.go @@ -16,7 +16,7 @@ limitations under the License. package constants -import kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" +import kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" const ( // QueueLabel is the label key in the workload that holds the queue name. diff --git a/pkg/controller/core/admissioncheck_controller.go b/pkg/controller/core/admissioncheck_controller.go index 326bc6b1da3..2f6ee429772 100644 --- a/pkg/controller/core/admissioncheck_controller.go +++ b/pkg/controller/core/admissioncheck_controller.go @@ -37,7 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/util/slices" diff --git a/pkg/controller/core/clusterqueue_controller.go b/pkg/controller/core/clusterqueue_controller.go index 7ff8acf76da..8774c220733 100644 --- a/pkg/controller/core/clusterqueue_controller.go +++ b/pkg/controller/core/clusterqueue_controller.go @@ -45,7 +45,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" @@ -358,7 +358,7 @@ func recordResourceMetrics(cq *kueue.ClusterQueue) { nominal := resource.QuantityToFloat(&r.NominalQuota) borrow := resource.QuantityToFloat(r.BorrowingLimit) lend := resource.QuantityToFloat(r.LendingLimit) - metrics.ReportClusterQueueQuotas(cq.Spec.Cohort, cq.Name, string(fq.Name), string(r.Name), nominal, borrow, lend) + metrics.ReportClusterQueueQuotas(cq.Spec.CohortName, cq.Name, string(fq.Name), string(r.Name), nominal, borrow, lend) } } } @@ -367,7 +367,7 @@ func recordResourceMetrics(cq *kueue.ClusterQueue) { fr := &cq.Status.FlavorsReservation[fri] for ri := range fr.Resources { r := &fr.Resources[ri] - metrics.ReportClusterQueueResourceReservations(cq.Spec.Cohort, cq.Name, string(fr.Name), string(r.Name), resource.QuantityToFloat(&r.Total)) + metrics.ReportClusterQueueResourceReservations(cq.Spec.CohortName, cq.Name, string(fr.Name), string(r.Name), resource.QuantityToFloat(&r.Total)) } } @@ -375,14 +375,14 @@ func recordResourceMetrics(cq *kueue.ClusterQueue) { fu := &cq.Status.FlavorsUsage[fui] for ri := range fu.Resources { r := &fu.Resources[ri] - metrics.ReportClusterQueueResourceUsage(cq.Spec.Cohort, cq.Name, string(fu.Name), string(r.Name), resource.QuantityToFloat(&r.Total)) + metrics.ReportClusterQueueResourceUsage(cq.Spec.CohortName, cq.Name, string(fu.Name), string(r.Name), resource.QuantityToFloat(&r.Total)) } } } func updateResourceMetrics(oldCq, newCq *kueue.ClusterQueue) { // if the cohort changed, drop all the old metrics - if oldCq.Spec.Cohort != newCq.Spec.Cohort { + if oldCq.Spec.CohortName != newCq.Spec.CohortName { metrics.ClearClusterQueueResourceMetrics(oldCq.Name) } else { // selective remove @@ -568,7 +568,7 @@ func (r *ClusterQueueReconciler) updateCqStatusIfChanged( if weightedShare == math.Inf(1) { weightedShare = math.NaN() } - metrics.ReportClusterQueueWeightedShare(cq.Name, string(cq.Spec.Cohort), weightedShare) + metrics.ReportClusterQueueWeightedShare(cq.Name, string(cq.Spec.CohortName), weightedShare) } if cq.Status.FairSharing == nil { cq.Status.FairSharing = &kueue.FairSharingStatus{} diff --git a/pkg/controller/core/clusterqueue_controller_test.go b/pkg/controller/core/clusterqueue_controller_test.go index 7dbb35aa29b..4ceb06213d6 100644 --- a/pkg/controller/core/clusterqueue_controller_test.go +++ b/pkg/controller/core/clusterqueue_controller_test.go @@ -26,13 +26,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/metrics" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingmetrics "sigs.k8s.io/kueue/pkg/util/testing/metrics" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestUpdateCqStatusIfChanged(t *testing.T) { @@ -278,7 +278,7 @@ func TestRecordResourceMetrics(t *testing.T) { Name: "name", }, Spec: kueue.ClusterQueueSpec{ - Cohort: "cohort", + CohortName: "cohort", ResourceGroups: []kueue.ResourceGroup{ { CoveredResources: []corev1.ResourceName{corev1.ResourceCPU}, @@ -380,7 +380,7 @@ func TestRecordResourceMetrics(t *testing.T) { }, updatedQueue: func() *kueue.ClusterQueue { ret := baseQueue.DeepCopy() - ret.Spec.Cohort = "cohort2" + ret.Spec.CohortName = "cohort2" return ret }(), wantUpdatedMetrics: cqMetrics{ diff --git a/pkg/controller/core/cohort_controller.go b/pkg/controller/core/cohort_controller.go index 77f03a9e91d..0d3587ebec7 100644 --- a/pkg/controller/core/cohort_controller.go +++ b/pkg/controller/core/cohort_controller.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/metrics" @@ -215,7 +215,7 @@ func (h *cohortCqHandler) Generic(ctx context.Context, e event.GenericEvent, q w ancestors, err := h.cache.ClusterQueueAncestors(cq) if err != nil { log := ctrl.LoggerFrom(ctx) - log.Error(err, "Failed getting ancestors for cohort", "cohort", cq.Spec.Cohort) + log.Error(err, "Failed getting ancestors for cohort", "cohort", cq.Spec.CohortName) } for _, ancestor := range ancestors { q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Name: string(ancestor)}}) diff --git a/pkg/controller/core/cohort_controller_test.go b/pkg/controller/core/cohort_controller_test.go index c400ca0ebdb..a071af0aaf3 100644 --- a/pkg/controller/core/cohort_controller_test.go +++ b/pkg/controller/core/cohort_controller_test.go @@ -28,12 +28,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/resources" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestCohortReconcileCohortNotFoundDelete(t *testing.T) { diff --git a/pkg/controller/core/indexer/indexer.go b/pkg/controller/core/indexer/indexer.go index 2c15b57d37a..8c5aeff1cea 100644 --- a/pkg/controller/core/indexer/indexer.go +++ b/pkg/controller/core/indexer/indexer.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/slices" ) diff --git a/pkg/controller/core/localqueue_controller.go b/pkg/controller/core/localqueue_controller.go index 9c0f886360e..0af72d5a774 100644 --- a/pkg/controller/core/localqueue_controller.go +++ b/pkg/controller/core/localqueue_controller.go @@ -41,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" @@ -319,7 +319,7 @@ func localQueueReferenceFromLocalQueue(lq *kueue.LocalQueue) metrics.LocalQueueR } func recordLocalQueueUsageMetrics(queue *kueue.LocalQueue) { - for _, flavor := range queue.Status.FlavorUsage { + for _, flavor := range queue.Status.FlavorsUsage { for _, r := range flavor.Resources { metrics.ReportLocalQueueResourceUsage(localQueueReferenceFromLocalQueue(queue), string(flavor.Name), string(r.Name), resource.QuantityToFloat(&r.Total)) } @@ -476,7 +476,7 @@ func (r *LocalQueueReconciler) UpdateStatusIfChanged( queue.Status.ReservingWorkloads = int32(stats.ReservingWorkloads) queue.Status.AdmittedWorkloads = int32(stats.AdmittedWorkloads) queue.Status.FlavorsReservation = stats.ReservedResources - queue.Status.FlavorUsage = stats.AdmittedResources + queue.Status.FlavorsUsage = stats.AdmittedResources queue.Status.Flavors = stats.Flavors if len(conditionStatus) != 0 && len(reason) != 0 && len(msg) != 0 { meta.SetStatusCondition(&queue.Status.Conditions, metav1.Condition{ diff --git a/pkg/controller/core/localqueue_controller_test.go b/pkg/controller/core/localqueue_controller_test.go index ecf9f2230a5..88bbe67752e 100644 --- a/pkg/controller/core/localqueue_controller_test.go +++ b/pkg/controller/core/localqueue_controller_test.go @@ -34,12 +34,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/pkg/controller/core/resourceflavor_controller.go b/pkg/controller/core/resourceflavor_controller.go index 9c14cc51e22..2bf1e45cf0e 100644 --- a/pkg/controller/core/resourceflavor_controller.go +++ b/pkg/controller/core/resourceflavor_controller.go @@ -37,7 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" ) diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index 0f1c3e1cf52..874744f116b 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -51,7 +51,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/core/indexer" @@ -562,7 +562,7 @@ func (r *WorkloadReconciler) reconcileMaxExecutionTime(ctx context.Context, wl * return 0, nil } - remainingTime := time.Duration(*wl.Spec.MaximumExecutionTimeSeconds-ptr.Deref(wl.Status.AccumulatedPastExexcutionTimeSeconds, 0))*time.Second - r.clock.Since(admittedCondition.LastTransitionTime.Time) + remainingTime := time.Duration(*wl.Spec.MaximumExecutionTimeSeconds-ptr.Deref(wl.Status.AccumulatedPastExecutionTimeSeconds, 0))*time.Second - r.clock.Since(admittedCondition.LastTransitionTime.Time) if remainingTime > 0 { return remainingTime, nil } @@ -570,8 +570,8 @@ func (r *WorkloadReconciler) reconcileMaxExecutionTime(ctx context.Context, wl * if !apimeta.IsStatusConditionTrue(wl.Status.Conditions, kueue.WorkloadDeactivationTarget) { err := workload.PatchAdmissionStatus(ctx, r.client, wl, r.clock, func() (*kueue.Workload, bool, error) { updated := workload.SetDeactivationTarget(wl, kueue.WorkloadMaximumExecutionTimeExceeded, "exceeding the maximum execution time") - if wl.Status.AccumulatedPastExexcutionTimeSeconds != nil { - wl.Status.AccumulatedPastExexcutionTimeSeconds = nil + if wl.Status.AccumulatedPastExecutionTimeSeconds != nil { + wl.Status.AccumulatedPastExecutionTimeSeconds = nil updated = true } return wl, updated, nil diff --git a/pkg/controller/core/workload_controller_test.go b/pkg/controller/core/workload_controller_test.go index 3e8c79fd9f6..f7c6c3be9ec 100644 --- a/pkg/controller/core/workload_controller_test.go +++ b/pkg/controller/core/workload_controller_test.go @@ -40,14 +40,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/dra" "sigs.k8s.io/kueue/pkg/features" utilqueue "sigs.k8s.io/kueue/pkg/util/queue" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/pkg/controller/jobframework/base_webhook_test.go b/pkg/controller/jobframework/base_webhook_test.go index f617b43e4df..ba9b13a47db 100644 --- a/pkg/controller/jobframework/base_webhook_test.go +++ b/pkg/controller/jobframework/base_webhook_test.go @@ -29,7 +29,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" mocks "sigs.k8s.io/kueue/internal/mocks/controller/jobframework" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" @@ -37,7 +37,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" utiljob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" ) diff --git a/pkg/controller/jobframework/defaults.go b/pkg/controller/jobframework/defaults.go index 65bd7198e91..73215ecb11f 100644 --- a/pkg/controller/jobframework/defaults.go +++ b/pkg/controller/jobframework/defaults.go @@ -26,7 +26,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/pkg/controller/jobframework/interface.go b/pkg/controller/jobframework/interface.go index aa91d3d8df2..6065edb716a 100644 --- a/pkg/controller/jobframework/interface.go +++ b/pkg/controller/jobframework/interface.go @@ -29,7 +29,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/podset" "sigs.k8s.io/kueue/pkg/util/admissioncheck" diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index 66d6f7ff5d2..19b9935a983 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -44,7 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/pkg/controller/jobframework/reconciler_test.go b/pkg/controller/jobframework/reconciler_test.go index 50e31eeef4e..a7aef3da6cb 100644 --- a/pkg/controller/jobframework/reconciler_test.go +++ b/pkg/controller/jobframework/reconciler_test.go @@ -38,14 +38,14 @@ import ( "sigs.k8s.io/jobset/api/jobset/v1alpha2" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" mocks "sigs.k8s.io/kueue/internal/mocks/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/kubeversion" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingaw "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" testingdeployment "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" diff --git a/pkg/controller/jobframework/setup_test.go b/pkg/controller/jobframework/setup_test.go index 8676ffd137f..836a22a665c 100644 --- a/pkg/controller/jobframework/setup_test.go +++ b/pkg/controller/jobframework/setup_test.go @@ -39,10 +39,10 @@ import ( ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestSetupControllers(t *testing.T) { diff --git a/pkg/controller/jobframework/tas.go b/pkg/controller/jobframework/tas.go index b024454ebf0..bbe886c3630 100644 --- a/pkg/controller/jobframework/tas.go +++ b/pkg/controller/jobframework/tas.go @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type podSetTopologyRequestBuilder struct { diff --git a/pkg/controller/jobframework/tas_test.go b/pkg/controller/jobframework/tas_test.go index 13dd0bea9c4..0646ce2313d 100644 --- a/pkg/controller/jobframework/tas_test.go +++ b/pkg/controller/jobframework/tas_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/utils/ptr" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) func TestPodSetTopologyRequestBuilder(t *testing.T) { diff --git a/pkg/controller/jobframework/tas_validation.go b/pkg/controller/jobframework/tas_validation.go index 0a1e41b2310..649ac9abdce 100644 --- a/pkg/controller/jobframework/tas_validation.go +++ b/pkg/controller/jobframework/tas_validation.go @@ -25,7 +25,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/orderedgroups" ) diff --git a/pkg/controller/jobs/appwrapper/appwrapper_controller.go b/pkg/controller/jobs/appwrapper/appwrapper_controller.go index cc3bac8cb38..8628ed87893 100644 --- a/pkg/controller/jobs/appwrapper/appwrapper_controller.go +++ b/pkg/controller/jobs/appwrapper/appwrapper_controller.go @@ -32,7 +32,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" @@ -75,8 +75,8 @@ func init() { // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=resourceflavors,verbs=get;list;watch // +kubebuilder:rbac:groups=kueue.x-k8s.io,resources=workloadpriorityclasses,verbs=get;list;watch // +kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers/finalizers,verbs=get;update -//+kubebuilder:webhook:path=/mutate-workload-codeflare-dev-v1beta2-appwrapper,mutating=true,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create,versions=v1beta2,name=mappwrapper.kb.io,admissionReviewVersions=v1 -//+kubebuilder:webhook:path=/validate-workload-codeflare-dev-v1beta2-appwrapper,mutating=false,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create;update,versions=v1beta2,name=vappwrapper.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-workload-codeflare-dev-v1beta2-appwrapper,mutating=true,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create,versions=v1beta2,name=mappwrapper.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-workload-codeflare-dev-v1beta2-appwrapper,mutating=false,failurePolicy=fail,sideEffects=None,groups=workload.codeflare.dev,resources=appwrappers,verbs=create;update,versions=v1beta2,name=vappwrapper.kb.io,admissionReviewVersions=v1 func NewJob() jobframework.GenericJob { return &AppWrapper{} diff --git a/pkg/controller/jobs/appwrapper/appwrapper_controller_test.go b/pkg/controller/jobs/appwrapper/appwrapper_controller_test.go index 16a19043fb2..4ab45291d9b 100644 --- a/pkg/controller/jobs/appwrapper/appwrapper_controller_test.go +++ b/pkg/controller/jobs/appwrapper/appwrapper_controller_test.go @@ -32,12 +32,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingappwrapper "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" utiltestingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingpytorchjob "sigs.k8s.io/kueue/pkg/util/testingjobs/pytorchjob" diff --git a/pkg/controller/jobs/appwrapper/appwrapper_multikueue_adapter.go b/pkg/controller/jobs/appwrapper/appwrapper_multikueue_adapter.go index a545ebd152e..e71097e1827 100644 --- a/pkg/controller/jobs/appwrapper/appwrapper_multikueue_adapter.go +++ b/pkg/controller/jobs/appwrapper/appwrapper_multikueue_adapter.go @@ -30,7 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/api" diff --git a/pkg/controller/jobs/appwrapper/appwrapper_multikueue_adapter_test.go b/pkg/controller/jobs/appwrapper/appwrapper_multikueue_adapter_test.go index c2d9718e41a..9b32922ca1e 100644 --- a/pkg/controller/jobs/appwrapper/appwrapper_multikueue_adapter_test.go +++ b/pkg/controller/jobs/appwrapper/appwrapper_multikueue_adapter_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/controller/jobs/deployment/deployment_webhook_test.go b/pkg/controller/jobs/deployment/deployment_webhook_test.go index 2896c576440..75cc8dc330d 100644 --- a/pkg/controller/jobs/deployment/deployment_webhook_test.go +++ b/pkg/controller/jobs/deployment/deployment_webhook_test.go @@ -32,7 +32,7 @@ import ( podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingdeployment "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" ) diff --git a/pkg/controller/jobs/job/job_controller.go b/pkg/controller/jobs/job/job_controller.go index 3f971305448..f94bd148884 100644 --- a/pkg/controller/jobs/job/job_controller.go +++ b/pkg/controller/jobs/job/job_controller.go @@ -37,7 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/controller/jobs/job/job_controller_test.go b/pkg/controller/jobs/job/job_controller_test.go index 76546cd6cb2..b5879f272ad 100644 --- a/pkg/controller/jobs/job/job_controller_test.go +++ b/pkg/controller/jobs/job/job_controller_test.go @@ -37,14 +37,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" utiltestingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/controller/jobs/job/job_multikueue_adapter.go b/pkg/controller/jobs/job/job_multikueue_adapter.go index acfe9bc85be..378709ab22c 100644 --- a/pkg/controller/jobs/job/job_multikueue_adapter.go +++ b/pkg/controller/jobs/job/job_multikueue_adapter.go @@ -32,7 +32,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/controller/jobs/job/job_multikueue_adapter_test.go b/pkg/controller/jobs/job/job_multikueue_adapter_test.go index 7db07fde01f..01e44675813 100644 --- a/pkg/controller/jobs/job/job_multikueue_adapter_test.go +++ b/pkg/controller/jobs/job/job_multikueue_adapter_test.go @@ -34,7 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/controller/jobs/job/job_webhook.go b/pkg/controller/jobs/job/job_webhook.go index cdb57bd67ca..b437ca3c107 100644 --- a/pkg/controller/jobs/job/job_webhook.go +++ b/pkg/controller/jobs/job/job_webhook.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/controller/jobs/job/job_webhook_test.go b/pkg/controller/jobs/job/job_webhook_test.go index c47a780d166..5c4e7c32cf7 100644 --- a/pkg/controller/jobs/job/job_webhook_test.go +++ b/pkg/controller/jobs/job/job_webhook_test.go @@ -33,14 +33,14 @@ import ( "k8s.io/utils/ptr" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingutil "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingmpijob "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" "sigs.k8s.io/kueue/pkg/workloadslicing" diff --git a/pkg/controller/jobs/jobset/jobset_controller.go b/pkg/controller/jobs/jobset/jobset_controller.go index 6745d7bbd32..c98cb00cfa4 100644 --- a/pkg/controller/jobs/jobset/jobset_controller.go +++ b/pkg/controller/jobs/jobset/jobset_controller.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" diff --git a/pkg/controller/jobs/jobset/jobset_controller_test.go b/pkg/controller/jobs/jobset/jobset_controller_test.go index 25bc02140ef..b033317de8e 100644 --- a/pkg/controller/jobs/jobset/jobset_controller_test.go +++ b/pkg/controller/jobs/jobset/jobset_controller_test.go @@ -31,13 +31,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" ) diff --git a/pkg/controller/jobs/jobset/jobset_multikueue_adapter.go b/pkg/controller/jobs/jobset/jobset_multikueue_adapter.go index 19c33ae3368..9f987b75702 100644 --- a/pkg/controller/jobs/jobset/jobset_multikueue_adapter.go +++ b/pkg/controller/jobs/jobset/jobset_multikueue_adapter.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/api" diff --git a/pkg/controller/jobs/jobset/jobset_multikueue_adapter_test.go b/pkg/controller/jobs/jobset/jobset_multikueue_adapter_test.go index 698300d945c..ebcb0016a6c 100644 --- a/pkg/controller/jobs/jobset/jobset_multikueue_adapter_test.go +++ b/pkg/controller/jobs/jobset/jobset_multikueue_adapter_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/controller/jobs/jobset/jobset_webhook.go b/pkg/controller/jobs/jobset/jobset_webhook.go index 5db6e2459de..0c20b90e959 100644 --- a/pkg/controller/jobs/jobset/jobset_webhook.go +++ b/pkg/controller/jobs/jobset/jobset_webhook.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/controller/jobs/jobset/jobset_webhook_test.go b/pkg/controller/jobs/jobset/jobset_webhook_test.go index 30c4821100c..b24ab3c4e13 100644 --- a/pkg/controller/jobs/jobset/jobset_webhook_test.go +++ b/pkg/controller/jobs/jobset/jobset_webhook_test.go @@ -27,13 +27,13 @@ import ( ctrl "sigs.k8s.io/controller-runtime" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingutil "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" ) diff --git a/pkg/controller/jobs/kubeflow/jobs/jaxjob/jaxjob_controller_test.go b/pkg/controller/jobs/kubeflow/jobs/jaxjob/jaxjob_controller_test.go index f8c684a0e26..b8bef95eef2 100644 --- a/pkg/controller/jobs/kubeflow/jobs/jaxjob/jaxjob_controller_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/jaxjob/jaxjob_controller_test.go @@ -26,10 +26,10 @@ import ( "k8s.io/component-base/featuregate" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingJAXjob "sigs.k8s.io/kueue/pkg/util/testingjobs/jaxjob" ) diff --git a/pkg/controller/jobs/kubeflow/jobs/jaxjob/jaxjob_multikueue_adapter_test.go b/pkg/controller/jobs/kubeflow/jobs/jaxjob/jaxjob_multikueue_adapter_test.go index 10d9e810014..37ed63c8d52 100644 --- a/pkg/controller/jobs/kubeflow/jobs/jaxjob/jaxjob_multikueue_adapter_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/jaxjob/jaxjob_multikueue_adapter_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" diff --git a/pkg/controller/jobs/kubeflow/jobs/paddlejob/paddlejob_controller_test.go b/pkg/controller/jobs/kubeflow/jobs/paddlejob/paddlejob_controller_test.go index 4febc624247..41c511b1903 100644 --- a/pkg/controller/jobs/kubeflow/jobs/paddlejob/paddlejob_controller_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/paddlejob/paddlejob_controller_test.go @@ -33,11 +33,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpaddlejob "sigs.k8s.io/kueue/pkg/util/testingjobs/paddlejob" ) diff --git a/pkg/controller/jobs/kubeflow/jobs/paddlejob/paddlejob_multikueue_adapter_test.go b/pkg/controller/jobs/kubeflow/jobs/paddlejob/paddlejob_multikueue_adapter_test.go index 0fc53118395..2acfe1500da 100644 --- a/pkg/controller/jobs/kubeflow/jobs/paddlejob/paddlejob_multikueue_adapter_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/paddlejob/paddlejob_multikueue_adapter_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" diff --git a/pkg/controller/jobs/kubeflow/jobs/pytorchjob/pytorch_multikueue_adapter_test.go b/pkg/controller/jobs/kubeflow/jobs/pytorchjob/pytorch_multikueue_adapter_test.go index 551f1e689d4..f421d46ea91 100644 --- a/pkg/controller/jobs/kubeflow/jobs/pytorchjob/pytorch_multikueue_adapter_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/pytorchjob/pytorch_multikueue_adapter_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" diff --git a/pkg/controller/jobs/kubeflow/jobs/pytorchjob/pytorchjob_controller_test.go b/pkg/controller/jobs/kubeflow/jobs/pytorchjob/pytorchjob_controller_test.go index 55949081216..14272c3443f 100644 --- a/pkg/controller/jobs/kubeflow/jobs/pytorchjob/pytorchjob_controller_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/pytorchjob/pytorchjob_controller_test.go @@ -25,10 +25,10 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpytorchjob "sigs.k8s.io/kueue/pkg/util/testingjobs/pytorchjob" ) diff --git a/pkg/controller/jobs/kubeflow/jobs/tfjob/tfjob_controller_test.go b/pkg/controller/jobs/kubeflow/jobs/tfjob/tfjob_controller_test.go index 1129a4d7d02..e3e28c07f2e 100644 --- a/pkg/controller/jobs/kubeflow/jobs/tfjob/tfjob_controller_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/tfjob/tfjob_controller_test.go @@ -25,10 +25,10 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingtfjob "sigs.k8s.io/kueue/pkg/util/testingjobs/tfjob" ) diff --git a/pkg/controller/jobs/kubeflow/jobs/tfjob/tfjob_multikueue_adapter_test.go b/pkg/controller/jobs/kubeflow/jobs/tfjob/tfjob_multikueue_adapter_test.go index 664e42691c0..548e431a161 100644 --- a/pkg/controller/jobs/kubeflow/jobs/tfjob/tfjob_multikueue_adapter_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/tfjob/tfjob_multikueue_adapter_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" diff --git a/pkg/controller/jobs/kubeflow/jobs/xgboostjob/xgboostjob_controller_test.go b/pkg/controller/jobs/kubeflow/jobs/xgboostjob/xgboostjob_controller_test.go index cec60528dd6..fa3c5b86d62 100644 --- a/pkg/controller/jobs/kubeflow/jobs/xgboostjob/xgboostjob_controller_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/xgboostjob/xgboostjob_controller_test.go @@ -33,11 +33,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingxgboostjob "sigs.k8s.io/kueue/pkg/util/testingjobs/xgboostjob" ) diff --git a/pkg/controller/jobs/kubeflow/jobs/xgboostjob/xgboostjob_multikueue_adapter_test.go b/pkg/controller/jobs/kubeflow/jobs/xgboostjob/xgboostjob_multikueue_adapter_test.go index fb3e8e41a17..f9c67390a5c 100644 --- a/pkg/controller/jobs/kubeflow/jobs/xgboostjob/xgboostjob_multikueue_adapter_test.go +++ b/pkg/controller/jobs/kubeflow/jobs/xgboostjob/xgboostjob_multikueue_adapter_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" diff --git a/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_controller.go b/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_controller.go index 712919dd604..bb7e3795dd8 100644 --- a/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_controller.go +++ b/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_controller.go @@ -26,7 +26,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" diff --git a/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_multikueue_adapter.go b/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_multikueue_adapter.go index cd564b88e3c..a1f52a65baf 100644 --- a/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_multikueue_adapter.go +++ b/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_multikueue_adapter.go @@ -29,7 +29,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" clientutil "sigs.k8s.io/kueue/pkg/util/client" diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_pod_reconciler.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_pod_reconciler.go index a8745faa1a1..34d7e32f5a0 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_pod_reconciler.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_pod_reconciler.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_pod_reconciler_test.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_pod_reconciler_test.go index 2f9b32e891d..47832a858dc 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_pod_reconciler_test.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_pod_reconciler_test.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" "sigs.k8s.io/kueue/pkg/util/testingjobs/leaderworkerset" diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go index ed080eb7799..70ad6b64a99 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go @@ -37,7 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/controller/jobframework" podcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/pod" diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler_test.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler_test.go index 32ab1772ac6..2fbe671ebe5 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler_test.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler_test.go @@ -31,13 +31,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/util/testingjobs/leaderworkerset" ) diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go index edbe8686c5b..4a2a51c7781 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook_test.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook_test.go index 65bc0c76f05..ea4e28c8f1e 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook_test.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" @@ -36,7 +36,7 @@ import ( podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingleaderworkerset "sigs.k8s.io/kueue/pkg/util/testingjobs/leaderworkerset" ) diff --git a/pkg/controller/jobs/mpijob/mpijob_controller.go b/pkg/controller/jobs/mpijob/mpijob_controller.go index bfbffd88209..63e8cb417bb 100644 --- a/pkg/controller/jobs/mpijob/mpijob_controller.go +++ b/pkg/controller/jobs/mpijob/mpijob_controller.go @@ -29,7 +29,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" diff --git a/pkg/controller/jobs/mpijob/mpijob_controller_test.go b/pkg/controller/jobs/mpijob/mpijob_controller_test.go index 77462b8914e..2fb69af0579 100644 --- a/pkg/controller/jobs/mpijob/mpijob_controller_test.go +++ b/pkg/controller/jobs/mpijob/mpijob_controller_test.go @@ -30,12 +30,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingmpijob "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" ) diff --git a/pkg/controller/jobs/mpijob/mpijob_multikueue_adapter.go b/pkg/controller/jobs/mpijob/mpijob_multikueue_adapter.go index 2f48266e131..dddf11029c0 100644 --- a/pkg/controller/jobs/mpijob/mpijob_multikueue_adapter.go +++ b/pkg/controller/jobs/mpijob/mpijob_multikueue_adapter.go @@ -30,7 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/api" diff --git a/pkg/controller/jobs/mpijob/mpijob_multikueue_adapter_test.go b/pkg/controller/jobs/mpijob/mpijob_multikueue_adapter_test.go index fb9dace46f6..fd01cd2e153 100644 --- a/pkg/controller/jobs/mpijob/mpijob_multikueue_adapter_test.go +++ b/pkg/controller/jobs/mpijob/mpijob_multikueue_adapter_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/controller/jobs/mpijob/mpijob_webhook.go b/pkg/controller/jobs/mpijob/mpijob_webhook.go index 346f05f6963..bfac8731e47 100644 --- a/pkg/controller/jobs/mpijob/mpijob_webhook.go +++ b/pkg/controller/jobs/mpijob/mpijob_webhook.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/controller/jobs/mpijob/mpijob_webhook_test.go b/pkg/controller/jobs/mpijob/mpijob_webhook_test.go index 6bfa721a5d9..f839a99e224 100644 --- a/pkg/controller/jobs/mpijob/mpijob_webhook_test.go +++ b/pkg/controller/jobs/mpijob/mpijob_webhook_test.go @@ -27,13 +27,13 @@ import ( "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingutil "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" ) diff --git a/pkg/controller/jobs/pod/event_handlers.go b/pkg/controller/jobs/pod/event_handlers.go index 2af453dde21..5a58bfbdf14 100644 --- a/pkg/controller/jobs/pod/event_handlers.go +++ b/pkg/controller/jobs/pod/event_handlers.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/util/expectations" ) diff --git a/pkg/controller/jobs/pod/pod_controller.go b/pkg/controller/jobs/pod/pod_controller.go index 6291b58fe99..693d29610e0 100644 --- a/pkg/controller/jobs/pod/pod_controller.go +++ b/pkg/controller/jobs/pod/pod_controller.go @@ -43,7 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" diff --git a/pkg/controller/jobs/pod/pod_controller_test.go b/pkg/controller/jobs/pod/pod_controller_test.go index 685610ee5b3..7af6a0eaf7f 100644 --- a/pkg/controller/jobs/pod/pod_controller_test.go +++ b/pkg/controller/jobs/pod/pod_controller_test.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" @@ -48,7 +48,7 @@ import ( "sigs.k8s.io/kueue/pkg/podset" utilpod "sigs.k8s.io/kueue/pkg/util/pod" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/pkg/workload" diff --git a/pkg/controller/jobs/pod/pod_multikueue_adapter.go b/pkg/controller/jobs/pod/pod_multikueue_adapter.go index be3e31f42b3..a665abbb4e0 100644 --- a/pkg/controller/jobs/pod/pod_multikueue_adapter.go +++ b/pkg/controller/jobs/pod/pod_multikueue_adapter.go @@ -30,7 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" diff --git a/pkg/controller/jobs/pod/pod_multikueue_adapter_test.go b/pkg/controller/jobs/pod/pod_multikueue_adapter_test.go index fc2d7593211..6d4585e94c4 100644 --- a/pkg/controller/jobs/pod/pod_multikueue_adapter_test.go +++ b/pkg/controller/jobs/pod/pod_multikueue_adapter_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/controller/jobs/pod/pod_webhook.go b/pkg/controller/jobs/pod/pod_webhook.go index 4f8ee2e4a41..82d5855a457 100644 --- a/pkg/controller/jobs/pod/pod_webhook.go +++ b/pkg/controller/jobs/pod/pod_webhook.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" ctrlconstants "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/pkg/controller/jobs/pod/pod_webhook_test.go b/pkg/controller/jobs/pod/pod_webhook_test.go index cfe3280dff6..4497ea5e70f 100644 --- a/pkg/controller/jobs/pod/pod_webhook_test.go +++ b/pkg/controller/jobs/pod/pod_webhook_test.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" @@ -44,7 +44,7 @@ import ( podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingmpijob "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" "sigs.k8s.io/kueue/pkg/util/testingjobs/paddlejob" diff --git a/pkg/controller/jobs/raycluster/raycluster_controller.go b/pkg/controller/jobs/raycluster/raycluster_controller.go index dc3c2f02cb6..4afaaee55c4 100644 --- a/pkg/controller/jobs/raycluster/raycluster_controller.go +++ b/pkg/controller/jobs/raycluster/raycluster_controller.go @@ -29,7 +29,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" diff --git a/pkg/controller/jobs/raycluster/raycluster_controller_test.go b/pkg/controller/jobs/raycluster/raycluster_controller_test.go index db63d738055..291abc4e2ff 100644 --- a/pkg/controller/jobs/raycluster/raycluster_controller_test.go +++ b/pkg/controller/jobs/raycluster/raycluster_controller_test.go @@ -34,13 +34,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingrayutil "sigs.k8s.io/kueue/pkg/util/testingjobs/raycluster" ) diff --git a/pkg/controller/jobs/raycluster/raycluster_multikueue_adapter.go b/pkg/controller/jobs/raycluster/raycluster_multikueue_adapter.go index 8a2e832d511..ec4150f72a1 100644 --- a/pkg/controller/jobs/raycluster/raycluster_multikueue_adapter.go +++ b/pkg/controller/jobs/raycluster/raycluster_multikueue_adapter.go @@ -30,7 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/api" diff --git a/pkg/controller/jobs/raycluster/raycluster_multikueue_adapter_test.go b/pkg/controller/jobs/raycluster/raycluster_multikueue_adapter_test.go index e37ffc1a041..64db571801f 100644 --- a/pkg/controller/jobs/raycluster/raycluster_multikueue_adapter_test.go +++ b/pkg/controller/jobs/raycluster/raycluster_multikueue_adapter_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/controller/jobs/raycluster/raycluster_webhook.go b/pkg/controller/jobs/raycluster/raycluster_webhook.go index 613dd24b056..e31ac1e20bb 100644 --- a/pkg/controller/jobs/raycluster/raycluster_webhook.go +++ b/pkg/controller/jobs/raycluster/raycluster_webhook.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/controller/jobs/raycluster/raycluster_webhook_test.go b/pkg/controller/jobs/raycluster/raycluster_webhook_test.go index 0dbe531914e..caff8a6c8b3 100644 --- a/pkg/controller/jobs/raycluster/raycluster_webhook_test.go +++ b/pkg/controller/jobs/raycluster/raycluster_webhook_test.go @@ -28,13 +28,13 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingrayutil "sigs.k8s.io/kueue/pkg/util/testingjobs/raycluster" ) diff --git a/pkg/controller/jobs/rayjob/rayjob_controller.go b/pkg/controller/jobs/rayjob/rayjob_controller.go index d1946f4a62f..c9710556bb7 100644 --- a/pkg/controller/jobs/rayjob/rayjob_controller.go +++ b/pkg/controller/jobs/rayjob/rayjob_controller.go @@ -30,7 +30,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" diff --git a/pkg/controller/jobs/rayjob/rayjob_controller_test.go b/pkg/controller/jobs/rayjob/rayjob_controller_test.go index 69a76b7f79a..97eb1a91fc1 100644 --- a/pkg/controller/jobs/rayjob/rayjob_controller_test.go +++ b/pkg/controller/jobs/rayjob/rayjob_controller_test.go @@ -27,11 +27,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/podset" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingrayutil "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" ) diff --git a/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter.go b/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter.go index 9976e0026a4..0d846b70364 100644 --- a/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter.go +++ b/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter.go @@ -30,7 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/api" diff --git a/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter_test.go b/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter_test.go index ef64aaf6c88..90514f5e1aa 100644 --- a/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter_test.go +++ b/pkg/controller/jobs/rayjob/rayjob_multikueue_adapter_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/controller/jobs/rayjob/rayjob_webhook.go b/pkg/controller/jobs/rayjob/rayjob_webhook.go index 4e1a222a6fa..98cbcbf195c 100644 --- a/pkg/controller/jobs/rayjob/rayjob_webhook.go +++ b/pkg/controller/jobs/rayjob/rayjob_webhook.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/controller/jobs/rayjob/rayjob_webhook_test.go b/pkg/controller/jobs/rayjob/rayjob_webhook_test.go index c908c97598f..81a088bb02b 100644 --- a/pkg/controller/jobs/rayjob/rayjob_webhook_test.go +++ b/pkg/controller/jobs/rayjob/rayjob_webhook_test.go @@ -28,13 +28,13 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingrayutil "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" ) diff --git a/pkg/controller/jobs/statefulset/statefulset_webhook.go b/pkg/controller/jobs/statefulset/statefulset_webhook.go index 6377de19012..6a7aa4ee08c 100644 --- a/pkg/controller/jobs/statefulset/statefulset_webhook.go +++ b/pkg/controller/jobs/statefulset/statefulset_webhook.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/pkg/controller/jobs/statefulset/statefulset_webhook_test.go b/pkg/controller/jobs/statefulset/statefulset_webhook_test.go index 2389b0b0c2f..c8d58b6e0ed 100644 --- a/pkg/controller/jobs/statefulset/statefulset_webhook_test.go +++ b/pkg/controller/jobs/statefulset/statefulset_webhook_test.go @@ -42,7 +42,7 @@ import ( podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingappwrapper "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" testingleaderworkerset "sigs.k8s.io/kueue/pkg/util/testingjobs/leaderworkerset" testingstatefulset "sigs.k8s.io/kueue/pkg/util/testingjobs/statefulset" diff --git a/pkg/controller/jobs/trainjob/trainjob_controller.go b/pkg/controller/jobs/trainjob/trainjob_controller.go index 1af7476d664..c4c8d37b756 100644 --- a/pkg/controller/jobs/trainjob/trainjob_controller.go +++ b/pkg/controller/jobs/trainjob/trainjob_controller.go @@ -41,7 +41,7 @@ import ( jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" jobsetapplyapi "sigs.k8s.io/jobset/client-go/applyconfiguration/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjobset "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" diff --git a/pkg/controller/jobs/trainjob/trainjob_controller_test.go b/pkg/controller/jobs/trainjob/trainjob_controller_test.go index 3753635b78e..837edec1564 100644 --- a/pkg/controller/jobs/trainjob/trainjob_controller_test.go +++ b/pkg/controller/jobs/trainjob/trainjob_controller_test.go @@ -31,12 +31,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/podset" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" ) diff --git a/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter.go b/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter.go index a48e306521c..5ac9db1ded1 100644 --- a/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter.go +++ b/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter.go @@ -30,7 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/api" diff --git a/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter_test.go b/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter_test.go index c73cfa37199..95b141d936f 100644 --- a/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter_test.go +++ b/pkg/controller/jobs/trainjob/trainjob_multikueue_adapter_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/controller/jobs/trainjob/trainjob_webhook_test.go b/pkg/controller/jobs/trainjob/trainjob_webhook_test.go index 44c476e91a5..b78a9a33d0c 100644 --- a/pkg/controller/jobs/trainjob/trainjob_webhook_test.go +++ b/pkg/controller/jobs/trainjob/trainjob_webhook_test.go @@ -28,13 +28,13 @@ import ( "k8s.io/utils/ptr" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" ) diff --git a/pkg/controller/tas/indexer/indexer.go b/pkg/controller/tas/indexer/indexer.go index d3eca5f6eff..98856749a37 100644 --- a/pkg/controller/tas/indexer/indexer.go +++ b/pkg/controller/tas/indexer/indexer.go @@ -24,7 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltas "sigs.k8s.io/kueue/pkg/util/tas" ) diff --git a/pkg/controller/tas/node_failure_controller.go b/pkg/controller/tas/node_failure_controller.go index 0f172be2d7e..065b2852032 100644 --- a/pkg/controller/tas/node_failure_controller.go +++ b/pkg/controller/tas/node_failure_controller.go @@ -44,7 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/controller/tas/node_failure_controller_test.go b/pkg/controller/tas/node_failure_controller_test.go index eeb38c80870..4bf26c64e20 100644 --- a/pkg/controller/tas/node_failure_controller_test.go +++ b/pkg/controller/tas/node_failure_controller_test.go @@ -33,12 +33,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" podcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" ) diff --git a/pkg/controller/tas/resource_flavor.go b/pkg/controller/tas/resource_flavor.go index e7165615dd0..17333d1b908 100644 --- a/pkg/controller/tas/resource_flavor.go +++ b/pkg/controller/tas/resource_flavor.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/pkg/controller/tas/topology_controller.go b/pkg/controller/tas/topology_controller.go index cea24de59b5..92604a57d9e 100644 --- a/pkg/controller/tas/topology_controller.go +++ b/pkg/controller/tas/topology_controller.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/pkg/controller/tas/topology_ungater.go b/pkg/controller/tas/topology_ungater.go index 7042cffa440..d9bc264da7c 100644 --- a/pkg/controller/tas/topology_ungater.go +++ b/pkg/controller/tas/topology_ungater.go @@ -41,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/core" diff --git a/pkg/controller/tas/topology_ungater_test.go b/pkg/controller/tas/topology_ungater_test.go index 92e40609775..58c6220f011 100644 --- a/pkg/controller/tas/topology_ungater_test.go +++ b/pkg/controller/tas/topology_ungater_test.go @@ -36,12 +36,12 @@ import ( jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/tas/indexer" utilpod "sigs.k8s.io/kueue/pkg/util/pod" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" _ "sigs.k8s.io/kueue/pkg/controller/jobs/job" diff --git a/pkg/controller/workloaddispatcher/incrementaldispatcher.go b/pkg/controller/workloaddispatcher/incrementaldispatcher.go index 76228f2fe52..f56bfdbee68 100644 --- a/pkg/controller/workloaddispatcher/incrementaldispatcher.go +++ b/pkg/controller/workloaddispatcher/incrementaldispatcher.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" kueueconfig "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utilmaps "sigs.k8s.io/kueue/pkg/util/maps" diff --git a/pkg/controller/workloaddispatcher/incrementaldispatcher_test.go b/pkg/controller/workloaddispatcher/incrementaldispatcher_test.go index 73c22243719..6caf9bb4a5f 100644 --- a/pkg/controller/workloaddispatcher/incrementaldispatcher_test.go +++ b/pkg/controller/workloaddispatcher/incrementaldispatcher_test.go @@ -36,11 +36,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" kueueconfig "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utilmaps "sigs.k8s.io/kueue/pkg/util/maps" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestIncrementalDispatcherReconciler_Reconcile(t *testing.T) { diff --git a/pkg/dra/claims.go b/pkg/dra/claims.go index 0781f204347..cfdac49331e 100644 --- a/pkg/dra/claims.go +++ b/pkg/dra/claims.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/resources" utilresource "sigs.k8s.io/kueue/pkg/util/resource" ) diff --git a/pkg/dra/claims_test.go b/pkg/dra/claims_test.go index 7e7cce60004..382c005b915 100644 --- a/pkg/dra/claims_test.go +++ b/pkg/dra/claims_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index b7de2ef408f..1d7f916d408 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -23,7 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/metrics" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/version" diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 157efa8dcf3..aa245cfb14e 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -22,7 +22,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing/metrics" "sigs.k8s.io/kueue/pkg/version" ) diff --git a/pkg/podset/podset.go b/pkg/podset/podset.go index 3ec3c516987..d5e89ae70cc 100644 --- a/pkg/podset/podset.go +++ b/pkg/podset/podset.go @@ -30,7 +30,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" utilmaps "sigs.k8s.io/kueue/pkg/util/maps" ) diff --git a/pkg/podset/podset_test.go b/pkg/podset/podset_test.go index b44bdecbf7c..db7e38992af 100644 --- a/pkg/podset/podset_test.go +++ b/pkg/podset/podset_test.go @@ -26,10 +26,10 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestFromAssignment(t *testing.T) { diff --git a/pkg/resources/resource.go b/pkg/resources/resource.go index 6a0342123fe..3c42cedc4f5 100644 --- a/pkg/resources/resource.go +++ b/pkg/resources/resource.go @@ -22,7 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type FlavorResource struct { diff --git a/pkg/scheduler/fair_sharing_iterator.go b/pkg/scheduler/fair_sharing_iterator.go index 951fded248e..562628a4c9f 100644 --- a/pkg/scheduler/fair_sharing_iterator.go +++ b/pkg/scheduler/fair_sharing_iterator.go @@ -24,7 +24,7 @@ import ( "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/priority" diff --git a/pkg/scheduler/flavorassigner/flavorassigner.go b/pkg/scheduler/flavorassigner/flavorassigner.go index eb06c8867ff..2306acc850c 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner.go +++ b/pkg/scheduler/flavorassigner/flavorassigner.go @@ -33,7 +33,7 @@ import ( "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" diff --git a/pkg/scheduler/flavorassigner/flavorassigner_test.go b/pkg/scheduler/flavorassigner/flavorassigner_test.go index 3b6eff74a55..92015fc80ee 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner_test.go +++ b/pkg/scheduler/flavorassigner/flavorassigner_test.go @@ -27,13 +27,13 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" preemptioncommon "sigs.k8s.io/kueue/pkg/scheduler/preemption/common" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) @@ -2822,7 +2822,7 @@ func TestAssignFlavors(t *testing.T) { cache.AddOrUpdateResourceFlavor(log, rf) } - if err := cache.AddOrUpdateCohort(utiltestingapi.MakeCohort(tc.clusterQueue.Spec.Cohort).Obj()); err != nil { + if err := cache.AddOrUpdateCohort(utiltestingapi.MakeCohort(tc.clusterQueue.Spec.CohortName).Obj()); err != nil { t.Fatalf("Failed to create a cohort") } diff --git a/pkg/scheduler/flavorassigner/podset_reducer.go b/pkg/scheduler/flavorassigner/podset_reducer.go index 16721bb7324..b856d8d2e2c 100644 --- a/pkg/scheduler/flavorassigner/podset_reducer.go +++ b/pkg/scheduler/flavorassigner/podset_reducer.go @@ -21,7 +21,7 @@ import ( "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) // PodSetReducer helper structure used to gradually walk down diff --git a/pkg/scheduler/flavorassigner/podset_reducer_test.go b/pkg/scheduler/flavorassigner/podset_reducer_test.go index 378692bd998..fae0d98454c 100644 --- a/pkg/scheduler/flavorassigner/podset_reducer_test.go +++ b/pkg/scheduler/flavorassigner/podset_reducer_test.go @@ -19,8 +19,8 @@ package flavorassigner import ( "testing" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestSearch(t *testing.T) { diff --git a/pkg/scheduler/flavorassigner/tas_flavorassigner.go b/pkg/scheduler/flavorassigner/tas_flavorassigner.go index c4c1e2f53aa..dd96bb58882 100644 --- a/pkg/scheduler/flavorassigner/tas_flavorassigner.go +++ b/pkg/scheduler/flavorassigner/tas_flavorassigner.go @@ -23,7 +23,7 @@ import ( "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/scheduler/preemption/classical/candidate_generator.go b/pkg/scheduler/preemption/classical/candidate_generator.go index 3e583c32d33..c83105a8023 100644 --- a/pkg/scheduler/preemption/classical/candidate_generator.go +++ b/pkg/scheduler/preemption/classical/candidate_generator.go @@ -26,7 +26,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/clock" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/resources" "sigs.k8s.io/kueue/pkg/workload" diff --git a/pkg/scheduler/preemption/classical/hierarchical_preemption.go b/pkg/scheduler/preemption/classical/hierarchical_preemption.go index 4bc0402eefb..9cedda3d3ea 100644 --- a/pkg/scheduler/preemption/classical/hierarchical_preemption.go +++ b/pkg/scheduler/preemption/classical/hierarchical_preemption.go @@ -20,7 +20,7 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/util/sets" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/resources" preemptioncommon "sigs.k8s.io/kueue/pkg/scheduler/preemption/common" diff --git a/pkg/scheduler/preemption/common/ordering.go b/pkg/scheduler/preemption/common/ordering.go index bffeda4d271..55c20d20bca 100644 --- a/pkg/scheduler/preemption/common/ordering.go +++ b/pkg/scheduler/preemption/common/ordering.go @@ -25,7 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" cmputil "sigs.k8s.io/kueue/pkg/util/cmp" "sigs.k8s.io/kueue/pkg/util/priority" "sigs.k8s.io/kueue/pkg/workload" diff --git a/pkg/scheduler/preemption/common/preemption_policy.go b/pkg/scheduler/preemption/common/preemption_policy.go index 0064be22134..0f15f80126a 100644 --- a/pkg/scheduler/preemption/common/preemption_policy.go +++ b/pkg/scheduler/preemption/common/preemption_policy.go @@ -17,7 +17,7 @@ limitations under the License. package preemptioncommon import ( - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/priority" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/scheduler/preemption/fairsharing/ordering.go b/pkg/scheduler/preemption/fairsharing/ordering.go index 471ce4a5d5e..e59973c13e5 100644 --- a/pkg/scheduler/preemption/fairsharing/ordering.go +++ b/pkg/scheduler/preemption/fairsharing/ordering.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/clock" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" preemptioncommon "sigs.k8s.io/kueue/pkg/scheduler/preemption/common" "sigs.k8s.io/kueue/pkg/workload" diff --git a/pkg/scheduler/preemption/policy.go b/pkg/scheduler/preemption/policy.go index 352b7287ce5..0ffbb926c0e 100644 --- a/pkg/scheduler/preemption/policy.go +++ b/pkg/scheduler/preemption/policy.go @@ -17,7 +17,7 @@ limitations under the License. package preemption import ( - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" ) diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index 9c721fb4132..45284b7d8ce 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -34,7 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/resources" diff --git a/pkg/scheduler/preemption/preemption_test.go b/pkg/scheduler/preemption/preemption_test.go index 67b6871dab3..65c60f66329 100644 --- a/pkg/scheduler/preemption/preemption_test.go +++ b/pkg/scheduler/preemption/preemption_test.go @@ -37,7 +37,7 @@ import ( "k8s.io/utils/ptr" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" @@ -48,7 +48,7 @@ import ( preemptioncommon "sigs.k8s.io/kueue/pkg/scheduler/preemption/common" utilslices "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index c5008773df1..92057a10132 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -38,7 +38,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/pkg/scheduler/scheduler_afs_test.go b/pkg/scheduler/scheduler_afs_test.go index 05047398659..facd16482bf 100644 --- a/pkg/scheduler/scheduler_afs_test.go +++ b/pkg/scheduler/scheduler_afs_test.go @@ -33,13 +33,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/routine" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestScheduleForAFS(t *testing.T) { diff --git a/pkg/scheduler/scheduler_tas_test.go b/pkg/scheduler/scheduler_tas_test.go index acfa0f657aa..ef3369d10bb 100644 --- a/pkg/scheduler/scheduler_tas_test.go +++ b/pkg/scheduler/scheduler_tas_test.go @@ -33,7 +33,7 @@ import ( testingclock "k8s.io/utils/clock/testing" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" @@ -42,7 +42,7 @@ import ( "sigs.k8s.io/kueue/pkg/util/slices" utiltas "sigs.k8s.io/kueue/pkg/util/tas" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/pkg/workload" diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index cb114f2230c..8560a4cb55d 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -41,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" @@ -52,7 +52,7 @@ import ( "sigs.k8s.io/kueue/pkg/util/limitrange" "sigs.k8s.io/kueue/pkg/util/routine" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/pkg/workloadslicing" ) diff --git a/pkg/util/admissioncheck/admissioncheck.go b/pkg/util/admissioncheck/admissioncheck.go index 1522f09dfb1..fe227968223 100644 --- a/pkg/util/admissioncheck/admissioncheck.go +++ b/pkg/util/admissioncheck/admissioncheck.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" ) diff --git a/pkg/util/admissioncheck/admissioncheck_test.go b/pkg/util/admissioncheck/admissioncheck_test.go index 0dad35606b7..a827e5d5efa 100644 --- a/pkg/util/admissioncheck/admissioncheck_test.go +++ b/pkg/util/admissioncheck/admissioncheck_test.go @@ -31,9 +31,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestConfigHelper(t *testing.T) { diff --git a/pkg/util/admissionfairsharing/admission_fair_sharing.go b/pkg/util/admissionfairsharing/admission_fair_sharing.go index b867af8a4c9..d16bdb2b587 100644 --- a/pkg/util/admissionfairsharing/admission_fair_sharing.go +++ b/pkg/util/admissionfairsharing/admission_fair_sharing.go @@ -22,7 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/resource" ) diff --git a/pkg/util/equality/podset.go b/pkg/util/equality/podset.go index 773771489fd..d07c326b732 100644 --- a/pkg/util/equality/podset.go +++ b/pkg/util/equality/podset.go @@ -21,7 +21,7 @@ import ( "k8s.io/apimachinery/pkg/api/equality" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) // TODO: Revisit this, maybe we should extend the check to everything that could potentially impact diff --git a/pkg/util/equality/podset_test.go b/pkg/util/equality/podset_test.go index bfc75c118e0..7b571f0537b 100644 --- a/pkg/util/equality/podset_test.go +++ b/pkg/util/equality/podset_test.go @@ -22,8 +22,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestComparePodSetSlices(t *testing.T) { diff --git a/pkg/util/podset/podset.go b/pkg/util/podset/podset.go index e45c974f09a..4f8160d01ec 100644 --- a/pkg/util/podset/podset.go +++ b/pkg/util/podset/podset.go @@ -16,7 +16,7 @@ limitations under the License. package podset -import kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" +import kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" func FindPodSetByName(podSets []kueue.PodSet, name kueue.PodSetReference) *kueue.PodSet { for _, podSet := range podSets { diff --git a/pkg/util/priority/priority.go b/pkg/util/priority/priority.go index 36fa5149966..060d896d8dd 100644 --- a/pkg/util/priority/priority.go +++ b/pkg/util/priority/priority.go @@ -24,7 +24,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" ) diff --git a/pkg/util/priority/priority_test.go b/pkg/util/priority/priority_test.go index 0e6d901c10a..6d8f005d5f9 100644 --- a/pkg/util/priority/priority_test.go +++ b/pkg/util/priority/priority_test.go @@ -26,10 +26,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestPriority(t *testing.T) { diff --git a/pkg/util/queue/cluster_queue.go b/pkg/util/queue/cluster_queue.go index 28bb497cff8..87cc6424919 100644 --- a/pkg/util/queue/cluster_queue.go +++ b/pkg/util/queue/cluster_queue.go @@ -17,7 +17,7 @@ package queue import ( "k8s.io/apimachinery/pkg/util/sets" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utilslices "sigs.k8s.io/kueue/pkg/util/slices" ) diff --git a/pkg/util/queue/local_queue.go b/pkg/util/queue/local_queue.go index beabb8287dc..ca2eeb57e16 100644 --- a/pkg/util/queue/local_queue.go +++ b/pkg/util/queue/local_queue.go @@ -18,7 +18,7 @@ import ( "fmt" "strings" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" ) diff --git a/pkg/util/tas/tas.go b/pkg/util/tas/tas.go index 2289d0c372a..f106d61db42 100644 --- a/pkg/util/tas/tas.go +++ b/pkg/util/tas/tas.go @@ -21,7 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type TopologyDomainID string diff --git a/pkg/util/testing/client.go b/pkg/util/testing/client.go index 84ee153cbbe..5f96d726452 100644 --- a/pkg/util/testing/client.go +++ b/pkg/util/testing/client.go @@ -30,7 +30,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/core/indexer" ) @@ -46,6 +47,7 @@ func NewClientBuilder(addToSchemes ...func(s *runtime.Scheme) error) *fake.Clien scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(kueue.AddToScheme(scheme)) + utilruntime.Must(kueuev1beta1.AddToScheme(scheme)) for i := range addToSchemes { utilruntime.Must(addToSchemes[i](scheme)) } diff --git a/pkg/util/testingjobs/job/wrappers.go b/pkg/util/testingjobs/job/wrappers.go index 29db73a3829..403dd5c57cf 100644 --- a/pkg/util/testingjobs/job/wrappers.go +++ b/pkg/util/testingjobs/job/wrappers.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/util/testing" ) diff --git a/pkg/util/testingjobs/pod/wrappers.go b/pkg/util/testingjobs/pod/wrappers.go index f6c30d55c97..4610bc726ab 100644 --- a/pkg/util/testingjobs/pod/wrappers.go +++ b/pkg/util/testingjobs/pod/wrappers.go @@ -28,7 +28,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" diff --git a/pkg/util/testingjobs/statefulset/wrappers.go b/pkg/util/testingjobs/statefulset/wrappers.go index 645b62283bf..869ac204d30 100644 --- a/pkg/util/testingjobs/statefulset/wrappers.go +++ b/pkg/util/testingjobs/statefulset/wrappers.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" diff --git a/pkg/visibility/api/v1beta1/pending_workloads_cq.go b/pkg/visibility/api/v1beta1/pending_workloads_cq.go index 389c84830e4..12fc8e0286a 100644 --- a/pkg/visibility/api/v1beta1/pending_workloads_cq.go +++ b/pkg/visibility/api/v1beta1/pending_workloads_cq.go @@ -26,7 +26,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" ctrl "sigs.k8s.io/controller-runtime" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" diff --git a/pkg/visibility/api/v1beta1/pending_workloads_cq_test.go b/pkg/visibility/api/v1beta1/pending_workloads_cq_test.go index 5f060a1d746..26f623666eb 100644 --- a/pkg/visibility/api/v1beta1/pending_workloads_cq_test.go +++ b/pkg/visibility/api/v1beta1/pending_workloads_cq_test.go @@ -27,12 +27,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestPendingWorkloadsInCQ(t *testing.T) { diff --git a/pkg/visibility/api/v1beta1/pending_workloads_lq.go b/pkg/visibility/api/v1beta1/pending_workloads_lq.go index 5fcc15ca2d6..227a4bcb515 100644 --- a/pkg/visibility/api/v1beta1/pending_workloads_lq.go +++ b/pkg/visibility/api/v1beta1/pending_workloads_lq.go @@ -27,7 +27,7 @@ import ( "k8s.io/apiserver/pkg/registry/rest" ctrl "sigs.k8s.io/controller-runtime" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" diff --git a/pkg/visibility/api/v1beta1/pending_workloads_lq_test.go b/pkg/visibility/api/v1beta1/pending_workloads_lq_test.go index 7944b8df714..367464d1b38 100644 --- a/pkg/visibility/api/v1beta1/pending_workloads_lq_test.go +++ b/pkg/visibility/api/v1beta1/pending_workloads_lq_test.go @@ -28,12 +28,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/endpoints/request" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestPendingWorkloadsInLQ(t *testing.T) { diff --git a/pkg/visibility/api/v1beta1/utils.go b/pkg/visibility/api/v1beta1/utils.go index a78d91f3185..f9f7a455f46 100644 --- a/pkg/visibility/api/v1beta1/utils.go +++ b/pkg/visibility/api/v1beta1/utils.go @@ -19,6 +19,7 @@ package v1beta1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" "sigs.k8s.io/kueue/pkg/workload" ) @@ -42,7 +43,7 @@ func newPendingWorkload(wlInfo *workload.Info, positionInLq int32, positionInCq }, PositionInClusterQueue: int32(positionInCq), Priority: *wlInfo.Obj.Spec.Priority, - LocalQueueName: wlInfo.Obj.Spec.QueueName, + LocalQueueName: kueuev1beta1.LocalQueueName(wlInfo.Obj.Spec.QueueName), PositionInLocalQueue: positionInLq, } } diff --git a/pkg/webhooks/clusterqueue_webhook.go b/pkg/webhooks/clusterqueue_webhook.go index d412bd9b37f..1824a1354c3 100644 --- a/pkg/webhooks/clusterqueue_webhook.go +++ b/pkg/webhooks/clusterqueue_webhook.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" ) @@ -51,7 +51,7 @@ func setupWebhookForClusterQueue(mgr ctrl.Manager) error { Complete() } -// +kubebuilder:webhook:path=/mutate-kueue-x-k8s-io-v1beta1-clusterqueue,mutating=true,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=clusterqueues,verbs=create,versions=v1beta1,name=mclusterqueue.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-kueue-x-k8s-io-v1beta2-clusterqueue,mutating=true,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=clusterqueues,verbs=create,versions=v1beta2,name=mclusterqueue.kb.io,admissionReviewVersions=v1 var _ webhook.CustomDefaulter = &ClusterQueueWebhook{} @@ -66,7 +66,7 @@ func (w *ClusterQueueWebhook) Default(ctx context.Context, obj runtime.Object) e return nil } -// +kubebuilder:webhook:path=/validate-kueue-x-k8s-io-v1beta1-clusterqueue,mutating=false,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=clusterqueues,verbs=create;update,versions=v1beta1,name=vclusterqueue.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-kueue-x-k8s-io-v1beta2-clusterqueue,mutating=false,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=clusterqueues,verbs=create;update,versions=v1beta2,name=vclusterqueue.kb.io,admissionReviewVersions=v1 var _ webhook.CustomValidator = &ClusterQueueWebhook{} @@ -99,7 +99,7 @@ func ValidateClusterQueue(cq *kueue.ClusterQueue) field.ErrorList { var allErrs field.ErrorList config := validationConfig{ - hasParent: cq.Spec.Cohort != "", + hasParent: cq.Spec.CohortName != "", enforceNominalGreaterThanLending: true, } allErrs = append(allErrs, validateResourceGroups(cq.Spec.ResourceGroups, config, path.Child("resourceGroups"), false)...) diff --git a/pkg/webhooks/clusterqueue_webhook_test.go b/pkg/webhooks/clusterqueue_webhook_test.go index 739cf09f8fc..95117bdde1f 100644 --- a/pkg/webhooks/clusterqueue_webhook_test.go +++ b/pkg/webhooks/clusterqueue_webhook_test.go @@ -28,9 +28,9 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestValidateClusterQueue(t *testing.T) { diff --git a/pkg/webhooks/cohort_webhook.go b/pkg/webhooks/cohort_webhook.go index 51148f1c0f0..56d869a69c4 100644 --- a/pkg/webhooks/cohort_webhook.go +++ b/pkg/webhooks/cohort_webhook.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type CohortWebhook struct{} @@ -41,7 +41,7 @@ func (w *CohortWebhook) Default(context.Context, runtime.Object) error { return nil } -//+kubebuilder:webhook:path=/validate-kueue-x-k8s-io-v1beta1-cohort,mutating=false,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=cohorts,verbs=create;update,versions=v1beta1,name=vcohort.kb.io,admissionReviewVersions=v1 +//+kubebuilder:webhook:path=/validate-kueue-x-k8s-io-v1beta2-cohort,mutating=false,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=cohorts,verbs=create;update,versions=v1beta2,name=vcohort.kb.io,admissionReviewVersions=v1 var _ webhook.CustomValidator = &CohortWebhook{} diff --git a/pkg/webhooks/cohort_webhook_test.go b/pkg/webhooks/cohort_webhook_test.go index cf6d154f44c..8391721302f 100644 --- a/pkg/webhooks/cohort_webhook_test.go +++ b/pkg/webhooks/cohort_webhook_test.go @@ -23,9 +23,9 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" "k8s.io/apimachinery/pkg/util/validation/field" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestValidateCohort(t *testing.T) { diff --git a/pkg/webhooks/common.go b/pkg/webhooks/common.go index 8375d754c6a..eb31946c95a 100644 --- a/pkg/webhooks/common.go +++ b/pkg/webhooks/common.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) func validateResourceName(name corev1.ResourceName, fldPath *field.Path) field.ErrorList { diff --git a/pkg/webhooks/resourceflavor_webhook.go b/pkg/webhooks/resourceflavor_webhook.go index 0ec302787a8..86aea1695dd 100644 --- a/pkg/webhooks/resourceflavor_webhook.go +++ b/pkg/webhooks/resourceflavor_webhook.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) type ResourceFlavorWebhook struct{} @@ -44,7 +44,7 @@ func setupWebhookForResourceFlavor(mgr ctrl.Manager) error { Complete() } -// +kubebuilder:webhook:path=/mutate-kueue-x-k8s-io-v1beta1-resourceflavor,mutating=true,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=resourceflavors,verbs=create,versions=v1beta1,name=mresourceflavor.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-kueue-x-k8s-io-v1beta2-resourceflavor,mutating=true,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=resourceflavors,verbs=create,versions=v1beta2,name=mresourceflavor.kb.io,admissionReviewVersions=v1 var _ webhook.CustomDefaulter = &ResourceFlavorWebhook{} @@ -60,7 +60,7 @@ func (w *ResourceFlavorWebhook) Default(ctx context.Context, obj runtime.Object) return nil } -// +kubebuilder:webhook:path=/validate-kueue-x-k8s-io-v1beta1-resourceflavor,mutating=false,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=resourceflavors,verbs=create;update,versions=v1beta1,name=vresourceflavor.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-kueue-x-k8s-io-v1beta2-resourceflavor,mutating=false,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=resourceflavors,verbs=create;update,versions=v1beta2,name=vresourceflavor.kb.io,admissionReviewVersions=v1 var _ webhook.CustomValidator = &ResourceFlavorWebhook{} diff --git a/pkg/webhooks/resourceflavor_webhook_test.go b/pkg/webhooks/resourceflavor_webhook_test.go index a7b88d639cb..6db8d4d118f 100644 --- a/pkg/webhooks/resourceflavor_webhook_test.go +++ b/pkg/webhooks/resourceflavor_webhook_test.go @@ -24,8 +24,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/validation/field" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestValidateResourceFlavor(t *testing.T) { diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go index 7c661634515..7531a778b97 100644 --- a/pkg/webhooks/webhooks.go +++ b/pkg/webhooks/webhooks.go @@ -19,7 +19,7 @@ package webhooks import ( ctrl "sigs.k8s.io/controller-runtime" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) // Setup sets up the webhooks for core controllers. It returns the name of the diff --git a/pkg/webhooks/workload_webhook.go b/pkg/webhooks/workload_webhook.go index 7bacc236d5d..1bd7eebc6b0 100644 --- a/pkg/webhooks/workload_webhook.go +++ b/pkg/webhooks/workload_webhook.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" utilslices "sigs.k8s.io/kueue/pkg/util/slices" @@ -52,7 +52,7 @@ func setupWebhookForWorkload(mgr ctrl.Manager) error { Complete() } -// +kubebuilder:webhook:path=/mutate-kueue-x-k8s-io-v1beta1-workload,mutating=true,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=workloads,verbs=create,versions=v1beta1,name=mworkload.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-kueue-x-k8s-io-v1beta2-workload,mutating=true,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=workloads,verbs=create,versions=v1beta2,name=mworkload.kb.io,admissionReviewVersions=v1 var _ webhook.CustomDefaulter = &WorkloadWebhook{} @@ -72,7 +72,7 @@ func (w *WorkloadWebhook) Default(ctx context.Context, obj runtime.Object) error return nil } -// +kubebuilder:webhook:path=/validate-kueue-x-k8s-io-v1beta1-workload,mutating=false,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=workloads;workloads/status,verbs=create;update,versions=v1beta1,name=vworkload.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-kueue-x-k8s-io-v1beta2-workload,mutating=false,failurePolicy=fail,sideEffects=None,groups=kueue.x-k8s.io,resources=workloads;workloads/status,verbs=create;update,versions=v1beta2,name=vworkload.kb.io,admissionReviewVersions=v1 var _ webhook.CustomValidator = &WorkloadWebhook{} diff --git a/pkg/webhooks/workload_webhook_test.go b/pkg/webhooks/workload_webhook_test.go index 3db54b92df1..dab56c982c3 100644 --- a/pkg/webhooks/workload_webhook_test.go +++ b/pkg/webhooks/workload_webhook_test.go @@ -27,10 +27,10 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/pkg/workloadslicing" ) diff --git a/pkg/workload/admissionchecks.go b/pkg/workload/admissionchecks.go index 38ee6d96810..4769739a348 100644 --- a/pkg/workload/admissionchecks.go +++ b/pkg/workload/admissionchecks.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/clock" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/admissioncheck" ) @@ -73,10 +73,10 @@ func SyncAdmittedCondition(w *kueue.Workload, now time.Time) bool { // in practice the oldCondition cannot be nil, however we should try to avoid nil ptr deref. if oldCondition != nil { d := int32(now.Sub(oldCondition.LastTransitionTime.Time).Seconds()) - if w.Status.AccumulatedPastExexcutionTimeSeconds != nil { - *w.Status.AccumulatedPastExexcutionTimeSeconds += d + if w.Status.AccumulatedPastExecutionTimeSeconds != nil { + *w.Status.AccumulatedPastExecutionTimeSeconds += d } else { - w.Status.AccumulatedPastExexcutionTimeSeconds = &d + w.Status.AccumulatedPastExecutionTimeSeconds = &d } } } diff --git a/pkg/workload/admissionchecks_test.go b/pkg/workload/admissionchecks_test.go index bfd84c65d3b..32256438af5 100644 --- a/pkg/workload/admissionchecks_test.go +++ b/pkg/workload/admissionchecks_test.go @@ -27,10 +27,10 @@ import ( testingclock "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestSyncAdmittedCondition(t *testing.T) { @@ -365,12 +365,12 @@ func TestSyncAdmittedCondition(t *testing.T) { } if tc.wantAdmittedTime > 0 { - if wl.Status.AccumulatedPastExexcutionTimeSeconds == nil { - t.Fatalf("Expecting AccumulatedPastExexcutionTimeSeconds not to be nil") + if wl.Status.AccumulatedPastExecutionTimeSeconds == nil { + t.Fatalf("Expecting AccumulatedPastExecutionTimeSeconds not to be nil") } - if diff := cmp.Diff(tc.wantAdmittedTime, *wl.Status.AccumulatedPastExexcutionTimeSeconds); diff != "" { - t.Errorf("Unexpected AccumulatedPastExexcutionTimeSeconds (- want/+ got):\n%s", diff) + if diff := cmp.Diff(tc.wantAdmittedTime, *wl.Status.AccumulatedPastExecutionTimeSeconds); diff != "" { + t.Errorf("Unexpected AccumulatedPastExecutionTimeSeconds (- want/+ got):\n%s", diff) } } }) diff --git a/pkg/workload/podsetscounts.go b/pkg/workload/podsetscounts.go index 0c430e12c37..022490d3ec2 100644 --- a/pkg/workload/podsetscounts.go +++ b/pkg/workload/podsetscounts.go @@ -19,7 +19,7 @@ package workload import ( "maps" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utilslices "sigs.k8s.io/kueue/pkg/util/slices" ) diff --git a/pkg/workload/podsetscounts_test.go b/pkg/workload/podsetscounts_test.go index ec7dcb6ee32..ea90b7522f7 100644 --- a/pkg/workload/podsetscounts_test.go +++ b/pkg/workload/podsetscounts_test.go @@ -21,7 +21,7 @@ import ( "github.com/google/go-cmp/cmp" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) func testPodSet(name string, count int32) kueue.PodSet { diff --git a/pkg/workload/resources.go b/pkg/workload/resources.go index badcbbae346..521edc3dc0d 100644 --- a/pkg/workload/resources.go +++ b/pkg/workload/resources.go @@ -27,7 +27,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/util/limitrange" "sigs.k8s.io/kueue/pkg/util/resource" diff --git a/pkg/workload/resources_test.go b/pkg/workload/resources_test.go index b69deb0136c..a68461064a3 100644 --- a/pkg/workload/resources_test.go +++ b/pkg/workload/resources_test.go @@ -26,12 +26,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/resources" "sigs.k8s.io/kueue/pkg/util/limitrange" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestAdjustResources(t *testing.T) { diff --git a/pkg/workload/usage.go b/pkg/workload/usage.go index 38f430471b5..c98ba7c3cb5 100644 --- a/pkg/workload/usage.go +++ b/pkg/workload/usage.go @@ -17,7 +17,7 @@ limitations under the License. package workload import ( - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/resources" ) diff --git a/pkg/workload/workload.go b/pkg/workload/workload.go index 10013ceb0cd..8c6edb7b928 100644 --- a/pkg/workload/workload.go +++ b/pkg/workload/workload.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" @@ -861,7 +861,7 @@ func admissionStatusPatch(w *kueue.Workload, wlCopy *kueue.Workload) { wlCopy.Status.Conditions = append(wlCopy.Status.Conditions, *existing.DeepCopy()) } } - wlCopy.Status.AccumulatedPastExexcutionTimeSeconds = w.Status.AccumulatedPastExexcutionTimeSeconds + wlCopy.Status.AccumulatedPastExecutionTimeSeconds = w.Status.AccumulatedPastExecutionTimeSeconds if w.Status.SchedulingStats != nil { if wlCopy.Status.SchedulingStats == nil { wlCopy.Status.SchedulingStats = &kueue.SchedulingStats{} diff --git a/pkg/workload/workload_test.go b/pkg/workload/workload_test.go index c05b11900b5..25f0c89ca03 100644 --- a/pkg/workload/workload_test.go +++ b/pkg/workload/workload_test.go @@ -33,14 +33,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/resources" "sigs.k8s.io/kueue/pkg/util/admissioncheck" qutil "sigs.k8s.io/kueue/pkg/util/queue" utiltas "sigs.k8s.io/kueue/pkg/util/tas" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) func TestNewInfo(t *testing.T) { diff --git a/pkg/workloadslicing/workloadslicing.go b/pkg/workloadslicing/workloadslicing.go index 56d2e7e1eb3..f5b2a4b1963 100644 --- a/pkg/workloadslicing/workloadslicing.go +++ b/pkg/workloadslicing/workloadslicing.go @@ -30,7 +30,7 @@ import ( "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/features" diff --git a/pkg/workloadslicing/workloadslicing_test.go b/pkg/workloadslicing/workloadslicing_test.go index a0ba333c9f3..ac4394559f0 100644 --- a/pkg/workloadslicing/workloadslicing_test.go +++ b/pkg/workloadslicing/workloadslicing_test.go @@ -39,14 +39,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/cache/hierarchy" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/scheduler/preemption" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/test/e2e/certmanager/certmanager_test.go b/test/e2e/certmanager/certmanager_test.go index 134f2e8cae2..2aa7a582f13 100644 --- a/test/e2e/certmanager/certmanager_test.go +++ b/test/e2e/certmanager/certmanager_test.go @@ -23,9 +23,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/certmanager/metrics_test.go b/test/e2e/certmanager/metrics_test.go index 550f7b29fd5..8826623f53c 100644 --- a/test/e2e/certmanager/metrics_test.go +++ b/test/e2e/certmanager/metrics_test.go @@ -26,9 +26,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobspod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/certmanager/visibility_test.go b/test/e2e/certmanager/visibility_test.go index 352c41af0d9..13d46fe147d 100644 --- a/test/e2e/certmanager/visibility_test.go +++ b/test/e2e/certmanager/visibility_test.go @@ -25,8 +25,8 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/customconfigs/managejobswithoutqueuename_test.go b/test/e2e/customconfigs/managejobswithoutqueuename_test.go index 27aea660325..a3b925011de 100644 --- a/test/e2e/customconfigs/managejobswithoutqueuename_test.go +++ b/test/e2e/customconfigs/managejobswithoutqueuename_test.go @@ -31,7 +31,7 @@ import ( leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" @@ -43,7 +43,7 @@ import ( "sigs.k8s.io/kueue/pkg/features" utilpod "sigs.k8s.io/kueue/pkg/util/pod" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" awtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" testingdeploy "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" diff --git a/test/e2e/customconfigs/objectretentionpolicies_test.go b/test/e2e/customconfigs/objectretentionpolicies_test.go index dcb42dc79d9..05a150ba198 100644 --- a/test/e2e/customconfigs/objectretentionpolicies_test.go +++ b/test/e2e/customconfigs/objectretentionpolicies_test.go @@ -27,11 +27,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/customconfigs/podintegrationautoenablement_test.go b/test/e2e/customconfigs/podintegrationautoenablement_test.go index 0ed80e32421..a4b15b88faa 100644 --- a/test/e2e/customconfigs/podintegrationautoenablement_test.go +++ b/test/e2e/customconfigs/podintegrationautoenablement_test.go @@ -25,11 +25,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/statefulset" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" testingsts "sigs.k8s.io/kueue/pkg/util/testingjobs/statefulset" "sigs.k8s.io/kueue/test/util" diff --git a/test/e2e/customconfigs/reconcile_test.go b/test/e2e/customconfigs/reconcile_test.go index b72d16ad801..01b08e42dfe 100644 --- a/test/e2e/customconfigs/reconcile_test.go +++ b/test/e2e/customconfigs/reconcile_test.go @@ -25,9 +25,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/test/util" diff --git a/test/e2e/customconfigs/waitforpodsready_test.go b/test/e2e/customconfigs/waitforpodsready_test.go index 42c14262fc7..2f04416c18d 100644 --- a/test/e2e/customconfigs/waitforpodsready_test.go +++ b/test/e2e/customconfigs/waitforpodsready_test.go @@ -28,10 +28,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingjobspod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/test/util" diff --git a/test/e2e/multikueue/e2e_test.go b/test/e2e/multikueue/e2e_test.go index ebc3de37bfe..9692b4a05ca 100644 --- a/test/e2e/multikueue/e2e_test.go +++ b/test/e2e/multikueue/e2e_test.go @@ -41,7 +41,7 @@ import ( jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" kueueconfig "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadaw "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" workloadjobset "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" @@ -55,7 +55,7 @@ import ( "sigs.k8s.io/kueue/pkg/util/admissioncheck" utilpod "sigs.k8s.io/kueue/pkg/util/pod" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingaw "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" testingdeployment "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" diff --git a/test/e2e/multikueue/suite_test.go b/test/e2e/multikueue/suite_test.go index 07c50f0b6b9..1832f2c02c9 100644 --- a/test/e2e/multikueue/suite_test.go +++ b/test/e2e/multikueue/suite_test.go @@ -45,7 +45,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/kubeversion" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/appwrapper_test.go b/test/e2e/singlecluster/appwrapper_test.go index 01c8ad821d9..90d56da15cb 100644 --- a/test/e2e/singlecluster/appwrapper_test.go +++ b/test/e2e/singlecluster/appwrapper_test.go @@ -25,10 +25,10 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" awtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" testingdeploy "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" utiltestingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" diff --git a/test/e2e/singlecluster/deployment_test.go b/test/e2e/singlecluster/deployment_test.go index 5afebdee95c..f39a408e343 100644 --- a/test/e2e/singlecluster/deployment_test.go +++ b/test/e2e/singlecluster/deployment_test.go @@ -24,11 +24,11 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/pod" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" deploymenttesting "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/e2e_test.go b/test/e2e/singlecluster/e2e_test.go index 736e6bd4e2e..9dd1deab927 100644 --- a/test/e2e/singlecluster/e2e_test.go +++ b/test/e2e/singlecluster/e2e_test.go @@ -29,12 +29,12 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/slices" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" diff --git a/test/e2e/singlecluster/e2e_v1beta2_test.go b/test/e2e/singlecluster/e2e_v1beta1_test.go similarity index 98% rename from test/e2e/singlecluster/e2e_v1beta2_test.go rename to test/e2e/singlecluster/e2e_v1beta1_test.go index 8b7175f9672..25437afae16 100644 --- a/test/e2e/singlecluster/e2e_v1beta2_test.go +++ b/test/e2e/singlecluster/e2e_v1beta1_test.go @@ -25,10 +25,10 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/fair_sharing_test.go b/test/e2e/singlecluster/fair_sharing_test.go index 11380099913..d696e219bfa 100644 --- a/test/e2e/singlecluster/fair_sharing_test.go +++ b/test/e2e/singlecluster/fair_sharing_test.go @@ -24,8 +24,8 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" jobtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/jaxjob_test.go b/test/e2e/singlecluster/jaxjob_test.go index ea6653403fd..976a390b267 100644 --- a/test/e2e/singlecluster/jaxjob_test.go +++ b/test/e2e/singlecluster/jaxjob_test.go @@ -25,9 +25,9 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/jaxjob" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" jaxjobtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/jaxjob" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/jobset_test.go b/test/e2e/singlecluster/jobset_test.go index 3a79aafe540..058839d4350 100644 --- a/test/e2e/singlecluster/jobset_test.go +++ b/test/e2e/singlecluster/jobset_test.go @@ -24,10 +24,10 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjobset "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/kuberay_test.go b/test/e2e/singlecluster/kuberay_test.go index 5ede4c3265b..6f00750072e 100644 --- a/test/e2e/singlecluster/kuberay_test.go +++ b/test/e2e/singlecluster/kuberay_test.go @@ -25,10 +25,10 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadraycluster "sigs.k8s.io/kueue/pkg/controller/jobs/raycluster" workloadrayjob "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingraycluster "sigs.k8s.io/kueue/pkg/util/testingjobs/raycluster" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" "sigs.k8s.io/kueue/test/util" diff --git a/test/e2e/singlecluster/kueuectl_test.go b/test/e2e/singlecluster/kueuectl_test.go index 376b15b723b..9c5754a4f8b 100644 --- a/test/e2e/singlecluster/kueuectl_test.go +++ b/test/e2e/singlecluster/kueuectl_test.go @@ -24,8 +24,8 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/leaderworkerset_test.go b/test/e2e/singlecluster/leaderworkerset_test.go index aa2c45aef4c..091db4951b8 100644 --- a/test/e2e/singlecluster/leaderworkerset_test.go +++ b/test/e2e/singlecluster/leaderworkerset_test.go @@ -27,13 +27,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" ctrlconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/leaderworkerset" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" leaderworkersettesting "sigs.k8s.io/kueue/pkg/util/testingjobs/leaderworkerset" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/metrics_test.go b/test/e2e/singlecluster/metrics_test.go index a6c26f60926..6dbb0a0ee12 100644 --- a/test/e2e/singlecluster/metrics_test.go +++ b/test/e2e/singlecluster/metrics_test.go @@ -28,10 +28,10 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/job" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingjobspod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/pkg/workload" diff --git a/test/e2e/singlecluster/pod_test.go b/test/e2e/singlecluster/pod_test.go index ef7e5d1d62f..b78052bcd42 100644 --- a/test/e2e/singlecluster/pod_test.go +++ b/test/e2e/singlecluster/pod_test.go @@ -28,13 +28,13 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/pod" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" podtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/pytorchjob_test.go b/test/e2e/singlecluster/pytorchjob_test.go index 103969ad557..cbe1ac3427c 100644 --- a/test/e2e/singlecluster/pytorchjob_test.go +++ b/test/e2e/singlecluster/pytorchjob_test.go @@ -25,9 +25,9 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/pytorchjob" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" pytorchjobtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/pytorchjob" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/statefulset_test.go b/test/e2e/singlecluster/statefulset_test.go index 0df4ee3b177..4a72d7df116 100644 --- a/test/e2e/singlecluster/statefulset_test.go +++ b/test/e2e/singlecluster/statefulset_test.go @@ -27,12 +27,12 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/statefulset" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" statefulsettesting "sigs.k8s.io/kueue/pkg/util/testingjobs/statefulset" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/tas_test.go b/test/e2e/singlecluster/tas_test.go index bb933118cd4..43f568aca10 100644 --- a/test/e2e/singlecluster/tas_test.go +++ b/test/e2e/singlecluster/tas_test.go @@ -28,14 +28,14 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" podcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/pod" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" workloadtrainjob "sigs.k8s.io/kueue/pkg/controller/jobs/trainjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" diff --git a/test/e2e/singlecluster/trainjob_test.go b/test/e2e/singlecluster/trainjob_test.go index d2a29db07e3..9efa0daa733 100644 --- a/test/e2e/singlecluster/trainjob_test.go +++ b/test/e2e/singlecluster/trainjob_test.go @@ -25,10 +25,10 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadtrainjob "sigs.k8s.io/kueue/pkg/controller/jobs/trainjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/singlecluster/visibility_test.go b/test/e2e/singlecluster/visibility_test.go index b5b0a567df5..d803939834e 100644 --- a/test/e2e/singlecluster/visibility_test.go +++ b/test/e2e/singlecluster/visibility_test.go @@ -29,10 +29,11 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" ) @@ -224,7 +225,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: highPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 0, - LocalQueueName: kueue.LocalQueueName(localQueueA.Name), + LocalQueueName: kueuev1beta1.LocalQueueName(localQueueA.Name), }, { ObjectMeta: metav1.ObjectMeta{ @@ -234,7 +235,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: midPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 1, - LocalQueueName: kueue.LocalQueueName(localQueueB.Name), + LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), }, { ObjectMeta: metav1.ObjectMeta{ @@ -244,7 +245,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: lowPriorityClass.Value, PositionInLocalQueue: 1, PositionInClusterQueue: 2, - LocalQueueName: kueue.LocalQueueName(localQueueB.Name), + LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), }, } gomega.Eventually(func(g gomega.Gomega) { @@ -345,7 +346,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: highPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 0, - LocalQueueName: kueue.LocalQueueName(localQueueA.Name), + LocalQueueName: kueuev1beta1.LocalQueueName(localQueueA.Name), }, } gomega.Eventually(func(g gomega.Gomega) { @@ -365,7 +366,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: midPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 1, - LocalQueueName: kueue.LocalQueueName(localQueueB.Name), + LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), }, { ObjectMeta: metav1.ObjectMeta{ @@ -375,7 +376,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: lowPriorityClass.Value, PositionInLocalQueue: 1, PositionInClusterQueue: 2, - LocalQueueName: kueue.LocalQueueName(localQueueB.Name), + LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), }, } gomega.Eventually(func(g gomega.Gomega) { @@ -438,7 +439,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: highPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 0, - LocalQueueName: kueue.LocalQueueName(localQueueA.Name), + LocalQueueName: kueuev1beta1.LocalQueueName(localQueueA.Name), }, } gomega.Eventually(func(g gomega.Gomega) { @@ -458,7 +459,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: midPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 1, - LocalQueueName: kueue.LocalQueueName(localQueueB.Name), + LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), }, { ObjectMeta: metav1.ObjectMeta{ @@ -468,7 +469,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: lowPriorityClass.Value, PositionInLocalQueue: 1, PositionInClusterQueue: 2, - LocalQueueName: kueue.LocalQueueName(localQueueB.Name), + LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), }, } gomega.Eventually(func(g gomega.Gomega) { diff --git a/test/e2e/tas/appwrapper_test.go b/test/e2e/tas/appwrapper_test.go index 3e6da15d990..f002731420e 100644 --- a/test/e2e/tas/appwrapper_test.go +++ b/test/e2e/tas/appwrapper_test.go @@ -25,9 +25,9 @@ import ( "k8s.io/apimachinery/pkg/fields" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" awtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" utiltestingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" diff --git a/test/e2e/tas/hotswap_test.go b/test/e2e/tas/hotswap_test.go index 90e22290c78..6ce198734f0 100644 --- a/test/e2e/tas/hotswap_test.go +++ b/test/e2e/tas/hotswap_test.go @@ -30,9 +30,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" "sigs.k8s.io/kueue/test/util" diff --git a/test/e2e/tas/job_test.go b/test/e2e/tas/job_test.go index 9f79ab3096e..2bae3bed4c9 100644 --- a/test/e2e/tas/job_test.go +++ b/test/e2e/tas/job_test.go @@ -28,10 +28,10 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" diff --git a/test/e2e/tas/jobset_test.go b/test/e2e/tas/jobset_test.go index d3f1a8a36c6..df3f6e34e09 100644 --- a/test/e2e/tas/jobset_test.go +++ b/test/e2e/tas/jobset_test.go @@ -28,9 +28,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/tas/leaderworkerset_test.go b/test/e2e/tas/leaderworkerset_test.go index 8a7c6bfe670..18fe68f40f0 100644 --- a/test/e2e/tas/leaderworkerset_test.go +++ b/test/e2e/tas/leaderworkerset_test.go @@ -28,9 +28,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" leaderworkersettesting "sigs.k8s.io/kueue/pkg/util/testingjobs/leaderworkerset" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/tas/mpijob_test.go b/test/e2e/tas/mpijob_test.go index 3cf68ff3628..ccd8585afb9 100644 --- a/test/e2e/tas/mpijob_test.go +++ b/test/e2e/tas/mpijob_test.go @@ -28,9 +28,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingmpijob "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/tas/pod_group_test.go b/test/e2e/tas/pod_group_test.go index 1c49ced77c9..7e82a6c88f3 100644 --- a/test/e2e/tas/pod_group_test.go +++ b/test/e2e/tas/pod_group_test.go @@ -23,9 +23,9 @@ import ( "k8s.io/apimachinery/pkg/fields" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/tas/pytorch_test.go b/test/e2e/tas/pytorch_test.go index 9833d0376fe..9da357209f9 100644 --- a/test/e2e/tas/pytorch_test.go +++ b/test/e2e/tas/pytorch_test.go @@ -27,9 +27,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpytorchjob "sigs.k8s.io/kueue/pkg/util/testingjobs/pytorchjob" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/tas/rayjob_test.go b/test/e2e/tas/rayjob_test.go index 28c1f71a797..eb7d9cfbede 100644 --- a/test/e2e/tas/rayjob_test.go +++ b/test/e2e/tas/rayjob_test.go @@ -28,9 +28,9 @@ import ( "k8s.io/utils/set" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/tas/statefulset_test.go b/test/e2e/tas/statefulset_test.go index 4fa97f5978a..0cf805c7e94 100644 --- a/test/e2e/tas/statefulset_test.go +++ b/test/e2e/tas/statefulset_test.go @@ -24,9 +24,9 @@ import ( "k8s.io/apimachinery/pkg/fields" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/util/testingjobs/statefulset" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/e2e/tas/trainjob_test.go b/test/e2e/tas/trainjob_test.go index 42a8dffc99f..6255454e21c 100644 --- a/test/e2e/tas/trainjob_test.go +++ b/test/e2e/tas/trainjob_test.go @@ -29,9 +29,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" "sigs.k8s.io/kueue/test/util" diff --git a/test/integration/framework/framework.go b/test/integration/framework/framework.go index 2d98e0f0759..58dadf7c00c 100644 --- a/test/integration/framework/framework.go +++ b/test/integration/framework/framework.go @@ -50,8 +50,8 @@ import ( jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" + kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" "sigs.k8s.io/kueue/test/util" ) @@ -86,7 +86,7 @@ func (f *Framework) Init() *rest.Config { err = kueue.AddToScheme(f.testEnv.Scheme) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) - err = kueuev1beta2.AddToScheme(f.testEnv.Scheme) + err = kueuev1beta1.AddToScheme(f.testEnv.Scheme) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) if len(f.WebhookPath) > 0 { @@ -123,7 +123,7 @@ func (f *Framework) SetupClient(cfg *rest.Config) (context.Context, client.Clien err = kueue.AddToScheme(f.scheme) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) - err = kueuev1beta2.AddToScheme(f.scheme) + err = kueuev1beta1.AddToScheme(f.scheme) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) err = awv1beta2.AddToScheme(f.scheme) diff --git a/test/integration/multikueue/dispatcher_test.go b/test/integration/multikueue/dispatcher_test.go index 9ddee6b0ec8..6e1348dedb1 100644 --- a/test/integration/multikueue/dispatcher_test.go +++ b/test/integration/multikueue/dispatcher_test.go @@ -32,12 +32,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" diff --git a/test/integration/multikueue/enabled_integration_test.go b/test/integration/multikueue/enabled_integration_test.go index 200edaff591..1a94471fa11 100644 --- a/test/integration/multikueue/enabled_integration_test.go +++ b/test/integration/multikueue/enabled_integration_test.go @@ -36,12 +36,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" workloadmpijob "sigs.k8s.io/kueue/pkg/controller/jobs/mpijob" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingmpijob "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" "sigs.k8s.io/kueue/test/util" diff --git a/test/integration/multikueue/external_job_test.go b/test/integration/multikueue/external_job_test.go index b90c0785ac2..2745e12d4e1 100644 --- a/test/integration/multikueue/external_job_test.go +++ b/test/integration/multikueue/external_job_test.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" @@ -44,7 +44,7 @@ import ( dispatcher "sigs.k8s.io/kueue/pkg/controller/workloaddispatcher" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/util" diff --git a/test/integration/multikueue/jobs_test.go b/test/integration/multikueue/jobs_test.go index 83e1514e319..35a6891b816 100644 --- a/test/integration/multikueue/jobs_test.go +++ b/test/integration/multikueue/jobs_test.go @@ -42,7 +42,7 @@ import ( jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadappwrapper "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" @@ -59,7 +59,7 @@ import ( "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingaw "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" diff --git a/test/integration/multikueue/no_gc_test.go b/test/integration/multikueue/no_gc_test.go index 93213cae937..960e77683a3 100644 --- a/test/integration/multikueue/no_gc_test.go +++ b/test/integration/multikueue/no_gc_test.go @@ -30,12 +30,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" diff --git a/test/integration/multikueue/setup_test.go b/test/integration/multikueue/setup_test.go index f790d1d66af..89e367f2692 100644 --- a/test/integration/multikueue/setup_test.go +++ b/test/integration/multikueue/setup_test.go @@ -31,9 +31,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go b/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go index 5b5e5b5295f..6c9d338d7f8 100644 --- a/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go +++ b/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go @@ -30,13 +30,13 @@ import ( autoscaling "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/admissionchecks/provisioning" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/admissioncheck" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" diff --git a/test/integration/singlecluster/controller/core/admissioncheck_controller_test.go b/test/integration/singlecluster/controller/core/admissioncheck_controller_test.go index b3f8ec218b8..95e1a611918 100644 --- a/test/integration/singlecluster/controller/core/admissioncheck_controller_test.go +++ b/test/integration/singlecluster/controller/core/admissioncheck_controller_test.go @@ -23,9 +23,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -90,7 +90,7 @@ var _ = ginkgo.Describe("AdmissionCheck controller", ginkgo.Ordered, ginkgo.Cont var cq kueue.ClusterQueue gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &cq)).To(gomega.Succeed()) - cq.Spec.Cohort = "foo-cohort" + cq.Spec.CohortName = "foo-cohort" g.Expect(k8sClient.Update(ctx, &cq)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) diff --git a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go index 7cb27ffbcf5..565d4771da1 100644 --- a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go +++ b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go @@ -26,11 +26,11 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" diff --git a/test/integration/singlecluster/controller/core/localqueue_controller_test.go b/test/integration/singlecluster/controller/core/localqueue_controller_test.go index 2979278865f..982b5909a53 100644 --- a/test/integration/singlecluster/controller/core/localqueue_controller_test.go +++ b/test/integration/singlecluster/controller/core/localqueue_controller_test.go @@ -24,10 +24,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -143,7 +143,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, }, FlavorsReservation: emptyUsage, - FlavorUsage: emptyUsage, + FlavorsUsage: emptyUsage, Flavors: []kueue.LocalQueueFlavorStatus{ {Name: flavorModelD, Resources: []corev1.ResourceName{"example.com/gpu"}}, {Name: flavorModelC, Resources: []corev1.ResourceName{"example.com/gpu"}}, @@ -170,7 +170,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, }, FlavorsReservation: emptyUsage, - FlavorUsage: emptyUsage, + FlavorsUsage: emptyUsage, Flavors: []kueue.LocalQueueFlavorStatus{ { Name: flavorModelD, @@ -274,7 +274,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, }, FlavorsReservation: emptyUsage, - FlavorUsage: emptyUsage, + FlavorsUsage: emptyUsage, Flavors: []kueue.LocalQueueFlavorStatus{ { Name: flavorModelD, @@ -340,7 +340,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, }, FlavorsReservation: fullUsage, - FlavorUsage: emptyUsage, + FlavorsUsage: emptyUsage, Flavors: []kueue.LocalQueueFlavorStatus{ { Name: flavorModelD, @@ -385,7 +385,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, }, FlavorsReservation: fullUsage, - FlavorUsage: fullUsage, + FlavorsUsage: fullUsage, Flavors: []kueue.LocalQueueFlavorStatus{ { Name: flavorModelD, @@ -427,7 +427,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, }, FlavorsReservation: emptyUsage, - FlavorUsage: emptyUsage, + FlavorsUsage: emptyUsage, Flavors: []kueue.LocalQueueFlavorStatus{ { Name: flavorModelD, @@ -508,7 +508,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, }, FlavorsReservation: emptyUsage, - FlavorUsage: emptyUsage, + FlavorsUsage: emptyUsage, Flavors: []kueue.LocalQueueFlavorStatus{ { Name: flavorModelD, @@ -568,7 +568,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, }, FlavorsReservation: fullUsage, - FlavorUsage: emptyUsage, + FlavorsUsage: emptyUsage, Flavors: []kueue.LocalQueueFlavorStatus{ { Name: flavorModelD, @@ -607,7 +607,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, }, FlavorsReservation: fullUsage, - FlavorUsage: fullUsage, + FlavorsUsage: fullUsage, Flavors: []kueue.LocalQueueFlavorStatus{ { Name: flavorModelD, diff --git a/test/integration/singlecluster/controller/core/resourceflavor_controller_test.go b/test/integration/singlecluster/controller/core/resourceflavor_controller_test.go index 75f4b0d91a8..5f29de2a5aa 100644 --- a/test/integration/singlecluster/controller/core/resourceflavor_controller_test.go +++ b/test/integration/singlecluster/controller/core/resourceflavor_controller_test.go @@ -22,8 +22,8 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -83,7 +83,7 @@ var _ = ginkgo.Describe("ResourceFlavor controller", ginkgo.Ordered, ginkgo.Cont var cq kueue.ClusterQueue gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &cq)).To(gomega.Succeed()) - cq.Spec.Cohort = "foo-cohort" + cq.Spec.CohortName = "foo-cohort" g.Expect(k8sClient.Update(ctx, &cq)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go index a39a491c0db..db193caad22 100644 --- a/test/integration/singlecluster/controller/core/workload_controller_test.go +++ b/test/integration/singlecluster/controller/core/workload_controller_test.go @@ -30,13 +30,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" "sigs.k8s.io/kueue/pkg/util/slices" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" ) @@ -511,14 +511,14 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn g.Expect(k8sClient.Get(ctx, key, wl)).To(gomega.Succeed()) g.Expect(workload.IsAdmitted(wl)).To(gomega.BeFalse()) g.Expect(workload.IsActive(wl)).To(gomega.BeTrue()) - g.Expect(ptr.Deref(wl.Status.AccumulatedPastExexcutionTimeSeconds, 0)).To(gomega.BeNumerically(">", int32(0))) + g.Expect(ptr.Deref(wl.Status.AccumulatedPastExecutionTimeSeconds, 0)).To(gomega.BeNumerically(">", int32(0))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.By("setting the accumulated admission time closer to maximum", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, key, wl)).To(gomega.Succeed()) - wl.Status.AccumulatedPastExexcutionTimeSeconds = ptr.To(*wl.Spec.MaximumExecutionTimeSeconds - 1) + wl.Status.AccumulatedPastExecutionTimeSeconds = ptr.To(*wl.Spec.MaximumExecutionTimeSeconds - 1) g.Expect(k8sClient.Status().Update(ctx, wl)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/dra/dra_test.go b/test/integration/singlecluster/controller/dra/dra_test.go index f06511d5dbd..dfc491fe99a 100644 --- a/test/integration/singlecluster/controller/dra/dra_test.go +++ b/test/integration/singlecluster/controller/dra/dra_test.go @@ -28,9 +28,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/controller/jobframework/setup/setup_controllers_test.go b/test/integration/singlecluster/controller/jobframework/setup/setup_controllers_test.go index bebf9b232c4..d574426ecb2 100644 --- a/test/integration/singlecluster/controller/jobframework/setup/setup_controllers_test.go +++ b/test/integration/singlecluster/controller/jobframework/setup/setup_controllers_test.go @@ -27,10 +27,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" diff --git a/test/integration/singlecluster/controller/jobframework/setup/suite_test.go b/test/integration/singlecluster/controller/jobframework/setup/suite_test.go index 46b53da294b..c7208a02c1f 100644 --- a/test/integration/singlecluster/controller/jobframework/setup/suite_test.go +++ b/test/integration/singlecluster/controller/jobframework/setup/suite_test.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/kueue/pkg/controller/jobframework" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" ) @@ -51,5 +52,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { // The integration manager is a shared state and that after enabled a framework // will remain enabled until the end of the test suite. gomega.Expect(jobframework.SetupControllers(ctx, mgr, ginkgo.GinkgoLogr, opts...)).NotTo(gomega.HaveOccurred()) + + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } diff --git a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go index 377d256c73c..785bdc88e97 100644 --- a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go @@ -36,12 +36,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadaw "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingaw "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" utiltestingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" diff --git a/test/integration/singlecluster/controller/jobs/appwrapper/suite_test.go b/test/integration/singlecluster/controller/jobs/appwrapper/suite_test.go index 730f0ffe624..8aa20919ea3 100644 --- a/test/integration/singlecluster/controller/jobs/appwrapper/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/appwrapper/suite_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -83,6 +84,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = appwrapper.SetupAppWrapperWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } diff --git a/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go b/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go index 24a4c5f394d..3c26c88ffc8 100644 --- a/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go @@ -30,13 +30,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjaxjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/jaxjob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjaxjob "sigs.k8s.io/kueue/pkg/util/testingjobs/jaxjob" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" "sigs.k8s.io/kueue/pkg/workload" diff --git a/test/integration/singlecluster/controller/jobs/jaxjob/suite_test.go b/test/integration/singlecluster/controller/jobs/jaxjob/suite_test.go index d879ce5859a..4de147f17b0 100644 --- a/test/integration/singlecluster/controller/jobs/jaxjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/jaxjob/suite_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -83,6 +84,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = jaxjob.SetupJAXJobWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } diff --git a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go index ea1d39e9740..fcb78a34a0f 100644 --- a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go @@ -41,13 +41,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" "sigs.k8s.io/kueue/pkg/workload" diff --git a/test/integration/singlecluster/controller/jobs/job/suite_test.go b/test/integration/singlecluster/controller/jobs/job/suite_test.go index f61be92ecb3..340cfec83a2 100644 --- a/test/integration/singlecluster/controller/jobs/job/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/job/suite_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" ) @@ -82,6 +83,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = job.SetupWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) jobframework.EnableIntegration(job.FrameworkName) } } diff --git a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go index a3c02f888e8..ccb16d6c734 100644 --- a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go @@ -35,12 +35,12 @@ import ( jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjobset "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" "sigs.k8s.io/kueue/pkg/workload" diff --git a/test/integration/singlecluster/controller/jobs/jobset/suite_test.go b/test/integration/singlecluster/controller/jobs/jobset/suite_test.go index 33683d8352a..ce3897ac88a 100644 --- a/test/integration/singlecluster/controller/jobs/jobset/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/jobset/suite_test.go @@ -36,6 +36,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -82,6 +83,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = jobset.SetupJobSetWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } diff --git a/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go b/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go index 0574eff44b5..859cdf7f0c0 100644 --- a/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go +++ b/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go @@ -31,12 +31,12 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go index c2da406a23d..c016a46b056 100644 --- a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go @@ -34,12 +34,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadmpijob "sigs.k8s.io/kueue/pkg/controller/jobs/mpijob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingmpijob "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" diff --git a/test/integration/singlecluster/controller/jobs/mpijob/suite_test.go b/test/integration/singlecluster/controller/jobs/mpijob/suite_test.go index 8eaeb86145c..058be895c5f 100644 --- a/test/integration/singlecluster/controller/jobs/mpijob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/mpijob/suite_test.go @@ -38,6 +38,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -88,6 +89,8 @@ func managerSetup(setupJobManager bool, opts ...jobframework.Option) framework.M err = mpijob.SetupMPIJobWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) jobframework.EnableIntegration(mpijob.FrameworkName) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) if setupJobManager { jobReconciler, _ := job.NewReconciler( diff --git a/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go b/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go index 7675e25faf0..122e181d018 100644 --- a/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go @@ -28,12 +28,12 @@ import ( "k8s.io/utils/ptr" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadpaddlejob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/paddlejob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingpaddlejob "sigs.k8s.io/kueue/pkg/util/testingjobs/paddlejob" "sigs.k8s.io/kueue/test/integration/framework" diff --git a/test/integration/singlecluster/controller/jobs/paddlejob/suite_test.go b/test/integration/singlecluster/controller/jobs/paddlejob/suite_test.go index 2c61fa214e7..0c893adc218 100644 --- a/test/integration/singlecluster/controller/jobs/paddlejob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/paddlejob/suite_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -83,6 +84,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = paddlejob.SetupPaddleJobWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } diff --git a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go index 7ab3449e5bf..29dfc6fb854 100644 --- a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go @@ -34,14 +34,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" podcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/pod" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" diff --git a/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go b/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go index 31aa0e64e26..15ce2c835f0 100644 --- a/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go @@ -30,13 +30,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadpytorchjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/pytorchjob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingpytorchjob "sigs.k8s.io/kueue/pkg/util/testingjobs/pytorchjob" "sigs.k8s.io/kueue/pkg/workload" diff --git a/test/integration/singlecluster/controller/jobs/pytorchjob/suite_test.go b/test/integration/singlecluster/controller/jobs/pytorchjob/suite_test.go index 6a09357b788..f44b2f74029 100644 --- a/test/integration/singlecluster/controller/jobs/pytorchjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/pytorchjob/suite_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -83,6 +84,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = pytorchjob.SetupPyTorchJobWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } diff --git a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go index 6f876cefb39..99fa3afd456 100644 --- a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go @@ -35,13 +35,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadraycluster "sigs.k8s.io/kueue/pkg/controller/jobs/raycluster" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingraycluster "sigs.k8s.io/kueue/pkg/util/testingjobs/raycluster" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" "sigs.k8s.io/kueue/pkg/workload" diff --git a/test/integration/singlecluster/controller/jobs/raycluster/suite_test.go b/test/integration/singlecluster/controller/jobs/raycluster/suite_test.go index 260d47c17b0..2ba8754a204 100644 --- a/test/integration/singlecluster/controller/jobs/raycluster/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/raycluster/suite_test.go @@ -36,6 +36,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobs/raycluster" "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -85,6 +86,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = raycluster.SetupRayClusterWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) jobframework.EnableIntegration(rayjob.FrameworkName) } } @@ -101,6 +104,9 @@ func managerAndSchedulerSetup(opts ...jobframework.Option) framework.ManagerSetu failedCtrl, err := core.SetupControllers(mgr, queues, cCache, &config.Configuration{}) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) + err = raycluster.SetupIndexes(ctx, mgr.GetFieldIndexer()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) r, _ := raycluster.NewReconciler(ctx, mgr.GetClient(), mgr.GetFieldIndexer(), diff --git a/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go b/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go index 2a49df48248..ef321691037 100644 --- a/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go @@ -32,12 +32,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadrayjob "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/controller/jobs/rayjob/suite_test.go b/test/integration/singlecluster/controller/jobs/rayjob/suite_test.go index b62b1e02517..25872f262ad 100644 --- a/test/integration/singlecluster/controller/jobs/rayjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/rayjob/suite_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -82,6 +83,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = rayjob.SetupRayJobWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } @@ -99,6 +102,9 @@ func managerAndSchedulerSetup(opts ...jobframework.Option) framework.ManagerSetu failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) + err = rayjob.SetupIndexes(ctx, mgr.GetFieldIndexer()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) r, _ := rayjob.NewReconciler(ctx, mgr.GetClient(), mgr.GetFieldIndexer(), diff --git a/test/integration/singlecluster/controller/jobs/tfjob/suite_test.go b/test/integration/singlecluster/controller/jobs/tfjob/suite_test.go index ed90ef2b868..0f34192c681 100644 --- a/test/integration/singlecluster/controller/jobs/tfjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/tfjob/suite_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -83,6 +84,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = tfjob.SetupTFJobWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } diff --git a/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go b/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go index 9505179327a..9c0cc2a4c7a 100644 --- a/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go @@ -28,12 +28,12 @@ import ( "k8s.io/utils/ptr" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadtfjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/tfjob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingtfjob "sigs.k8s.io/kueue/pkg/util/testingjobs/tfjob" "sigs.k8s.io/kueue/test/integration/framework" diff --git a/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go b/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go index 74c130e70c4..fc70e4dc4e7 100644 --- a/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/trainjob/suite_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -83,6 +84,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = trainjob.SetupTrainJobWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } diff --git a/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go b/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go index d4f914fb90e..bcb55822838 100644 --- a/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go @@ -34,12 +34,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadtrainjob "sigs.k8s.io/kueue/pkg/controller/jobs/trainjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" diff --git a/test/integration/singlecluster/controller/jobs/xgboostjob/suite_test.go b/test/integration/singlecluster/controller/jobs/xgboostjob/suite_test.go index 54b1646269a..0d9e371a231 100644 --- a/test/integration/singlecluster/controller/jobs/xgboostjob/suite_test.go +++ b/test/integration/singlecluster/controller/jobs/xgboostjob/suite_test.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/tas" tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/webhooks" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -83,6 +84,8 @@ func managerSetup(opts ...jobframework.Option) framework.ManagerSetup { gomega.Expect(err).NotTo(gomega.HaveOccurred()) err = xgboostjob.SetupXGBoostJobWebhook(mgr, opts...) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } } diff --git a/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go b/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go index a54790d7b44..68aa61668dd 100644 --- a/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go @@ -28,12 +28,12 @@ import ( "k8s.io/utils/ptr" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadxgboostjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/xgboostjob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingxgboostjob "sigs.k8s.io/kueue/pkg/util/testingjobs/xgboostjob" "sigs.k8s.io/kueue/test/integration/framework" diff --git a/test/integration/singlecluster/conversion/conversions_v1beta2_test.go b/test/integration/singlecluster/conversion/conversions_v1beta1_test.go similarity index 96% rename from test/integration/singlecluster/conversion/conversions_v1beta2_test.go rename to test/integration/singlecluster/conversion/conversions_v1beta1_test.go index 17e2d1dda49..bc8ffd30813 100644 --- a/test/integration/singlecluster/conversion/conversions_v1beta2_test.go +++ b/test/integration/singlecluster/conversion/conversions_v1beta1_test.go @@ -28,10 +28,10 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/controller/constants" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" @@ -114,7 +114,7 @@ var _ = ginkgo.Describe("v1beta2 conversions", ginkgo.Ordered, ginkgo.ContinueOn gomega.Eventually(func(g gomega.Gomega) { var updatedCq kueue.ClusterQueue g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueues[0]), &updatedCq)).To(gomega.Succeed()) - g.Expect(updatedCq.Spec.CohortName).Should(gomega.Equal(kueue.CohortReference("cohort"))) + g.Expect(updatedCq.Spec.Cohort).Should(gomega.Equal(kueue.CohortReference("cohort"))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("await for the LocalQueue to be ready") @@ -151,7 +151,7 @@ var _ = ginkgo.Describe("v1beta2 conversions", ginkgo.Ordered, ginkgo.ContinueOn }, }, FlavorsReservation: emptyUsage, - FlavorsUsage: emptyUsage, + FlavorUsage: emptyUsage, }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors"))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -191,7 +191,7 @@ var _ = ginkgo.Describe("v1beta2 conversions", ginkgo.Ordered, ginkgo.ContinueOn AdmittedWorkloads: 1, PendingWorkloads: 0, FlavorsReservation: partUsage, - FlavorsUsage: partUsage, + FlavorUsage: partUsage, }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors", "Conditions"))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -239,7 +239,7 @@ var _ = ginkgo.Describe("v1beta2 conversions", ginkgo.Ordered, ginkgo.ContinueOn }, }, FlavorsReservation: fullUsage, - FlavorsUsage: fullUsage, + FlavorUsage: fullUsage, }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors"))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -289,7 +289,7 @@ var _ = ginkgo.Describe("v1beta2 conversions", ginkgo.Ordered, ginkgo.ContinueOn Message: "Can submit new workloads to localQueue", }, }, - }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors", "FlavorsReservation", "FlavorsUsage"))) + }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors", "FlavorsReservation", "FlavorUsage"))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("Creating workload1") @@ -308,7 +308,7 @@ var _ = ginkgo.Describe("v1beta2 conversions", ginkgo.Ordered, ginkgo.ContinueOn ReservingWorkloads: 1, AdmittedWorkloads: 1, PendingWorkloads: 0, - }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors", "FlavorsReservation", "FlavorsUsage", "Conditions"))) + }, util.IgnoreConditionTimestampsAndObservedGeneration, cmpopts.IgnoreFields(kueue.LocalQueueStatus{}, "Flavors", "FlavorsReservation", "FlavorUsage", "Conditions"))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("Creating workload2") @@ -329,7 +329,7 @@ var _ = ginkgo.Describe("v1beta2 conversions", ginkgo.Ordered, ginkgo.ContinueOn g.Expect(apimeta.IsStatusConditionTrue(updatedWl.Status.Conditions, kueue.WorkloadQuotaReserved)).Should(gomega.BeFalse()) g.Expect(apimeta.IsStatusConditionTrue(updatedWl.Status.Conditions, kueue.WorkloadEvicted)).Should(gomega.BeTrue()) g.Expect(updatedWl.Spec.MaximumExecutionTimeSeconds).ShouldNot(gomega.BeNil()) - g.Expect(updatedWl.Status.AccumulatedPastExecutionTimeSeconds).ShouldNot(gomega.BeNil()) + g.Expect(updatedWl.Status.AccumulatedPastExexcutionTimeSeconds).ShouldNot(gomega.BeNil()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/integration/singlecluster/importer/importer_test.go b/test/integration/singlecluster/importer/importer_test.go index 7168888a326..0db62d51557 100644 --- a/test/integration/singlecluster/importer/importer_test.go +++ b/test/integration/singlecluster/importer/importer_test.go @@ -17,10 +17,15 @@ limitations under the License. package importer import ( + "context" + "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/component-base/metrics/testutil" "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" @@ -99,7 +104,8 @@ var _ = ginkgo.Describe("Importer", func() { gomega.Expect(k8sClient.Get(ctx, wl1LookupKey, wl1)).To(gomega.Succeed()) gomega.Expect(k8sClient.Get(ctx, wl2LookupKey, wl2)).To(gomega.Succeed()) - util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, wl1, wl2) + ginkgo.By("Verify workload2 is correct") + exepectWorkladsToBeAdmitted(ctx, k8sClient, wl1, wl2) }) wl1UID := wl1.UID @@ -108,7 +114,7 @@ var _ = ginkgo.Describe("Importer", func() { ginkgo.By("Starting kueue, the cluster queue status should account for the imported Workloads", func() { fwk.StartManager(ctx, cfg, managerAndSchedulerSetup) - util.ExpectClusterQueueStatusMetric(cq, metrics.CQStatusActive) + expectClusterQueueStatusMetric(cq, metrics.CQStatusActive) gomega.Eventually(func(g gomega.Gomega) { updatedQueue := &kueue.ClusterQueue{} g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cq), updatedQueue)).To(gomega.Succeed()) @@ -135,15 +141,15 @@ var _ = ginkgo.Describe("Importer", func() { g.Expect(k8sClient.Get(ctx, wl3LookupKey, wl3)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) - util.ExpectWorkloadsToBePending(ctx, k8sClient, wl3) - util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, wl1, wl2) + expectWorkloadsToBePending(ctx, k8sClient, wl3) + exepectWorkladsToBeAdmitted(ctx, k8sClient, wl1, wl2) }) ginkgo.By("By finishing an imported pod, the new one's Workload should be admitted", func() { util.SetPodsPhase(ctx, k8sClient, corev1.PodSucceeded, pod2) util.ExpectWorkloadToFinish(ctx, k8sClient, wl2LookupKey) - util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, wl1, wl3) + exepectWorkladsToBeAdmitted(ctx, k8sClient, wl1, wl3) }) ginkgo.By("Checking the imported Workloads are not recreated", func() { @@ -155,3 +161,50 @@ var _ = ginkgo.Describe("Importer", func() { }) }) }) + +func exepectWorkladsToBeAdmitted(ctx context.Context, k8sClient client.Client, wls ...*kueue.Workload) { + gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { + admitted := 0 + var updatedWorkload kueue.Workload + for _, wl := range wls { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedWorkload)).To(gomega.Succeed()) + if apimeta.IsStatusConditionTrue(updatedWorkload.Status.Conditions, kueue.WorkloadAdmitted) { + admitted++ + } + } + g.Expect(admitted).Should(gomega.Equal(len(wls)), "Not enough workloads are admitted") + }, util.Timeout, util.Interval).Should(gomega.Succeed()) +} + +func expectWorkloadsToBePending(ctx context.Context, k8sClient client.Client, wls ...*kueue.Workload) { + gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { + pending := 0 + var updatedWorkload kueue.Workload + for _, wl := range wls { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedWorkload)).To(gomega.Succeed()) + cond := apimeta.FindStatusCondition(updatedWorkload.Status.Conditions, kueue.WorkloadQuotaReserved) + if cond == nil { + continue + } + if cond.Status == metav1.ConditionFalse && cond.Reason == "Pending" { + pending++ + } + } + g.Expect(pending).Should(gomega.Equal(len(wls)), "Not enough workloads are pending") + }, util.Timeout, util.Interval).Should(gomega.Succeed()) +} + +func expectClusterQueueStatusMetric(cq *kueue.ClusterQueue, status metrics.ClusterQueueStatus) { + for i, s := range metrics.CQStatuses { + var wantV float64 + if metrics.CQStatuses[i] == status { + wantV = 1 + } + metric := metrics.ClusterQueueByStatus.WithLabelValues(cq.Name, string(s)) + gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { + v, err := testutil.GetGaugeMetricValue(metric) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(v).Should(gomega.Equal(wantV), "cluster_queue_status with status=%s", s) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + } +} diff --git a/test/integration/singlecluster/kueuectl/create_test.go b/test/integration/singlecluster/kueuectl/create_test.go index 4a5af61b82a..4e7f9a09e90 100644 --- a/test/integration/singlecluster/kueuectl/create_test.go +++ b/test/integration/singlecluster/kueuectl/create_test.go @@ -31,10 +31,10 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -168,7 +168,7 @@ var _ = ginkgo.Describe("Kueuectl Create", ginkgo.Ordered, ginkgo.ContinueOnFail gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cqName, Namespace: ns.Name}, &createdQueue)).To(gomega.Succeed()) g.Expect(createdQueue.Name).Should(gomega.Equal(cqName)) - g.Expect(createdQueue.Spec.Cohort).Should(gomega.BeEmpty()) + g.Expect(createdQueue.Spec.CohortName).Should(gomega.BeEmpty()) g.Expect(createdQueue.Spec.QueueingStrategy).Should(gomega.Equal(kueue.BestEffortFIFO)) g.Expect(*createdQueue.Spec.NamespaceSelector).Should(gomega.Equal(metav1.LabelSelector{})) g.Expect(createdQueue.Spec.Preemption.ReclaimWithinCohort).Should(gomega.Equal(kueue.PreemptionPolicyNever)) @@ -204,7 +204,7 @@ var _ = ginkgo.Describe("Kueuectl Create", ginkgo.Ordered, ginkgo.ContinueOnFail gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cqName, Namespace: ns.Name}, &createdQueue)).To(gomega.Succeed()) g.Expect(createdQueue.Name).Should(gomega.Equal(cqName)) - g.Expect(createdQueue.Spec.Cohort).Should(gomega.Equal(kueue.CohortReference("cohort"))) + g.Expect(createdQueue.Spec.CohortName).Should(gomega.Equal(kueue.CohortReference("cohort"))) g.Expect(createdQueue.Spec.QueueingStrategy).Should(gomega.Equal(kueue.StrictFIFO)) g.Expect(*createdQueue.Spec.NamespaceSelector).Should(gomega.Equal(metav1.LabelSelector{ MatchLabels: map[string]string{"fooX": "barX", "fooY": "barY"}, @@ -392,7 +392,7 @@ var _ = ginkgo.Describe("Kueuectl Create", ginkgo.Ordered, ginkgo.ContinueOnFail gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cqName, Namespace: ns.Name}, &createdQueue)).To(gomega.Succeed()) g.Expect(createdQueue.Name).Should(gomega.Equal(cqName)) - g.Expect(createdQueue.Spec.Cohort).Should(gomega.Equal(kueue.CohortReference("cohort"))) + g.Expect(createdQueue.Spec.CohortName).Should(gomega.Equal(kueue.CohortReference("cohort"))) g.Expect(createdQueue.Spec.QueueingStrategy).Should(gomega.Equal(kueue.StrictFIFO)) g.Expect(*createdQueue.Spec.NamespaceSelector).Should(gomega.Equal(metav1.LabelSelector{ MatchLabels: map[string]string{"fooX": "barX", "fooY": "barY"}, diff --git a/test/integration/singlecluster/kueuectl/list_test.go b/test/integration/singlecluster/kueuectl/list_test.go index 83ecfbce516..3afd2d5c178 100644 --- a/test/integration/singlecluster/kueuectl/list_test.go +++ b/test/integration/singlecluster/kueuectl/list_test.go @@ -28,10 +28,10 @@ import ( "k8s.io/cli-runtime/pkg/genericiooptions" testingclock "k8s.io/utils/clock/testing" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app" "sigs.k8s.io/kueue/cmd/kueuectl/app/list" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/kueuectl/passthrough_test.go b/test/integration/singlecluster/kueuectl/passthrough_test.go index e28cb4c6a25..99a638a19f0 100644 --- a/test/integration/singlecluster/kueuectl/passthrough_test.go +++ b/test/integration/singlecluster/kueuectl/passthrough_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/kueuectl/resume_test.go b/test/integration/singlecluster/kueuectl/resume_test.go index d844998e7be..43742da0bc8 100644 --- a/test/integration/singlecluster/kueuectl/resume_test.go +++ b/test/integration/singlecluster/kueuectl/resume_test.go @@ -28,9 +28,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/kueuectl/stop_test.go b/test/integration/singlecluster/kueuectl/stop_test.go index d0ce45e5b04..74f218fff0d 100644 --- a/test/integration/singlecluster/kueuectl/stop_test.go +++ b/test/integration/singlecluster/kueuectl/stop_test.go @@ -28,9 +28,9 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go b/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go index f963bcd9476..01d0d967613 100644 --- a/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go +++ b/test/integration/singlecluster/scheduler/fairsharing/fair_sharing_test.go @@ -30,11 +30,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" ) @@ -396,7 +396,7 @@ var _ = ginkgo.Describe("Scheduler", ginkgo.Ordered, ginkgo.ContinueOnFailure, f ginkgo.By("checking the weighted share metric") gomega.Eventually(func(g gomega.Gomega) { - metric := metrics.ClusterQueueWeightedShare.WithLabelValues(cqA.Name, string(cqA.Spec.Cohort)) + metric := metrics.ClusterQueueWeightedShare.WithLabelValues(cqA.Name, string(cqA.Spec.CohortName)) v, err := testutil.GetGaugeMetricValue(metric) g.Expect(err).ToNot(gomega.HaveOccurred()) g.Expect(math.IsNaN(v)).Should(gomega.BeTrue()) diff --git a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go index 1082d86e146..26e070038aa 100644 --- a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go +++ b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go @@ -31,8 +31,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" config "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" diff --git a/test/integration/singlecluster/scheduler/preemption_test.go b/test/integration/singlecluster/scheduler/preemption_test.go index 6d7fc69ed85..93325336f7f 100644 --- a/test/integration/singlecluster/scheduler/preemption_test.go +++ b/test/integration/singlecluster/scheduler/preemption_test.go @@ -28,11 +28,11 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/scheduler/preemption" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/pkg/workloadslicing" "sigs.k8s.io/kueue/test/util" diff --git a/test/integration/singlecluster/scheduler/scheduler_test.go b/test/integration/singlecluster/scheduler/scheduler_test.go index ff361cf67bd..2630ae906cf 100644 --- a/test/integration/singlecluster/scheduler/scheduler_test.go +++ b/test/integration/singlecluster/scheduler/scheduler_test.go @@ -29,12 +29,12 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" @@ -742,7 +742,7 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.It("Should re-enqueue by the delete event of workload belonging to the same Cohort", func() { fooCQ := utiltestingapi.MakeClusterQueue("foo-clusterqueue"). - Cohort(cq.Spec.Cohort). + Cohort(cq.Spec.CohortName). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("on-demand").Resource(corev1.ResourceCPU, "5").Obj()). Obj() util.MustCreate(ctx, k8sClient, fooCQ) @@ -827,7 +827,7 @@ var _ = ginkgo.Describe("Scheduler", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cq), updatedCq)).Should(gomega.Succeed()) - updatedCq.Spec.Cohort = "cohort" + updatedCq.Spec.CohortName = "cohort" updatedCq.Spec.ResourceGroups[0].Flavors[0].Resources[0] = kueue.ResourceQuota{ Name: corev1.ResourceCPU, NominalQuota: resource.MustParse("6"), @@ -1155,7 +1155,7 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.By("checking the workload gets admitted when a fallback ClusterQueue gets added") fallbackClusterQueue := utiltestingapi.MakeClusterQueue("fallback-cq"). - Cohort(prodCQ.Spec.Cohort). + Cohort(prodCQ.Spec.CohortName). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("spot-tainted").Resource(corev1.ResourceCPU, "5").Obj(), // prod-cq can't borrow this due to its borrowingLimit *utiltestingapi.MakeFlavorQuotas("on-demand").Resource(corev1.ResourceCPU, "5").Obj(), @@ -1481,7 +1481,7 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.By("cq switches and workload admitted") gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cq), cq)).Should(gomega.Succeed()) - cq.Spec.Cohort = "cohort" + cq.Spec.CohortName = "cohort" g.Expect(k8sClient.Update(ctx, cq)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) expectAdmission := utiltestingapi.MakeAdmission(cq.Name).PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName).Assignment(corev1.ResourceCPU, "on-demand", "10").Obj()).Obj() @@ -1571,7 +1571,7 @@ var _ = ginkgo.Describe("Scheduler", func() { ginkgo.By("checking the workload gets admitted when another ClusterQueue gets added") devCQ := utiltestingapi.MakeClusterQueue("dev-cq"). - Cohort(prodCQ.Spec.Cohort). + Cohort(prodCQ.Spec.CohortName). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("on-demand").Resource(corev1.ResourceCPU, "5", "", "4").Obj(), ).Obj() diff --git a/test/integration/singlecluster/scheduler/workload_controller_test.go b/test/integration/singlecluster/scheduler/workload_controller_test.go index ab1dc0b1788..cf3df01a4f9 100644 --- a/test/integration/singlecluster/scheduler/workload_controller_test.go +++ b/test/integration/singlecluster/scheduler/workload_controller_test.go @@ -27,10 +27,10 @@ import ( "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/slices" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" diff --git a/test/integration/singlecluster/tas/tas_test.go b/test/integration/singlecluster/tas/tas_test.go index 2473d8d45fa..f14338fcfa3 100644 --- a/test/integration/singlecluster/tas/tas_test.go +++ b/test/integration/singlecluster/tas/tas_test.go @@ -34,14 +34,14 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/admissionchecks/provisioning" "sigs.k8s.io/kueue/pkg/controller/tas" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utiltas "sigs.k8s.io/kueue/pkg/util/tas" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" diff --git a/test/integration/singlecluster/webhook/core/admissioncheck_test.go b/test/integration/singlecluster/webhook/core/admissioncheck_test.go index 239ebb08e11..4b1b3993638 100644 --- a/test/integration/singlecluster/webhook/core/admissioncheck_test.go +++ b/test/integration/singlecluster/webhook/core/admissioncheck_test.go @@ -28,9 +28,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/webhook/core/clusterqueue_test.go b/test/integration/singlecluster/webhook/core/clusterqueue_test.go index 92102439de5..284b7e4a6b9 100644 --- a/test/integration/singlecluster/webhook/core/clusterqueue_test.go +++ b/test/integration/singlecluster/webhook/core/clusterqueue_test.go @@ -31,9 +31,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/webhook/core/cohort_test.go b/test/integration/singlecluster/webhook/core/cohort_test.go index fe6c7d0fb9f..cf2e37e9fd8 100644 --- a/test/integration/singlecluster/webhook/core/cohort_test.go +++ b/test/integration/singlecluster/webhook/core/cohort_test.go @@ -29,9 +29,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/webhook/core/localqueue_test.go b/test/integration/singlecluster/webhook/core/localqueue_test.go index d9718e5e2b9..534caa64f9a 100644 --- a/test/integration/singlecluster/webhook/core/localqueue_test.go +++ b/test/integration/singlecluster/webhook/core/localqueue_test.go @@ -24,15 +24,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) const queueName = "queue-test" var _ = ginkgo.Describe("Queue validating webhook", ginkgo.Ordered, func() { + var _ = ginkgo.BeforeEach(func() { + ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "core-") + }) + var _ = ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { managerSetup(ctx, mgr) diff --git a/test/integration/singlecluster/webhook/core/resourceflavor_test.go b/test/integration/singlecluster/webhook/core/resourceflavor_test.go index a0423cd450e..e957d7910ed 100644 --- a/test/integration/singlecluster/webhook/core/resourceflavor_test.go +++ b/test/integration/singlecluster/webhook/core/resourceflavor_test.go @@ -28,9 +28,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/manager" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) diff --git a/test/integration/singlecluster/webhook/core/suite_test.go b/test/integration/singlecluster/webhook/core/suite_test.go index e140608b7fb..61dac107703 100644 --- a/test/integration/singlecluster/webhook/core/suite_test.go +++ b/test/integration/singlecluster/webhook/core/suite_test.go @@ -66,9 +66,6 @@ func managerSetup(ctx context.Context, mgr manager.Manager) { err := indexer.Setup(ctx, mgr.GetFieldIndexer()) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - failedWebhook, err := webhooks.Setup(mgr) - gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) - cCache := schdcache.New(mgr.GetClient()) queues := qcache.NewManager(mgr.GetClient(), cCache) @@ -77,4 +74,7 @@ func managerSetup(ctx context.Context, mgr manager.Manager) { failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) + + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) } diff --git a/test/integration/singlecluster/webhook/core/workload_test.go b/test/integration/singlecluster/webhook/core/workload_test.go index dabd56fa0fc..6ba2543e964 100644 --- a/test/integration/singlecluster/webhook/core/workload_test.go +++ b/test/integration/singlecluster/webhook/core/workload_test.go @@ -32,11 +32,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/pkg/workloadslicing" "sigs.k8s.io/kueue/test/util" @@ -49,15 +49,15 @@ const ( podSetsMaxItems = 8 ) -var _ = ginkgo.BeforeEach(func() { - ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "core-") -}) +var _ = ginkgo.Describe("Workload defaulting webhook", ginkgo.Ordered, func() { + var _ = ginkgo.BeforeEach(func() { + ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "core-") + }) -var _ = ginkgo.AfterEach(func() { - gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) -}) + var _ = ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) -var _ = ginkgo.Describe("Workload defaulting webhook", ginkgo.Ordered, func() { ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { managerSetup(ctx, mgr) @@ -114,6 +114,13 @@ var _ = ginkgo.Describe("Workload defaulting webhook", ginkgo.Ordered, func() { }) var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { + var _ = ginkgo.BeforeEach(func() { + ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "core-") + }) + + var _ = ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { managerSetup(ctx, mgr) @@ -1060,6 +1067,13 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }) var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher AllAtOnce", ginkgo.Ordered, func() { + var _ = ginkgo.BeforeEach(func() { + ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "core-") + }) + + var _ = ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { managerSetup(ctx, mgr) @@ -1164,6 +1178,12 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al }) var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Incremental", ginkgo.Ordered, func() { + var _ = ginkgo.BeforeEach(func() { + ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "core-") + }) + var _ = ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, func(ctx context.Context, mgr manager.Manager) { managerSetup(ctx, mgr) diff --git a/test/integration/singlecluster/webhook/jobs/jobset_webhook_test.go b/test/integration/singlecluster/webhook/jobs/jobset_webhook_test.go index dd645eddf59..788e9d20de3 100644 --- a/test/integration/singlecluster/webhook/jobs/jobset_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/jobset_webhook_test.go @@ -21,7 +21,7 @@ import ( "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" "sigs.k8s.io/kueue/pkg/util/testing" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" diff --git a/test/integration/singlecluster/webhook/jobs/paddlejob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/paddlejob_webhook_test.go index 945492ef5e3..1c01c4b81a5 100644 --- a/test/integration/singlecluster/webhook/jobs/paddlejob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/paddlejob_webhook_test.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/paddlejob" "sigs.k8s.io/kueue/pkg/util/testing" testingjobspaddlejob "sigs.k8s.io/kueue/pkg/util/testingjobs/paddlejob" diff --git a/test/integration/singlecluster/webhook/jobs/pytorchjob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/pytorchjob_webhook_test.go index b49334ae6da..ef756f82b61 100644 --- a/test/integration/singlecluster/webhook/jobs/pytorchjob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/pytorchjob_webhook_test.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/pytorchjob" "sigs.k8s.io/kueue/pkg/util/testing" testingjobspytorchjob "sigs.k8s.io/kueue/pkg/util/testingjobs/pytorchjob" diff --git a/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go b/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go index d6f1471dab0..4109aab05fb 100644 --- a/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go @@ -26,14 +26,14 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/raycluster" workloadrayjob "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob" "sigs.k8s.io/kueue/pkg/util/testing" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingraycluster "sigs.k8s.io/kueue/pkg/util/testingjobs/raycluster" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" "sigs.k8s.io/kueue/pkg/webhooks" diff --git a/test/integration/singlecluster/webhook/jobs/tfjob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/tfjob_webhook_test.go index 9cbb211cada..34eaad0a794 100644 --- a/test/integration/singlecluster/webhook/jobs/tfjob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/tfjob_webhook_test.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/tfjob" "sigs.k8s.io/kueue/pkg/util/testing" testingjobstfjob "sigs.k8s.io/kueue/pkg/util/testingjobs/tfjob" diff --git a/test/integration/singlecluster/webhook/jobs/xgboostjob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/xgboostjob_webhook_test.go index d6fa2145f2d..001360f90e9 100644 --- a/test/integration/singlecluster/webhook/jobs/xgboostjob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/xgboostjob_webhook_test.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/xgboostjob" "sigs.k8s.io/kueue/pkg/util/testing" testingjobsxgboostjob "sigs.k8s.io/kueue/pkg/util/testingjobs/xgboostjob" diff --git a/test/performance/scheduler/minimalkueue/main.go b/test/performance/scheduler/minimalkueue/main.go index df95d6bbc01..919750a583e 100644 --- a/test/performance/scheduler/minimalkueue/main.go +++ b/test/performance/scheduler/minimalkueue/main.go @@ -38,7 +38,7 @@ import ( configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueuealpha "sigs.k8s.io/kueue/apis/kueue/v1alpha1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" diff --git a/test/performance/scheduler/runner/controller/controller.go b/test/performance/scheduler/runner/controller/controller.go index e7caeccbae6..4fefd877deb 100644 --- a/test/performance/scheduler/runner/controller/controller.go +++ b/test/performance/scheduler/runner/controller/controller.go @@ -37,7 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/performance/scheduler/runner/generator" diff --git a/test/performance/scheduler/runner/generator/generator.go b/test/performance/scheduler/runner/generator/generator.go index 772d26b91d6..7a1540c759c 100644 --- a/test/performance/scheduler/runner/generator/generator.go +++ b/test/performance/scheduler/runner/generator/generator.go @@ -29,8 +29,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" ) const ( diff --git a/test/performance/scheduler/runner/generator/generator_test.go b/test/performance/scheduler/runner/generator/generator_test.go index 0f5b010b681..498bedd218b 100644 --- a/test/performance/scheduler/runner/generator/generator_test.go +++ b/test/performance/scheduler/runner/generator/generator_test.go @@ -23,7 +23,7 @@ import ( "github.com/google/go-cmp/cmp" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" ) func TestLoadConfig(t *testing.T) { diff --git a/test/performance/scheduler/runner/main.go b/test/performance/scheduler/runner/main.go index 1c67c8a3930..a866b37a187 100644 --- a/test/performance/scheduler/runner/main.go +++ b/test/performance/scheduler/runner/main.go @@ -47,7 +47,7 @@ import ( configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueuealpha "sigs.k8s.io/kueue/apis/kueue/v1alpha1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" "sigs.k8s.io/kueue/test/performance/scheduler/runner/controller" "sigs.k8s.io/kueue/test/performance/scheduler/runner/generator" diff --git a/test/performance/scheduler/runner/recorder/recorder.go b/test/performance/scheduler/runner/recorder/recorder.go index e0591492304..1e86c93517b 100644 --- a/test/performance/scheduler/runner/recorder/recorder.go +++ b/test/performance/scheduler/runner/recorder/recorder.go @@ -28,7 +28,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/yaml" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/test/performance/scheduler/runner/generator" ) @@ -451,7 +451,7 @@ func (r *Recorder) RecordCQState(cq *kueue.ClusterQueue) { Time: time.Now(), Name: kueue.ClusterQueueReference(cq.Name), ClassName: cq.Labels[generator.ClassLabel], - Cohort: cq.Spec.Cohort, + Cohort: cq.Spec.CohortName, UID: cq.UID, CPUReservation: cpuReserved, diff --git a/test/util/e2e.go b/test/util/e2e.go index 1132070d03f..07025aec8c5 100644 --- a/test/util/e2e.go +++ b/test/util/e2e.go @@ -50,8 +50,8 @@ import ( "sigs.k8s.io/yaml" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" + kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" kueueclientset "sigs.k8s.io/kueue/client-go/clientset/versioned" visibilityv1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta1" @@ -143,7 +143,7 @@ func CreateClientUsingCluster(kContext string) (client.WithWatch, *rest.Config, err = cmv1.AddToScheme(scheme.Scheme) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) - err = kueuev1beta2.AddToScheme(scheme.Scheme) + err = kueuev1beta1.AddToScheme(scheme.Scheme) gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) err = visibility.AddToScheme(scheme.Scheme) diff --git a/test/util/util.go b/test/util/util.go index e84704e5ba4..28498eb77e8 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -66,7 +66,7 @@ import ( jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/metrics" "sigs.k8s.io/kueue/pkg/scheduler/preemption" @@ -729,7 +729,7 @@ func ExpectClusterQueueStatusMetric(cq *kueue.ClusterQueue, status metrics.Clust } func ExpectClusterQueueWeightedShareMetric(cq *kueue.ClusterQueue, value float64) { - metric := metrics.ClusterQueueWeightedShare.WithLabelValues(cq.Name, string(cq.Spec.Cohort)) + metric := metrics.ClusterQueueWeightedShare.WithLabelValues(cq.Name, string(cq.Spec.CohortName)) gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { v, err := testutil.GetGaugeMetricValue(metric) g.Expect(err).ToNot(gomega.HaveOccurred()) @@ -756,7 +756,7 @@ func ExpectLocalQueueResourceReservationsMetric(queue *kueue.LocalQueue, flavorN } func ExpectCQResourceNominalQuota(cq *kueue.ClusterQueue, flavor, resource string, value float64) { - metric := metrics.ClusterQueueResourceNominalQuota.WithLabelValues(string(cq.Spec.Cohort), cq.Name, flavor, resource) + metric := metrics.ClusterQueueResourceNominalQuota.WithLabelValues(string(cq.Spec.CohortName), cq.Name, flavor, resource) gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { v, err := testutil.GetGaugeMetricValue(metric) g.Expect(err).ToNot(gomega.HaveOccurred()) @@ -765,7 +765,7 @@ func ExpectCQResourceNominalQuota(cq *kueue.ClusterQueue, flavor, resource strin } func ExpectCQResourceBorrowingQuota(cq *kueue.ClusterQueue, flavor, resource string, value float64) { - metric := metrics.ClusterQueueResourceBorrowingLimit.WithLabelValues(string(cq.Spec.Cohort), cq.Name, flavor, resource) + metric := metrics.ClusterQueueResourceBorrowingLimit.WithLabelValues(string(cq.Spec.CohortName), cq.Name, flavor, resource) gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { v, err := testutil.GetGaugeMetricValue(metric) g.Expect(err).ToNot(gomega.HaveOccurred()) @@ -774,7 +774,7 @@ func ExpectCQResourceBorrowingQuota(cq *kueue.ClusterQueue, flavor, resource str } func ExpectCQResourceReservations(cq *kueue.ClusterQueue, flavor, resource string, value float64) { - metric := metrics.ClusterQueueResourceReservations.WithLabelValues(string(cq.Spec.Cohort), cq.Name, flavor, resource) + metric := metrics.ClusterQueueResourceReservations.WithLabelValues(string(cq.Spec.CohortName), cq.Name, flavor, resource) gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { v, err := testutil.GetGaugeMetricValue(metric) g.Expect(err).ToNot(gomega.HaveOccurred()) diff --git a/test/util/util_scheduling.go b/test/util/util_scheduling.go index 8f98f0cb19e..d7d2e972201 100644 --- a/test/util/util_scheduling.go +++ b/test/util/util_scheduling.go @@ -27,7 +27,7 @@ import ( "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" - kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) From 4d627e622f466e5304a0fa4401c0a3fc7492f843 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Mon, 27 Oct 2025 18:11:36 +0530 Subject: [PATCH 032/119] Split preemptions unit tests. (#7403) --- .../preemption/preemption_fair_test.go | 954 +++++++ .../preemption_hierarchical_test.go | 1287 +++++++++ pkg/scheduler/preemption/preemption_test.go | 2441 +---------------- 3 files changed, 2383 insertions(+), 2299 deletions(-) create mode 100644 pkg/scheduler/preemption/preemption_fair_test.go create mode 100644 pkg/scheduler/preemption/preemption_hierarchical_test.go diff --git a/pkg/scheduler/preemption/preemption_fair_test.go b/pkg/scheduler/preemption/preemption_fair_test.go new file mode 100644 index 00000000000..38ef43f220d --- /dev/null +++ b/pkg/scheduler/preemption/preemption_fair_test.go @@ -0,0 +1,954 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preemption + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + clocktesting "k8s.io/utils/clock/testing" + "k8s.io/utils/ptr" + + config "sigs.k8s.io/kueue/apis/config/v1beta2" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" + "sigs.k8s.io/kueue/pkg/constants" + "sigs.k8s.io/kueue/pkg/scheduler/flavorassigner" + utilslices "sigs.k8s.io/kueue/pkg/util/slices" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + "sigs.k8s.io/kueue/pkg/workload" +) + +func TestFairPreemptions(t *testing.T) { + now := time.Now() + flavors := []*kueue.ResourceFlavor{ + utiltestingapi.MakeResourceFlavor("default").Obj(), + } + baseCQs := []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + BorrowWithinCohort: &kueue.BorrowWithinCohort{ + Policy: kueue.BorrowWithinCohortPolicyLowerPriority, + MaxPriorityThreshold: ptr.To[int32](-3), + }, + }). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + BorrowWithinCohort: &kueue.BorrowWithinCohort{ + Policy: kueue.BorrowWithinCohortPolicyLowerPriority, + MaxPriorityThreshold: ptr.To[int32](-3), + }, + }). + Obj(), + utiltestingapi.MakeClusterQueue("c"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + BorrowWithinCohort: &kueue.BorrowWithinCohort{ + Policy: kueue.BorrowWithinCohortPolicyLowerPriority, + MaxPriorityThreshold: ptr.To[int32](-3), + }, + }). + Obj(), + utiltestingapi.MakeClusterQueue("preemptible"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "0").Obj()). + Obj(), + } + unitWl := *utiltestingapi.MakeWorkload("unit", "").Request(corev1.ResourceCPU, "1") + cases := map[string]struct { + clusterQueues []*kueue.ClusterQueue + cohorts []*kueue.Cohort + strategies []config.PreemptionStrategy + admitted []kueue.Workload + incoming *kueue.Workload + targetCQ kueue.ClusterQueueReference + wantPreempted sets.Set[string] + }{ + "reclaim nominal from user using the most": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("c1").SimpleReserveQuota("c", "default", now).Obj(), + }, + incoming: unitWl.Clone().Name("c_incoming").Obj(), + targetCQ: "c", + wantPreempted: sets.New(targetKeyReason("/b1", kueue.InCohortFairSharingReason)), + }, + "can reclaim from queue using less, if taking the latest workload from user using the most isn't enough": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("a2", "").Request(corev1.ResourceCPU, "1").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("c_incoming", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), + targetCQ: "c", + wantPreempted: sets.New(targetKeyReason("/a1", kueue.InCohortFairSharingReason)), // attempts to preempt b1, but it's not enough. + }, + "reclaim borrowable quota from user using the most": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("c1").SimpleReserveQuota("c", "default", now).Obj(), + }, + incoming: unitWl.Clone().Name("a_incoming").Obj(), + targetCQ: "a", + wantPreempted: sets.New(targetKeyReason("/b1", kueue.InCohortFairSharingReason)), + }, + "preempt one from each CQ borrowing": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "0.5").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("a2", "").Request(corev1.ResourceCPU, "0.5").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("a3", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "0.5").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "0.5").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b3", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("c_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), + targetCQ: "c", + wantPreempted: sets.New( + targetKeyReason("/a1", kueue.InCohortFairSharingReason), + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + ), + }, + "can't preempt when everyone under nominal": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("c1").SimpleReserveQuota("c", "default", now).Obj(), + *unitWl.Clone().Name("c2").SimpleReserveQuota("c", "default", now).Obj(), + *unitWl.Clone().Name("c3").SimpleReserveQuota("c", "default", now).Obj(), + }, + incoming: unitWl.Clone().Name("c_incoming").Obj(), + targetCQ: "c", + }, + "can't preempt when it would switch the imbalance": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), + targetCQ: "a", + }, + "can preempt lower priority workloads from same CQ": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1_low").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2_low").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a4").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/a1_low", kueue.InClusterQueueReason), + targetKeyReason("/a2_low", kueue.InClusterQueueReason), + ), + }, + "can preempt a combination of same CQ and highest user": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a_low").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b6").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/a_low", kueue.InClusterQueueReason), + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + ), + }, + "preempt huge workload if there is no other option, as long as the target CQ gets a lower share": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "9").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), + targetCQ: "a", + wantPreempted: sets.New(targetKeyReason("/b1", kueue.InCohortFairSharingReason)), + }, + "can't preempt huge workload if the incoming is also huge": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "7").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "5").Obj(), + targetCQ: "a", + }, + "can't preempt 2 smaller workloads if the incoming is huge": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b3", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "6").Obj(), + targetCQ: "a", + }, + "preempt from target and others even if over nominal": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1_low", "").Priority(-1).Request(corev1.ResourceCPU, "2").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("a2_low", "").Priority(-1).Request(corev1.ResourceCPU, "1").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "4").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/a1_low", kueue.InClusterQueueReason), + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + ), + }, + "prefer to preempt workloads that don't make the target CQ have the biggest share": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "1").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b3", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("c1", "").Request(corev1.ResourceCPU, "1").SimpleReserveQuota("c", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "3.5").Obj(), + targetCQ: "a", + // It would have been possible to preempt "/b1" under rule S2-b, but S2-a was possible first. + wantPreempted: sets.New(targetKeyReason("/b2", kueue.InCohortFairSharingReason)), + }, + "preempt from different cluster queues if the end result has a smaller max share": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "2.5").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("c1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("c", "default", now).Obj(), + *utiltestingapi.MakeWorkload("c2", "").Request(corev1.ResourceCPU, "2.5").SimpleReserveQuota("c", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "3.5").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + targetKeyReason("/c1", kueue.InCohortFairSharingReason), + ), + }, + "scenario above does not flap": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "3.5").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "2.5").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("c2", "").Request(corev1.ResourceCPU, "2.5").SimpleReserveQuota("c", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("b_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), + targetCQ: "b", + }, + "cannot preempt if it would make the candidate CQ go under nominal after preempting one element": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("c1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("c", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "4").Obj(), + targetCQ: "a", + }, + // preemption.borrowWithinCohort does not affect how + // we handle Fair Sharing preemptions. Lower priority + // workloads are not preempted unless + // DominantResourceShare value indicates that they + // should be preempted. + "workloads under priority threshold not capriciously preempted": { + clusterQueues: baseCQs, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("preemptible1").Priority(-3).SimpleReserveQuota("preemptible", "default", now).Obj(), + *unitWl.Clone().Name("preemptible2").Priority(-3).SimpleReserveQuota("preemptible", "default", now).Obj(), + *unitWl.Clone().Name("preemptible3").Priority(-3).SimpleReserveQuota("preemptible", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), + targetCQ: "a", + wantPreempted: nil, + }, + "preempt lower priority first, even if big": { + clusterQueues: baseCQs, + strategies: []config.PreemptionStrategy{config.LessThanInitialShare}, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b_low", "").Priority(0).Request(corev1.ResourceCPU, "5").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b_high", "").Priority(1).Request(corev1.ResourceCPU, "1").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "1").Obj(), + targetCQ: "a", + wantPreempted: sets.New(targetKeyReason("/b_low", kueue.InCohortFairSharingReason)), + }, + "preempt workload that doesn't transfer the imbalance, even if high priority": { + clusterQueues: baseCQs, + strategies: []config.PreemptionStrategy{config.LessThanOrEqualToFinalShare}, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b_low", "").Priority(0).Request(corev1.ResourceCPU, "5").SimpleReserveQuota("b", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b_high", "").Priority(1).Request(corev1.ResourceCPU, "1").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "1").Obj(), + targetCQ: "a", + wantPreempted: sets.New(targetKeyReason("/b_high", kueue.InCohortFairSharingReason)), + }, + "CQ with higher weight can preempt more": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + FairWeight(resource.MustParse("2")). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("c"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b6").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + targetKeyReason("/b2", kueue.InCohortFairSharingReason), + ), + }, + "can preempt anything borrowing from CQ with 0 weight": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + FairWeight(resource.MustParse("0")). + Obj(), + utiltestingapi.MakeClusterQueue("c"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b6").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "3").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + targetKeyReason("/b2", kueue.InCohortFairSharingReason), + targetKeyReason("/b3", kueue.InCohortFairSharingReason), + ), + }, + "can't preempt nominal from CQ with 0 weight": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("all"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + FairWeight(resource.MustParse("0")). + Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: unitWl.Clone().Name("a_incoming").Obj(), + targetCQ: "a", + }, + "can't preempt nominal from Cohort with 0 weight": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("left-cq"). + Cohort("root"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "0").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("right-cq"). + Cohort("right-cohort"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "0").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + FairWeight(resource.MustParse("0")). + Obj(), + }, + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("right-cohort"). + FairWeight(resource.MustParse("0")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1").Obj()). + Parent("root").Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("right-1").SimpleReserveQuota("right-cq", "default", now).Obj(), + }, + incoming: unitWl.Clone().Name("left-1").Obj(), + wantPreempted: sets.New[string](), + targetCQ: "left-cq", + }, + "can preempt within cluster queue when no cohort": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), + }, + incoming: unitWl.Clone().Name("a_incoming").Priority(1000).Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/a1", kueue.InClusterQueueReason), + ), + }, + // Each Cohort provides 5 capacity, while each CQ + // provides 1 capacity. + // + // Here is a representation of the tree the 3 tuple + // is: (Usage,Quota,SubtreeQuota) + // + // ROOT(20,5,20) + // / | \ + // LEFT(5,5,7) c(5,1,1) RIGHT(10,5,7) + // / \ / \ + // a(0,1,1) b(5,1,1) d(5,1,1) e(5,1,1) + // + // We show how ClusterQueue a is able to preempt + // workloads in all of these ClusterQueues. We set + // FairWeight of a, and LEFT, to 2.0, to make this + // possible. We set FairWeight of e to 0.99, to make + // preemptions deterministic + "hierarchical preemption": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("LEFT"). + FairWeight(resource.MustParse("2")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("LEFT"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1").Obj()). + Obj(), + utiltestingapi.MakeClusterQueue("c"). + Cohort("ROOT"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1").Obj()). + Obj(), + utiltestingapi.MakeClusterQueue("d"). + Cohort("RIGHT"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1").Obj()). + Obj(), + utiltestingapi.MakeClusterQueue("e"). + Cohort("RIGHT"). + // for determinism, we slightly prefer preemptions from e + // compared to d. + FairWeight(resource.MustParse("0.99")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1").Obj()). + Obj(), + }, + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("ROOT"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "5").Obj()). + Obj(), + utiltestingapi.MakeCohort("LEFT"). + FairWeight(resource.MustParse("2")). + Parent("ROOT"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "5").Obj()). + Obj(), + utiltestingapi.MakeCohort("RIGHT"). + Parent("ROOT"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "5").Obj()). + Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("b1").Priority(1).SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b2").Priority(2).SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b3").Priority(3).SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b4").Priority(4).SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("b5").Priority(5).SimpleReserveQuota("b", "default", now).Obj(), + *unitWl.Clone().Name("c1").Priority(1).SimpleReserveQuota("c", "default", now).Obj(), + *unitWl.Clone().Name("c2").Priority(2).SimpleReserveQuota("c", "default", now).Obj(), + *unitWl.Clone().Name("c3").Priority(3).SimpleReserveQuota("c", "default", now).Obj(), + *unitWl.Clone().Name("c4").Priority(4).SimpleReserveQuota("c", "default", now).Obj(), + *unitWl.Clone().Name("c5").Priority(5).SimpleReserveQuota("c", "default", now).Obj(), + *unitWl.Clone().Name("d1").Priority(1).SimpleReserveQuota("d", "default", now).Obj(), + *unitWl.Clone().Name("d2").Priority(2).SimpleReserveQuota("d", "default", now).Obj(), + *unitWl.Clone().Name("d3").Priority(3).SimpleReserveQuota("d", "default", now).Obj(), + *unitWl.Clone().Name("d4").Priority(4).SimpleReserveQuota("d", "default", now).Obj(), + *unitWl.Clone().Name("d5").Priority(5).SimpleReserveQuota("d", "default", now).Obj(), + *unitWl.Clone().Name("e1").Priority(1).SimpleReserveQuota("e", "default", now).Obj(), + *unitWl.Clone().Name("e2").Priority(2).SimpleReserveQuota("e", "default", now).Obj(), + *unitWl.Clone().Name("e3").Priority(3).SimpleReserveQuota("e", "default", now).Obj(), + *unitWl.Clone().Name("e4").Priority(4).SimpleReserveQuota("e", "default", now).Obj(), + *unitWl.Clone().Name("e5").Priority(5).SimpleReserveQuota("e", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("unit", "").Request(corev1.ResourceCPU, "5").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + targetKeyReason("/b2", kueue.InCohortFairSharingReason), + targetKeyReason("/c1", kueue.InCohortFairSharingReason), + targetKeyReason("/c2", kueue.InCohortFairSharingReason), + targetKeyReason("/e1", kueue.InCohortFairSharingReason)), + }, + // though ClusterQueue b is borrowing, its Cohort is not, + // and therefore it cannot be preempted. + // ROOT + // / \ + // a(3,5,5) RIGHT(1,1,1) + // \ + // b(1,0,0) + // + // incoming workload to a, of size 5, must + // preempt its own workloads, despite its fair weight + // being high, and RIGHT/b having a low weight. + "borrowing cq in non-borrowing cohort is protected": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("ROOT"). + FairWeight(resource.MustParse("10")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "5").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("RIGHT"). + FairWeight(resource.MustParse("0.1")). + Obj(), + }, + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("ROOT").Obj(), + utiltestingapi.MakeCohort("RIGHT").Parent("ROOT"). + FairWeight(resource.MustParse("0.1")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "1").Obj()). + Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("b1").Priority(-1).SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("unit", "").Request(corev1.ResourceCPU, "5").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/a1", kueue.InClusterQueueReason), + targetKeyReason("/a2", kueue.InClusterQueueReason), + targetKeyReason("/a3", kueue.InClusterQueueReason), + ), + }, + // Preempting the small workload would bring + // RIGHT to a DRS of 0, and we can't even consider the + // higher priority workload. incoming workload to a, + // of size 4, must preempt its own workloads + // ROOT + // / \ + // a(3,5,5) RIGHT(4,3,3) + // \ + // b(4,0,0) + // + "forced to preempt within clusterqueue because borrowing workload too important": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("ROOT"). + FairWeight(resource.MustParse("10")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "5").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("RIGHT"). + FairWeight(resource.MustParse("0.1")). + Obj(), + }, + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("ROOT").Obj(), + utiltestingapi.MakeCohort("RIGHT").Parent("ROOT"). + FairWeight(resource.MustParse("0.1")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "3").Obj()). + Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("a1").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a2").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), + *unitWl.Clone().Name("a3").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), + *utiltestingapi.MakeWorkload("b1", "").Priority(100).Request(corev1.ResourceCPU, "4").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: utiltestingapi.MakeWorkload("unit", "").Request(corev1.ResourceCPU, "4").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/a1", kueue.InClusterQueueReason), + targetKeyReason("/a2", kueue.InClusterQueueReason), + targetKeyReason("/a3", kueue.InClusterQueueReason), + ), + }, + // ROOT + // / | \ + // A B C + // | | | + // AA BB CC + // | | | + // AAA BBB CCC + // | | | + // a b CCCC + // cq-a wants capacity, cq-b uses capacity, and + // Cohort-CCCC provides capacity + // + // Organization A is more important than organization + // B, indicated by relative FairSharing value, so the + // preemption is possible. + "deep preemption": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("AAA"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "0").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("BBB"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "0").Obj()). + Obj(), + }, + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("ROOT").Obj(), + utiltestingapi.MakeCohort("A").Parent("ROOT"). + // we are comparing + // almostLCAs=(A,B). In order + // for the preemption to + // occur, we indicate that A + // is more important than B. + FairWeight(resource.MustParse("1.01")). + Obj(), + utiltestingapi.MakeCohort("AA").Parent("A"). + Obj(), + utiltestingapi.MakeCohort("AAA").Parent("AA"). + Obj(), + utiltestingapi.MakeCohort("B").Parent("ROOT"). + FairWeight(resource.MustParse("0.99")). + Obj(), + utiltestingapi.MakeCohort("BB").Parent("B"). + Obj(), + utiltestingapi.MakeCohort("BBB").Parent("BB"). + Obj(), + utiltestingapi.MakeCohort("C").Parent("ROOT"). + Obj(), + utiltestingapi.MakeCohort("CC").Parent("C"). + Obj(), + utiltestingapi.MakeCohort("CCC").Parent("CC"). + Obj(), + utiltestingapi.MakeCohort("CCCC").Parent("CCC"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "1").Obj()). + Obj(), + }, + admitted: []kueue.Workload{ + unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Workload, + }, + incoming: unitWl.Clone().Name("a1").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + ), + }, + "cq with zero weight can reclaim nominal quota": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("ROOT"). + FairWeight(resource.MustParse("0.0")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("ROOT"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "0").Obj()). + FairWeight(resource.MustParse("1.0")). + Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: unitWl.Clone().Name("a1").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + ), + }, + "cohort with zero weight can reclaim nominal quota": { + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("a"). + Cohort("A"). + FairWeight(resource.MustParse("0.0")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "0").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("b"). + Cohort("ROOT"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "0").Obj()). + FairWeight(resource.MustParse("1.0")). + Obj(), + }, + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("A"). + Parent("ROOT"). + FairWeight(resource.MustParse("0.0")). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1").Obj()).Obj(), + }, + admitted: []kueue.Workload{ + *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), + }, + incoming: unitWl.Clone().Name("a1").Obj(), + targetCQ: "a", + wantPreempted: sets.New( + targetKeyReason("/b1", kueue.InCohortFairSharingReason), + ), + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ctx, log := utiltesting.ContextWithLog(t) + // Set name as UID so that candidates sorting is predictable. + for i := range tc.admitted { + tc.admitted[i].UID = types.UID(tc.admitted[i].Name) + } + cl := utiltesting.NewClientBuilder(). + WithLists(&kueue.WorkloadList{Items: tc.admitted}). + Build() + cqCache := schdcache.New(cl) + for _, flv := range flavors { + cqCache.AddOrUpdateResourceFlavor(log, flv) + } + for _, cq := range tc.clusterQueues { + if err := cqCache.AddClusterQueue(ctx, cq); err != nil { + t.Fatalf("Couldn't add ClusterQueue to cache: %v", err) + } + } + for _, cohort := range tc.cohorts { + if err := cqCache.AddOrUpdateCohort(cohort); err != nil { + t.Fatalf("Couldn't add Cohort to cache: %v", err) + } + } + + broadcaster := record.NewBroadcaster() + scheme := runtime.NewScheme() + recorder := broadcaster.NewRecorder(scheme, corev1.EventSource{Component: constants.AdmissionName}) + preemptor := New(cl, workload.Ordering{}, recorder, config.FairSharing{ + Enable: true, + PreemptionStrategies: tc.strategies, + }, false, clocktesting.NewFakeClock(now)) + + beforeSnapshot, err := cqCache.Snapshot(ctx) + if err != nil { + t.Fatalf("unexpected error while building snapshot: %v", err) + } + snapshotWorkingCopy, err := cqCache.Snapshot(ctx) + if err != nil { + t.Fatalf("unexpected error while building snapshot: %v", err) + } + wlInfo := workload.NewInfo(tc.incoming) + wlInfo.ClusterQueue = tc.targetCQ + targets := preemptor.GetTargets(log, *wlInfo, singlePodSetAssignment( + flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", Mode: flavorassigner.Preempt, + }, + }, + ), snapshotWorkingCopy) + gotTargets := sets.New(utilslices.Map(targets, func(t **Target) string { + return targetKeyReason(workload.Key((*t).WorkloadInfo.Obj), (*t).Reason) + })...) + if diff := cmp.Diff(tc.wantPreempted, gotTargets, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("Issued preemptions (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff(beforeSnapshot, snapshotWorkingCopy, snapCmpOpts); diff != "" { + t.Errorf("Snapshot was modified (-initial,+end):\n%s", diff) + } + }) + } +} diff --git a/pkg/scheduler/preemption/preemption_hierarchical_test.go b/pkg/scheduler/preemption/preemption_hierarchical_test.go new file mode 100644 index 00000000000..cec87bb50d0 --- /dev/null +++ b/pkg/scheduler/preemption/preemption_hierarchical_test.go @@ -0,0 +1,1287 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preemption + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + clocktesting "k8s.io/utils/clock/testing" + "k8s.io/utils/ptr" + + config "sigs.k8s.io/kueue/apis/config/v1beta2" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" + "sigs.k8s.io/kueue/pkg/constants" + "sigs.k8s.io/kueue/pkg/scheduler/flavorassigner" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + "sigs.k8s.io/kueue/pkg/workload" +) + +func TestHierarchicalPreemptions(t *testing.T) { + now := time.Now() + flavors := []*kueue.ResourceFlavor{ + utiltestingapi.MakeResourceFlavor("default").Obj(), + } + cases := map[string]struct { + clusterQueues []*kueue.ClusterQueue + cohorts []*kueue.Cohort + admitted []kueue.Workload + incoming *kueue.Workload + targetCQ kueue.ClusterQueueReference + assignment flavorassigner.Assignment + wantPreempted sets.Set[string] + }{ + // + // R + // / | + // C(2) q_borrowing(0) + // / + // q + "preempt with hierarchical advantage": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted2", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New(targetKeyReason("/admitted2", kueue.InCohortReclamationReason)), + }, + // + // R + // / | \ + // C(2) q_borrowing(0) q_nominal(2) + // / + // q + "avoid queues within nominal quota": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q_nominal"). + Cohort("r").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted1", ""). + Priority(-10). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_nominal"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted2", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New(targetKeyReason("/admitted2", kueue.InCohortReclamationReason)), + }, + // + // R(0) + // / | + // C(2) q_borrowing(0) + // / + // q(0) + "preempt multiple with hierarchical advantage": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted1", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted2", ""). + Priority(2). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New( + targetKeyReason("/admitted1", kueue.InCohortReclamationReason), + targetKeyReason("/admitted2", kueue.InCohortReclamationReason)), + }, + // + // R(0) + // / + // C(3) + // / \ + // q(0) q_same_cohort(0) + "preempt in cohort and own CQ": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + BorrowWithinCohort: &kueue.BorrowWithinCohort{ + Policy: kueue.BorrowWithinCohortPolicyLowerPriority, + MaxPriorityThreshold: ptr.To[int32](0), + }, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_same_cohort"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_not_preemptible", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_preemptible", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_own_queue", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(1). + Request(corev1.ResourceCPU, "2"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New( + targetKeyReason("/admitted_preemptible", kueue.InCohortReclaimWhileBorrowingReason), + targetKeyReason("/admitted_own_queue", kueue.InClusterQueueReason)), + }, + // + // R(0) + // / | + // C(2) q_borrowing(0) + // / + // q(0) + "prefer to preempt hierarchical candidate": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_queue", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New( + targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason)), + }, + // + // R(0) + // / | + // C(2) q_nominal(2) + // / \ + // q(0) q_same_cohort(0) + "forced to preempt priority candidate": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + BorrowWithinCohort: &kueue.BorrowWithinCohort{ + Policy: kueue.BorrowWithinCohortPolicyLowerPriority, + MaxPriorityThreshold: ptr.To[int32](0), + }, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_nominal"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_same_cohort"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_nominal", ""). + Priority(-10). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_nominal"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). + Priority(-1). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New( + targetKeyReason("/admitted_same_cohort", kueue.InCohortReclaimWhileBorrowingReason)), + }, + // + // R(0) + // / | + // C(2) q_borrowing(0) + // / \ + // q(4) q_same_cohort(0) + // + "incoming workload fits in CQ nominal quota": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "4").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_same_cohort"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(10). + Request(corev1.ResourceCPU, "3"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). + Priority(10). + Request(corev1.ResourceCPU, "3"). + ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "4"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New( + targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason), + targetKeyReason("/admitted_same_cohort", kueue.InCohortReclamationReason)), + }, + // + // R(1) + // / | + // C(4) q_borrowing(0) + // / \ + // q(0) q_same_cohort(0) + // + "preempt hierarchical and priority candidates": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, + BorrowWithinCohort: &kueue.BorrowWithinCohort{ + Policy: kueue.BorrowWithinCohortPolicyLowerPriority, + MaxPriorityThreshold: ptr.To[int32](0), + }, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_same_cohort"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(-1). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort_preemptible", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_not_preemptible", ""). + Priority(1). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "3"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New( + targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason), + targetKeyReason("/admitted_same_cohort_preemptible", kueue.InCohortReclaimWhileBorrowingReason)), + }, + // + // R(1) + // / | + // C(4) q_borrowing(0) + // / \ + // q(0) q_same_cohort(0) + // + "preempt hierarchical candidates and inside CQ": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "1"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, + BorrowWithinCohort: &kueue.BorrowWithinCohort{ + Policy: kueue.BorrowWithinCohortPolicyLowerPriority, + MaxPriorityThreshold: ptr.To[int32](0), + }, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_same_cohort"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(-1). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_queue_preemptible", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_not_preemptible", ""). + Priority(1). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "3"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New( + targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason), + targetKeyReason("/admitted_same_queue_preemptible", kueue.InClusterQueueReason)), + }, + // + // R(0) + // / | \ + // C(3) q_borrowing(0) q_nominal(2) + // / + // q(0) + "reclaim nominal quota from lowest priority workload, excluding non-borrowing": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_nominal"). + Cohort("r").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing_prio_8", ""). + Priority(8). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_prio_9", ""). + Priority(9). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_prio_10", ""). + Priority(9). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_nominal", ""). + Priority(-2). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_nominal"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New( + targetKeyReason("/admitted_borrowing_prio_8", kueue.InCohortReclamationReason)), + }, + // + // R + // / \ + // C(2) C_other(2) + // / \ | + // q(0) q_same_cohort(0) q_other(0) + "infeasible preemption all available workloads in pruned subtrees": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c_other"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q_other"). + Cohort("c_other").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_same_cohort"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_other_1", ""). + Priority(-10). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_other"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_other_2", ""). + Priority(-10). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_other"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New[string](), + }, + // + // R(3CPU, 0Gi) + // / | + // C(4CPU,4Gi) q_borrowing(0) + // / \ + // q(0) q_same_cohort(0) + "hiearchical preemption with multiple resources": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Resource(corev1.ResourceMemory, "4Gi"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_same_cohort"). + Cohort("c").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(0). + Request(corev1.ResourceCPU, "3"). + Request(corev1.ResourceMemory, "1Gi"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3"). + Assignment(corev1.ResourceMemory, "default", "1Gi").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "3Gi"). + ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "3Gi").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(-2). + Request(corev1.ResourceCPU, "2"). + Request(corev1.ResourceMemory, "1Gi"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + corev1.ResourceMemory: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New(targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason)), + }, + // + // R(0) + // / | + // C(2) q_borrowing(0) + // / \ + // q(0) q_same_cohort(0) + // + "prefer to preempt evicted workloads": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + BorrowWithinCohort: &kueue.BorrowWithinCohort{ + Policy: kueue.BorrowWithinCohortPolicyLowerPriority, + MaxPriorityThreshold: ptr.To[int32](0), + }, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_same_cohort"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(-10). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("evicted_same_cohort", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(now), + }). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New[string](), + }, + // + // R(0) + // / | + // C(2) q_borrowing(0) + // / + // q(3, lending limit 2) + // + "respect lending limits": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "2"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q"). + Cohort("c").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "3", "", "2").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + utiltestingapi.MakeClusterQueue("q_borrowing"). + Cohort("r").ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default").Obj()). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }).Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(0). + Request(corev1.ResourceCPU, "4"). + ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(-2). + Request(corev1.ResourceCPU, "5"). + Obj(), + targetCQ: "q", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New( + targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason)), + }, + // r + // / \ + // c11 c12 + // / | \ \ + // c21 c22 c23 q1 + // / | | | + // c31 c32 q3 q2 + // / | + // q5 q4 + // quotas: + // 4: c11, c12, c21, c22, c23, c32, c31 + // 0: q1, q3, q4, q5 + "reclaim in complex hierarchy": { + cohorts: []*kueue.Cohort{ + utiltestingapi.MakeCohort("r").Obj(), + utiltestingapi.MakeCohort("c11"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c12"). + Parent("r"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c21"). + Parent("c11"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c22"). + Parent("c11"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c23"). + Parent("c11"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c31"). + Parent("c21"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Obj()).Obj(), + utiltestingapi.MakeCohort("c32"). + Parent("c21"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "4"). + Obj()).Obj(), + }, + clusterQueues: []*kueue.ClusterQueue{ + utiltestingapi.MakeClusterQueue("q1"). + Cohort("c12"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("q2"). + Cohort("c23"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("q3"). + Cohort("c22"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("q4"). + Cohort("c32"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + utiltestingapi.MakeClusterQueue("q5"). + Cohort("c31"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), + }, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing_1", ""). + Priority(-6). + Request(corev1.ResourceCPU, "4"). + ReserveQuota(utiltestingapi.MakeAdmission("q1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_2", ""). + Priority(-5). + Request(corev1.ResourceCPU, "4"). + ReserveQuota(utiltestingapi.MakeAdmission("q1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_3", ""). + Priority(-9). + Request(corev1.ResourceCPU, "4"). + ReserveQuota(utiltestingapi.MakeAdmission("q2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_4", ""). + Priority(-10). + Request(corev1.ResourceCPU, "4"). + ReserveQuota(utiltestingapi.MakeAdmission("q2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_5", ""). + Priority(-4). + Request(corev1.ResourceCPU, "4"). + ReserveQuota(utiltestingapi.MakeAdmission("q3"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_6", ""). + Priority(-3). + Request(corev1.ResourceCPU, "3"). + ReserveQuota(utiltestingapi.MakeAdmission("q3"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_7", ""). + Priority(4). + Request(corev1.ResourceCPU, "2"). + ReserveQuota(utiltestingapi.MakeAdmission("q4"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_8", ""). + Priority(2). + Request(corev1.ResourceCPU, "3"). + ReserveQuota(utiltestingapi.MakeAdmission("q4"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). + Obj(), + }, + incoming: utiltestingapi.MakeWorkload("incoming", ""). + Priority(-2). + Request(corev1.ResourceCPU, "7"). + Obj(), + targetCQ: "q5", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + // only one of workloads from q2 will be preempted because + // after preempting the first one, the usage of cohort + // c23 will be back within nominal quota + wantPreempted: sets.New( + targetKeyReason("/admitted_borrowing_1", kueue.InCohortReclamationReason), + targetKeyReason("/admitted_borrowing_4", kueue.InCohortReclamationReason)), + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ctx, log := utiltesting.ContextWithLog(t) + cl := utiltesting.NewClientBuilder(). + WithLists(&kueue.WorkloadList{Items: tc.admitted}). + Build() + + cqCache := schdcache.New(cl) + for _, flv := range flavors { + cqCache.AddOrUpdateResourceFlavor(log, flv) + } + for _, cq := range tc.clusterQueues { + if err := cqCache.AddClusterQueue(ctx, cq); err != nil { + t.Fatalf("Couldn't add ClusterQueue to cache: %v", err) + } + } + for _, cohort := range tc.cohorts { + if err := cqCache.AddOrUpdateCohort(cohort); err != nil { + t.Fatalf("Couldn't add Cohort to cache: %v", err) + } + } + + var lock sync.Mutex + gotPreempted := sets.New[string]() + broadcaster := record.NewBroadcaster() + scheme := runtime.NewScheme() + if err := kueue.AddToScheme(scheme); err != nil { + t.Fatalf("Failed adding kueue scheme: %v", err) + } + recorder := broadcaster.NewRecorder(scheme, corev1.EventSource{Component: constants.AdmissionName}) + preemptor := New(cl, workload.Ordering{}, recorder, config.FairSharing{}, false, clocktesting.NewFakeClock(now)) + preemptor.applyPreemption = func(ctx context.Context, w *kueue.Workload, reason, _ string) error { + lock.Lock() + gotPreempted.Insert(targetKeyReason(workload.Key(w), reason)) + lock.Unlock() + return nil + } + beforeSnapshot, err := cqCache.Snapshot(ctx) + if err != nil { + t.Fatalf("unexpected error while building snapshot: %v", err) + } + // make a working copy of the snapshotWorkingCopy than preemption can temporarily modify + snapshotWorkingCopy, err := cqCache.Snapshot(ctx) + if err != nil { + t.Fatalf("unexpected error while building snapshot: %v", err) + } + wlInfo := workload.NewInfo(tc.incoming) + wlInfo.ClusterQueue = tc.targetCQ + targets := preemptor.GetTargets(log, *wlInfo, tc.assignment, snapshotWorkingCopy) + _, err = preemptor.IssuePreemptions(ctx, wlInfo, targets) + if err != nil { + t.Fatalf("Failed doing preemption") + } + if diff := cmp.Diff(tc.wantPreempted, gotPreempted, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("Issued preemptions (-want,+got):\n%s", diff) + } + if diff := cmp.Diff(beforeSnapshot, snapshotWorkingCopy, snapCmpOpts); diff != "" { + t.Errorf("Snapshot was modified (-initial,+end):\n%s", diff) + } + }) + } +} diff --git a/pkg/scheduler/preemption/preemption_test.go b/pkg/scheduler/preemption/preemption_test.go index 65c60f66329..067c3d40a96 100644 --- a/pkg/scheduler/preemption/preemption_test.go +++ b/pkg/scheduler/preemption/preemption_test.go @@ -27,10 +27,8 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" clocktesting "k8s.io/utils/clock/testing" @@ -2299,2321 +2297,166 @@ func TestPreemption(t *testing.T) { } } -func TestFairPreemptions(t *testing.T) { +func targetKeyReason(key workload.Reference, reason string) string { + return fmt.Sprintf("%s:%s", key, reason) +} +func TestCandidatesOrdering(t *testing.T) { now := time.Now() - flavors := []*kueue.ResourceFlavor{ - utiltestingapi.MakeResourceFlavor("default").Obj(), - } - baseCQs := []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - BorrowWithinCohort: &kueue.BorrowWithinCohort{ - Policy: kueue.BorrowWithinCohortPolicyLowerPriority, - MaxPriorityThreshold: ptr.To[int32](-3), - }, - }). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - BorrowWithinCohort: &kueue.BorrowWithinCohort{ - Policy: kueue.BorrowWithinCohortPolicyLowerPriority, - MaxPriorityThreshold: ptr.To[int32](-3), - }, - }). - Obj(), - utiltestingapi.MakeClusterQueue("c"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - BorrowWithinCohort: &kueue.BorrowWithinCohort{ - Policy: kueue.BorrowWithinCohortPolicyLowerPriority, - MaxPriorityThreshold: ptr.To[int32](-3), - }, - }). - Obj(), - utiltestingapi.MakeClusterQueue("preemptible"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "0").Obj()). - Obj(), - } - unitWl := *utiltestingapi.MakeWorkload("unit", "").Request(corev1.ResourceCPU, "1") + + preemptorCq := "preemptor" + + wlLowUsageLq := workload.NewInfo(utiltestingapi.MakeWorkload("low_lq_usage", ""). + Queue("low_usage_lq"). + ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). + Priority(1). + Obj()) + wlLowUsageLq.LocalQueueFSUsage = ptr.To(0.1) + + wlMidUsageLq := workload.NewInfo(utiltestingapi.MakeWorkload("mid_lq_usage", ""). + Queue("mid_usage_lq"). + ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). + Priority(10). + Obj()) + wlMidUsageLq.LocalQueueFSUsage = ptr.To(0.5) + + wlHighUsageLqDifCQ := workload.NewInfo(utiltestingapi.MakeWorkload("high_lq_usage_different_cq", ""). + Queue("high_usage_lq_different_cq"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("different_cq").Obj(), now). + Priority(1). + Obj()) + wlHighUsageLqDifCQ.LocalQueueFSUsage = ptr.To(1.0) + cases := map[string]struct { - clusterQueues []*kueue.ClusterQueue - cohorts []*kueue.Cohort - strategies []config.PreemptionStrategy - admitted []kueue.Workload - incoming *kueue.Workload - targetCQ kueue.ClusterQueueReference - wantPreempted sets.Set[string] + candidates []workload.Info + wantCandidates []workload.Reference + admissionFairSharingEnabled bool }{ - "reclaim nominal from user using the most": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("c1").SimpleReserveQuota("c", "default", now).Obj(), - }, - incoming: unitWl.Clone().Name("c_incoming").Obj(), - targetCQ: "c", - wantPreempted: sets.New(targetKeyReason("/b1", kueue.InCohortFairSharingReason)), - }, - "can reclaim from queue using less, if taking the latest workload from user using the most isn't enough": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("a2", "").Request(corev1.ResourceCPU, "1").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("c_incoming", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), - targetCQ: "c", - wantPreempted: sets.New(targetKeyReason("/a1", kueue.InCohortFairSharingReason)), // attempts to preempt b1, but it's not enough. - }, - "reclaim borrowable quota from user using the most": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("c1").SimpleReserveQuota("c", "default", now).Obj(), - }, - incoming: unitWl.Clone().Name("a_incoming").Obj(), - targetCQ: "a", - wantPreempted: sets.New(targetKeyReason("/b1", kueue.InCohortFairSharingReason)), - }, - "preempt one from each CQ borrowing": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "0.5").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("a2", "").Request(corev1.ResourceCPU, "0.5").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("a3", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "0.5").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "0.5").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b3", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("c_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), - targetCQ: "c", - wantPreempted: sets.New( - targetKeyReason("/a1", kueue.InCohortFairSharingReason), - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - ), - }, - "can't preempt when everyone under nominal": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("c1").SimpleReserveQuota("c", "default", now).Obj(), - *unitWl.Clone().Name("c2").SimpleReserveQuota("c", "default", now).Obj(), - *unitWl.Clone().Name("c3").SimpleReserveQuota("c", "default", now).Obj(), - }, - incoming: unitWl.Clone().Name("c_incoming").Obj(), - targetCQ: "c", - }, - "can't preempt when it would switch the imbalance": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), - targetCQ: "a", - }, - "can preempt lower priority workloads from same CQ": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1_low").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2_low").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a4").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/a1_low", kueue.InClusterQueueReason), - targetKeyReason("/a2_low", kueue.InClusterQueueReason), - ), - }, - "can preempt a combination of same CQ and highest user": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a_low").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b6").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/a_low", kueue.InClusterQueueReason), - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - ), - }, - "preempt huge workload if there is no other option, as long as the target CQ gets a lower share": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "9").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), - targetCQ: "a", - wantPreempted: sets.New(targetKeyReason("/b1", kueue.InCohortFairSharingReason)), - }, - "can't preempt huge workload if the incoming is also huge": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "7").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "5").Obj(), - targetCQ: "a", - }, - "can't preempt 2 smaller workloads if the incoming is huge": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b3", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "6").Obj(), - targetCQ: "a", - }, - "preempt from target and others even if over nominal": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a1_low", "").Priority(-1).Request(corev1.ResourceCPU, "2").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("a2_low", "").Priority(-1).Request(corev1.ResourceCPU, "1").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), + "workloads sorted by priority": { + candidates: []workload.Info{ + *workload.NewInfo(utiltestingapi.MakeWorkload("high", ""). + ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). + Priority(10). + Obj()), + *workload.NewInfo(utiltestingapi.MakeWorkload("low", ""). + ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). + Priority(-10). + Obj()), }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "4").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/a1_low", kueue.InClusterQueueReason), - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - ), + wantCandidates: []workload.Reference{"low", "high"}, }, - "prefer to preempt workloads that don't make the target CQ have the biggest share": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "1").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b3", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("c1", "").Request(corev1.ResourceCPU, "1").SimpleReserveQuota("c", "default", now).Obj(), + "evicted workload first": { + candidates: []workload.Info{ + *workload.NewInfo(utiltestingapi.MakeWorkload("other", ""). + ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). + Priority(10). + Obj()), + *workload.NewInfo(utiltestingapi.MakeWorkload("evicted", ""). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(now), + }). + Obj()), }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "3.5").Obj(), - targetCQ: "a", - // It would have been possible to preempt "/b1" under rule S2-b, but S2-a was possible first. - wantPreempted: sets.New(targetKeyReason("/b2", kueue.InCohortFairSharingReason)), + wantCandidates: []workload.Reference{"evicted", "other"}, }, - "preempt from different cluster queues if the end result has a smaller max share": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "2.5").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("c1", "").Request(corev1.ResourceCPU, "2").SimpleReserveQuota("c", "default", now).Obj(), - *utiltestingapi.MakeWorkload("c2", "").Request(corev1.ResourceCPU, "2.5").SimpleReserveQuota("c", "default", now).Obj(), + "workload from different CQ first": { + candidates: []workload.Info{ + *workload.NewInfo(utiltestingapi.MakeWorkload("preemptorCq", ""). + ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). + Priority(10). + Obj()), + *workload.NewInfo(utiltestingapi.MakeWorkload("other", ""). + ReserveQuotaAt(utiltestingapi.MakeAdmission("other").Obj(), now). + Priority(10). + Obj()), }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "3.5").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - targetKeyReason("/c1", kueue.InCohortFairSharingReason), - ), + wantCandidates: []workload.Reference{"other", "preemptorCq"}, }, - "scenario above does not flap": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "3.5").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "2.5").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("c2", "").Request(corev1.ResourceCPU, "2.5").SimpleReserveQuota("c", "default", now).Obj(), + "old workloads last": { + candidates: []workload.Info{ + *workload.NewInfo(utiltestingapi.MakeWorkload("older", ""). + ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now.Add(-time.Second)). + Obj()), + *workload.NewInfo(utiltestingapi.MakeWorkload("younger", ""). + ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now.Add(time.Second)). + Obj()), + *workload.NewInfo(utiltestingapi.MakeWorkload("current", ""). + ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). + Obj()), }, - incoming: utiltestingapi.MakeWorkload("b_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), - targetCQ: "b", + wantCandidates: []workload.Reference{"younger", "current", "older"}, }, - "cannot preempt if it would make the candidate CQ go under nominal after preempting one element": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("b1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b2", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("c1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("c", "default", now).Obj(), + "workloads with higher LQ usage first": { + candidates: []workload.Info{ + *wlLowUsageLq, + *wlMidUsageLq, }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "4").Obj(), - targetCQ: "a", + wantCandidates: []workload.Reference{"mid_lq_usage", "low_lq_usage"}, + admissionFairSharingEnabled: true, }, - // preemption.borrowWithinCohort does not affect how - // we handle Fair Sharing preemptions. Lower priority - // workloads are not preempted unless - // DominantResourceShare value indicates that they - // should be preempted. - "workloads under priority threshold not capriciously preempted": { - clusterQueues: baseCQs, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("preemptible1").Priority(-3).SimpleReserveQuota("preemptible", "default", now).Obj(), - *unitWl.Clone().Name("preemptible2").Priority(-3).SimpleReserveQuota("preemptible", "default", now).Obj(), - *unitWl.Clone().Name("preemptible3").Priority(-3).SimpleReserveQuota("preemptible", "default", now).Obj(), + "workloads from different CQ are sorted based on priority and timestamp": { + candidates: []workload.Info{ + *wlMidUsageLq, + *wlHighUsageLqDifCQ, }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), - targetCQ: "a", - wantPreempted: nil, + wantCandidates: []workload.Reference{"high_lq_usage_different_cq", "mid_lq_usage"}, + admissionFairSharingEnabled: true, + }} + + _, log := utiltesting.ContextWithLog(t) + for _, tc := range cases { + features.SetFeatureGateDuringTest(t, features.AdmissionFairSharing, tc.admissionFairSharingEnabled) + slices.SortFunc(tc.candidates, func(a, b workload.Info) int { + return preemptioncommon.CandidatesOrdering(log, tc.admissionFairSharingEnabled, &a, &b, kueue.ClusterQueueReference(preemptorCq), now) + }) + got := utilslices.Map(tc.candidates, func(c *workload.Info) workload.Reference { + return workload.Reference(c.Obj.Name) + }) + if diff := cmp.Diff(tc.wantCandidates, got); diff != "" { + t.Errorf("Sorted with wrong order (-want,+got):\n%s", diff) + } + } +} + +func singlePodSetAssignment(assignments flavorassigner.ResourceAssignment) flavorassigner.Assignment { + return flavorassigner.Assignment{ + PodSets: []flavorassigner.PodSetAssignment{{ + Name: kueue.DefaultPodSetName, + Flavors: assignments, + Count: 1, + }}, + } +} + +func TestPreemptionMessage(t *testing.T) { + cases := []struct { + preemptor *kueue.Workload + reason string + want string + }{ + { + preemptor: &kueue.Workload{}, + want: "Preempted to accommodate a workload (UID: UNKNOWN, JobUID: UNKNOWN) due to UNKNOWN", }, - "preempt lower priority first, even if big": { - clusterQueues: baseCQs, - strategies: []config.PreemptionStrategy{config.LessThanInitialShare}, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b_low", "").Priority(0).Request(corev1.ResourceCPU, "5").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b_high", "").Priority(1).Request(corev1.ResourceCPU, "1").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "1").Obj(), - targetCQ: "a", - wantPreempted: sets.New(targetKeyReason("/b_low", kueue.InCohortFairSharingReason)), + { + preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid"}}, + want: "Preempted to accommodate a workload (UID: uid, JobUID: UNKNOWN) due to UNKNOWN", }, - "preempt workload that doesn't transfer the imbalance, even if high priority": { - clusterQueues: baseCQs, - strategies: []config.PreemptionStrategy{config.LessThanOrEqualToFinalShare}, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a1", "").Request(corev1.ResourceCPU, "3").SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b_low", "").Priority(0).Request(corev1.ResourceCPU, "5").SimpleReserveQuota("b", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b_high", "").Priority(1).Request(corev1.ResourceCPU, "1").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "1").Obj(), - targetCQ: "a", - wantPreempted: sets.New(targetKeyReason("/b_high", kueue.InCohortFairSharingReason)), + { + preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid", Labels: map[string]string{controllerconstants.JobUIDLabel: "juid"}}}, + want: "Preempted to accommodate a workload (UID: uid, JobUID: juid) due to UNKNOWN", }, - "CQ with higher weight can preempt more": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - FairWeight(resource.MustParse("2")). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("c"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b6").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "2").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - targetKeyReason("/b2", kueue.InCohortFairSharingReason), - ), - }, - "can preempt anything borrowing from CQ with 0 weight": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - FairWeight(resource.MustParse("0")). - Obj(), - utiltestingapi.MakeClusterQueue("c"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b4").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b5").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b6").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("a_incoming", "").Request(corev1.ResourceCPU, "3").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - targetKeyReason("/b2", kueue.InCohortFairSharingReason), - targetKeyReason("/b3", kueue.InCohortFairSharingReason), - ), - }, - "can't preempt nominal from CQ with 0 weight": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("all"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - FairWeight(resource.MustParse("0")). - Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: unitWl.Clone().Name("a_incoming").Obj(), - targetCQ: "a", - }, - "can't preempt nominal from Cohort with 0 weight": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("left-cq"). - Cohort("root"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "0").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("right-cq"). - Cohort("right-cohort"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "0").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - FairWeight(resource.MustParse("0")). - Obj(), - }, - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("right-cohort"). - FairWeight(resource.MustParse("0")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1").Obj()). - Parent("root").Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("right-1").SimpleReserveQuota("right-cq", "default", now).Obj(), - }, - incoming: unitWl.Clone().Name("left-1").Obj(), - wantPreempted: sets.New[string](), - targetCQ: "left-cq", - }, - "can preempt within cluster queue when no cohort": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - }). - Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").SimpleReserveQuota("a", "default", now).Obj(), - }, - incoming: unitWl.Clone().Name("a_incoming").Priority(1000).Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/a1", kueue.InClusterQueueReason), - ), - }, - // Each Cohort provides 5 capacity, while each CQ - // provides 1 capacity. - // - // Here is a representation of the tree the 3 tuple - // is: (Usage,Quota,SubtreeQuota) - // - // ROOT(20,5,20) - // / | \ - // LEFT(5,5,7) c(5,1,1) RIGHT(10,5,7) - // / \ / \ - // a(0,1,1) b(5,1,1) d(5,1,1) e(5,1,1) - // - // We show how ClusterQueue a is able to preempt - // workloads in all of these ClusterQueues. We set - // FairWeight of a, and LEFT, to 2.0, to make this - // possible. We set FairWeight of e to 0.99, to make - // preemptions deterministic - "hierarchical preemption": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("LEFT"). - FairWeight(resource.MustParse("2")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("LEFT"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1").Obj()). - Obj(), - utiltestingapi.MakeClusterQueue("c"). - Cohort("ROOT"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1").Obj()). - Obj(), - utiltestingapi.MakeClusterQueue("d"). - Cohort("RIGHT"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1").Obj()). - Obj(), - utiltestingapi.MakeClusterQueue("e"). - Cohort("RIGHT"). - // for determinism, we slightly prefer preemptions from e - // compared to d. - FairWeight(resource.MustParse("0.99")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1").Obj()). - Obj(), - }, - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("ROOT"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "5").Obj()). - Obj(), - utiltestingapi.MakeCohort("LEFT"). - FairWeight(resource.MustParse("2")). - Parent("ROOT"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "5").Obj()). - Obj(), - utiltestingapi.MakeCohort("RIGHT"). - Parent("ROOT"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "5").Obj()). - Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("b1").Priority(1).SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b2").Priority(2).SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b3").Priority(3).SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b4").Priority(4).SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("b5").Priority(5).SimpleReserveQuota("b", "default", now).Obj(), - *unitWl.Clone().Name("c1").Priority(1).SimpleReserveQuota("c", "default", now).Obj(), - *unitWl.Clone().Name("c2").Priority(2).SimpleReserveQuota("c", "default", now).Obj(), - *unitWl.Clone().Name("c3").Priority(3).SimpleReserveQuota("c", "default", now).Obj(), - *unitWl.Clone().Name("c4").Priority(4).SimpleReserveQuota("c", "default", now).Obj(), - *unitWl.Clone().Name("c5").Priority(5).SimpleReserveQuota("c", "default", now).Obj(), - *unitWl.Clone().Name("d1").Priority(1).SimpleReserveQuota("d", "default", now).Obj(), - *unitWl.Clone().Name("d2").Priority(2).SimpleReserveQuota("d", "default", now).Obj(), - *unitWl.Clone().Name("d3").Priority(3).SimpleReserveQuota("d", "default", now).Obj(), - *unitWl.Clone().Name("d4").Priority(4).SimpleReserveQuota("d", "default", now).Obj(), - *unitWl.Clone().Name("d5").Priority(5).SimpleReserveQuota("d", "default", now).Obj(), - *unitWl.Clone().Name("e1").Priority(1).SimpleReserveQuota("e", "default", now).Obj(), - *unitWl.Clone().Name("e2").Priority(2).SimpleReserveQuota("e", "default", now).Obj(), - *unitWl.Clone().Name("e3").Priority(3).SimpleReserveQuota("e", "default", now).Obj(), - *unitWl.Clone().Name("e4").Priority(4).SimpleReserveQuota("e", "default", now).Obj(), - *unitWl.Clone().Name("e5").Priority(5).SimpleReserveQuota("e", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("unit", "").Request(corev1.ResourceCPU, "5").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - targetKeyReason("/b2", kueue.InCohortFairSharingReason), - targetKeyReason("/c1", kueue.InCohortFairSharingReason), - targetKeyReason("/c2", kueue.InCohortFairSharingReason), - targetKeyReason("/e1", kueue.InCohortFairSharingReason)), - }, - // though ClusterQueue b is borrowing, its Cohort is not, - // and therefore it cannot be preempted. - // ROOT - // / \ - // a(3,5,5) RIGHT(1,1,1) - // \ - // b(1,0,0) - // - // incoming workload to a, of size 5, must - // preempt its own workloads, despite its fair weight - // being high, and RIGHT/b having a low weight. - "borrowing cq in non-borrowing cohort is protected": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("ROOT"). - FairWeight(resource.MustParse("10")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "5").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - }). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("RIGHT"). - FairWeight(resource.MustParse("0.1")). - Obj(), - }, - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("ROOT").Obj(), - utiltestingapi.MakeCohort("RIGHT").Parent("ROOT"). - FairWeight(resource.MustParse("0.1")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "1").Obj()). - Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("b1").Priority(-1).SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("unit", "").Request(corev1.ResourceCPU, "5").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/a1", kueue.InClusterQueueReason), - targetKeyReason("/a2", kueue.InClusterQueueReason), - targetKeyReason("/a3", kueue.InClusterQueueReason), - ), - }, - // Preempting the small workload would bring - // RIGHT to a DRS of 0, and we can't even consider the - // higher priority workload. incoming workload to a, - // of size 4, must preempt its own workloads - // ROOT - // / \ - // a(3,5,5) RIGHT(4,3,3) - // \ - // b(4,0,0) - // - "forced to preempt within clusterqueue because borrowing workload too important": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("ROOT"). - FairWeight(resource.MustParse("10")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "5").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - }). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("RIGHT"). - FairWeight(resource.MustParse("0.1")). - Obj(), - }, - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("ROOT").Obj(), - utiltestingapi.MakeCohort("RIGHT").Parent("ROOT"). - FairWeight(resource.MustParse("0.1")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "3").Obj()). - Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("a1").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a2").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), - *unitWl.Clone().Name("a3").Priority(-1).SimpleReserveQuota("a", "default", now).Obj(), - *utiltestingapi.MakeWorkload("b1", "").Priority(100).Request(corev1.ResourceCPU, "4").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: utiltestingapi.MakeWorkload("unit", "").Request(corev1.ResourceCPU, "4").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/a1", kueue.InClusterQueueReason), - targetKeyReason("/a2", kueue.InClusterQueueReason), - targetKeyReason("/a3", kueue.InClusterQueueReason), - ), - }, - // ROOT - // / | \ - // A B C - // | | | - // AA BB CC - // | | | - // AAA BBB CCC - // | | | - // a b CCCC - // cq-a wants capacity, cq-b uses capacity, and - // Cohort-CCCC provides capacity - // - // Organization A is more important than organization - // B, indicated by relative FairSharing value, so the - // preemption is possible. - "deep preemption": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("AAA"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "0").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("BBB"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "0").Obj()). - Obj(), - }, - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("ROOT").Obj(), - utiltestingapi.MakeCohort("A").Parent("ROOT"). - // we are comparing - // almostLCAs=(A,B). In order - // for the preemption to - // occur, we indicate that A - // is more important than B. - FairWeight(resource.MustParse("1.01")). - Obj(), - utiltestingapi.MakeCohort("AA").Parent("A"). - Obj(), - utiltestingapi.MakeCohort("AAA").Parent("AA"). - Obj(), - utiltestingapi.MakeCohort("B").Parent("ROOT"). - FairWeight(resource.MustParse("0.99")). - Obj(), - utiltestingapi.MakeCohort("BB").Parent("B"). - Obj(), - utiltestingapi.MakeCohort("BBB").Parent("BB"). - Obj(), - utiltestingapi.MakeCohort("C").Parent("ROOT"). - Obj(), - utiltestingapi.MakeCohort("CC").Parent("C"). - Obj(), - utiltestingapi.MakeCohort("CCC").Parent("CC"). - Obj(), - utiltestingapi.MakeCohort("CCCC").Parent("CCC"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "1").Obj()). - Obj(), - }, - admitted: []kueue.Workload{ - unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Workload, - }, - incoming: unitWl.Clone().Name("a1").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - ), - }, - "cq with zero weight can reclaim nominal quota": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("ROOT"). - FairWeight(resource.MustParse("0.0")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("ROOT"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "0").Obj()). - FairWeight(resource.MustParse("1.0")). - Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: unitWl.Clone().Name("a1").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - ), - }, - "cohort with zero weight can reclaim nominal quota": { - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("a"). - Cohort("A"). - FairWeight(resource.MustParse("0.0")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "0").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("b"). - Cohort("ROOT"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "0").Obj()). - FairWeight(resource.MustParse("1.0")). - Obj(), - }, - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("A"). - Parent("ROOT"). - FairWeight(resource.MustParse("0.0")). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1").Obj()).Obj(), - }, - admitted: []kueue.Workload{ - *unitWl.Clone().Name("b1").SimpleReserveQuota("b", "default", now).Obj(), - }, - incoming: unitWl.Clone().Name("a1").Obj(), - targetCQ: "a", - wantPreempted: sets.New( - targetKeyReason("/b1", kueue.InCohortFairSharingReason), - ), - }, - } - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - ctx, log := utiltesting.ContextWithLog(t) - // Set name as UID so that candidates sorting is predictable. - for i := range tc.admitted { - tc.admitted[i].UID = types.UID(tc.admitted[i].Name) - } - cl := utiltesting.NewClientBuilder(). - WithLists(&kueue.WorkloadList{Items: tc.admitted}). - Build() - cqCache := schdcache.New(cl) - for _, flv := range flavors { - cqCache.AddOrUpdateResourceFlavor(log, flv) - } - for _, cq := range tc.clusterQueues { - if err := cqCache.AddClusterQueue(ctx, cq); err != nil { - t.Fatalf("Couldn't add ClusterQueue to cache: %v", err) - } - } - for _, cohort := range tc.cohorts { - if err := cqCache.AddOrUpdateCohort(cohort); err != nil { - t.Fatalf("Couldn't add Cohort to cache: %v", err) - } - } - - broadcaster := record.NewBroadcaster() - scheme := runtime.NewScheme() - recorder := broadcaster.NewRecorder(scheme, corev1.EventSource{Component: constants.AdmissionName}) - preemptor := New(cl, workload.Ordering{}, recorder, config.FairSharing{ - Enable: true, - PreemptionStrategies: tc.strategies, - }, false, clocktesting.NewFakeClock(now)) - - beforeSnapshot, err := cqCache.Snapshot(ctx) - if err != nil { - t.Fatalf("unexpected error while building snapshot: %v", err) - } - snapshotWorkingCopy, err := cqCache.Snapshot(ctx) - if err != nil { - t.Fatalf("unexpected error while building snapshot: %v", err) - } - wlInfo := workload.NewInfo(tc.incoming) - wlInfo.ClusterQueue = tc.targetCQ - targets := preemptor.GetTargets(log, *wlInfo, singlePodSetAssignment( - flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", Mode: flavorassigner.Preempt, - }, - }, - ), snapshotWorkingCopy) - gotTargets := sets.New(utilslices.Map(targets, func(t **Target) string { - return targetKeyReason(workload.Key((*t).WorkloadInfo.Obj), (*t).Reason) - })...) - if diff := cmp.Diff(tc.wantPreempted, gotTargets, cmpopts.EquateEmpty()); diff != "" { - t.Errorf("Issued preemptions (-want,+got):\n%s", diff) - } - - if diff := cmp.Diff(beforeSnapshot, snapshotWorkingCopy, snapCmpOpts); diff != "" { - t.Errorf("Snapshot was modified (-initial,+end):\n%s", diff) - } - }) - } -} - -func targetKeyReason(key workload.Reference, reason string) string { - return fmt.Sprintf("%s:%s", key, reason) -} -func TestCandidatesOrdering(t *testing.T) { - now := time.Now() - - preemptorCq := "preemptor" - - wlLowUsageLq := workload.NewInfo(utiltestingapi.MakeWorkload("low_lq_usage", ""). - Queue("low_usage_lq"). - ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). - Priority(1). - Obj()) - wlLowUsageLq.LocalQueueFSUsage = ptr.To(0.1) - - wlMidUsageLq := workload.NewInfo(utiltestingapi.MakeWorkload("mid_lq_usage", ""). - Queue("mid_usage_lq"). - ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). - Priority(10). - Obj()) - wlMidUsageLq.LocalQueueFSUsage = ptr.To(0.5) - - wlHighUsageLqDifCQ := workload.NewInfo(utiltestingapi.MakeWorkload("high_lq_usage_different_cq", ""). - Queue("high_usage_lq_different_cq"). - ReserveQuotaAt(utiltestingapi.MakeAdmission("different_cq").Obj(), now). - Priority(1). - Obj()) - wlHighUsageLqDifCQ.LocalQueueFSUsage = ptr.To(1.0) - - cases := map[string]struct { - candidates []workload.Info - wantCandidates []workload.Reference - admissionFairSharingEnabled bool - }{ - "workloads sorted by priority": { - candidates: []workload.Info{ - *workload.NewInfo(utiltestingapi.MakeWorkload("high", ""). - ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). - Priority(10). - Obj()), - *workload.NewInfo(utiltestingapi.MakeWorkload("low", ""). - ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). - Priority(-10). - Obj()), - }, - wantCandidates: []workload.Reference{"low", "high"}, - }, - "evicted workload first": { - candidates: []workload.Info{ - *workload.NewInfo(utiltestingapi.MakeWorkload("other", ""). - ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). - Priority(10). - Obj()), - *workload.NewInfo(utiltestingapi.MakeWorkload("evicted", ""). - SetOrReplaceCondition(metav1.Condition{ - Type: kueue.WorkloadEvicted, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(now), - }). - Obj()), - }, - wantCandidates: []workload.Reference{"evicted", "other"}, - }, - "workload from different CQ first": { - candidates: []workload.Info{ - *workload.NewInfo(utiltestingapi.MakeWorkload("preemptorCq", ""). - ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). - Priority(10). - Obj()), - *workload.NewInfo(utiltestingapi.MakeWorkload("other", ""). - ReserveQuotaAt(utiltestingapi.MakeAdmission("other").Obj(), now). - Priority(10). - Obj()), - }, - wantCandidates: []workload.Reference{"other", "preemptorCq"}, - }, - "old workloads last": { - candidates: []workload.Info{ - *workload.NewInfo(utiltestingapi.MakeWorkload("older", ""). - ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now.Add(-time.Second)). - Obj()), - *workload.NewInfo(utiltestingapi.MakeWorkload("younger", ""). - ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now.Add(time.Second)). - Obj()), - *workload.NewInfo(utiltestingapi.MakeWorkload("current", ""). - ReserveQuotaAt(utiltestingapi.MakeAdmission(preemptorCq).Obj(), now). - Obj()), - }, - wantCandidates: []workload.Reference{"younger", "current", "older"}, - }, - "workloads with higher LQ usage first": { - candidates: []workload.Info{ - *wlLowUsageLq, - *wlMidUsageLq, - }, - wantCandidates: []workload.Reference{"mid_lq_usage", "low_lq_usage"}, - admissionFairSharingEnabled: true, - }, - "workloads from different CQ are sorted based on priority and timestamp": { - candidates: []workload.Info{ - *wlMidUsageLq, - *wlHighUsageLqDifCQ, - }, - wantCandidates: []workload.Reference{"high_lq_usage_different_cq", "mid_lq_usage"}, - admissionFairSharingEnabled: true, - }} - - _, log := utiltesting.ContextWithLog(t) - for _, tc := range cases { - features.SetFeatureGateDuringTest(t, features.AdmissionFairSharing, tc.admissionFairSharingEnabled) - slices.SortFunc(tc.candidates, func(a, b workload.Info) int { - return preemptioncommon.CandidatesOrdering(log, tc.admissionFairSharingEnabled, &a, &b, kueue.ClusterQueueReference(preemptorCq), now) - }) - got := utilslices.Map(tc.candidates, func(c *workload.Info) workload.Reference { - return workload.Reference(c.Obj.Name) - }) - if diff := cmp.Diff(tc.wantCandidates, got); diff != "" { - t.Errorf("Sorted with wrong order (-want,+got):\n%s", diff) - } - } -} - -func singlePodSetAssignment(assignments flavorassigner.ResourceAssignment) flavorassigner.Assignment { - return flavorassigner.Assignment{ - PodSets: []flavorassigner.PodSetAssignment{{ - Name: kueue.DefaultPodSetName, - Flavors: assignments, - Count: 1, - }}, - } -} - -func TestPreemptionMessage(t *testing.T) { - cases := []struct { - preemptor *kueue.Workload - reason string - want string - }{ - { - preemptor: &kueue.Workload{}, - want: "Preempted to accommodate a workload (UID: UNKNOWN, JobUID: UNKNOWN) due to UNKNOWN", - }, - { - preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid"}}, - want: "Preempted to accommodate a workload (UID: uid, JobUID: UNKNOWN) due to UNKNOWN", - }, - { - preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid", Labels: map[string]string{controllerconstants.JobUIDLabel: "juid"}}}, - want: "Preempted to accommodate a workload (UID: uid, JobUID: juid) due to UNKNOWN", - }, - { - preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid", Labels: map[string]string{controllerconstants.JobUIDLabel: "juid"}}}, - reason: kueue.InClusterQueueReason, - want: "Preempted to accommodate a workload (UID: uid, JobUID: juid) due to prioritization in the ClusterQueue", - }, - } - for _, tc := range cases { - got := preemptionMessage(tc.preemptor, tc.reason) - if got != tc.want { - t.Errorf("preemptionMessage(preemptor=kueue.Workload{UID:%v, Labels:%v}, reason=%q) returned %q, want %q", tc.preemptor.UID, tc.preemptor.Labels, tc.reason, got, tc.want) - } - } -} - -func TestHierarchicalPreemptions(t *testing.T) { - now := time.Now() - flavors := []*kueue.ResourceFlavor{ - utiltestingapi.MakeResourceFlavor("default").Obj(), - } - cases := map[string]struct { - clusterQueues []*kueue.ClusterQueue - cohorts []*kueue.Cohort - admitted []kueue.Workload - incoming *kueue.Workload - targetCQ kueue.ClusterQueueReference - assignment flavorassigner.Assignment - wantPreempted sets.Set[string] - }{ - // - // R - // / | - // C(2) q_borrowing(0) - // / - // q - "preempt with hierarchical advantage": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted2", ""). - Priority(0). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "2"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/admitted2", kueue.InCohortReclamationReason)), - }, - // - // R - // / | \ - // C(2) q_borrowing(0) q_nominal(2) - // / - // q - "avoid queues within nominal quota": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q_nominal"). - Cohort("r").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted1", ""). - Priority(-10). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_nominal"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted2", ""). - Priority(0). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "2"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/admitted2", kueue.InCohortReclamationReason)), - }, - // - // R(0) - // / | - // C(2) q_borrowing(0) - // / - // q(0) - "preempt multiple with hierarchical advantage": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted1", ""). - Priority(1). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted2", ""). - Priority(2). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "2"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New( - targetKeyReason("/admitted1", kueue.InCohortReclamationReason), - targetKeyReason("/admitted2", kueue.InCohortReclamationReason)), - }, - // - // R(0) - // / - // C(3) - // / \ - // q(0) q_same_cohort(0) - "preempt in cohort and own CQ": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - BorrowWithinCohort: &kueue.BorrowWithinCohort{ - Policy: kueue.BorrowWithinCohortPolicyLowerPriority, - MaxPriorityThreshold: ptr.To[int32](0), - }, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_same_cohort"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_not_preemptible", ""). - Priority(1). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_preemptible", ""). - Priority(0). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_own_queue", ""). - Priority(-1). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(1). - Request(corev1.ResourceCPU, "2"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New( - targetKeyReason("/admitted_preemptible", kueue.InCohortReclaimWhileBorrowingReason), - targetKeyReason("/admitted_own_queue", kueue.InClusterQueueReason)), - }, - // - // R(0) - // / | - // C(2) q_borrowing(0) - // / - // q(0) - "prefer to preempt hierarchical candidate": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_borrowing", ""). - Priority(1). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_same_queue", ""). - Priority(-2). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "1"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason)), - }, - // - // R(0) - // / | - // C(2) q_nominal(2) - // / \ - // q(0) q_same_cohort(0) - "forced to preempt priority candidate": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - BorrowWithinCohort: &kueue.BorrowWithinCohort{ - Policy: kueue.BorrowWithinCohortPolicyLowerPriority, - MaxPriorityThreshold: ptr.To[int32](0), - }, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_nominal"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_same_cohort"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_nominal", ""). - Priority(-10). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_nominal"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). - Priority(-1). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "2"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New( - targetKeyReason("/admitted_same_cohort", kueue.InCohortReclaimWhileBorrowingReason)), - }, - // - // R(0) - // / | - // C(2) q_borrowing(0) - // / \ - // q(4) q_same_cohort(0) - // - "incoming workload fits in CQ nominal quota": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU, "4").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_same_cohort"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_borrowing", ""). - Priority(10). - Request(corev1.ResourceCPU, "3"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). - Priority(10). - Request(corev1.ResourceCPU, "3"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "4"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason), - targetKeyReason("/admitted_same_cohort", kueue.InCohortReclamationReason)), - }, - // - // R(1) - // / | - // C(4) q_borrowing(0) - // / \ - // q(0) q_same_cohort(0) - // - "preempt hierarchical and priority candidates": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, - BorrowWithinCohort: &kueue.BorrowWithinCohort{ - Policy: kueue.BorrowWithinCohortPolicyLowerPriority, - MaxPriorityThreshold: ptr.To[int32](0), - }, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_same_cohort"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_borrowing", ""). - Priority(-1). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_same_cohort_preemptible", ""). - Priority(-1). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_not_preemptible", ""). - Priority(1). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "3"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason), - targetKeyReason("/admitted_same_cohort_preemptible", kueue.InCohortReclaimWhileBorrowingReason)), - }, - // - // R(1) - // / | - // C(4) q_borrowing(0) - // / \ - // q(0) q_same_cohort(0) - // - "preempt hierarchical candidates and inside CQ": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "1"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, - BorrowWithinCohort: &kueue.BorrowWithinCohort{ - Policy: kueue.BorrowWithinCohortPolicyLowerPriority, - MaxPriorityThreshold: ptr.To[int32](0), - }, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_same_cohort"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_borrowing", ""). - Priority(-1). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_same_queue_preemptible", ""). - Priority(-1). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_not_preemptible", ""). - Priority(1). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "3"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason), - targetKeyReason("/admitted_same_queue_preemptible", kueue.InClusterQueueReason)), - }, - // - // R(0) - // / | \ - // C(3) q_borrowing(0) q_nominal(2) - // / - // q(0) - "reclaim nominal quota from lowest priority workload, excluding non-borrowing": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_nominal"). - Cohort("r").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_borrowing_prio_8", ""). - Priority(8). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_prio_9", ""). - Priority(9). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_prio_10", ""). - Priority(9). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_nominal", ""). - Priority(-2). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_nominal"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "1"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing_prio_8", kueue.InCohortReclamationReason)), - }, - // - // R - // / \ - // C(2) C_other(2) - // / \ | - // q(0) q_same_cohort(0) q_other(0) - "infeasible preemption all available workloads in pruned subtrees": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c_other"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q_other"). - Cohort("c_other").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_same_cohort"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_other_1", ""). - Priority(-10). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_other"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_other_2", ""). - Priority(-10). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_other"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). - Priority(0). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "2"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New[string](), - }, - // - // R(3CPU, 0Gi) - // / | - // C(4CPU,4Gi) q_borrowing(0) - // / \ - // q(0) q_same_cohort(0) - "hiearchical preemption with multiple resources": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Resource(corev1.ResourceMemory, "4Gi"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_same_cohort"). - Cohort("c").ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_borrowing", ""). - Priority(0). - Request(corev1.ResourceCPU, "3"). - Request(corev1.ResourceMemory, "1Gi"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3"). - Assignment(corev1.ResourceMemory, "default", "1Gi").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). - Priority(-2). - Request(corev1.ResourceCPU, "1"). - Request(corev1.ResourceMemory, "3Gi"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1"). - Assignment(corev1.ResourceMemory, "default", "3Gi").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(-2). - Request(corev1.ResourceCPU, "2"). - Request(corev1.ResourceMemory, "1Gi"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - corev1.ResourceMemory: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason)), - }, - // - // R(0) - // / | - // C(2) q_borrowing(0) - // / \ - // q(0) q_same_cohort(0) - // - "prefer to preempt evicted workloads": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - BorrowWithinCohort: &kueue.BorrowWithinCohort{ - Policy: kueue.BorrowWithinCohortPolicyLowerPriority, - MaxPriorityThreshold: ptr.To[int32](0), - }, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_same_cohort"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_borrowing", ""). - Priority(-10). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("evicted_same_cohort", ""). - Priority(-1). - Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). - SetOrReplaceCondition(metav1.Condition{ - Type: kueue.WorkloadEvicted, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(now), - }). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(0). - Request(corev1.ResourceCPU, "1"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New[string](), - }, - // - // R(0) - // / | - // C(2) q_borrowing(0) - // / - // q(3, lending limit 2) - // - "respect lending limits": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "2"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q"). - Cohort("c").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "3", "", "2").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - utiltestingapi.MakeClusterQueue("q_borrowing"). - Cohort("r").ResourceGroup( - *utiltestingapi.MakeFlavorQuotas("default").Obj()). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }).Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_borrowing", ""). - Priority(0). - Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(-2). - Request(corev1.ResourceCPU, "5"). - Obj(), - targetCQ: "q", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason)), - }, - // r - // / \ - // c11 c12 - // / | \ \ - // c21 c22 c23 q1 - // / | | | - // c31 c32 q3 q2 - // / | - // q5 q4 - // quotas: - // 4: c11, c12, c21, c22, c23, c32, c31 - // 0: q1, q3, q4, q5 - "reclaim in complex hierarchy": { - cohorts: []*kueue.Cohort{ - utiltestingapi.MakeCohort("r").Obj(), - utiltestingapi.MakeCohort("c11"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c12"). - Parent("r"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c21"). - Parent("c11"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c22"). - Parent("c11"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c23"). - Parent("c11"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c31"). - Parent("c21"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Obj()).Obj(), - utiltestingapi.MakeCohort("c32"). - Parent("c21"). - ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default"). - Resource(corev1.ResourceCPU, "4"). - Obj()).Obj(), - }, - clusterQueues: []*kueue.ClusterQueue{ - utiltestingapi.MakeClusterQueue("q1"). - Cohort("c12"). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("q2"). - Cohort("c23"). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("q3"). - Cohort("c22"). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("q4"). - Cohort("c32"). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - utiltestingapi.MakeClusterQueue("q5"). - Cohort("c31"). - Preemption(kueue.ClusterQueuePreemption{ - WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, - ReclaimWithinCohort: kueue.PreemptionPolicyAny, - }). - Obj(), - }, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("admitted_borrowing_1", ""). - Priority(-6). - Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q1"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_2", ""). - Priority(-5). - Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q1"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_3", ""). - Priority(-9). - Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q2"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_4", ""). - Priority(-10). - Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q2"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_5", ""). - Priority(-4). - Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q3"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_6", ""). - Priority(-3). - Request(corev1.ResourceCPU, "3"). - ReserveQuota(utiltestingapi.MakeAdmission("q3"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_7", ""). - Priority(4). - Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q4"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). - Obj(), - *utiltestingapi.MakeWorkload("admitted_borrowing_8", ""). - Priority(2). - Request(corev1.ResourceCPU, "3"). - ReserveQuota(utiltestingapi.MakeAdmission("q4"). - PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). - Obj(), - }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). - Priority(-2). - Request(corev1.ResourceCPU, "7"). - Obj(), - targetCQ: "q5", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - // only one of workloads from q2 will be preempted because - // after preempting the first one, the usage of cohort - // c23 will be back within nominal quota - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing_1", kueue.InCohortReclamationReason), - targetKeyReason("/admitted_borrowing_4", kueue.InCohortReclamationReason)), + { + preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid", Labels: map[string]string{controllerconstants.JobUIDLabel: "juid"}}}, + reason: kueue.InClusterQueueReason, + want: "Preempted to accommodate a workload (UID: uid, JobUID: juid) due to prioritization in the ClusterQueue", }, } - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - ctx, log := utiltesting.ContextWithLog(t) - cl := utiltesting.NewClientBuilder(). - WithLists(&kueue.WorkloadList{Items: tc.admitted}). - Build() - - cqCache := schdcache.New(cl) - for _, flv := range flavors { - cqCache.AddOrUpdateResourceFlavor(log, flv) - } - for _, cq := range tc.clusterQueues { - if err := cqCache.AddClusterQueue(ctx, cq); err != nil { - t.Fatalf("Couldn't add ClusterQueue to cache: %v", err) - } - } - for _, cohort := range tc.cohorts { - if err := cqCache.AddOrUpdateCohort(cohort); err != nil { - t.Fatalf("Couldn't add Cohort to cache: %v", err) - } - } - - var lock sync.Mutex - gotPreempted := sets.New[string]() - broadcaster := record.NewBroadcaster() - scheme := runtime.NewScheme() - if err := kueue.AddToScheme(scheme); err != nil { - t.Fatalf("Failed adding kueue scheme: %v", err) - } - recorder := broadcaster.NewRecorder(scheme, corev1.EventSource{Component: constants.AdmissionName}) - preemptor := New(cl, workload.Ordering{}, recorder, config.FairSharing{}, false, clocktesting.NewFakeClock(now)) - preemptor.applyPreemption = func(ctx context.Context, w *kueue.Workload, reason, _ string) error { - lock.Lock() - gotPreempted.Insert(targetKeyReason(workload.Key(w), reason)) - lock.Unlock() - return nil - } - beforeSnapshot, err := cqCache.Snapshot(ctx) - if err != nil { - t.Fatalf("unexpected error while building snapshot: %v", err) - } - // make a working copy of the snapshotWorkingCopy than preemption can temporarily modify - snapshotWorkingCopy, err := cqCache.Snapshot(ctx) - if err != nil { - t.Fatalf("unexpected error while building snapshot: %v", err) - } - wlInfo := workload.NewInfo(tc.incoming) - wlInfo.ClusterQueue = tc.targetCQ - targets := preemptor.GetTargets(log, *wlInfo, tc.assignment, snapshotWorkingCopy) - _, err = preemptor.IssuePreemptions(ctx, wlInfo, targets) - if err != nil { - t.Fatalf("Failed doing preemption") - } - if diff := cmp.Diff(tc.wantPreempted, gotPreempted, cmpopts.EquateEmpty()); diff != "" { - t.Errorf("Issued preemptions (-want,+got):\n%s", diff) - } - if diff := cmp.Diff(beforeSnapshot, snapshotWorkingCopy, snapCmpOpts); diff != "" { - t.Errorf("Snapshot was modified (-initial,+end):\n%s", diff) - } - }) + for _, tc := range cases { + got := preemptionMessage(tc.preemptor, tc.reason) + if got != tc.want { + t.Errorf("preemptionMessage(preemptor=kueue.Workload{UID:%v, Labels:%v}, reason=%q) returned %q, want %q", tc.preemptor.UID, tc.preemptor.Labels, tc.reason, got, tc.want) + } } } From 72f0e69081de21396cc454d829ed454894454828 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 07:17:36 -0700 Subject: [PATCH 033/119] Bump cypress/base from 22.20.0 to 22.21.0 in /hack/cypress (#7402) Bumps cypress/base from 22.20.0 to 22.21.0. --- updated-dependencies: - dependency-name: cypress/base dependency-version: 22.21.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- hack/cypress/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/cypress/Dockerfile b/hack/cypress/Dockerfile index 682d1a2de1c..840a695759f 100644 --- a/hack/cypress/Dockerfile +++ b/hack/cypress/Dockerfile @@ -1 +1 @@ -FROM cypress/base:22.20.0 +FROM cypress/base:22.21.0 From ecc5785a121f5ff9a9e62b15dd86198a8158ac94 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 27 Oct 2025 10:53:37 -0400 Subject: [PATCH 034/119] update documentation to use v1beta2 (#7409) --- keps/2941-DRA/README.md | 8 +++---- keps/973-workload-priority/README.md | 4 ++-- keps/976-plain-pods/README.md | 6 ++--- .../docs/concepts/admission_check/_index.md | 6 ++--- .../admission_check/provisioning_request.md | 2 +- .../docs/concepts/admission_fair_sharing.md | 4 ++-- .../content/en/docs/concepts/cluster_queue.md | 22 +++++++++---------- site/content/en/docs/concepts/cohort.md | 14 ++++++------ site/content/en/docs/concepts/local_queue.md | 2 +- site/content/en/docs/concepts/workload.md | 2 +- .../docs/concepts/workload_priority_class.md | 4 ++-- .../tasks/manage/administer_cluster_quotas.md | 22 +++++++++---------- .../manage/run_job_with_workload_priority.md | 4 ++-- .../manage/setup_object_retention_policy.md | 8 +++---- .../tasks/manage/setup_wait_for_pods_ready.md | 6 ++--- site/content/en/docs/tasks/run/jobs.md | 2 +- .../troubleshooting/troubleshooting_jobs.md | 6 ++--- .../provisioning.md | 2 +- .../zh-CN/docs/concepts/admission_check.md | 6 ++--- .../docs/concepts/admission_fair_sharing.md | 4 ++-- .../zh-CN/docs/concepts/cluster_queue.md | 22 +++++++++---------- site/content/zh-CN/docs/concepts/cohort.md | 14 ++++++------ .../zh-CN/docs/concepts/local_queue.md | 2 +- site/content/zh-CN/docs/concepts/workload.md | 2 +- .../docs/concepts/workload_priority_class.md | 4 ++-- .../tasks/manage/administer_cluster_quotas.md | 22 +++++++++---------- .../manage/run_job_with_workload_priority.md | 4 ++-- .../manage/setup_object_retention_policy.md | 8 +++---- .../tasks/manage/setup_wait_for_pods_ready.md | 6 ++--- site/content/zh-CN/docs/tasks/run/jobs.md | 2 +- .../troubleshooting/troubleshooting_jobs.md | 6 ++--- site/static/examples/admin/minimal-cq.yaml | 2 +- .../examples/admin/resource-flavor-empty.yaml | 2 +- .../admin/resource-flavor-taints.yaml | 2 +- .../admin/resource-flavor-tolerations.yaml | 2 +- .../admin/single-clusterqueue-setup.yaml | 6 ++--- .../admission-fair-sharing-setup.yaml | 8 +++---- .../examples/multikueue/multikueue-setup.yaml | 12 +++++----- .../provisioning/provisioning-setup.yaml | 10 ++++----- .../examples/sample-admission-check.yaml | 4 ++-- .../examples/tas/sample-gpu-queues.yaml | 8 +++---- site/static/examples/tas/sample-queues.yaml | 8 +++---- 42 files changed, 145 insertions(+), 145 deletions(-) diff --git a/keps/2941-DRA/README.md b/keps/2941-DRA/README.md index a12cbfab5af..9ae0080e018 100644 --- a/keps/2941-DRA/README.md +++ b/keps/2941-DRA/README.md @@ -299,14 +299,14 @@ data: - ts-shard-gpus.example.com - sp-shared-gpus.example.com --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "default-gpu-flavor" spec: # No changed needed here --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "gpus-cluster-queue" @@ -477,7 +477,7 @@ data: - gpu.example.com # Maps gpu.example.com -> whole-gpus --- # Step 2: Define ClusterQueue with DRA resource quotas -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "gpus-cluster-queue" @@ -536,7 +536,7 @@ spec: resourceClaimTemplateName: single-gpu # Must use template, not direct claim --- # Step 5: Resulting Workload status after admission -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: job-job0-6f46e diff --git a/keps/973-workload-priority/README.md b/keps/973-workload-priority/README.md index 2a03a497210..f0d2ee9d66a 100644 --- a/keps/973-workload-priority/README.md +++ b/keps/973-workload-priority/README.md @@ -156,7 +156,7 @@ This label is always mutable because it might be useful for the preemption. ```yaml # sample-priority-class.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: WorkloadPriorityClass metadata: name: sample-priority @@ -190,7 +190,7 @@ To distinguish, when using `WorkloadPriorityClass`, a `priorityClassSource` fiel When using `PriorityClass`, a `priorityClassSource` field has the `scheduling.k8s.io/priorityclass` value. ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: job-sample-job-7f173 diff --git a/keps/976-plain-pods/README.md b/keps/976-plain-pods/README.md index 9b37e82a847..bc8886b1ed2 100644 --- a/keps/976-plain-pods/README.md +++ b/keps/976-plain-pods/README.md @@ -402,7 +402,7 @@ The simplest case we want to support is single Pod jobs. These Pods only have th The Workload for the Pod in [story 1](#story-1) would look as follows: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: pod-foo @@ -473,7 +473,7 @@ indicating the reason. The Workload for the Pod in [story 2](#story-2) would look as follows: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: pod-group @@ -526,7 +526,7 @@ The template for the initial pods can be left empty, as it can be populated by K The Workload for the Pod in [story 3](#story-3) would look as follows: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: pod-group diff --git a/site/content/en/docs/concepts/admission_check/_index.md b/site/content/en/docs/concepts/admission_check/_index.md index 7fb4dd665c9..f7c3dd3f175 100644 --- a/site/content/en/docs/concepts/admission_check/_index.md +++ b/site/content/en/docs/concepts/admission_check/_index.md @@ -23,7 +23,7 @@ AdmissionCheck is a non-namespaced API object used to define details about an ad An AdmissionCheck object looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: AdmissionCheck metadata: name: prov-test @@ -53,7 +53,7 @@ Only one of the above-mentioned fields can be specified at the time. ##### Using `.spec.admissionChecks` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -66,7 +66,7 @@ spec: ##### Using `.spec.admissionCheckStrategy` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" diff --git a/site/content/en/docs/concepts/admission_check/provisioning_request.md b/site/content/en/docs/concepts/admission_check/provisioning_request.md index 108f1f4c5b4..f3d63996da1 100644 --- a/site/content/en/docs/concepts/admission_check/provisioning_request.md +++ b/site/content/en/docs/concepts/admission_check/provisioning_request.md @@ -68,7 +68,7 @@ It enables you to set `provisioningClassName`, `managedResources`, and `paramete A `ProvisioningRequestConfig` looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ProvisioningRequestConfig metadata: name: prov-test-config diff --git a/site/content/en/docs/concepts/admission_fair_sharing.md b/site/content/en/docs/concepts/admission_fair_sharing.md index 2eaad88f10c..3bd5853f121 100644 --- a/site/content/en/docs/concepts/admission_fair_sharing.md +++ b/site/content/en/docs/concepts/admission_fair_sharing.md @@ -63,7 +63,7 @@ admissionFairSharing: Enable Admission Fair Sharing by adding an AdmissionScope to your ClusterQueue: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: sample-queue @@ -79,7 +79,7 @@ spec: You can define a `fairSharing` section in your LocalQueue to adjust its weight in the fair sharing calculation (defaults to `1`): ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: name: team-a-queue diff --git a/site/content/en/docs/concepts/cluster_queue.md b/site/content/en/docs/concepts/cluster_queue.md index 10500e051da..465e2e2235c 100644 --- a/site/content/en/docs/concepts/cluster_queue.md +++ b/site/content/en/docs/concepts/cluster_queue.md @@ -18,7 +18,7 @@ Only [batch administrators](/docs/tasks#batch-administrator) should create `Clus A sample ClusterQueue looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -92,7 +92,7 @@ through the network or simply you wish to track their quota independently of oth An example of a ClusterQueue with multiple resource groups looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -252,7 +252,7 @@ by setting a [`flavorFungibility`](/docs/concepts/cluster_queue#flavorfungibilit Assume you created the following two ClusterQueues: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -271,7 +271,7 @@ spec: ``` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -309,7 +309,7 @@ you can set the `.spec.resourcesGroup[*].flavors[*].resource[*].borrowingLimit` As an example, assume you created the following two ClusterQueues: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -327,7 +327,7 @@ spec: ``` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -369,7 +369,7 @@ You can disable it by setting the `LendingLimit` feature gate. Check the [Instal As an example, assume you created the following two ClusterQueues: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -386,7 +386,7 @@ spec: ``` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -422,7 +422,7 @@ A configuration for a ClusterQueue that enables preemption looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -487,7 +487,7 @@ process by configuring the `flavorFungibility` field. A configuration for a ClusterQueue that configures this behavior looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -541,7 +541,7 @@ guide for details on feature gate configuration. StopPolicy allows a cluster administrator to temporary stop the admission of workloads within a ClusterQueue by setting its value in the [spec](/docs/reference/kueue.v1beta1/#kueue-x-k8s-io-v1beta1-ClusterQueueSpec) like: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" diff --git a/site/content/en/docs/concepts/cohort.md b/site/content/en/docs/concepts/cohort.md index cf1b0f2e272..e4676bd99dd 100644 --- a/site/content/en/docs/concepts/cohort.md +++ b/site/content/en/docs/concepts/cohort.md @@ -10,7 +10,7 @@ description: > Cohorts give you the ability to organize your Quotas. ClusterQueues within the same Cohort (or same CohortTree for [Hierarchical Cohorts](#hierarchical-cohorts)) can share resources with each other. The simplest possible Cohort is the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "hello-cohort" @@ -18,7 +18,7 @@ metadata: A ClusterQueue may join this Cohort by referencing it: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "my-cluster-queue" @@ -38,7 +38,7 @@ it. Additionally, this quota may also be lent out to parent Cohort(s), subject to LendingLimit. ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "hello-cohort" @@ -58,7 +58,7 @@ In order for a ClusterQueue to borrow resources from its Cohort, it **must** define nominal quota for the desired Resource and Flavor - even if this value is 0. ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "my-cluster-queue" @@ -87,12 +87,12 @@ These Borrowing and Lending Limits can be specified for Cohorts, as well as for Here is a simple CohortTree, with three Cohorts: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "root-cohort" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "important-org" @@ -101,7 +101,7 @@ spec: fairSharing: weight: "0.75" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "regular-org" diff --git a/site/content/en/docs/concepts/local_queue.md b/site/content/en/docs/concepts/local_queue.md index bc9ad1c219c..e8f90e2ea06 100644 --- a/site/content/en/docs/concepts/local_queue.md +++ b/site/content/en/docs/concepts/local_queue.md @@ -14,7 +14,7 @@ from which resources are allocated to run its Workloads. A `LocalQueue` definition looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: team-a diff --git a/site/content/en/docs/concepts/workload.md b/site/content/en/docs/concepts/workload.md index 005ca414049..75239f0edbb 100644 --- a/site/content/en/docs/concepts/workload.md +++ b/site/content/en/docs/concepts/workload.md @@ -23,7 +23,7 @@ the decisions and statuses. The manifest for a Workload looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: sample-job diff --git a/site/content/en/docs/concepts/workload_priority_class.md b/site/content/en/docs/concepts/workload_priority_class.md index 9047ccda3d3..5ff4ccef54d 100644 --- a/site/content/en/docs/concepts/workload_priority_class.md +++ b/site/content/en/docs/concepts/workload_priority_class.md @@ -14,7 +14,7 @@ This feature is useful for these cases: A sample WorkloadPriorityClass looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: WorkloadPriorityClass metadata: name: sample-priority @@ -47,7 +47,7 @@ a `priorityClassSource` field has the `kueue.x-k8s.io/workloadpriorityclass` val When using `PriorityClass`, a `priorityClassSource` field has the `scheduling.k8s.io/priorityclass` value. ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: job-sample-job-7f173 diff --git a/site/content/en/docs/tasks/manage/administer_cluster_quotas.md b/site/content/en/docs/tasks/manage/administer_cluster_quotas.md index d8074d21a94..22bf9ef12fe 100644 --- a/site/content/en/docs/tasks/manage/administer_cluster_quotas.md +++ b/site/content/en/docs/tasks/manage/administer_cluster_quotas.md @@ -41,7 +41,7 @@ Write the manifest for the ClusterQueue. It should look similar to the following ```yaml # cluster-queue.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -83,7 +83,7 @@ Write the manifest for the ResourceFlavor. It should look similar to the followi ```yaml # default-flavor.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "default-flavor" @@ -110,7 +110,7 @@ Write the manifest for the LocalQueue. It should look similar to the following: ```yaml # default-user-queue.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" @@ -140,7 +140,7 @@ following: ```yaml # flavor-x86.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "x86" @@ -151,7 +151,7 @@ spec: ```yaml # flavor-arm.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "arm" @@ -178,7 +178,7 @@ look similar to the following: ```yaml # cluster-queue.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -226,7 +226,7 @@ ClusterQueues `team-a-cq` and `team-b-cq`. ```yaml # team-a-cq.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -247,7 +247,7 @@ spec: ``` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -289,7 +289,7 @@ tenants look like the following: ```yaml # team-a-cq.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -318,7 +318,7 @@ spec: ```yaml # team-b-cq.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -347,7 +347,7 @@ spec: ```yaml # shared-cq.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "shared-cq" diff --git a/site/content/en/docs/tasks/manage/run_job_with_workload_priority.md b/site/content/en/docs/tasks/manage/run_job_with_workload_priority.md index a4e821dc976..f717b590ac4 100644 --- a/site/content/en/docs/tasks/manage/run_job_with_workload_priority.md +++ b/site/content/en/docs/tasks/manage/run_job_with_workload_priority.md @@ -25,7 +25,7 @@ Make sure the following conditions are met: The WorkloadPriorityClass should be created first. ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: WorkloadPriorityClass metadata: name: sample-priority @@ -66,7 +66,7 @@ Workload's `Priority` field is always mutable because it might be useful for the Workload's `PriorityClassSource` and `PriorityClassName` fields are immutable. ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: job-sample-job-7f173 diff --git a/site/content/en/docs/tasks/manage/setup_object_retention_policy.md b/site/content/en/docs/tasks/manage/setup_object_retention_policy.md index 59f64a33839..207111de9e7 100644 --- a/site/content/en/docs/tasks/manage/setup_object_retention_policy.md +++ b/site/content/en/docs/tasks/manage/setup_object_retention_policy.md @@ -68,7 +68,7 @@ It contains the following optional fields: 1. **Submit** a simple Workload that should finish normally: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: successful-cq @@ -82,7 +82,7 @@ spec: - name: cpu nominalQuota: "2" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: default @@ -138,7 +138,7 @@ kubectl get jobs -n default 2. **Submit** a Workload that requests more than is available on the node: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: limited-cq @@ -152,7 +152,7 @@ spec: - name: cpu nominalQuota: "100" # more than is available on the node --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: default diff --git a/site/content/en/docs/tasks/manage/setup_wait_for_pods_ready.md b/site/content/en/docs/tasks/manage/setup_wait_for_pods_ready.md index 18b9067f114..0154ef945f1 100644 --- a/site/content/en/docs/tasks/manage/setup_wait_for_pods_ready.md +++ b/site/content/en/docs/tasks/manage/setup_wait_for_pods_ready.md @@ -150,12 +150,12 @@ our cluster, in order to simulate issues with provisioning. Save the following cluster queues configuration as `cluster-queues.yaml`: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "default-flavor" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -169,7 +169,7 @@ spec: - name: "memory" nominalQuota: 16858Mi # double the value of allocatable memory in the cluster --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" diff --git a/site/content/en/docs/tasks/run/jobs.md b/site/content/en/docs/tasks/run/jobs.md index 0fc59f28a3b..244e692c342 100644 --- a/site/content/en/docs/tasks/run/jobs.md +++ b/site/content/en/docs/tasks/run/jobs.md @@ -97,7 +97,7 @@ Name: sample-job-sl4bm Namespace: default Labels: Annotations: -API Version: kueue.x-k8s.io/v1beta1 +API Version: kueue.x-k8s.io/v1beta2 Kind: Workload Metadata: ... diff --git a/site/content/en/docs/tasks/troubleshooting/troubleshooting_jobs.md b/site/content/en/docs/tasks/troubleshooting/troubleshooting_jobs.md index 28f3d7a231c..56043f8b32f 100644 --- a/site/content/en/docs/tasks/troubleshooting/troubleshooting_jobs.md +++ b/site/content/en/docs/tasks/troubleshooting/troubleshooting_jobs.md @@ -81,7 +81,7 @@ kubectl describe workload -n my-namespace job-my-job-19797 The output should be similar to the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload ... status: @@ -135,7 +135,7 @@ kubectl get workload -n my-namespace my-workload -o yaml If your Job is admitted, the Workload should have a status similar to the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload ... status: @@ -195,7 +195,7 @@ $ kubectl get jobs job-0-9-size-6 -o json | jq -r .spec.template.spec.containers If your ClusterQueue does not have a definition for the `requests`, Kueue cannot admit the job. For the job above, you should define `cpu` quotas under `resourceGroups`. A ClusterQueue defining `cpu` quota looks like the following: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" diff --git a/site/content/zh-CN/docs/admission-check-controllers/provisioning.md b/site/content/zh-CN/docs/admission-check-controllers/provisioning.md index 6d647e3c590..fc717db4e2e 100644 --- a/site/content/zh-CN/docs/admission-check-controllers/provisioning.md +++ b/site/content/zh-CN/docs/admission-check-controllers/provisioning.md @@ -46,7 +46,7 @@ Kueue 为您的作业创建 ProvisioningRequests 有两种配置方式: 一个 `ProvisioningRequestConfig` 如下所示: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ProvisioningRequestConfig metadata: name: prov-test-config diff --git a/site/content/zh-CN/docs/concepts/admission_check.md b/site/content/zh-CN/docs/concepts/admission_check.md index 4815a6a9f6e..1ac7e330810 100644 --- a/site/content/zh-CN/docs/concepts/admission_check.md +++ b/site/content/zh-CN/docs/concepts/admission_check.md @@ -20,7 +20,7 @@ AdmissionCheck 是一个非命名空间的 API 对象,用于定义准入检查 AdmissionCheck 对象示例如下: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: AdmissionCheck metadata: name: prov-test @@ -49,7 +49,7 @@ spec: ##### 使用 `.spec.admissionChecks` {#example-admission-checks} ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -62,7 +62,7 @@ spec: ##### 使用 `.spec.admissionCheckStrategy` {#example-admission-check-strategy} ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" diff --git a/site/content/zh-CN/docs/concepts/admission_fair_sharing.md b/site/content/zh-CN/docs/concepts/admission_fair_sharing.md index 58ded129276..01e60d19bbe 100644 --- a/site/content/zh-CN/docs/concepts/admission_fair_sharing.md +++ b/site/content/zh-CN/docs/concepts/admission_fair_sharing.md @@ -56,7 +56,7 @@ admissionFairSharing: 通过在 ClusterQueue 中添加 AdmissionScope 来启用准入公平共享: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: sample-queue @@ -72,7 +72,7 @@ spec: 您可以在 LocalQueue 中定义 `fairSharing` 部分,以调整其在公平共享计算中的权重(默认为 `1`): ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: name: team-a-queue diff --git a/site/content/zh-CN/docs/concepts/cluster_queue.md b/site/content/zh-CN/docs/concepts/cluster_queue.md index c12438a34b2..75e2727a93c 100644 --- a/site/content/zh-CN/docs/concepts/cluster_queue.md +++ b/site/content/zh-CN/docs/concepts/cluster_queue.md @@ -16,7 +16,7 @@ ClusterQueue 是一个集群范围的对象,用于管理一组资源池,如 一个示例 ClusterQueue 如下所示: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -79,7 +79,7 @@ Kueue 会自动计算一个 Workload 需要的 Pod 数量。 一个包含多个资源组的 ClusterQueue 示例如下: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -231,7 +231,7 @@ ClusterQueue 借用配额。 假设你创建了以下两个 ClusterQueues: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -250,7 +250,7 @@ spec: ``` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -288,7 +288,7 @@ ClusterQueue `team-a-cq` 可以根据以下情况接纳工作负载: 例如,假设你创建了以下两个 ClusterQueues: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -306,7 +306,7 @@ spec: ``` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -347,7 +347,7 @@ ClusterQueue 可以借用所有名义配额从 cohort 中的所有 ClusterQueues 例如,假设你创建了以下两个 ClusterQueues: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -364,7 +364,7 @@ spec: ``` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -398,7 +398,7 @@ ClusterQueue 的策略。 一个配置 ClusterQueue 以启用预留的示例如下: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -451,7 +451,7 @@ Kueue 按顺序评估 ClusterQueue 中的规格。你可以影响是否优先 一个配置 ClusterQueue 以配置此行为的示例如下: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -482,7 +482,7 @@ spec: StopPolicy 允许集群管理员通过在 [spec](/docs/reference/kueue.v1beta1/#kueue-x-k8s-io-v1beta1-ClusterQueueSpec) 中设置其值来临时停止 ClusterQueue 中工作负载的接纳,如下所示: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" diff --git a/site/content/zh-CN/docs/concepts/cohort.md b/site/content/zh-CN/docs/concepts/cohort.md index 8100125aa4e..c76d0b5ccff 100644 --- a/site/content/zh-CN/docs/concepts/cohort.md +++ b/site/content/zh-CN/docs/concepts/cohort.md @@ -13,7 +13,7 @@ Cohort 使你能够组织你的配额。同一个 Cohort(或对于[分层 Coho 最简单的 Cohort 如下: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "hello-cohort" @@ -22,7 +22,7 @@ metadata: ClusterQueue 可以通过引用来加入此 Cohort: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "my-cluster-queue" @@ -40,7 +40,7 @@ spec: ClusterQueue 的共享池。此外,此配额也可能根据 LendingLimit 借给父 Cohort。 ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "hello-cohort" @@ -59,7 +59,7 @@ spec: Flavor 定义 nominal 配额 - 即使该值为 0。 ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "my-cluster-queue" @@ -90,12 +90,12 @@ Cohort 的组合称为 **CohortTree**。 这是一个简单的 CohortTree,包含三个 Cohort: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "root-cohort" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "important-org" @@ -104,7 +104,7 @@ spec: fairSharing: weight: "0.75" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Cohort metadata: name: "regular-org" diff --git a/site/content/zh-CN/docs/concepts/local_queue.md b/site/content/zh-CN/docs/concepts/local_queue.md index fb7f82f865e..32e6495158d 100644 --- a/site/content/zh-CN/docs/concepts/local_queue.md +++ b/site/content/zh-CN/docs/concepts/local_queue.md @@ -14,7 +14,7 @@ description: > `LocalQueue` 定义如下所示: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: team-a diff --git a/site/content/zh-CN/docs/concepts/workload.md b/site/content/zh-CN/docs/concepts/workload.md index 06ab141e0d5..278d1814d71 100644 --- a/site/content/zh-CN/docs/concepts/workload.md +++ b/site/content/zh-CN/docs/concepts/workload.md @@ -21,7 +21,7 @@ Workload 对象。Kueue 自动为每个 Job 对象创建一个 Workload,并同 Workload 的清单如下所示: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: sample-job diff --git a/site/content/zh-CN/docs/concepts/workload_priority_class.md b/site/content/zh-CN/docs/concepts/workload_priority_class.md index ffa7299c0a5..d133f8714a6 100644 --- a/site/content/zh-CN/docs/concepts/workload_priority_class.md +++ b/site/content/zh-CN/docs/concepts/workload_priority_class.md @@ -14,7 +14,7 @@ description: > 一个示例 WorkloadPriorityClass 如下所示: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: WorkloadPriorityClass metadata: name: sample-priority @@ -46,7 +46,7 @@ Kueue 会为上述 Job 生成如下 `Workload`。 字段为 `scheduling.k8s.io/priorityclass`。 ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: job-sample-job-7f173 diff --git a/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md b/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md index 5ac6202af4c..5f681cedb41 100644 --- a/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md +++ b/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md @@ -38,7 +38,7 @@ kubectl apply -f examples/admin/single-clusterqueue-setup.yaml ```yaml # cluster-queue.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -79,7 +79,7 @@ ClusterQueue 目前还不能使用,因为 `default` 风味尚未定义。 ```yaml # default-flavor.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "default-flavor" @@ -106,7 +106,7 @@ kubectl apply -f default-flavor.yaml ```yaml # default-user-queue.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" @@ -135,7 +135,7 @@ kubectl apply -f default-user-queue.yaml ```yaml # flavor-x86.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "x86" @@ -146,7 +146,7 @@ spec: ```yaml # flavor-arm.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "arm" @@ -171,7 +171,7 @@ ResourceFlavor 中设置的标签应与你的节点标签一致。 ```yaml # cluster-queue.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -218,7 +218,7 @@ kubectl apply -f cluster-queue.yaml ```yaml # team-a-cq.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -239,7 +239,7 @@ spec: ``` ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -280,7 +280,7 @@ kubectl apply -f team-a-cq.yaml -f team-b-cq.yaml ```yaml # team-a-cq.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-a-cq" @@ -309,7 +309,7 @@ spec: ```yaml # team-b-cq.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "team-b-cq" @@ -338,7 +338,7 @@ spec: ```yaml # shared-cq.yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "shared-cq" diff --git a/site/content/zh-CN/docs/tasks/manage/run_job_with_workload_priority.md b/site/content/zh-CN/docs/tasks/manage/run_job_with_workload_priority.md index b0eabe004ef..485f5078c88 100644 --- a/site/content/zh-CN/docs/tasks/manage/run_job_with_workload_priority.md +++ b/site/content/zh-CN/docs/tasks/manage/run_job_with_workload_priority.md @@ -26,7 +26,7 @@ description: > 首先应该创建 WorkloadPriorityClass。 ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: WorkloadPriorityClass metadata: name: sample-priority @@ -68,7 +68,7 @@ Kueue 为上述作业生成了以下 `Workload`。 工作负载的 `PriorityClassSource` 和 `PriorityClassName` 字段是不可变的。 ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload metadata: name: job-sample-job-7f173 diff --git a/site/content/zh-CN/docs/tasks/manage/setup_object_retention_policy.md b/site/content/zh-CN/docs/tasks/manage/setup_object_retention_policy.md index 59f64a33839..207111de9e7 100644 --- a/site/content/zh-CN/docs/tasks/manage/setup_object_retention_policy.md +++ b/site/content/zh-CN/docs/tasks/manage/setup_object_retention_policy.md @@ -68,7 +68,7 @@ It contains the following optional fields: 1. **Submit** a simple Workload that should finish normally: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: successful-cq @@ -82,7 +82,7 @@ spec: - name: cpu nominalQuota: "2" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: default @@ -138,7 +138,7 @@ kubectl get jobs -n default 2. **Submit** a Workload that requests more than is available on the node: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: limited-cq @@ -152,7 +152,7 @@ spec: - name: cpu nominalQuota: "100" # more than is available on the node --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: default diff --git a/site/content/zh-CN/docs/tasks/manage/setup_wait_for_pods_ready.md b/site/content/zh-CN/docs/tasks/manage/setup_wait_for_pods_ready.md index ef342b57978..23eb7310949 100644 --- a/site/content/zh-CN/docs/tasks/manage/setup_wait_for_pods_ready.md +++ b/site/content/zh-CN/docs/tasks/manage/setup_wait_for_pods_ready.md @@ -138,12 +138,12 @@ echo $TOTAL_ALLOCATABLE 将以下 cluster queues 配置保存为 `cluster-queues.yaml`: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "default-flavor" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -157,7 +157,7 @@ spec: - name: "memory" nominalQuota: 16858Mi # double the value of allocatable memory in the cluster --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" diff --git a/site/content/zh-CN/docs/tasks/run/jobs.md b/site/content/zh-CN/docs/tasks/run/jobs.md index 15fafdf400b..72d5a6b377e 100644 --- a/site/content/zh-CN/docs/tasks/run/jobs.md +++ b/site/content/zh-CN/docs/tasks/run/jobs.md @@ -95,7 +95,7 @@ Name: sample-job-sl4bm Namespace: default Labels: Annotations: -API Version: kueue.x-k8s.io/v1beta1 +API Version: kueue.x-k8s.io/v1beta2 Kind: Workload Metadata: ... diff --git a/site/content/zh-CN/docs/tasks/troubleshooting/troubleshooting_jobs.md b/site/content/zh-CN/docs/tasks/troubleshooting/troubleshooting_jobs.md index 82b5c5cef28..cbcacd7a6e9 100644 --- a/site/content/zh-CN/docs/tasks/troubleshooting/troubleshooting_jobs.md +++ b/site/content/zh-CN/docs/tasks/troubleshooting/troubleshooting_jobs.md @@ -83,7 +83,7 @@ kubectl describe workload -n my-namespace job-my-job-19797 输出应该类似于以下内容: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload ... status: @@ -135,7 +135,7 @@ kubectl get workload -n my-namespace my-workload -o yaml 如果你的 Job 被准入,Workload 应该有类似以下的状态: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Workload ... status: @@ -200,7 +200,7 @@ kubectl get jobs job-0-9-size-6 -o json | jq -r .spec.template.spec.containers[0 定义 `cpu` 配额的 ClusterQueue 如下所示: ```yaml -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" diff --git a/site/static/examples/admin/minimal-cq.yaml b/site/static/examples/admin/minimal-cq.yaml index 4718cc38871..b5cd26091f6 100644 --- a/site/static/examples/admin/minimal-cq.yaml +++ b/site/static/examples/admin/minimal-cq.yaml @@ -1,4 +1,4 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue-2" diff --git a/site/static/examples/admin/resource-flavor-empty.yaml b/site/static/examples/admin/resource-flavor-empty.yaml index 6cbccf3b778..174636939f9 100644 --- a/site/static/examples/admin/resource-flavor-empty.yaml +++ b/site/static/examples/admin/resource-flavor-empty.yaml @@ -1,4 +1,4 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: default-flavor diff --git a/site/static/examples/admin/resource-flavor-taints.yaml b/site/static/examples/admin/resource-flavor-taints.yaml index f04d54e505d..2247c7680e7 100644 --- a/site/static/examples/admin/resource-flavor-taints.yaml +++ b/site/static/examples/admin/resource-flavor-taints.yaml @@ -1,4 +1,4 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "spot" diff --git a/site/static/examples/admin/resource-flavor-tolerations.yaml b/site/static/examples/admin/resource-flavor-tolerations.yaml index 564cce5efe6..9591ecd12c7 100644 --- a/site/static/examples/admin/resource-flavor-tolerations.yaml +++ b/site/static/examples/admin/resource-flavor-tolerations.yaml @@ -1,4 +1,4 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "spot" diff --git a/site/static/examples/admin/single-clusterqueue-setup.yaml b/site/static/examples/admin/single-clusterqueue-setup.yaml index a46b19df5bf..2ee6dd52daf 100644 --- a/site/static/examples/admin/single-clusterqueue-setup.yaml +++ b/site/static/examples/admin/single-clusterqueue-setup.yaml @@ -1,9 +1,9 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "default-flavor" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -19,7 +19,7 @@ spec: - name: "memory" nominalQuota: 36Gi --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" diff --git a/site/static/examples/admission-fs/admission-fair-sharing-setup.yaml b/site/static/examples/admission-fs/admission-fair-sharing-setup.yaml index 1a47838e44f..27b63b730ea 100644 --- a/site/static/examples/admission-fs/admission-fair-sharing-setup.yaml +++ b/site/static/examples/admission-fs/admission-fair-sharing-setup.yaml @@ -1,9 +1,9 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "default-flavor" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -20,7 +20,7 @@ spec: - name: "cpu" nominalQuota: 9 --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" @@ -30,7 +30,7 @@ spec: fairSharing: weight: "1" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" diff --git a/site/static/examples/multikueue/multikueue-setup.yaml b/site/static/examples/multikueue/multikueue-setup.yaml index 0ee4847293f..d2f0367084f 100644 --- a/site/static/examples/multikueue/multikueue-setup.yaml +++ b/site/static/examples/multikueue/multikueue-setup.yaml @@ -1,9 +1,9 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "default-flavor" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -21,7 +21,7 @@ spec: admissionChecks: - sample-multikueue --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" @@ -29,7 +29,7 @@ metadata: spec: clusterQueue: "cluster-queue" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: AdmissionCheck metadata: name: sample-multikueue @@ -40,7 +40,7 @@ spec: kind: MultiKueueConfig name: multikueue-test --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: MultiKueueConfig metadata: name: multikueue-test @@ -48,7 +48,7 @@ spec: clusters: - multikueue-test-worker1 --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: MultiKueueCluster metadata: name: multikueue-test-worker1 diff --git a/site/static/examples/provisioning/provisioning-setup.yaml b/site/static/examples/provisioning/provisioning-setup.yaml index 2140b1f69f5..0896f3dd776 100644 --- a/site/static/examples/provisioning/provisioning-setup.yaml +++ b/site/static/examples/provisioning/provisioning-setup.yaml @@ -1,9 +1,9 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ResourceFlavor metadata: name: "default-flavor" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "cluster-queue" @@ -25,7 +25,7 @@ spec: - name: "sample-prov" onFlavors: [default-flavor] # optional if the admission check targets all flavors. --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" @@ -33,7 +33,7 @@ metadata: spec: clusterQueue: "cluster-queue" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: AdmissionCheck metadata: name: sample-prov @@ -44,7 +44,7 @@ spec: kind: ProvisioningRequestConfig name: prov-test-config --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ProvisioningRequestConfig metadata: name: prov-test-config diff --git a/site/static/examples/sample-admission-check.yaml b/site/static/examples/sample-admission-check.yaml index a933cb8c38a..3fa16f17255 100644 --- a/site/static/examples/sample-admission-check.yaml +++ b/site/static/examples/sample-admission-check.yaml @@ -1,4 +1,4 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: AdmissionCheck metadata: name: prov-test @@ -11,7 +11,7 @@ spec: --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ProvisioningRequestConfig metadata: name: prov-test-config diff --git a/site/static/examples/tas/sample-gpu-queues.yaml b/site/static/examples/tas/sample-gpu-queues.yaml index c75fd6b74d0..2931662e37f 100644 --- a/site/static/examples/tas/sample-gpu-queues.yaml +++ b/site/static/examples/tas/sample-gpu-queues.yaml @@ -1,4 +1,4 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Topology metadata: name: "default" @@ -9,7 +9,7 @@ spec: - nodeLabel: "kubernetes.io/hostname" # Taints on nodes are only respected if this label is the lowest topology level. --- kind: ResourceFlavor -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 metadata: name: "tas-flavor" spec: @@ -21,7 +21,7 @@ spec: operator: "Exists" effect: "NoSchedule" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "tas-cluster-queue" @@ -39,7 +39,7 @@ spec: - name: "nvidia.com/gpu" nominalQuota: 16 --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" diff --git a/site/static/examples/tas/sample-queues.yaml b/site/static/examples/tas/sample-queues.yaml index 6d1c01740ea..f3d31b8853e 100644 --- a/site/static/examples/tas/sample-queues.yaml +++ b/site/static/examples/tas/sample-queues.yaml @@ -1,4 +1,4 @@ -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: Topology metadata: name: "default" @@ -9,7 +9,7 @@ spec: - nodeLabel: "kubernetes.io/hostname" --- kind: ResourceFlavor -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 metadata: name: "tas-flavor" spec: @@ -17,7 +17,7 @@ spec: cloud.provider.com/node-group: "tas-group" topologyName: "default" --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: ClusterQueue metadata: name: "tas-cluster-queue" @@ -33,7 +33,7 @@ spec: - name: "memory" nominalQuota: 100Gi --- -apiVersion: kueue.x-k8s.io/v1beta1 +apiVersion: kueue.x-k8s.io/v1beta2 kind: LocalQueue metadata: namespace: "default" From 1b6910c8cc83bb709d9a4308f5d87068065460da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Mon, 27 Oct 2025 16:21:37 +0100 Subject: [PATCH 035/119] Helm: request conversion webhooks only for types requiring it (#7410) --- .../templates/crd/kueue.x-k8s.io_admissionchecks.yaml | 10 ---------- charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml | 10 ---------- .../crd/kueue.x-k8s.io_multikueueclusters.yaml | 10 ---------- .../crd/kueue.x-k8s.io_multikueueconfigs.yaml | 10 ---------- .../crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml | 10 ---------- .../templates/crd/kueue.x-k8s.io_resourceflavors.yaml | 10 ---------- .../kueue/templates/crd/kueue.x-k8s.io_topologies.yaml | 10 ---------- .../crd/kueue.x-k8s.io_workloadpriorityclasses.yaml | 10 ---------- hack/processing-plan.yaml | 1 + 9 files changed, 1 insertion(+), 80 deletions(-) diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml index 57110b47a5b..40b3a352f13 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml @@ -12,16 +12,6 @@ metadata: controller-gen.kubebuilder.io/version: v0.19.0 name: admissionchecks.kueue.x-k8s.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kueue.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1 group: kueue.x-k8s.io names: kind: AdmissionCheck diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml index 102e130327b..8c0f6241669 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml @@ -12,16 +12,6 @@ metadata: controller-gen.kubebuilder.io/version: v0.19.0 name: cohorts.kueue.x-k8s.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kueue.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1 group: kueue.x-k8s.io names: kind: Cohort diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml index 9d502ea76c6..1a61f31ba5d 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml @@ -12,16 +12,6 @@ metadata: controller-gen.kubebuilder.io/version: v0.19.0 name: multikueueclusters.kueue.x-k8s.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kueue.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1 group: kueue.x-k8s.io names: kind: MultiKueueCluster diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml index a500a70dece..6041c805dbe 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml @@ -12,16 +12,6 @@ metadata: controller-gen.kubebuilder.io/version: v0.19.0 name: multikueueconfigs.kueue.x-k8s.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kueue.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1 group: kueue.x-k8s.io names: kind: MultiKueueConfig diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml index d5dd6a9f0ac..3cb8e02c602 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml @@ -12,16 +12,6 @@ metadata: controller-gen.kubebuilder.io/version: v0.19.0 name: provisioningrequestconfigs.kueue.x-k8s.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kueue.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1 group: kueue.x-k8s.io names: kind: ProvisioningRequestConfig diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_resourceflavors.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_resourceflavors.yaml index 42842918c4e..90cae4cc6e0 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_resourceflavors.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_resourceflavors.yaml @@ -12,16 +12,6 @@ metadata: controller-gen.kubebuilder.io/version: v0.19.0 name: resourceflavors.kueue.x-k8s.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kueue.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1 group: kueue.x-k8s.io names: kind: ResourceFlavor diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_topologies.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_topologies.yaml index 6ff25be3664..bd9a463bea2 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_topologies.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_topologies.yaml @@ -12,16 +12,6 @@ metadata: controller-gen.kubebuilder.io/version: v0.19.0 name: topologies.kueue.x-k8s.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kueue.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1 group: kueue.x-k8s.io names: kind: Topology diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml index 6ce58990421..4ddb00e4d03 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml @@ -12,16 +12,6 @@ metadata: controller-gen.kubebuilder.io/version: v0.19.0 name: workloadpriorityclasses.kueue.x-k8s.io spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - name: '{{ include "kueue.fullname" . }}-webhook-service' - namespace: '{{ .Release.Namespace }}' - path: /convert - conversionReviewVersions: - - v1 group: kueue.x-k8s.io names: kind: WorkloadPriorityClass diff --git a/hack/processing-plan.yaml b/hack/processing-plan.yaml index 5c7e3593ac4..a1eb3b5223b 100644 --- a/hack/processing-plan.yaml +++ b/hack/processing-plan.yaml @@ -8,6 +8,7 @@ files: operations: - type: INSERT_OBJECT key: .spec + onFileCondition: '{"Workload": true, "LocalQueue": true, "ClusterQueue": true}[.spec.names.kind]' value: | conversion: strategy: Webhook From b4973f78a3bd4e674ecd00d9cc57aebbfa175ca5 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 27 Oct 2025 14:31:36 -0400 Subject: [PATCH 036/119] replace cohort with cohortName for v1beta2 docs (#7412) --- site/content/en/docs/concepts/cluster_queue.md | 12 ++++++------ site/content/en/docs/concepts/cohort.md | 4 ++-- .../docs/tasks/manage/administer_cluster_quotas.md | 10 +++++----- site/content/zh-CN/docs/concepts/cluster_queue.md | 12 ++++++------ site/content/zh-CN/docs/concepts/cohort.md | 4 ++-- .../docs/tasks/manage/administer_cluster_quotas.md | 10 +++++----- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/site/content/en/docs/concepts/cluster_queue.md b/site/content/en/docs/concepts/cluster_queue.md index 465e2e2235c..85dae3cd429 100644 --- a/site/content/en/docs/concepts/cluster_queue.md +++ b/site/content/en/docs/concepts/cluster_queue.md @@ -258,7 +258,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -277,7 +277,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -315,7 +315,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -333,7 +333,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -375,7 +375,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: @@ -392,7 +392,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: diff --git a/site/content/en/docs/concepts/cohort.md b/site/content/en/docs/concepts/cohort.md index e4676bd99dd..2d61bd14add 100644 --- a/site/content/en/docs/concepts/cohort.md +++ b/site/content/en/docs/concepts/cohort.md @@ -23,7 +23,7 @@ kind: ClusterQueue metadata: name: "my-cluster-queue" spec: - cohort: "hello-cohort" + cohortName: "hello-cohort" ``` ## Configuring Quotas @@ -63,7 +63,7 @@ kind: ClusterQueue metadata: name: "my-cluster-queue" spec: - cohort: "hello-cohort" + cohortName: "hello-cohort" resourceGroups: - coveredResources: ["cpu"] flavors: diff --git a/site/content/en/docs/tasks/manage/administer_cluster_quotas.md b/site/content/en/docs/tasks/manage/administer_cluster_quotas.md index 22bf9ef12fe..28f6521ee41 100644 --- a/site/content/en/docs/tasks/manage/administer_cluster_quotas.md +++ b/site/content/en/docs/tasks/manage/administer_cluster_quotas.md @@ -232,7 +232,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -253,7 +253,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -295,7 +295,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: @@ -324,7 +324,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: @@ -353,7 +353,7 @@ metadata: name: "shared-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: diff --git a/site/content/zh-CN/docs/concepts/cluster_queue.md b/site/content/zh-CN/docs/concepts/cluster_queue.md index 75e2727a93c..185810db980 100644 --- a/site/content/zh-CN/docs/concepts/cluster_queue.md +++ b/site/content/zh-CN/docs/concepts/cluster_queue.md @@ -237,7 +237,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # 匹配所有命名空间。 - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -256,7 +256,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} # 匹配所有命名空间。 - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -294,7 +294,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # 匹配所有命名空间。 - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -312,7 +312,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} # 匹配所有命名空间。 - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -353,7 +353,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # 匹配所有命名空间。 - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: @@ -370,7 +370,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} # 匹配所有命名空间。 - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: diff --git a/site/content/zh-CN/docs/concepts/cohort.md b/site/content/zh-CN/docs/concepts/cohort.md index c76d0b5ccff..35ea5a88e74 100644 --- a/site/content/zh-CN/docs/concepts/cohort.md +++ b/site/content/zh-CN/docs/concepts/cohort.md @@ -27,7 +27,7 @@ kind: ClusterQueue metadata: name: "my-cluster-queue" spec: - cohort: "hello-cohort" + cohortName: "hello-cohort" ``` ## 配置配额 @@ -64,7 +64,7 @@ kind: ClusterQueue metadata: name: "my-cluster-queue" spec: - cohort: "hello-cohort" + cohortName: "hello-cohort" resourceGroups: - coveredResources: ["cpu"] flavors: diff --git a/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md b/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md index 5f681cedb41..fac636a35a1 100644 --- a/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md +++ b/site/content/zh-CN/docs/tasks/manage/administer_cluster_quotas.md @@ -224,7 +224,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -245,7 +245,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu", "memory"] flavors: @@ -286,7 +286,7 @@ metadata: name: "team-a-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: @@ -315,7 +315,7 @@ metadata: name: "team-b-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: @@ -344,7 +344,7 @@ metadata: name: "shared-cq" spec: namespaceSelector: {} # match all. - cohort: "team-ab" + cohortName: "team-ab" resourceGroups: - coveredResources: ["cpu"] flavors: From 7c1f588bdd2a80340b9898842741697973dc40be Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 27 Oct 2025 14:59:35 -0400 Subject: [PATCH 037/119] add v1beta2 api gen docs (#7414) --- hack/genref/config.yaml | 15 +- .../en/docs/reference/kueue.v1beta2.md | 3370 +++++++++++++++++ 2 files changed, 3381 insertions(+), 4 deletions(-) create mode 100644 site/content/en/docs/reference/kueue.v1beta2.md diff --git a/hack/genref/config.yaml b/hack/genref/config.yaml index 3d2339f1af6..3a008a8c8e1 100644 --- a/hack/genref/config.yaml +++ b/hack/genref/config.yaml @@ -8,15 +8,22 @@ apis: package: sigs.k8s.io/kueue path: apis/kueue/v1beta1 + - name: kueue + title: Kueue API + package: sigs.k8s.io/kueue + path: apis/kueue/v1beta2 + - name: kueue-config title: Kueue Configuration API package: sigs.k8s.io/kueue path: apis/config/v1beta1 - - name: kueue-alpha - title: Kueue Alpha API - package: sigs.k8s.io/kueue - path: apis/kueue/v1alpha1 + # code fails if there are no types in alpha apis + # leave in in case alpha apis are added + #- name: kueue-alpha + # title: Kueue Alpha API + # package: sigs.k8s.io/kueue + # path: apis/kueue/v1alpha1 externalPackages: - match: ^k8s\.io/(api|apimachinery/pkg/apis)/ diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md new file mode 100644 index 00000000000..563503ba4fd --- /dev/null +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -0,0 +1,3370 @@ +--- +title: Kueue API +content_type: tool-reference +package: kueue.x-k8s.io/v1beta2 +auto_generated: true +description: Generated API reference documentation for kueue.x-k8s.io/v1beta2. +--- + + +## Resource Types + + +- [AdmissionCheck](#kueue-x-k8s-io-v1beta2-AdmissionCheck) +- [ClusterQueue](#kueue-x-k8s-io-v1beta2-ClusterQueue) +- [Cohort](#kueue-x-k8s-io-v1beta2-Cohort) +- [LocalQueue](#kueue-x-k8s-io-v1beta2-LocalQueue) +- [MultiKueueCluster](#kueue-x-k8s-io-v1beta2-MultiKueueCluster) +- [MultiKueueConfig](#kueue-x-k8s-io-v1beta2-MultiKueueConfig) +- [ProvisioningRequestConfig](#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfig) +- [ResourceFlavor](#kueue-x-k8s-io-v1beta2-ResourceFlavor) +- [Topology](#kueue-x-k8s-io-v1beta2-Topology) +- [Workload](#kueue-x-k8s-io-v1beta2-Workload) +- [WorkloadPriorityClass](#kueue-x-k8s-io-v1beta2-WorkloadPriorityClass) + + +## `AdmissionCheck` {#kueue-x-k8s-io-v1beta2-AdmissionCheck} + + +**Appears in:** + + + +

AdmissionCheck is the Schema for the admissionchecks API

+ + + + + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
AdmissionCheck
spec [Required]
+AdmissionCheckSpec +
+

spec is the specification of the AdmissionCheck.

+
status [Required]
+AdmissionCheckStatus +
+

status is the status of the AdmissionCheck.

+
+ +## `ClusterQueue` {#kueue-x-k8s-io-v1beta2-ClusterQueue} + + +**Appears in:** + + + +

ClusterQueue is the Schema for the clusterQueue API.

+ + + + + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
ClusterQueue
spec [Required]
+ClusterQueueSpec +
+

spec is the specification of the ClusterQueue.

+
status [Required]
+ClusterQueueStatus +
+

status is the status of the ClusterQueue.

+
+ +## `Cohort` {#kueue-x-k8s-io-v1beta2-Cohort} + + +**Appears in:** + + + +

Cohort defines the Cohorts API.

+

Hierarchical Cohorts (any Cohort which has a parent) are compatible +with Fair Sharing as of v0.11. Using these features together in +V0.9 and V0.10 is unsupported, and results in undefined behavior.

+ + + + + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
Cohort
spec [Required]
+CohortSpec +
+

spec is the specification of the Cohort.

+
status [Required]
+CohortStatus +
+

status is the status of the Cohort.

+
+ +## `LocalQueue` {#kueue-x-k8s-io-v1beta2-LocalQueue} + + +**Appears in:** + + + +

LocalQueue is the Schema for the localQueues API

+ + + + + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
LocalQueue
spec [Required]
+LocalQueueSpec +
+

spec is the specification of the LocalQueue.

+
status [Required]
+LocalQueueStatus +
+

status is the status of the LocalQueue.

+
+ +## `MultiKueueCluster` {#kueue-x-k8s-io-v1beta2-MultiKueueCluster} + + +**Appears in:** + + + +

MultiKueueCluster is the Schema for the multikueue API

+ + + + + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
MultiKueueCluster
spec [Required]
+MultiKueueClusterSpec +
+

spec is the specification of the MultiKueueCluster.

+
status [Required]
+MultiKueueClusterStatus +
+

status is the status of the MultiKueueCluster.

+
+ +## `MultiKueueConfig` {#kueue-x-k8s-io-v1beta2-MultiKueueConfig} + + +**Appears in:** + + + +

MultiKueueConfig is the Schema for the multikueue API

+ + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
MultiKueueConfig
spec [Required]
+MultiKueueConfigSpec +
+

spec is the specification of the MultiKueueConfig.

+
+ +## `ProvisioningRequestConfig` {#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfig} + + +**Appears in:** + + + +

ProvisioningRequestConfig is the Schema for the provisioningrequestconfig API

+ + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
ProvisioningRequestConfig
spec [Required]
+ProvisioningRequestConfigSpec +
+

spec is the specification of the ProvisioningRequestConfig.

+
+ +## `ResourceFlavor` {#kueue-x-k8s-io-v1beta2-ResourceFlavor} + + +**Appears in:** + + + +

ResourceFlavor is the Schema for the resourceflavors API.

+ + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
ResourceFlavor
spec [Required]
+ResourceFlavorSpec +
+

spec is the specification of the ResourceFlavor.

+
+ +## `Topology` {#kueue-x-k8s-io-v1beta2-Topology} + + +**Appears in:** + + + +

Topology is the Schema for the topology API

+ + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
Topology
spec [Required]
+TopologySpec +
+

spec is the specification of the Topology.

+
+ +## `Workload` {#kueue-x-k8s-io-v1beta2-Workload} + + +**Appears in:** + + + +

Workload is the Schema for the workloads API

+ + + + + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
Workload
spec [Required]
+WorkloadSpec +
+

spec is the specification of the Workload.

+
status [Required]
+WorkloadStatus +
+

status is the status of the Workload.

+
+ +## `WorkloadPriorityClass` {#kueue-x-k8s-io-v1beta2-WorkloadPriorityClass} + + +**Appears in:** + + + +

WorkloadPriorityClass is the Schema for the workloadPriorityClass API

+ + + + + + + + + + + + + + + + + +
FieldDescription
apiVersion
string
kueue.x-k8s.io/v1beta2
kind
string
WorkloadPriorityClass
value [Required]
+int32 +
+

value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads +receive when jobs have the name of this class in their workloadPriorityClass label. +Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created.

+
description
+string +
+

description is an arbitrary string that usually provides guidelines on +when this workloadPriorityClass should be used.

+
+ +## `Admission` {#kueue-x-k8s-io-v1beta2-Admission} + + +**Appears in:** + +- [WorkloadStatus](#kueue-x-k8s-io-v1beta2-WorkloadStatus) + + + + + + + + + + + + + + + +
FieldDescription
clusterQueue [Required]
+ClusterQueueReference +
+

clusterQueue is the name of the ClusterQueue that admitted this workload.

+
podSetAssignments [Required]
+[]PodSetAssignment +
+

podSetAssignments hold the admission results for each of the .spec.podSets entries.

+
+ +## `AdmissionCheckParametersReference` {#kueue-x-k8s-io-v1beta2-AdmissionCheckParametersReference} + + +**Appears in:** + +- [AdmissionCheckSpec](#kueue-x-k8s-io-v1beta2-AdmissionCheckSpec) + + + + + + + + + + + + + + + + + + +
FieldDescription
apiGroup [Required]
+string +
+

apiGroup is the group for the resource being referenced.

+
kind [Required]
+string +
+

kind is the type of the resource being referenced.

+
name [Required]
+string +
+

name is the name of the resource being referenced.

+
+ +## `AdmissionCheckReference` {#kueue-x-k8s-io-v1beta2-AdmissionCheckReference} + +(Alias of `string`) + +**Appears in:** + +- [AdmissionCheckState](#kueue-x-k8s-io-v1beta2-AdmissionCheckState) + +- [AdmissionCheckStrategyRule](#kueue-x-k8s-io-v1beta2-AdmissionCheckStrategyRule) + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + + +

AdmissionCheckReference is the name of an AdmissionCheck.

+ + + + +## `AdmissionCheckSpec` {#kueue-x-k8s-io-v1beta2-AdmissionCheckSpec} + + +**Appears in:** + +- [AdmissionCheck](#kueue-x-k8s-io-v1beta2-AdmissionCheck) + + +

AdmissionCheckSpec defines the desired state of AdmissionCheck

+ + + + + + + + + + + + + + + + + +
FieldDescription
controllerName [Required]
+string +
+

controllerName identifies the controller that processes the AdmissionCheck, +not necessarily a Kubernetes Pod or Deployment name. Cannot be empty.

+
retryDelayMinutes
+int64 +
+

retryDelayMinutes specifies how long to keep the workload suspended after +a failed check (after it transitioned to False). When the delay period has passed, the check +state goes to "Unknown". The default is 15 min. +Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2.

+
parameters
+AdmissionCheckParametersReference +
+

parameters identifies a configuration with additional parameters for the +check.

+
+ +## `AdmissionCheckState` {#kueue-x-k8s-io-v1beta2-AdmissionCheckState} + + +**Appears in:** + +- [WorkloadStatus](#kueue-x-k8s-io-v1beta2-WorkloadStatus) + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+AdmissionCheckReference +
+

name identifies the admission check.

+
state [Required]
+CheckState +
+

state of the admissionCheck, one of Pending, Ready, Retry, Rejected

+
lastTransitionTime [Required]
+k8s.io/apimachinery/pkg/apis/meta/v1.Time +
+

lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.

+
message [Required]
+string +
+

message is a human readable message indicating details about the transition. +This may be an empty string.

+
podSetUpdates
+[]PodSetUpdate +
+

podSetUpdates contains a list of pod set modifications suggested by AdmissionChecks.

+
+ +## `AdmissionCheckStatus` {#kueue-x-k8s-io-v1beta2-AdmissionCheckStatus} + + +**Appears in:** + +- [AdmissionCheck](#kueue-x-k8s-io-v1beta2-AdmissionCheck) + + +

AdmissionCheckStatus defines the observed state of AdmissionCheck

+ + + + + + + + + + + +
FieldDescription
conditions
+[]k8s.io/apimachinery/pkg/apis/meta/v1.Condition +
+

conditions hold the latest available observations of the AdmissionCheck +current state.

+
+ +## `AdmissionCheckStrategyRule` {#kueue-x-k8s-io-v1beta2-AdmissionCheckStrategyRule} + + +**Appears in:** + +- [AdmissionChecksStrategy](#kueue-x-k8s-io-v1beta2-AdmissionChecksStrategy) + + +

AdmissionCheckStrategyRule defines rules for a single AdmissionCheck

+ + + + + + + + + + + + + + +
FieldDescription
name [Required]
+AdmissionCheckReference +
+

name is an AdmissionCheck's name.

+
onFlavors
+[]ResourceFlavorReference +
+

onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. +If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue.

+
+ +## `AdmissionChecksStrategy` {#kueue-x-k8s-io-v1beta2-AdmissionChecksStrategy} + + +**Appears in:** + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + + +

AdmissionChecksStrategy defines a strategy for a AdmissionCheck.

+ + + + + + + + + + + +
FieldDescription
admissionChecks [Required]
+[]AdmissionCheckStrategyRule +
+

admissionChecks is a list of strategies for AdmissionChecks

+
+ +## `AdmissionFairSharingStatus` {#kueue-x-k8s-io-v1beta2-AdmissionFairSharingStatus} + + +**Appears in:** + +- [FairSharingStatus](#kueue-x-k8s-io-v1beta2-FairSharingStatus) + + + + + + + + + + + + + + + +
FieldDescription
consumedResources [Required]
+k8s.io/api/core/v1.ResourceList +
+

consumedResources represents the aggregated usage of resources over time, +with decaying function applied. +The value is populated if usage consumption functionality is enabled in Kueue config.

+
lastUpdate [Required]
+k8s.io/apimachinery/pkg/apis/meta/v1.Time +
+

lastUpdate is the time when share and consumed resources were updated.

+
+ +## `AdmissionMode` {#kueue-x-k8s-io-v1beta2-AdmissionMode} + +(Alias of `string`) + +**Appears in:** + +- [AdmissionScope](#kueue-x-k8s-io-v1beta2-AdmissionScope) + + + + + +## `AdmissionScope` {#kueue-x-k8s-io-v1beta2-AdmissionScope} + + +**Appears in:** + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + + + + + + + + + + + + +
FieldDescription
admissionMode [Required]
+AdmissionMode +
+

admissionMode indicates which mode for AdmissionFairSharing should be used +in the AdmissionScope. Possible values are:

+
    +
  • UsageBasedAdmissionFairSharing
  • +
  • NoAdmissionFairSharing
  • +
+
+ +## `BorrowWithinCohort` {#kueue-x-k8s-io-v1beta2-BorrowWithinCohort} + + +**Appears in:** + +- [ClusterQueuePreemption](#kueue-x-k8s-io-v1beta2-ClusterQueuePreemption) + + +

BorrowWithinCohort contains configuration which allows to preempt workloads +within cohort while borrowing. It only works with Classical Preemption, +not with Fair Sharing.

+ + + + + + + + + + + + + + +
FieldDescription
policy [Required]
+BorrowWithinCohortPolicy +
+

policy determines the policy for preemption to reclaim quota within cohort while borrowing. +Possible values are:

+
    +
  • Never (default): do not allow for preemption, in other +ClusterQueues within the cohort, for a borrowing workload.
  • +
  • LowerPriority: allow preemption, in other ClusterQueues +within the cohort, for a borrowing workload, but only if +the preempted workloads are of lower priority.
  • +
+
maxPriorityThreshold
+int32 +
+

maxPriorityThreshold allows to restrict the set of workloads which +might be preempted by a borrowing workload, to only workloads with +priority less than or equal to the specified threshold priority. +When the threshold is not specified, then any workload satisfying the +policy can be preempted by the borrowing workload.

+
+ +## `BorrowWithinCohortPolicy` {#kueue-x-k8s-io-v1beta2-BorrowWithinCohortPolicy} + +(Alias of `string`) + +**Appears in:** + +- [BorrowWithinCohort](#kueue-x-k8s-io-v1beta2-BorrowWithinCohort) + + + + + +## `CheckState` {#kueue-x-k8s-io-v1beta2-CheckState} + +(Alias of `string`) + +**Appears in:** + +- [AdmissionCheckState](#kueue-x-k8s-io-v1beta2-AdmissionCheckState) + + + + + +## `ClusterQueuePendingWorkload` {#kueue-x-k8s-io-v1beta2-ClusterQueuePendingWorkload} + + +**Appears in:** + +- [ClusterQueuePendingWorkloadsStatus](#kueue-x-k8s-io-v1beta2-ClusterQueuePendingWorkloadsStatus) + + +

ClusterQueuePendingWorkload contains the information identifying a pending workload +in the cluster queue.

+ + + + + + + + + + + + + + +
FieldDescription
name [Required]
+string +
+

name indicates the name of the pending workload.

+
namespace [Required]
+string +
+

namespace indicates the name of the pending workload.

+
+ +## `ClusterQueuePendingWorkloadsStatus` {#kueue-x-k8s-io-v1beta2-ClusterQueuePendingWorkloadsStatus} + + +**Appears in:** + +- [ClusterQueueStatus](#kueue-x-k8s-io-v1beta2-ClusterQueueStatus) + + + + + + + + + + + + + + + +
FieldDescription
clusterQueuePendingWorkload
+[]ClusterQueuePendingWorkload +
+

clusterQueuePendingWorkload contains the list of top pending workloads.

+
lastChangeTime [Required]
+k8s.io/apimachinery/pkg/apis/meta/v1.Time +
+

lastChangeTime indicates the time of the last change of the structure.

+
+ +## `ClusterQueuePreemption` {#kueue-x-k8s-io-v1beta2-ClusterQueuePreemption} + + +**Appears in:** + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + + +

ClusterQueuePreemption contains policies to preempt Workloads from this +ClusterQueue or the ClusterQueue's cohort.

+

Preemption may be configured to work in the following scenarios:

+
    +
  • When a Workload fits within the nominal quota of the ClusterQueue, but +the quota is currently borrowed by other ClusterQueues in the cohort. +We preempt workloads in other ClusterQueues to allow this ClusterQueue to +reclaim its nominal quota. Configured using reclaimWithinCohort.
  • +
  • When a Workload doesn't fit within the nominal quota of the ClusterQueue +and there are admitted Workloads in the ClusterQueue with lower priority. +Configured using withinClusterQueue.
  • +
  • When a Workload may fit while both borrowing and preempting +low priority workloads in the Cohort. Configured using borrowWithinCohort.
  • +
  • When FairSharing is enabled, to maintain fair distribution of +unused resources. See FairSharing documentation.
  • +
+

The preemption algorithm tries to find a minimal set of Workloads to +preempt to accomomdate the pending Workload, preempting Workloads with +lower priority first.

+ + + + + + + + + + + + + + + + + +
FieldDescription
reclaimWithinCohort [Required]
+PreemptionPolicy +
+

reclaimWithinCohort determines whether a pending Workload can preempt +Workloads from other ClusterQueues in the cohort that are using more than +their nominal quota. The possible values are:

+
    +
  • Never (default): do not preempt Workloads in the cohort.
  • +
  • LowerPriority: Classic Preemption if the pending Workload +fits within the nominal quota of its ClusterQueue, only preempt +Workloads in the cohort that have lower priority than the pending +Workload. Fair Sharing only preempt Workloads in the cohort that +have lower priority than the pending Workload and that satisfy the +Fair Sharing preemptionStategies.
  • +
  • Any: Classic Preemption if the pending Workload fits within +the nominal quota of its ClusterQueue, preempt any Workload in the +cohort, irrespective of priority. Fair Sharing preempt Workloads +in the cohort that satisfy the Fair Sharing preemptionStrategies.
  • +
+
borrowWithinCohort [Required]
+BorrowWithinCohort +
+

borrowWithinCohort determines whether a pending Workload can preempt +Workloads from other ClusterQueues in the cohort if the workload requires borrowing. +May only be configured with Classical Preemption, and not with Fair Sharing.

+
withinClusterQueue [Required]
+PreemptionPolicy +
+

withinClusterQueue determines whether a pending Workload that doesn't fit +within the nominal quota for its ClusterQueue, can preempt active Workloads in +the ClusterQueue. The possible values are:

+
    +
  • Never (default): do not preempt Workloads in the ClusterQueue.
  • +
  • LowerPriority: only preempt Workloads in the ClusterQueue that have +lower priority than the pending Workload.
  • +
  • LowerOrNewerEqualPriority: only preempt Workloads in the ClusterQueue that +either have a lower priority than the pending workload or equal priority +and are newer than the pending workload.
  • +
+
+ +## `ClusterQueueReference` {#kueue-x-k8s-io-v1beta2-ClusterQueueReference} + +(Alias of `string`) + +**Appears in:** + +- [Admission](#kueue-x-k8s-io-v1beta2-Admission) + +- [LocalQueueSpec](#kueue-x-k8s-io-v1beta2-LocalQueueSpec) + + +

ClusterQueueReference is the name of the ClusterQueue. +It must be a DNS (RFC 1123) and has the maximum length of 253 characters.

+ + + + +## `ClusterQueueSpec` {#kueue-x-k8s-io-v1beta2-ClusterQueueSpec} + + +**Appears in:** + +- [ClusterQueue](#kueue-x-k8s-io-v1beta2-ClusterQueue) + + +

ClusterQueueSpec defines the desired state of ClusterQueue

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
resourceGroups [Required]
+[]ResourceGroup +
+

resourceGroups describes groups of resources. +Each resource group defines the list of resources and a list of flavors +that provide quotas for these resources. +Each resource and each flavor can only form part of one resource group. +resourceGroups can be up to 16, with a max of 256 total flavors across all groups.

+
cohortName [Required]
+CohortReference +
+

cohortName that this ClusterQueue belongs to. CQs that belong to the +same cohort can borrow unused resources from each other.

+

A CQ can be a member of a single borrowing cohort. A workload submitted +to a queue referencing this CQ can borrow quota from any CQ in the cohort. +Only quota for the [resource, flavor] pairs listed in the CQ can be +borrowed. +If empty, this ClusterQueue cannot borrow from any other ClusterQueue and +vice versa.

+

A cohort is a name that links CQs together, but it doesn't reference any +object.

+
queueingStrategy [Required]
+QueueingStrategy +
+

queueingStrategy indicates the queueing strategy of the workloads +across the queues in this ClusterQueue. +Current Supported Strategies:

+
    +
  • StrictFIFO: workloads are ordered strictly by creation time. +Older workloads that can't be admitted will block admitting newer +workloads even if they fit available quota.
  • +
  • BestEffortFIFO: workloads are ordered by creation time, +however older workloads that can't be admitted will not block +admitting newer workloads that fit existing quota.
  • +
+
namespaceSelector [Required]
+k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector +
+

namespaceSelector defines which namespaces are allowed to submit workloads to +this clusterQueue. Beyond this basic support for policy, a policy agent like +Gatekeeper should be used to enforce more advanced policies. +Defaults to null which is a nothing selector (no namespaces eligible). +If set to an empty selector {}, then all namespaces are eligible.

+
flavorFungibility [Required]
+FlavorFungibility +
+

flavorFungibility defines whether a workload should try the next flavor +before borrowing or preempting in the flavor being evaluated.

+
preemption [Required]
+ClusterQueuePreemption +
+

preemption defines the preemption policies.

+
admissionChecks
+[]AdmissionCheckReference +
+

admissionChecks lists the AdmissionChecks required by this ClusterQueue. +Cannot be used along with AdmissionCheckStrategy.

+
admissionChecksStrategy
+AdmissionChecksStrategy +
+

admissionChecksStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. +This property cannot be used in conjunction with the 'admissionChecks' property.

+
stopPolicy
+StopPolicy +
+

stopPolicy - if set to a value different from None, the ClusterQueue is considered Inactive, no new reservation being +made.

+

Depending on its value, its associated workloads will:

+
    +
  • None - Workloads are admitted
  • +
  • HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation.
  • +
  • Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation.
  • +
+
fairSharing
+FairSharing +
+

fairSharing defines the properties of the ClusterQueue when +participating in FairSharing. The values are only relevant +if FairSharing is enabled in the Kueue configuration.

+
admissionScope
+AdmissionScope +
+

admissionScope indicates whether ClusterQueue uses the Admission Fair Sharing

+
+ +## `ClusterQueueStatus` {#kueue-x-k8s-io-v1beta2-ClusterQueueStatus} + + +**Appears in:** + +- [ClusterQueue](#kueue-x-k8s-io-v1beta2-ClusterQueue) + + +

ClusterQueueStatus defines the observed state of ClusterQueue

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
conditions
+[]k8s.io/apimachinery/pkg/apis/meta/v1.Condition +
+

conditions hold the latest available observations of the ClusterQueue +current state.

+
flavorsReservation
+[]FlavorUsage +
+

flavorsReservation are the reserved quotas, by flavor, currently in use by the +workloads assigned to this ClusterQueue.

+
flavorsUsage
+[]FlavorUsage +
+

flavorsUsage are the used quotas, by flavor, currently in use by the +workloads admitted in this ClusterQueue.

+
pendingWorkloads
+int32 +
+

pendingWorkloads is the number of workloads currently waiting to be +admitted to this clusterQueue.

+
reservingWorkloads
+int32 +
+

reservingWorkloads is the number of workloads currently reserving quota in this +clusterQueue.

+
admittedWorkloads
+int32 +
+

admittedWorkloads is the number of workloads currently admitted to this +clusterQueue and haven't finished yet.

+
pendingWorkloadsStatus
+ClusterQueuePendingWorkloadsStatus +
+

pendingWorkloadsStatus contains the information exposed about the current +status of the pending workloads in the cluster queue. +Deprecated: This field is no longer effective since v0.14.0, which means Kueue no longer stores and updates information. +You can migrate to VisibilityOnDemand +(https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) +instead.

+
fairSharing
+FairSharingStatus +
+

fairSharing contains the current state for this ClusterQueue +when participating in Fair Sharing. +This is recorded only when Fair Sharing is enabled in the Kueue configuration.

+
+ +## `CohortReference` {#kueue-x-k8s-io-v1beta2-CohortReference} + +(Alias of `string`) + +**Appears in:** + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + +- [CohortSpec](#kueue-x-k8s-io-v1beta2-CohortSpec) + + +

CohortReference is the name of the Cohort.

+

Validation of a cohort name is equivalent to that of object names: +subdomain in DNS (RFC 1123).

+ + + + +## `CohortSpec` {#kueue-x-k8s-io-v1beta2-CohortSpec} + + +**Appears in:** + +- [Cohort](#kueue-x-k8s-io-v1beta2-Cohort) + + +

CohortSpec defines the desired state of Cohort

+ + + + + + + + + + + + + + + + + +
FieldDescription
parentName [Required]
+CohortReference +
+

parentName references the name of the Cohort's parent, if +any. It satisfies one of three cases:

+
    +
  1. Unset. This Cohort is the root of its Cohort tree.
  2. +
  3. References a non-existent Cohort. We use default Cohort (no borrowing/lending limits).
  4. +
  5. References an existent Cohort.
  6. +
+

If a cycle is created, we disable all members of the +Cohort, including ClusterQueues, until the cycle is +removed. We prevent further admission while the cycle +exists.

+
resourceGroups [Required]
+[]ResourceGroup +
+

resourceGroups describes groupings of Resources and +Flavors. Each ResourceGroup defines a list of Resources +and a list of Flavors which provide quotas for these +Resources. Each Resource and each Flavor may only form part +of one ResourceGroup. There may be up to 16 ResourceGroups +within a Cohort.

+

Please note that nominalQuota defined at the Cohort level +represents additional resources on top of those defined by +ClusterQueues within the Cohort. The Cohort's nominalQuota +may be thought of as a shared pool for the ClusterQueues +within it. Additionally, this quota may also be lent out to +parent Cohort(s), subject to LendingLimit.

+

BorrowingLimit limits how much members of this Cohort +subtree can borrow from the parent subtree.

+

LendingLimit limits how much members of this Cohort subtree +can lend to the parent subtree.

+

Borrowing and Lending limits must only be set when the +Cohort has a parent. Otherwise, the Cohort create/update +will be rejected by the webhook.

+
fairSharing
+FairSharing +
+

fairSharing defines the properties of the Cohort when +participating in FairSharing. The values are only relevant +if FairSharing is enabled in the Kueue configuration.

+
+ +## `CohortStatus` {#kueue-x-k8s-io-v1beta2-CohortStatus} + + +**Appears in:** + +- [Cohort](#kueue-x-k8s-io-v1beta2-Cohort) + + +

CohortStatus defines the observed state of Cohort.

+ + + + + + + + + + + +
FieldDescription
fairSharing
+FairSharingStatus +
+

fairSharing contains the current state for this Cohort +when participating in Fair Sharing. +The is recorded only when Fair Sharing is enabled in the Kueue configuration.

+
+ +## `DelayedTopologyRequestState` {#kueue-x-k8s-io-v1beta2-DelayedTopologyRequestState} + +(Alias of `string`) + +**Appears in:** + +- [PodSetAssignment](#kueue-x-k8s-io-v1beta2-PodSetAssignment) + + +

DelayedTopologyRequestState indicates the state of the delayed TopologyRequest.

+ + + + +## `EvictionUnderlyingCause` {#kueue-x-k8s-io-v1beta2-EvictionUnderlyingCause} + +(Alias of `string`) + +**Appears in:** + +- [WorkloadSchedulingStatsEviction](#kueue-x-k8s-io-v1beta2-WorkloadSchedulingStatsEviction) + + +

EvictionUnderlyingCause represents the underlying cause of a workload eviction.

+ + + + +## `FairSharing` {#kueue-x-k8s-io-v1beta2-FairSharing} + + +**Appears in:** + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + +- [CohortSpec](#kueue-x-k8s-io-v1beta2-CohortSpec) + +- [LocalQueueSpec](#kueue-x-k8s-io-v1beta2-LocalQueueSpec) + + +

FairSharing contains the properties of the ClusterQueue or Cohort, +when participating in FairSharing.

+

Fair Sharing is compatible with Hierarchical Cohorts (any Cohort +which has a parent) as of v0.11. Using these features together in +V0.9 and V0.10 is unsupported, and results in undefined behavior.

+ + + + + + + + + + + +
FieldDescription
weight [Required]
+k8s.io/apimachinery/pkg/api/resource.Quantity +
+

weight gives a comparative advantage to this ClusterQueue +or Cohort when competing for unused resources in the +Cohort. The share is based on the dominant resource usage +above nominal quotas for each resource, divided by the +weight. Admission prioritizes scheduling workloads from +ClusterQueues and Cohorts with the lowest share and +preempting workloads from the ClusterQueues and Cohorts +with the highest share. A zero weight implies infinite +share value, meaning that this Node will always be at +disadvantage against other ClusterQueues and Cohorts. +When not 0, Weight must be greater than 10^-9.

+
+ +## `FairSharingStatus` {#kueue-x-k8s-io-v1beta2-FairSharingStatus} + + +**Appears in:** + +- [ClusterQueueStatus](#kueue-x-k8s-io-v1beta2-ClusterQueueStatus) + +- [CohortStatus](#kueue-x-k8s-io-v1beta2-CohortStatus) + +- [LocalQueueStatus](#kueue-x-k8s-io-v1beta2-LocalQueueStatus) + + +

FairSharingStatus contains the information about the current status of Fair Sharing.

+ + + + + + + + + + + + + + +
FieldDescription
weightedShare [Required]
+int64 +
+

weightedShare represents the maximum of the ratios of usage +above nominal quota to the lendable resources in the +Cohort, among all the resources provided by the Node, and +divided by the weight. If zero, it means that the usage of +the Node is below the nominal quota. If the Node has a +weight of zero and is borrowing, this will return +9223372036854775807, the maximum possible share value.

+
admissionFairSharingStatus
+AdmissionFairSharingStatus +
+

admissionFairSharingStatus represents information relevant to the Admission Fair Sharing

+
+ +## `FlavorFungibility` {#kueue-x-k8s-io-v1beta2-FlavorFungibility} + + +**Appears in:** + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + + +

FlavorFungibility determines whether a workload should try the next flavor +before borrowing or preempting in current flavor.

+ + + + + + + + + + + + + + +
FieldDescription
whenCanBorrow [Required]
+FlavorFungibilityPolicy +
+

whenCanBorrow determines whether a workload should try the next flavor +before borrowing in current flavor. The possible values are:

+
    +
  • MayStopSearch (default): stop the search for candidate flavors if workload +fits or requires borrowing to fit.
  • +
  • TryNextFlavor: try next flavor if workload requires borrowing to fit.
  • +
  • Borrow (deprecated): old name for MayStopSearch; please use new name.
  • +
+
whenCanPreempt [Required]
+FlavorFungibilityPolicy +
+

whenCanPreempt determines whether a workload should try the next flavor +before borrowing in current flavor. The possible values are:

+
    +
  • MayStopSearch: stop the search for candidate flavors if workload fits or requires +preemption to fit.
  • +
  • TryNextFlavor (default): try next flavor if workload requires preemption +to fit in current flavor.
  • +
  • Preempt (deprecated): old name for MayStopSearch; please use new name.
  • +
+
+ +## `FlavorFungibilityPolicy` {#kueue-x-k8s-io-v1beta2-FlavorFungibilityPolicy} + +(Alias of `string`) + +**Appears in:** + +- [FlavorFungibility](#kueue-x-k8s-io-v1beta2-FlavorFungibility) + + + + + +## `FlavorQuotas` {#kueue-x-k8s-io-v1beta2-FlavorQuotas} + + +**Appears in:** + +- [ResourceGroup](#kueue-x-k8s-io-v1beta2-ResourceGroup) + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+ResourceFlavorReference +
+

name of this flavor. The name should match the .metadata.name of a +ResourceFlavor. If a matching ResourceFlavor does not exist, the +ClusterQueue will have an Active condition set to False.

+
resources [Required]
+[]ResourceQuota +
+

resources is the list of quotas for this flavor per resource. +There could be up to 64 resources.

+
+ +## `FlavorUsage` {#kueue-x-k8s-io-v1beta2-FlavorUsage} + + +**Appears in:** + +- [ClusterQueueStatus](#kueue-x-k8s-io-v1beta2-ClusterQueueStatus) + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+ResourceFlavorReference +
+

name of the flavor.

+
resources [Required]
+[]ResourceUsage +
+

resources lists the quota usage for the resources in this flavor.

+
+ +## `KubeConfig` {#kueue-x-k8s-io-v1beta2-KubeConfig} + + +**Appears in:** + +- [MultiKueueClusterSpec](#kueue-x-k8s-io-v1beta2-MultiKueueClusterSpec) + + + + + + + + + + + + + + + +
FieldDescription
location [Required]
+string +
+

location of the KubeConfig.

+

If LocationType is Secret then Location is the name of the secret inside the namespace in +which the kueue controller manager is running. The config should be stored in the "kubeconfig" key.

+
locationType [Required]
+LocationType +
+

locationType of the KubeConfig.

+
+ +## `LocalQueueFlavorStatus` {#kueue-x-k8s-io-v1beta2-LocalQueueFlavorStatus} + + +**Appears in:** + +- [LocalQueueStatus](#kueue-x-k8s-io-v1beta2-LocalQueueStatus) + + +

Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.

+ + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+ResourceFlavorReference +
+

name of the flavor.

+
resources
+[]k8s.io/api/core/v1.ResourceName +
+

resources used in the flavor.

+
nodeLabels
+map[string]string +
+

nodeLabels are labels that associate the ResourceFlavor with Nodes that +have the same labels.

+
nodeTaints
+[]k8s.io/api/core/v1.Taint +
+

nodeTaints are taints that the nodes associated with this ResourceFlavor +have.

+
topology
+TopologyInfo +
+

topology is the topology that associated with this ResourceFlavor.

+

This is a beta field and requires enabling the TopologyAwareScheduling +feature gate.

+
+ +## `LocalQueueFlavorUsage` {#kueue-x-k8s-io-v1beta2-LocalQueueFlavorUsage} + + +**Appears in:** + +- [LocalQueueStatus](#kueue-x-k8s-io-v1beta2-LocalQueueStatus) + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+ResourceFlavorReference +
+

name of the flavor.

+
resources [Required]
+[]LocalQueueResourceUsage +
+

resources lists the quota usage for the resources in this flavor.

+
+ +## `LocalQueueName` {#kueue-x-k8s-io-v1beta2-LocalQueueName} + +(Alias of `string`) + +**Appears in:** + +- [WorkloadSpec](#kueue-x-k8s-io-v1beta2-WorkloadSpec) + + +

LocalQueueName is the name of the LocalQueue. +It must be a DNS (RFC 1123) and has the maximum length of 253 characters.

+ + + + +## `LocalQueueResourceUsage` {#kueue-x-k8s-io-v1beta2-LocalQueueResourceUsage} + + +**Appears in:** + +- [LocalQueueFlavorUsage](#kueue-x-k8s-io-v1beta2-LocalQueueFlavorUsage) + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+k8s.io/api/core/v1.ResourceName +
+

name of the resource.

+
total [Required]
+k8s.io/apimachinery/pkg/api/resource.Quantity +
+

total is the total quantity of used quota.

+
+ +## `LocalQueueSpec` {#kueue-x-k8s-io-v1beta2-LocalQueueSpec} + + +**Appears in:** + +- [LocalQueue](#kueue-x-k8s-io-v1beta2-LocalQueue) + + +

LocalQueueSpec defines the desired state of LocalQueue

+ + + + + + + + + + + + + + + + + +
FieldDescription
clusterQueue [Required]
+ClusterQueueReference +
+

clusterQueue is a reference to a clusterQueue that backs this localQueue.

+
stopPolicy
+StopPolicy +
+

stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, +no new reservation being made.

+

Depending on its value, its associated workloads will:

+
    +
  • None - Workloads are admitted
  • +
  • HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation.
  • +
  • Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation.
  • +
+
fairSharing
+FairSharing +
+

fairSharing defines the properties of the LocalQueue when +participating in AdmissionFairSharing. The values are only relevant +if AdmissionFairSharing is enabled in the Kueue configuration.

+
+ +## `LocalQueueStatus` {#kueue-x-k8s-io-v1beta2-LocalQueueStatus} + + +**Appears in:** + +- [LocalQueue](#kueue-x-k8s-io-v1beta2-LocalQueue) + + +

LocalQueueStatus defines the observed state of LocalQueue

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
conditions
+[]k8s.io/apimachinery/pkg/apis/meta/v1.Condition +
+

conditions hold the latest available observations of the LocalQueue +current state.

+
pendingWorkloads
+int32 +
+

pendingWorkloads is the number of Workloads in the LocalQueue not yet admitted to a ClusterQueue

+
reservingWorkloads
+int32 +
+

reservingWorkloads is the number of workloads in this LocalQueue +reserving quota in a ClusterQueue and that haven't finished yet.

+
admittedWorkloads
+int32 +
+

admittedWorkloads is the number of workloads in this LocalQueue +admitted to a ClusterQueue and that haven't finished yet.

+
flavorsReservation
+[]LocalQueueFlavorUsage +
+

flavorsReservation are the reserved quotas, by flavor currently in use by the +workloads assigned to this LocalQueue.

+
flavorsUsage
+[]LocalQueueFlavorUsage +
+

flavorsUsage are the used quotas, by flavor currently in use by the +workloads assigned to this LocalQueue.

+
flavors
+[]LocalQueueFlavorStatus +
+

flavors lists all currently available ResourceFlavors in specified ClusterQueue. +Deprecated: Flavors is deprecated and marked for removal in v1beta2.

+
fairSharing
+FairSharingStatus +
+

fairSharing contains the information about the current status of fair sharing.

+
+ +## `LocationType` {#kueue-x-k8s-io-v1beta2-LocationType} + +(Alias of `string`) + +**Appears in:** + +- [KubeConfig](#kueue-x-k8s-io-v1beta2-KubeConfig) + + + + + +## `MultiKueueClusterSpec` {#kueue-x-k8s-io-v1beta2-MultiKueueClusterSpec} + + +**Appears in:** + +- [MultiKueueCluster](#kueue-x-k8s-io-v1beta2-MultiKueueCluster) + + + + + + + + + + + + +
FieldDescription
kubeConfig [Required]
+KubeConfig +
+

kubeConfig is information on how to connect to the cluster.

+
+ +## `MultiKueueClusterStatus` {#kueue-x-k8s-io-v1beta2-MultiKueueClusterStatus} + + +**Appears in:** + +- [MultiKueueCluster](#kueue-x-k8s-io-v1beta2-MultiKueueCluster) + + + + + + + + + + + + +
FieldDescription
conditions
+[]k8s.io/apimachinery/pkg/apis/meta/v1.Condition +
+

conditions hold the latest available observations of the MultiKueueCluster +current state.

+
+ +## `MultiKueueConfigSpec` {#kueue-x-k8s-io-v1beta2-MultiKueueConfigSpec} + + +**Appears in:** + +- [MultiKueueConfig](#kueue-x-k8s-io-v1beta2-MultiKueueConfig) + + +

MultiKueueConfigSpec defines the desired state of MultiKueueConfig

+ + + + + + + + + + + +
FieldDescription
clusters [Required]
+[]string +
+

clusters is a list of MultiKueueClusters names where the workloads from the ClusterQueue should be distributed.

+
+ +## `Parameter` {#kueue-x-k8s-io-v1beta2-Parameter} + +(Alias of `string`) + +**Appears in:** + +- [ProvisioningRequestConfigSpec](#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfigSpec) + + +

Parameter is limited to 255 characters.

+ + + + +## `PodSet` {#kueue-x-k8s-io-v1beta2-PodSet} + + +**Appears in:** + +- [WorkloadSpec](#kueue-x-k8s-io-v1beta2-WorkloadSpec) + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+PodSetReference +
+

name is the PodSet name.

+
template [Required]
+k8s.io/api/core/v1.PodTemplateSpec +
+

template is the Pod template.

+

The only allowed fields in template.metadata are labels and annotations.

+

If requests are omitted for a container or initContainer, +they default to the limits if they are explicitly specified for the +container or initContainer.

+

During admission, the rules in nodeSelector and +nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution that match +the keys in the nodeLabels from the ResourceFlavors considered for this +Workload are used to filter the ResourceFlavors that can be assigned to +this podSet.

+
count [Required]
+int32 +
+

count is the number of pods for the spec.

+
minCount
+int32 +
+

minCount is the minimum number of pods for the spec acceptable +if the workload supports partial admission.

+

If not provided, partial admission for the current PodSet is not +enabled.

+

Only one podSet within the workload can use this.

+

This is an alpha field and requires enabling PartialAdmission feature gate.

+
topologyRequest
+PodSetTopologyRequest +
+

topologyRequest defines the topology request for the PodSet.

+
+ +## `PodSetAssignment` {#kueue-x-k8s-io-v1beta2-PodSetAssignment} + + +**Appears in:** + +- [Admission](#kueue-x-k8s-io-v1beta2-Admission) + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+PodSetReference +
+

name is the name of the podSet. It should match one of the names in .spec.podSets.

+
flavors [Required]
+map[ResourceName]ResourceFlavorReference +
+

flavors are the flavors assigned to the workload for each resource.

+
resourceUsage [Required]
+k8s.io/api/core/v1.ResourceList +
+

resourceUsage keeps track of the total resources all the pods in the podset need to run.

+

Beside what is provided in podSet's specs, this calculation takes into account +the LimitRange defaults and RuntimeClass overheads at the moment of admission. +This field will not change in case of quota reclaim.

+
count
+int32 +
+

count is the number of pods taken into account at admission time. +This field will not change in case of quota reclaim. +Value could be missing for Workloads created before this field was added, +in that case spec.podSets[*].count value will be used.

+
topologyAssignment
+TopologyAssignment +
+

topologyAssignment indicates the topology assignment divided into +topology domains corresponding to the lowest level of the topology. +The assignment specifies the number of Pods to be scheduled per topology +domain and specifies the node selectors for each topology domain, in the +following way: the node selector keys are specified by the levels field +(same for all domains), and the corresponding node selector value is +specified by the domains.values subfield. If the TopologySpec.Levels field contains +"kubernetes.io/hostname" label, topologyAssignment will contain data only for +this label, and omit higher levels in the topology

+

Example:

+

topologyAssignment: +levels:

+
    +
  • cloud.provider.com/topology-block
  • +
  • cloud.provider.com/topology-rack +domains:
  • +
  • values: [block-1, rack-1] +count: 4
  • +
  • values: [block-1, rack-2] +count: 2
  • +
+

Here:

+
    +
  • 4 Pods are to be scheduled on nodes matching the node selector: +cloud.provider.com/topology-block: block-1 +cloud.provider.com/topology-rack: rack-1
  • +
  • 2 Pods are to be scheduled on nodes matching the node selector: +cloud.provider.com/topology-block: block-1 +cloud.provider.com/topology-rack: rack-2
  • +
+

Example: +Below there is an equivalent of the above example assuming, Topology +object defines kubernetes.io/hostname as the lowest level in topology. +Hence we omit higher level of topologies, since the hostname label +is sufficient to explicitly identify a proper node.

+

topologyAssignment: +levels:

+
    +
  • kubernetes.io/hostname +domains:
  • +
  • values: [hostname-1] +count: 4
  • +
  • values: [hostname-2] +count: 2
  • +
+
delayedTopologyRequest
+DelayedTopologyRequestState +
+

delayedTopologyRequest indicates the topology assignment is delayed. +Topology assignment might be delayed in case there is ProvisioningRequest +AdmissionCheck used. +Kueue schedules the second pass of scheduling for each workload with at +least one PodSet which has delayedTopologyRequest=true and without +topologyAssignment.

+
+ +## `PodSetReference` {#kueue-x-k8s-io-v1beta2-PodSetReference} + +(Alias of `string`) + +**Appears in:** + +- [PodSet](#kueue-x-k8s-io-v1beta2-PodSet) + +- [PodSetAssignment](#kueue-x-k8s-io-v1beta2-PodSetAssignment) + +- [PodSetRequest](#kueue-x-k8s-io-v1beta2-PodSetRequest) + +- [PodSetUpdate](#kueue-x-k8s-io-v1beta2-PodSetUpdate) + +- [ReclaimablePod](#kueue-x-k8s-io-v1beta2-ReclaimablePod) + + +

PodSetReference is the name of a PodSet.

+ + + + +## `PodSetRequest` {#kueue-x-k8s-io-v1beta2-PodSetRequest} + + +**Appears in:** + +- [WorkloadStatus](#kueue-x-k8s-io-v1beta2-WorkloadStatus) + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+PodSetReference +
+

name is the name of the podSet. It should match one of the names in .spec.podSets.

+
resources
+k8s.io/api/core/v1.ResourceList +
+

resources is the total resources all the pods in the podset need to run.

+

Beside what is provided in podSet's specs, this value also takes into account +the LimitRange defaults and RuntimeClass overheads at the moment of consideration +and the application of resource.excludeResourcePrefixes and resource.transformations.

+
+ +## `PodSetTopologyRequest` {#kueue-x-k8s-io-v1beta2-PodSetTopologyRequest} + + +**Appears in:** + +- [PodSet](#kueue-x-k8s-io-v1beta2-PodSet) + + +

PodSetTopologyRequest defines the topology request for a PodSet.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
required
+string +
+

required indicates the topology level required by the PodSet, as +indicated by the kueue.x-k8s.io/podset-required-topology PodSet +annotation.

+
preferred
+string +
+

preferred indicates the topology level preferred by the PodSet, as +indicated by the kueue.x-k8s.io/podset-preferred-topology PodSet +annotation.

+
unconstrained
+bool +
+

unconstrained indicates that Kueue has the freedom to schedule the PodSet within +the entire available capacity, without constraints on the compactness of the placement. +This is indicated by the kueue.x-k8s.io/podset-unconstrained-topology PodSet annotation.

+
podIndexLabel [Required]
+string +
+

podIndexLabel indicates the name of the label indexing the pods. +For example, in the context of

+
    +
  • kubernetes job this is: kubernetes.io/job-completion-index
  • +
  • JobSet: kubernetes.io/job-completion-index (inherited from Job)
  • +
  • Kubeflow: training.kubeflow.org/replica-index
  • +
+
subGroupIndexLabel [Required]
+string +
+

subGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) +within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index.

+
subGroupCount [Required]
+int32 +
+

subGroupCount indicates the count of replicated Jobs (groups) within a PodSet. +For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas.

+
podSetGroupName
+string +
+

podSetGroupName indicates the name of the group of PodSets to which this PodSet belongs to. +PodSets with the same PodSetGroupName should be assigned the same ResourceFlavor

+
podSetSliceRequiredTopology
+string +
+

podSetSliceRequiredTopology indicates the topology level required by the PodSet slice, as +indicated by the kueue.x-k8s.io/podset-slice-required-topology annotation.

+
podSetSliceSize
+int32 +
+

podSetSliceSize indicates the size of a subgroup of pods in a PodSet for which +Kueue finds a requested topology domain on a level defined +in kueue.x-k8s.io/podset-slice-required-topology annotation.

+
+ +## `PodSetUpdate` {#kueue-x-k8s-io-v1beta2-PodSetUpdate} + + +**Appears in:** + +- [AdmissionCheckState](#kueue-x-k8s-io-v1beta2-AdmissionCheckState) + + +

PodSetUpdate contains a list of pod set modifications suggested by AdmissionChecks. +The modifications should be additive only - modifications of already existing keys +or having the same key provided by multiple AdmissionChecks is not allowed and will +result in failure during workload admission.

+ + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+PodSetReference +
+

name of the PodSet to modify. Should match to one of the Workload's PodSets.

+
labels
+map[string]string +
+

labels of the PodSet to modify.

+
annotations
+map[string]string +
+

annotations of the PodSet to modify.

+
nodeSelector
+map[string]string +
+

nodeSelector of the PodSet to modify.

+
tolerations
+[]k8s.io/api/core/v1.Toleration +
+

tolerations of the PodSet to modify.

+
+ +## `PreemptionPolicy` {#kueue-x-k8s-io-v1beta2-PreemptionPolicy} + +(Alias of `string`) + +**Appears in:** + +- [ClusterQueuePreemption](#kueue-x-k8s-io-v1beta2-ClusterQueuePreemption) + + + + + +## `ProvisioningRequestConfigPodSetMergePolicy` {#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfigPodSetMergePolicy} + +(Alias of `string`) + +**Appears in:** + +- [ProvisioningRequestConfigSpec](#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfigSpec) + + + + + +## `ProvisioningRequestConfigSpec` {#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfigSpec} + + +**Appears in:** + +- [ProvisioningRequestConfig](#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfig) + + +

ProvisioningRequestConfigSpec defines the desired state of ProvisioningRequestConfig

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
provisioningClassName [Required]
+string +
+

provisioningClassName describes the different modes of provisioning the resources. +Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details.

+
parameters
+map[string]Parameter +
+

parameters contains all other parameters classes may require.

+
managedResources
+[]k8s.io/api/core/v1.ResourceName +
+

managedResources contains the list of resources managed by the autoscaling.

+

If empty, all resources are considered managed.

+

If not empty, the ProvisioningRequest will contain only the podsets that are +requesting at least one of them.

+

If none of the workloads podsets is requesting at least a managed resource, +the workload is considered ready.

+
retryStrategy
+ProvisioningRequestRetryStrategy +
+

retryStrategy defines strategy for retrying ProvisioningRequest. +If null, then the default configuration is applied with the following parameter values: +backoffLimitCount: 3 +backoffBaseSeconds: 60 - 1 min +backoffMaxSeconds: 1800 - 30 mins

+

To switch off retry mechanism +set retryStrategy.backoffLimitCount to 0.

+
podSetUpdates
+ProvisioningRequestPodSetUpdates +
+

podSetUpdates specifies the update of the workload's PodSetUpdates which +are used to target the provisioned nodes.

+
podSetMergePolicy
+ProvisioningRequestConfigPodSetMergePolicy +
+

podSetMergePolicy specifies the policy for merging PodSets before being passed +to the cluster autoscaler.

+
+ +## `ProvisioningRequestPodSetUpdates` {#kueue-x-k8s-io-v1beta2-ProvisioningRequestPodSetUpdates} + + +**Appears in:** + +- [ProvisioningRequestConfigSpec](#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfigSpec) + + + + + + + + + + + + +
FieldDescription
nodeSelector
+[]ProvisioningRequestPodSetUpdatesNodeSelector +
+

nodeSelector specifies the list of updates for the NodeSelector.

+
+ +## `ProvisioningRequestPodSetUpdatesNodeSelector` {#kueue-x-k8s-io-v1beta2-ProvisioningRequestPodSetUpdatesNodeSelector} + + +**Appears in:** + +- [ProvisioningRequestPodSetUpdates](#kueue-x-k8s-io-v1beta2-ProvisioningRequestPodSetUpdates) + + + + + + + + + + + + + + + +
FieldDescription
key [Required]
+string +
+

key specifies the key for the NodeSelector.

+
valueFromProvisioningClassDetail [Required]
+string +
+

valueFromProvisioningClassDetail specifies the key of the +ProvisioningRequest.status.provisioningClassDetails from which the value +is used for the update.

+
+ +## `ProvisioningRequestRetryStrategy` {#kueue-x-k8s-io-v1beta2-ProvisioningRequestRetryStrategy} + + +**Appears in:** + +- [ProvisioningRequestConfigSpec](#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfigSpec) + + + + + + + + + + + + + + + + + + +
FieldDescription
backoffLimitCount
+int32 +
+

backoffLimitCount defines the maximum number of re-queuing retries. +Once the number is reached, the workload is deactivated (.spec.activate=false).

+

Every backoff duration is about "b*2^(n-1)+Rand" where:

+
    +
  • "b" represents the base set by "BackoffBaseSeconds" parameter,
  • +
  • "n" represents the "workloadStatus.requeueState.count",
  • +
  • "Rand" represents the random jitter. +During this time, the workload is taken as an inadmissible and +other workloads will have a chance to be admitted. +By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).
  • +
+

Defaults to 3.

+
backoffBaseSeconds
+int32 +
+

backoffBaseSeconds defines the base for the exponential backoff for +re-queuing an evicted workload.

+

Defaults to 60.

+
backoffMaxSeconds
+int32 +
+

backoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.

+

Defaults to 1800.

+
+ +## `QueueingStrategy` {#kueue-x-k8s-io-v1beta2-QueueingStrategy} + +(Alias of `string`) + +**Appears in:** + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + + + + + +## `ReclaimablePod` {#kueue-x-k8s-io-v1beta2-ReclaimablePod} + + +**Appears in:** + +- [WorkloadStatus](#kueue-x-k8s-io-v1beta2-WorkloadStatus) + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+PodSetReference +
+

name is the PodSet name.

+
count [Required]
+int32 +
+

count is the number of pods for which the requested resources are no longer needed.

+
+ +## `RequeueState` {#kueue-x-k8s-io-v1beta2-RequeueState} + + +**Appears in:** + +- [WorkloadStatus](#kueue-x-k8s-io-v1beta2-WorkloadStatus) + + + + + + + + + + + + + + + +
FieldDescription
count
+int32 +
+

count records the number of times a workload has been re-queued +When a deactivated (.spec.activate=false) workload is reactivated (.spec.activate=true), +this count would be reset to null.

+
requeueAt
+k8s.io/apimachinery/pkg/apis/meta/v1.Time +
+

requeueAt records the time when a workload will be re-queued. +When a deactivated (.spec.activate=false) workload is reactivated (.spec.activate=true), +this time would be reset to null.

+
+ +## `ResourceFlavorReference` {#kueue-x-k8s-io-v1beta2-ResourceFlavorReference} + +(Alias of `string`) + +**Appears in:** + +- [AdmissionCheckStrategyRule](#kueue-x-k8s-io-v1beta2-AdmissionCheckStrategyRule) + +- [FlavorQuotas](#kueue-x-k8s-io-v1beta2-FlavorQuotas) + +- [FlavorUsage](#kueue-x-k8s-io-v1beta2-FlavorUsage) + +- [LocalQueueFlavorStatus](#kueue-x-k8s-io-v1beta2-LocalQueueFlavorStatus) + +- [LocalQueueFlavorUsage](#kueue-x-k8s-io-v1beta2-LocalQueueFlavorUsage) + +- [PodSetAssignment](#kueue-x-k8s-io-v1beta2-PodSetAssignment) + + +

ResourceFlavorReference is the name of the ResourceFlavor.

+ + + + +## `ResourceFlavorSpec` {#kueue-x-k8s-io-v1beta2-ResourceFlavorSpec} + + +**Appears in:** + +- [ResourceFlavor](#kueue-x-k8s-io-v1beta2-ResourceFlavor) + + +

ResourceFlavorSpec defines the desired state of the ResourceFlavor

+ + + + + + + + + + + + + + + + + + + + +
FieldDescription
nodeLabels
+map[string]string +
+

nodeLabels are labels that associate the ResourceFlavor with Nodes that +have the same labels. +When a Workload is admitted, its podsets can only get assigned +ResourceFlavors whose nodeLabels match the nodeSelector and nodeAffinity +fields. +Once a ResourceFlavor is assigned to a podSet, the ResourceFlavor's +nodeLabels should be injected into the pods of the Workload by the +controller that integrates with the Workload object.

+

nodeLabels can be up to 8 elements.

+
nodeTaints
+[]k8s.io/api/core/v1.Taint +
+

nodeTaints are taints that the nodes associated with this ResourceFlavor +have. +Workloads' podsets must have tolerations for these nodeTaints in order to +get assigned this ResourceFlavor during admission. +When this ResourceFlavor has also set the matching tolerations (in .spec.tolerations), +then the nodeTaints are not considered during admission. +Only the 'NoSchedule' and 'NoExecute' taint effects are evaluated, +while 'PreferNoSchedule' is ignored.

+

An example of a nodeTaint is +cloud.provider.com/preemptible="true":NoSchedule

+

nodeTaints can be up to 8 elements.

+
tolerations
+[]k8s.io/api/core/v1.Toleration +
+

tolerations are extra tolerations that will be added to the pods admitted in +the quota associated with this resource flavor.

+

An example of a toleration is +cloud.provider.com/preemptible="true":NoSchedule

+

tolerations can be up to 8 elements.

+
topologyName
+TopologyReference +
+

topologyName indicates topology for the TAS ResourceFlavor. +When specified, it enables scraping of the topology information from the +nodes matching to the Resource Flavor node labels.

+
+ +## `ResourceGroup` {#kueue-x-k8s-io-v1beta2-ResourceGroup} + + +**Appears in:** + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + +- [CohortSpec](#kueue-x-k8s-io-v1beta2-CohortSpec) + + + + + + + + + + + + + + + +
FieldDescription
coveredResources [Required]
+[]k8s.io/api/core/v1.ResourceName +
+

coveredResources is the list of resources covered by the flavors in this +group. +Examples: cpu, memory, vendor.com/gpu. +The list cannot be empty and it can contain up to 64 resources. With a total +of up to 256 covered resources across all resource groups in the ClusterQueue.

+
flavors [Required]
+[]FlavorQuotas +
+

flavors is the list of flavors that provide the resources of this group. +Typically, different flavors represent different hardware models +(e.g., gpu models, cpu architectures) or pricing models (on-demand vs spot +cpus). +Each flavor MUST list all the resources listed for this group in the same +order as the .resources field. +The list cannot be empty and it can contain up to 64 flavors, with a max of +256 total flavors across all resource groups in the ClusterQueue.

+
+ +## `ResourceQuota` {#kueue-x-k8s-io-v1beta2-ResourceQuota} + + +**Appears in:** + +- [FlavorQuotas](#kueue-x-k8s-io-v1beta2-FlavorQuotas) + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+k8s.io/api/core/v1.ResourceName +
+

name of this resource.

+
nominalQuota [Required]
+k8s.io/apimachinery/pkg/api/resource.Quantity +
+

nominalQuota is the quantity of this resource that is available for +Workloads admitted by this ClusterQueue at a point in time. +The nominalQuota must be non-negative. +nominalQuota should represent the resources in the cluster available for +running jobs (after discounting resources consumed by system components +and pods not managed by kueue). In an autoscaled cluster, nominalQuota +should account for resources that can be provided by a component such as +Kubernetes cluster-autoscaler.

+

If the ClusterQueue belongs to a cohort, the sum of the quotas for each +(flavor, resource) combination defines the maximum quantity that can be +allocated by a ClusterQueue in the cohort.

+
borrowingLimit
+k8s.io/apimachinery/pkg/api/resource.Quantity +
+

borrowingLimit is the maximum amount of quota for the [flavor, resource] +combination that this ClusterQueue is allowed to borrow from the unused +quota of other ClusterQueues in the same cohort. +In total, at a given time, Workloads in a ClusterQueue can consume a +quantity of quota equal to nominalQuota+borrowingLimit, assuming the other +ClusterQueues in the cohort have enough unused quota. +If null, it means that there is no borrowing limit. +If not null, it must be non-negative. +borrowingLimit must be null if spec.cohort is empty.

+
lendingLimit
+k8s.io/apimachinery/pkg/api/resource.Quantity +
+

lendingLimit is the maximum amount of unused quota for the [flavor, resource] +combination that this ClusterQueue can lend to other ClusterQueues in the same cohort. +In total, at a given time, ClusterQueue reserves for its exclusive use +a quantity of quota equals to nominalQuota - lendingLimit. +If null, it means that there is no lending limit, meaning that +all the nominalQuota can be borrowed by other clusterQueues in the cohort. +If not null, it must be non-negative. +lendingLimit must be null if spec.cohort is empty. +This field is in beta stage and is enabled by default.

+
+ +## `ResourceUsage` {#kueue-x-k8s-io-v1beta2-ResourceUsage} + + +**Appears in:** + +- [FlavorUsage](#kueue-x-k8s-io-v1beta2-FlavorUsage) + + + + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+k8s.io/api/core/v1.ResourceName +
+

name of the resource

+
total [Required]
+k8s.io/apimachinery/pkg/api/resource.Quantity +
+

total is the total quantity of used quota, including the amount borrowed +from the cohort.

+
borrowed [Required]
+k8s.io/apimachinery/pkg/api/resource.Quantity +
+

borrowed is quantity of quota that is borrowed from the cohort. In other +words, it's the used quota that is over the nominalQuota.

+
+ +## `SchedulingStats` {#kueue-x-k8s-io-v1beta2-SchedulingStats} + + +**Appears in:** + +- [WorkloadStatus](#kueue-x-k8s-io-v1beta2-WorkloadStatus) + + + + + + + + + + + + +
FieldDescription
evictions
+[]WorkloadSchedulingStatsEviction +
+

evictions tracks eviction statistics by reason and underlyingCause.

+
+ +## `StopPolicy` {#kueue-x-k8s-io-v1beta2-StopPolicy} + +(Alias of `string`) + +**Appears in:** + +- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) + +- [LocalQueueSpec](#kueue-x-k8s-io-v1beta2-LocalQueueSpec) + + + + + +## `TopologyAssignment` {#kueue-x-k8s-io-v1beta2-TopologyAssignment} + + +**Appears in:** + +- [PodSetAssignment](#kueue-x-k8s-io-v1beta2-PodSetAssignment) + + + + + + + + + + + + + + + +
FieldDescription
levels [Required]
+[]string +
+

levels is an ordered list of keys denoting the levels of the assigned +topology (i.e. node label keys), from the highest to the lowest level of +the topology.

+
domains [Required]
+[]TopologyDomainAssignment +
+

domains is a list of topology assignments split by topology domains at +the lowest level of the topology.

+
+ +## `TopologyDomainAssignment` {#kueue-x-k8s-io-v1beta2-TopologyDomainAssignment} + + +**Appears in:** + +- [TopologyAssignment](#kueue-x-k8s-io-v1beta2-TopologyAssignment) + + + + + + + + + + + + + + + +
FieldDescription
values [Required]
+[]string +
+

values is an ordered list of node selector values describing a topology +domain. The values correspond to the consecutive topology levels, from +the highest to the lowest.

+
count [Required]
+int32 +
+

count indicates the number of Pods to be scheduled in the topology +domain indicated by the values field.

+
+ +## `TopologyInfo` {#kueue-x-k8s-io-v1beta2-TopologyInfo} + + +**Appears in:** + +- [LocalQueueFlavorStatus](#kueue-x-k8s-io-v1beta2-LocalQueueFlavorStatus) + + + + + + + + + + + + + + + +
FieldDescription
name [Required]
+TopologyReference +
+

name is the name of the topology.

+
levels [Required]
+[]string +
+

levels define the levels of topology.

+
+ +## `TopologyLevel` {#kueue-x-k8s-io-v1beta2-TopologyLevel} + + +**Appears in:** + +- [TopologySpec](#kueue-x-k8s-io-v1beta2-TopologySpec) + + +

TopologyLevel defines the desired state of TopologyLevel

+ + + + + + + + + + + +
FieldDescription
nodeLabel [Required]
+string +
+

nodeLabel indicates the name of the node label for a specific topology +level.

+

Examples:

+
    +
  • cloud.provider.com/topology-block
  • +
  • cloud.provider.com/topology-rack
  • +
+
+ +## `TopologyReference` {#kueue-x-k8s-io-v1beta2-TopologyReference} + +(Alias of `string`) + +**Appears in:** + +- [ResourceFlavorSpec](#kueue-x-k8s-io-v1beta2-ResourceFlavorSpec) + +- [TopologyInfo](#kueue-x-k8s-io-v1beta2-TopologyInfo) + + +

TopologyReference is the name of the Topology.

+ + + + +## `TopologySpec` {#kueue-x-k8s-io-v1beta2-TopologySpec} + + +**Appears in:** + +- [Topology](#kueue-x-k8s-io-v1beta2-Topology) + + +

TopologySpec defines the desired state of Topology

+ + + + + + + + + + + +
FieldDescription
levels [Required]
+[]TopologyLevel +
+

levels define the levels of topology.

+
+ +## `UnhealthyNode` {#kueue-x-k8s-io-v1beta2-UnhealthyNode} + + +**Appears in:** + +- [WorkloadStatus](#kueue-x-k8s-io-v1beta2-WorkloadStatus) + + + + + + + + + + + + +
FieldDescription
name [Required]
+string +
+

name is the name of the unhealthy node.

+
+ +## `WorkloadSchedulingStatsEviction` {#kueue-x-k8s-io-v1beta2-WorkloadSchedulingStatsEviction} + + +**Appears in:** + +- [SchedulingStats](#kueue-x-k8s-io-v1beta2-SchedulingStats) + + + + + + + + + + + + + + + + + + +
FieldDescription
reason [Required]
+string +
+

reason specifies the programmatic identifier for the eviction cause.

+
underlyingCause [Required]
+EvictionUnderlyingCause +
+

underlyingCause specifies a finer-grained explanation that complements the eviction reason. +This may be an empty string.

+
count [Required]
+int32 +
+

count tracks the number of evictions for this reason and detailed reason.

+
+ +## `WorkloadSpec` {#kueue-x-k8s-io-v1beta2-WorkloadSpec} + + +**Appears in:** + +- [Workload](#kueue-x-k8s-io-v1beta2-Workload) + + +

WorkloadSpec defines the desired state of Workload

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
podSets [Required]
+[]PodSet +
+

podSets is a list of sets of homogeneous pods, each described by a Pod spec +and a count. +There must be at least one element and at most 8. +podSets cannot be changed.

+
queueName [Required]
+LocalQueueName +
+

queueName is the name of the LocalQueue the Workload is associated with. +queueName cannot be changed while .status.admission is not null.

+
priorityClassName [Required]
+string +
+

priorityClassName is the name of the PriorityClass the Workload is associated with. +If specified, indicates the workload's priority. +"system-node-critical" and "system-cluster-critical" are two special +keywords which indicate the highest priorities with the former being +the highest priority. Any other name must be defined by creating a +PriorityClass object with that name. If not specified, the workload +priority will be default or zero if there is no default.

+
priority [Required]
+int32 +
+

priority determines the order of access to the resources managed by the +ClusterQueue where the workload is queued. +The priority value is populated from PriorityClassName. +The higher the value, the higher the priority. +If priorityClassName is specified, priority must not be null.

+
priorityClassSource [Required]
+string +
+

priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. +Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. +When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value.

+
active [Required]
+bool +
+

active determines if a workload can be admitted into a queue. +Changing active from true to false will evict any running workloads. +Possible values are:

+
    +
  • false: indicates that a workload should never be admitted and evicts running workloads
  • +
  • true: indicates that a workload can be evaluated for admission into it's respective queue.
  • +
+

Defaults to true

+
maximumExecutionTimeSeconds
+int32 +
+

maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, +the workload can be admitted before it's automatically deactivated.

+

If unspecified, no execution time limit is enforced on the Workload.

+
+ +## `WorkloadStatus` {#kueue-x-k8s-io-v1beta2-WorkloadStatus} + + +**Appears in:** + +- [Workload](#kueue-x-k8s-io-v1beta2-Workload) + + +

WorkloadStatus defines the observed state of Workload

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
conditions
+[]k8s.io/apimachinery/pkg/apis/meta/v1.Condition +
+

conditions hold the latest available observations of the Workload +current state.

+

The type of the condition could be:

+
    +
  • Admitted: the Workload was admitted through a ClusterQueue.
  • +
  • Finished: the associated workload finished running (failed or succeeded).
  • +
  • PodsReady: at least .spec.podSets[*].count Pods are ready or have +succeeded.
  • +
+
admission
+Admission +
+

admission holds the parameters of the admission of the workload by a +ClusterQueue. admission can be set back to null, but its fields cannot be +changed once set.

+
requeueState
+RequeueState +
+

requeueState holds the re-queue state +when a workload meets Eviction with PodsReadyTimeout reason.

+
reclaimablePods
+[]ReclaimablePod +
+

reclaimablePods keeps track of the number pods within a podset for which +the resource reservation is no longer needed.

+
admissionChecks
+[]AdmissionCheckState +
+

admissionChecks list all the admission checks required by the workload and the current status

+
resourceRequests
+[]PodSetRequest +
+

resourceRequests provides a detailed view of the resources that were +requested by a non-admitted workload when it was considered for admission. +If admission is non-null, resourceRequests will be empty because +admission.resourceUsage contains the detailed information.

+
accumulatedPastExecutionTimeSeconds
+int32 +
+

accumulatedPastExecutionTimeSeconds holds the total time, in seconds, the workload spent +in Admitted state, in the previous Admit - Evict cycles.

+
schedulingStats
+SchedulingStats +
+

schedulingStats tracks scheduling statistics

+
nominatedClusterNames
+[]string +
+

nominatedClusterNames specifies the list of cluster names that have been nominated for scheduling. +This field is mutually exclusive with the .status.clusterName field, and is reset when +status.clusterName is set. +This field is optional.

+
clusterName
+string +
+

clusterName is the name of the cluster where the workload is currently assigned.

+

With ElasticJobs, this field may also indicate the cluster where the original (old) workload +was assigned, providing placement context for new scaled-up workloads. This supports +affinity or propagation policies across workload slices.

+

This field is reset after the Workload is evicted.

+
unhealthyNodes
+[]UnhealthyNode +
+

unhealthyNodes holds the failed nodes running at least one pod of this workload +when Topology-Aware Scheduling is used. This field should not be set by the users. +It indicates Kueue's scheduler is searching for replacements of the failed nodes. +Requires enabling the TASFailedNodeReplacement feature gate.

+
+ \ No newline at end of file From 7b7882894032bb7e04c0e1987330e3a99f44f370 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 27 Oct 2025 15:25:35 -0400 Subject: [PATCH 038/119] formatting issue: add space after comments for apigeneration tags (#7413) --- apis/kueue/v1beta1/cohort_types.go | 4 ++-- apis/kueue/v1beta1/topology_types.go | 2 +- apis/kueue/v1beta1/types.go | 2 +- apis/kueue/v1beta2/cohort_types.go | 4 ++-- apis/kueue/v1beta2/topology_types.go | 2 +- apis/kueue/v1beta2/types.go | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apis/kueue/v1beta1/cohort_types.go b/apis/kueue/v1beta1/cohort_types.go index b7c11d9809e..e6ea5f77b6a 100644 --- a/apis/kueue/v1beta1/cohort_types.go +++ b/apis/kueue/v1beta1/cohort_types.go @@ -58,8 +58,8 @@ type CohortSpec struct { // Cohort has a parent. Otherwise, the Cohort create/update // will be rejected by the webhook. // - //+listType=atomic - //+kubebuilder:validation:MaxItems=16 + // +listType=atomic + // +kubebuilder:validation:MaxItems=16 ResourceGroups []ResourceGroup `json:"resourceGroups,omitempty"` // fairSharing defines the properties of the Cohort when diff --git a/apis/kueue/v1beta1/topology_types.go b/apis/kueue/v1beta1/topology_types.go index 2af83e0e886..f323dbc39b2 100644 --- a/apis/kueue/v1beta1/topology_types.go +++ b/apis/kueue/v1beta1/topology_types.go @@ -139,7 +139,7 @@ type Topology struct { Spec TopologySpec `json:"spec,omitempty"` } -//+kubebuilder:object:root=true +// +kubebuilder:object:root=true // TopologyList contains a list of Topology type TopologyList struct { diff --git a/apis/kueue/v1beta1/types.go b/apis/kueue/v1beta1/types.go index 0ffdd5898ae..425064e99d2 100644 --- a/apis/kueue/v1beta1/types.go +++ b/apis/kueue/v1beta1/types.go @@ -20,4 +20,4 @@ This file is needed for kubernetes/code-generator/kube_codegen.sh script used in package v1beta1 -//+genclient +// +genclient diff --git a/apis/kueue/v1beta2/cohort_types.go b/apis/kueue/v1beta2/cohort_types.go index 2a8eb26e12d..8e2a6d34b95 100644 --- a/apis/kueue/v1beta2/cohort_types.go +++ b/apis/kueue/v1beta2/cohort_types.go @@ -58,8 +58,8 @@ type CohortSpec struct { // Cohort has a parent. Otherwise, the Cohort create/update // will be rejected by the webhook. // - //+listType=atomic - //+kubebuilder:validation:MaxItems=16 + // +listType=atomic + // +kubebuilder:validation:MaxItems=16 ResourceGroups []ResourceGroup `json:"resourceGroups,omitempty"` // fairSharing defines the properties of the Cohort when diff --git a/apis/kueue/v1beta2/topology_types.go b/apis/kueue/v1beta2/topology_types.go index 07c6cf2e1f4..4ecbd596f8c 100644 --- a/apis/kueue/v1beta2/topology_types.go +++ b/apis/kueue/v1beta2/topology_types.go @@ -138,7 +138,7 @@ type Topology struct { Spec TopologySpec `json:"spec,omitempty"` } -//+kubebuilder:object:root=true +// +kubebuilder:object:root=true // TopologyList contains a list of Topology type TopologyList struct { diff --git a/apis/kueue/v1beta2/types.go b/apis/kueue/v1beta2/types.go index b73f086d336..a19d888acdd 100644 --- a/apis/kueue/v1beta2/types.go +++ b/apis/kueue/v1beta2/types.go @@ -20,4 +20,4 @@ This file is needed for kubernetes/code-generator/kube_codegen.sh script used in package v1beta2 -//+genclient +// +genclient From 361f52371296c1d2f2eb80385c38aeb2492f987f Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Tue, 28 Oct 2025 13:37:35 +0530 Subject: [PATCH 039/119] Replace preemtion stub with interceptor function in TestPreemption. (#7400) --- pkg/scheduler/preemption/preemption_test.go | 2959 +++++++++++++++---- 1 file changed, 2418 insertions(+), 541 deletions(-) diff --git a/pkg/scheduler/preemption/preemption_test.go b/pkg/scheduler/preemption/preemption_test.go index 067c3d40a96..53e1fe50ac2 100644 --- a/pkg/scheduler/preemption/preemption_test.go +++ b/pkg/scheduler/preemption/preemption_test.go @@ -17,10 +17,8 @@ limitations under the License. package preemption import ( - "context" "fmt" "slices" - "sync" "testing" "time" @@ -29,10 +27,10 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" clocktesting "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" @@ -62,7 +60,7 @@ var snapCmpOpts = cmp.Options{ } func TestPreemption(t *testing.T) { - now := time.Now() + now := time.Now().Truncate(time.Second) flavors := []*kueue.ResourceFlavor{ utiltestingapi.MakeResourceFlavor("default").Obj(), utiltestingapi.MakeResourceFlavor("alpha").Obj(), @@ -275,6 +273,10 @@ func TestPreemption(t *testing.T) { ). Obj(), } + baseIncomingWl := utiltestingapi.MakeWorkload("in", ""). + UID("wl-in"). + Label(controllerconstants.JobUIDLabel, "job-in") + cases := map[string]struct { clusterQueues []*kueue.ClusterQueue cohorts []*kueue.Cohort @@ -282,7 +284,8 @@ func TestPreemption(t *testing.T) { incoming *kueue.Workload targetCQ kueue.ClusterQueueReference assignment flavorassigner.Assignment - wantPreempted sets.Set[string] + wantPreempted int + wantWorkloads []kueue.Workload disableLendingLimit bool }{ "preempt lowest priority": { @@ -324,7 +327,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "2"). Obj(), @@ -335,7 +338,59 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/low", kueue.InClusterQueueReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("high", ""). + Priority(1). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("mid", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, }, "preempt multiple": { clusterQueues: defaultClusterQueues, @@ -376,7 +431,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "3"). Obj(), @@ -387,9 +442,75 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/low", kueue.InClusterQueueReason), targetKeyReason("/mid", kueue.InClusterQueueReason)), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("high", ""). + Priority(1). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("mid", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, - "no preemption for low priority": { clusterQueues: defaultClusterQueues, admitted: []kueue.Workload{ @@ -428,6 +549,31 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("mid", ""). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, }, "not enough low priority workloads": { clusterQueues: defaultClusterQueues, @@ -466,6 +612,31 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("mid", ""). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, }, "some free quota, preempt low priority": { clusterQueues: defaultClusterQueues, @@ -506,7 +677,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "2"). Obj(), @@ -517,7 +688,59 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/low", kueue.InClusterQueueReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("high", ""). + Priority(1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("mid", ""). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, }, "minimal set excludes low priority": { clusterQueues: defaultClusterQueues, @@ -558,7 +781,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "2"). Obj(), @@ -569,7 +792,59 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/mid", kueue.InClusterQueueReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("high", ""). + Priority(1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("mid", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, "only preempt workloads using the chosen flavor": { clusterQueues: defaultClusterQueues, @@ -610,7 +885,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "1"). Request(corev1.ResourceMemory, "2Gi"). @@ -626,16 +901,68 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/mid", kueue.InClusterQueueReason)), - }, - "reclaim quota from borrower": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("c1-low", ""). - Priority(-1). - Request(corev1.ResourceCPU, "3"). + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("high", ""). + Priority(1). + Request(corev1.ResourceMemory, "1Gi"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c1"). + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceMemory, "beta", "1Gi"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("low", ""). + Priority(-1). + Request(corev1.ResourceMemory, "2Gi"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceMemory, "alpha", "2Gi"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("mid", ""). + Request(corev1.ResourceMemory, "1Gi"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceMemory, "beta", "1Gi"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, + }, + "reclaim quota from borrower": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "3000m"). Obj()). @@ -667,7 +994,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "3"). Obj(), @@ -678,7 +1005,59 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/c2-mid", kueue.InCohortReclamationReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-high", ""). + Priority(1). + Request(corev1.ResourceCPU, "6"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "6000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-mid", ""). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, "reclaim quota if workload requests 0 resources for a resource at nominal quota": { clusterQueues: defaultClusterQueues, @@ -699,7 +1078,7 @@ func TestPreemption(t *testing.T) { SimpleReserveQuota("c2", "default", now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "3"). Request(corev1.ResourceMemory, "0"). @@ -715,7 +1094,39 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Fit, }, }), - wantPreempted: sets.New(targetKeyReason("/c2-mid", kueue.InCohortReclamationReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + Request(corev1.ResourceMemory, "3Gi"). + SimpleReserveQuota("c1", "default", now). + Obj(), + *utiltestingapi.MakeWorkload("c2-high", ""). + Priority(1). + Request(corev1.ResourceCPU, "6"). + SimpleReserveQuota("c2", "default", now). + Obj(), + *utiltestingapi.MakeWorkload("c2-mid", ""). + Request(corev1.ResourceCPU, "3"). + SimpleReserveQuota("c2", "default", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, "no workloads borrowing": { clusterQueues: defaultClusterQueues, @@ -745,7 +1156,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "4"). Obj(), @@ -756,6 +1167,32 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-high", ""). + Priority(1). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-low-1", ""). + Priority(-1). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, }, "not enough workloads borrowing": { clusterQueues: defaultClusterQueues, @@ -797,7 +1234,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "4"). Obj(), @@ -808,6 +1245,44 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-high", ""). + Priority(1). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-low-1", ""). + Priority(-1). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-low-2", ""). + Priority(-1). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, }, "preempting locally and borrowing other resources in cohort, without cohort candidates": { clusterQueues: defaultClusterQueues, @@ -849,7 +1324,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "4"). Request(corev1.ResourceMemory, "5Gi"). @@ -865,13 +1340,10 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/c1-low", kueue.InClusterQueueReason)), - }, - "preempting locally and borrowing same resource in cohort": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("c1-med", ""). - Priority(0). + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-low", ""). + Priority(-1). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( utiltestingapi.MakeAdmission("c1"). @@ -881,12 +1353,27 @@ func TestPreemption(t *testing.T) { Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), - *utiltestingapi.MakeWorkload("c1-low", ""). - Priority(-1). + *utiltestingapi.MakeWorkload("c2-high-2", ""). + Priority(1). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c1"). + utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). @@ -907,77 +1394,62 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Priority(1). - Request(corev1.ResourceCPU, "4"). - Obj(), - targetCQ: "c1", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/c1-low", kueue.InClusterQueueReason)), }, - "preempting locally and borrowing same resource in cohort; no borrowing limit in the cohort": { + "preempting locally and borrowing same resource in cohort": { clusterQueues: defaultClusterQueues, admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("d1-med", ""). + *utiltestingapi.MakeWorkload("c1-med", ""). Priority(0). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("d1"). + utiltestingapi.MakeAdmission("c1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("d1-low", ""). + *utiltestingapi.MakeWorkload("c1-low", ""). Priority(-1). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("d1"). + utiltestingapi.MakeAdmission("c1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("d2-low-1", ""). + *utiltestingapi.MakeWorkload("c2-low-1", ""). Priority(-1). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("d2"). + utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "4"). Obj(), - targetCQ: "d1", + targetCQ: "c1", assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ Name: "default", Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/d1-low", kueue.InClusterQueueReason)), - }, - "preempting locally and borrowing other resources in cohort, with cohort candidates": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("c1-med", ""). - Priority(0). + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-low", ""). + Priority(-1). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( utiltestingapi.MakeAdmission("c1"). @@ -987,155 +1459,159 @@ func TestPreemption(t *testing.T) { Obj(), now, ). - Obj(), - *utiltestingapi.MakeWorkload("c2-low-1", ""). - Priority(-1). - Request(corev1.ResourceCPU, "5"). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("c1-med", ""). + Priority(0). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c2"). + utiltestingapi.MakeAdmission("c1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "5000m"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("c2-low-2", ""). + *utiltestingapi.MakeWorkload("c2-low-1", ""). Priority(-1). - Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1000m"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("c2-low-3", ""). - Priority(-1). - Request(corev1.ResourceCPU, "1"). + }, + }, + "preempting locally and borrowing same resource in cohort; no borrowing limit in the cohort": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("d1-med", ""). + Priority(0). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c2"). + utiltestingapi.MakeAdmission("d1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1000m"). + Assignment(corev1.ResourceCPU, "default", "4"). Obj()). Obj(), now, ). Obj(), - }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Priority(1). - Request(corev1.ResourceCPU, "2"). - Request(corev1.ResourceMemory, "5Gi"). - Obj(), - targetCQ: "c1", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - corev1.ResourceMemory: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/c1-med", kueue.InClusterQueueReason)), - }, - "preempting locally and not borrowing same resource in 1-queue cohort": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("l1-med", ""). - Priority(0). + *utiltestingapi.MakeWorkload("d1-low", ""). + Priority(-1). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("l1"). + utiltestingapi.MakeAdmission("d1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4000m"). + Assignment(corev1.ResourceCPU, "default", "4"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("l1-low", ""). + *utiltestingapi.MakeWorkload("d2-low-1", ""). Priority(-1). - Request(corev1.ResourceCPU, "2"). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("l1"). + utiltestingapi.MakeAdmission("d2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2000m"). + Assignment(corev1.ResourceCPU, "default", "4"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "4"). Obj(), - targetCQ: "l1", + targetCQ: "d1", assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ Name: "default", Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/l1-med", kueue.InClusterQueueReason)), - }, - "do not reclaim borrowed quota from same priority for withinCohort=ReclaimFromLowerPriority": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("c1", ""). - Request(corev1.ResourceCPU, "2"). + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("d1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c1"). + utiltestingapi.MakeAdmission("d1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2000m"). + Assignment(corev1.ResourceCPU, "default", "4"). Obj()). Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), - *utiltestingapi.MakeWorkload("c2-1", ""). + *utiltestingapi.MakeWorkload("d1-med", ""). + Priority(0). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c2"). + utiltestingapi.MakeAdmission("d1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4000m"). + Assignment(corev1.ResourceCPU, "default", "4"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("c2-2", ""). + *utiltestingapi.MakeWorkload("d2-low-1", ""). + Priority(-1). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c2"). + utiltestingapi.MakeAdmission("d2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4000m"). + Assignment(corev1.ResourceCPU, "default", "4"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Request(corev1.ResourceCPU, "4"). - Obj(), - targetCQ: "c1", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), }, - "reclaim borrowed quota from same priority for withinCohort=ReclaimFromAny": { + "preempting locally and borrowing other resources in cohort, with cohort candidates": { clusterQueues: defaultClusterQueues, admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("c1-1", ""). + *utiltestingapi.MakeWorkload("c1-med", ""). + Priority(0). Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( utiltestingapi.MakeAdmission("c1"). @@ -1146,408 +1622,1620 @@ func TestPreemption(t *testing.T) { now, ). Obj(), - *utiltestingapi.MakeWorkload("c1-2", ""). - Priority(1). - Request(corev1.ResourceCPU, "4"). + *utiltestingapi.MakeWorkload("c2-low-1", ""). + Priority(-1). + Request(corev1.ResourceCPU, "5"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c1"). + utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4000m"). + Assignment(corev1.ResourceCPU, "default", "5000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("c2", ""). - Request(corev1.ResourceCPU, "2"). + *utiltestingapi.MakeWorkload("c2-low-2", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2000m"). + Assignment(corev1.ResourceCPU, "default", "1000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-low-3", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1000m"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Request(corev1.ResourceCPU, "4"). + incoming: baseIncomingWl.Clone(). + Priority(1). + Request(corev1.ResourceCPU, "2"). + Request(corev1.ResourceMemory, "5Gi"). Obj(), - targetCQ: "c2", + targetCQ: "c1", assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ Name: "default", Mode: flavorassigner.Preempt, }, + corev1.ResourceMemory: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, }), - wantPreempted: sets.New(targetKeyReason("/c1-1", kueue.InCohortReclamationReason)), - }, - "preempt from all ClusterQueues in cohort": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("c1-low", ""). - Priority(-1). - Request(corev1.ResourceCPU, "3"). + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-med", ""). + Priority(0). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( utiltestingapi.MakeAdmission("c1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3000m"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), - *utiltestingapi.MakeWorkload("c1-mid", ""). - Request(corev1.ResourceCPU, "2"). + *utiltestingapi.MakeWorkload("c2-low-1", ""). + Priority(-1). + Request(corev1.ResourceCPU, "5"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c1"). + utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2000m"). + Assignment(corev1.ResourceCPU, "default", "5000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("c2-low", ""). + *utiltestingapi.MakeWorkload("c2-low-2", ""). Priority(-1). - Request(corev1.ResourceCPU, "3"). + Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3000m"). + Assignment(corev1.ResourceCPU, "default", "1000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("c2-mid", ""). - Request(corev1.ResourceCPU, "4"). + *utiltestingapi.MakeWorkload("c2-low-3", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4000m"). + Assignment(corev1.ResourceCPU, "default", "1000m"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Request(corev1.ResourceCPU, "4"). - Obj(), - targetCQ: "c1", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/c1-low", kueue.InClusterQueueReason), targetKeyReason("/c2-low", kueue.InCohortReclamationReason)), }, - "can't preempt workloads in ClusterQueue for withinClusterQueue=Never": { + "preempting locally and not borrowing same resource in 1-queue cohort": { clusterQueues: defaultClusterQueues, admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("c2-low", ""). + *utiltestingapi.MakeWorkload("l1-med", ""). + Priority(0). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("l1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("l1-low", ""). Priority(-1). - Request(corev1.ResourceCPU, "3"). + Request(corev1.ResourceCPU, "2"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("c2"). + utiltestingapi.MakeAdmission("l1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3000m"). + Assignment(corev1.ResourceCPU, "default", "2000m"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "4"). Obj(), - targetCQ: "c2", + targetCQ: "l1", assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ Name: "default", Mode: flavorassigner.Preempt, }, }), - }, - "each podset preempts a different flavor": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("low-alpha", ""). + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("l1-low", ""). Priority(-1). - Request(corev1.ResourceMemory, "2Gi"). + Request(corev1.ResourceCPU, "2"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("standalone"). + utiltestingapi.MakeAdmission("l1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceMemory, "alpha", "2Gi"). + Assignment(corev1.ResourceCPU, "default", "2000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("low-beta", ""). - Priority(-1). - Request(corev1.ResourceMemory, "2Gi"). + *utiltestingapi.MakeWorkload("l1-med", ""). + Priority(0). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("standalone"). + utiltestingapi.MakeAdmission("l1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceMemory, "beta", "2Gi"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - PodSets( - *utiltestingapi.MakePodSet("launcher", 1). - Request(corev1.ResourceMemory, "2Gi").Obj(), - *utiltestingapi.MakePodSet("workers", 2). - Request(corev1.ResourceMemory, "1Gi").Obj(), - ). - Obj(), - targetCQ: "standalone", - assignment: flavorassigner.Assignment{ - PodSets: []flavorassigner.PodSetAssignment{ - { - Name: "launcher", - Flavors: flavorassigner.ResourceAssignment{ - corev1.ResourceMemory: { - Name: "alpha", - Mode: flavorassigner.Preempt, - }, - }, - Count: 1, - }, - { - Name: "workers", - Flavors: flavorassigner.ResourceAssignment{ - corev1.ResourceMemory: { - Name: "beta", - Mode: flavorassigner.Preempt, - }, - }, - Count: 2, - }, - }, - }, - wantPreempted: sets.New(targetKeyReason("/low-alpha", kueue.InClusterQueueReason), targetKeyReason("/low-beta", kueue.InClusterQueueReason)), }, - "preempt newer workloads with the same priority": { + "do not reclaim borrowed quota from same priority for withinCohort=ReclaimFromLowerPriority": { clusterQueues: defaultClusterQueues, admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("wl1", ""). - Priority(2). + *utiltestingapi.MakeWorkload("c1", ""). Request(corev1.ResourceCPU, "2"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("preventStarvation"). + utiltestingapi.MakeAdmission("c1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2"). + Assignment(corev1.ResourceCPU, "default", "2000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("wl2", ""). - Priority(1). - Creation(now). - Request(corev1.ResourceCPU, "2"). + *utiltestingapi.MakeWorkload("c2-1", ""). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("preventStarvation"). + utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), - now.Add(time.Second), + now, ). Obj(), - *utiltestingapi.MakeWorkload("wl3", ""). - Priority(1). - Creation(now). - Request(corev1.ResourceCPU, "2"). + *utiltestingapi.MakeWorkload("c2-2", ""). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("preventStarvation"). + utiltestingapi.MakeAdmission("c2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Priority(1). - Creation(now.Add(-15 * time.Second)). - PodSets( - *utiltestingapi.MakePodSet("launcher", 1). - Request(corev1.ResourceCPU, "2").Obj(), - ). + incoming: baseIncomingWl.Clone(). + Request(corev1.ResourceCPU, "4"). Obj(), - targetCQ: "preventStarvation", - assignment: flavorassigner.Assignment{ - PodSets: []flavorassigner.PodSetAssignment{ - { - Name: "launcher", - Flavors: flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: { - Name: "default", - Mode: flavorassigner.Preempt, - }, - }, - }, + targetCQ: "c1", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, }, + }), + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-1", ""). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-2", ""). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), }, - wantPreempted: sets.New(targetKeyReason("/wl2", kueue.InClusterQueueReason)), }, - "use BorrowWithinCohort; allow preempting a lower-priority workload from another ClusterQueue while borrowing": { + "reclaim borrowed quota from same priority for withinCohort=ReclaimFromAny": { clusterQueues: defaultClusterQueues, admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a_best_effort_low", ""). - Priority(-1). - Request(corev1.ResourceCPU, "10"). + *utiltestingapi.MakeWorkload("c1-1", ""). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("a_best_effort"). + utiltestingapi.MakeAdmission("c1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "10"). + Assignment(corev1.ResourceCPU, "default", "4000m"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("b_best_effort_low", ""). - Priority(-1). - Request(corev1.ResourceCPU, "1"). + *utiltestingapi.MakeWorkload("c1-2", ""). + Priority(1). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("b_best_effort"). + utiltestingapi.MakeAdmission("c1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Request(corev1.ResourceCPU, "10"). + incoming: baseIncomingWl.Clone(). + Request(corev1.ResourceCPU, "4"). Obj(), - targetCQ: "a_standard", + targetCQ: "c2", assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ Name: "default", Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/a_best_effort_low", kueue.InCohortReclaimWhileBorrowingReason)), - }, - "use BorrowWithinCohort; don't allow preempting a lower-priority workload with priority above MaxPriorityThreshold, if borrowing is required even after the preemption": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("b_standard", ""). + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-1", ""). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("c1-2", ""). Priority(1). - Request(corev1.ResourceCPU, "10"). + Request(corev1.ResourceCPU, "4"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("b_standard"). + utiltestingapi.MakeAdmission("c1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "10000m"). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Priority(2). - Request(corev1.ResourceCPU, "10"). - Obj(), - targetCQ: "a_standard", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), }, - "use BorrowWithinCohort; allow preempting a lower-priority workload with priority above MaxPriorityThreshold, if borrowing is not required after the preemption": { + "preempt from all ClusterQueues in cohort": { clusterQueues: defaultClusterQueues, admitted: []kueue.Workload{ - // this admitted workload consumes all resources so it needs to be preempted to run a new workload - *utiltestingapi.MakeWorkload("b_standard", ""). - Priority(1). - Request(corev1.ResourceCPU, "13"). + *utiltestingapi.MakeWorkload("c1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("b_standard"). + utiltestingapi.MakeAdmission("c1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c1-mid", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-mid", ""). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Request(corev1.ResourceCPU, "4"). + Obj(), + targetCQ: "c1", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("c1-mid", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("c2-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("c2-mid", ""). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + }, + "can't preempt workloads in ClusterQueue for withinClusterQueue=Never": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c2-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Priority(1). + Request(corev1.ResourceCPU, "4"). + Obj(), + targetCQ: "c2", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("c2-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("c2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + }, + "each podset preempts a different flavor": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("low-alpha", ""). + Priority(-1). + Request(corev1.ResourceMemory, "2Gi"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceMemory, "alpha", "2Gi"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("low-beta", ""). + Priority(-1). + Request(corev1.ResourceMemory, "2Gi"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceMemory, "beta", "2Gi"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + PodSets( + *utiltestingapi.MakePodSet("launcher", 1). + Request(corev1.ResourceMemory, "2Gi").Obj(), + *utiltestingapi.MakePodSet("workers", 2). + Request(corev1.ResourceMemory, "1Gi").Obj(), + ). + Obj(), + targetCQ: "standalone", + assignment: flavorassigner.Assignment{ + PodSets: []flavorassigner.PodSetAssignment{ + { + Name: "launcher", + Flavors: flavorassigner.ResourceAssignment{ + corev1.ResourceMemory: { + Name: "alpha", + Mode: flavorassigner.Preempt, + }, + }, + Count: 1, + }, + { + Name: "workers", + Flavors: flavorassigner.ResourceAssignment{ + corev1.ResourceMemory: { + Name: "beta", + Mode: flavorassigner.Preempt, + }, + }, + Count: 2, + }, + }, + }, + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("low-alpha", ""). + Priority(-1). + Request(corev1.ResourceMemory, "2Gi"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceMemory, "alpha", "2Gi"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("low-beta", ""). + Priority(-1). + Request(corev1.ResourceMemory, "2Gi"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("standalone"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceMemory, "beta", "2Gi"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, + }, + "preempt newer workloads with the same priority": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("wl1", ""). + Priority(2). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("preventStarvation"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("wl2", ""). + Priority(1). + Creation(now). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("preventStarvation"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2"). + Obj()). + Obj(), + now.Add(time.Second), + ). + Obj(), + *utiltestingapi.MakeWorkload("wl3", ""). + Priority(1). + Creation(now). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("preventStarvation"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Priority(1). + Creation(now.Add(-15 * time.Second)). + PodSets( + *utiltestingapi.MakePodSet("launcher", 1). + Request(corev1.ResourceCPU, "2").Obj(), + ). + Obj(), + targetCQ: "preventStarvation", + assignment: flavorassigner.Assignment{ + PodSets: []flavorassigner.PodSetAssignment{ + { + Name: "launcher", + Flavors: flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: { + Name: "default", + Mode: flavorassigner.Preempt, + }, + }, + }, + }, + }, + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("wl1", ""). + Priority(2). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("preventStarvation"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("wl2", ""). + Priority(1). + Creation(now). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("preventStarvation"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2"). + Obj()). + Obj(), + now.Add(time.Second), + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("wl3", ""). + Priority(1). + Creation(now). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("preventStarvation"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + }, + "use BorrowWithinCohort; allow preempting a lower-priority workload from another ClusterQueue while borrowing": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a_best_effort_low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "10"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_best_effort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "10"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b_best_effort_low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_best_effort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Request(corev1.ResourceCPU, "10"). + Obj(), + targetCQ: "a_standard", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a_best_effort_low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "10"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_best_effort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "10"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclaimWhileBorrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("b_best_effort_low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_best_effort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + }, + "use BorrowWithinCohort; don't allow preempting a lower-priority workload with priority above MaxPriorityThreshold, if borrowing is required even after the preemption": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("b_standard", ""). + Priority(1). + Request(corev1.ResourceCPU, "10"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "10000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Priority(2). + Request(corev1.ResourceCPU, "10"). + Obj(), + targetCQ: "a_standard", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("b_standard", ""). + Priority(1). + Request(corev1.ResourceCPU, "10"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "10000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + }, + "use BorrowWithinCohort; allow preempting a lower-priority workload with priority above MaxPriorityThreshold, if borrowing is not required after the preemption": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + // this admitted workload consumes all resources so it needs to be preempted to run a new workload + *utiltestingapi.MakeWorkload("b_standard", ""). + Priority(1). + Request(corev1.ResourceCPU, "13"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "13000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + // this is a small workload which can be admitted without borrowing, if the b_standard workload is preempted + Priority(2). + Request(corev1.ResourceCPU, "1"). + Obj(), + targetCQ: "a_standard", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + // this admitted workload consumes all resources so it needs to be preempted to run a new workload + *utiltestingapi.MakeWorkload("b_standard", ""). + Priority(1). + Request(corev1.ResourceCPU, "13"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "13000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, + }, + "use BorrowWithinCohort; don't allow for preemption of lower-priority workload from the same ClusterQueue": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a_standard", ""). + Priority(1). + Request(corev1.ResourceCPU, "13"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "13000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Priority(2). + Request(corev1.ResourceCPU, "1"). + Obj(), + targetCQ: "a_standard", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a_standard", ""). + Priority(1). + Request(corev1.ResourceCPU, "13"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "13000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + }, + "use BorrowWithinCohort; only preempt from CQ if no workloads below threshold and already above nominal": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a_standard_1", ""). + Priority(1). + Request(corev1.ResourceCPU, "10"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "10"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("a_standard_2", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b_standard_1", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b_standard_2", ""). + Priority(2). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Priority(3). + Request(corev1.ResourceCPU, "1"). + Obj(), + targetCQ: "b_standard", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a_standard_1", ""). + Priority(1). + Request(corev1.ResourceCPU, "10"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "10"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("a_standard_2", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b_standard_1", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("b_standard_2", ""). + Priority(2). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + }, + "use BorrowWithinCohort; preempt from CQ and from other CQs with workloads below threshold": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("b_standard_high", ""). + Priority(2). + Request(corev1.ResourceCPU, "10"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "10"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b_standard_mid", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("a_best_effort_low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_best_effort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("a_best_effort_lower", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_best_effort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Priority(2). + Request(corev1.ResourceCPU, "2"). + Obj(), + targetCQ: "b_standard", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a_best_effort_low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_best_effort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("a_best_effort_lower", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a_best_effort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclaimWhileBorrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("b_standard_high", ""). + Priority(2). + Request(corev1.ResourceCPU, "10"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "10"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b_standard_mid", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b_standard"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, + }, + "reclaim quota from lender": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("lend1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("lend2-mid", ""). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("lend2-high", ""). + Priority(1). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Priority(1). + Request(corev1.ResourceCPU, "3"). + Obj(), + targetCQ: "lend1", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("lend1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("lend2-high", ""). + Priority(1). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("lend2-mid", ""). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, + }, + "preempt from all ClusterQueues in cohort-lend": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("lend1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("lend1-mid", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("lend2-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("lend2-mid", ""). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + incoming: baseIncomingWl.Clone(). + Request(corev1.ResourceCPU, "4"). + Obj(), + targetCQ: "lend1", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("lend1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("lend1-mid", ""). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2000m"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("lend2-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3000m"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("lend2-mid", ""). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4000m"). + Obj()). + Obj(), + now, + ). + Obj(), + }, + }, + "cannot preempt from other ClusterQueues if exceeds requestable quota including lending limit": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("lend2-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "10"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("lend2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "13000m"). + Assignment(corev1.ResourceCPU, "default", "10"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - // this is a small workload which can be admitted without borrowing, if the b_standard workload is preempted - Priority(2). - Request(corev1.ResourceCPU, "1"). + incoming: baseIncomingWl.Clone(). + Request(corev1.ResourceCPU, "9"). Obj(), - targetCQ: "a_standard", + targetCQ: "lend1", assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ Name: "default", Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/b_standard", kueue.InCohortReclamationReason)), - }, - "use BorrowWithinCohort; don't allow for preemption of lower-priority workload from the same ClusterQueue": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a_standard", ""). - Priority(1). - Request(corev1.ResourceCPU, "13"). + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("lend2-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "10"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("a_standard"). + utiltestingapi.MakeAdmission("lend2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "13000m"). + Assignment(corev1.ResourceCPU, "default", "10"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Priority(2). - Request(corev1.ResourceCPU, "1"). - Obj(), - targetCQ: "a_standard", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), }, - "use BorrowWithinCohort; only preempt from CQ if no workloads below threshold and already above nominal": { + "preemptions from cq when target queue is exhausted for the single requested resource": { clusterQueues: defaultClusterQueues, admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("a_standard_1", ""). - Priority(1). - Request(corev1.ResourceCPU, "10"). + *utiltestingapi.MakeWorkload("a1", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("a_standard"). + utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "10"). + Assignment(corev1.ResourceCPU, "default", "1"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("a_standard_2", ""). - Priority(1). + *utiltestingapi.MakeWorkload("a2", ""). + Priority(-2). Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("a_standard"). + utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). Obj()). @@ -1555,11 +3243,11 @@ func TestPreemption(t *testing.T) { now, ). Obj(), - *utiltestingapi.MakeWorkload("b_standard_1", ""). - Priority(1). + *utiltestingapi.MakeWorkload("a3", ""). + Priority(-1). Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("b_standard"). + utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). Obj()). @@ -1567,11 +3255,35 @@ func TestPreemption(t *testing.T) { now, ). Obj(), - *utiltestingapi.MakeWorkload("b_standard_2", ""). - Priority(2). + *utiltestingapi.MakeWorkload("b1", ""). + Priority(0). Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("b_standard"). + utiltestingapi.MakeAdmission("b"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b2", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b3", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). Obj()). @@ -1580,51 +3292,78 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Priority(3). - Request(corev1.ResourceCPU, "1"). + incoming: baseIncomingWl.Clone(). + Request(corev1.ResourceCPU, "2"). + Priority(0). Obj(), - targetCQ: "b_standard", + targetCQ: "a", assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ Name: "default", Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/b_standard_1", kueue.InClusterQueueReason)), - }, - "use BorrowWithinCohort; preempt from CQ and from other CQs with workloads below threshold": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("b_standard_high", ""). - Priority(2). - Request(corev1.ResourceCPU, "10"). + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("b_standard"). + utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "10"). + Assignment(corev1.ResourceCPU, "default", "1"). Obj()). Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), - *utiltestingapi.MakeWorkload("b_standard_mid", ""). - Priority(1). + *utiltestingapi.MakeWorkload("a2", ""). + Priority(-2). Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("b_standard"). + utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). Obj()). Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), - *utiltestingapi.MakeWorkload("a_best_effort_low", ""). + *utiltestingapi.MakeWorkload("a3", ""). Priority(-1). Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("a_best_effort"). + utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). Obj()). @@ -1632,11 +3371,11 @@ func TestPreemption(t *testing.T) { now, ). Obj(), - *utiltestingapi.MakeWorkload("a_best_effort_lower", ""). - Priority(-2). + *utiltestingapi.MakeWorkload("b1", ""). + Priority(0). Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("a_best_effort"). + utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). Obj()). @@ -1644,196 +3383,205 @@ func TestPreemption(t *testing.T) { now, ). Obj(), - }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Priority(2). - Request(corev1.ResourceCPU, "2"). - Obj(), - targetCQ: "b_standard", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/b_standard_mid", kueue.InClusterQueueReason), targetKeyReason("/a_best_effort_lower", kueue.InCohortReclaimWhileBorrowingReason)), - }, - "reclaim quota from lender": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("lend1-low", ""). - Priority(-1). - Request(corev1.ResourceCPU, "3"). + *utiltestingapi.MakeWorkload("b2", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("lend1"). + utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3000m"). + Assignment(corev1.ResourceCPU, "default", "1"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("lend2-mid", ""). - Request(corev1.ResourceCPU, "3"). + *utiltestingapi.MakeWorkload("b3", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("lend2"). + utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3000m"). + Assignment(corev1.ResourceCPU, "default", "1"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("lend2-high", ""). - Priority(1). - Request(corev1.ResourceCPU, "4"). + }, + }, + "preemptions from cq when target queue is exhausted for two requested resources": { + clusterQueues: defaultClusterQueues, + admitted: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("lend2"). + utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4000m"). + Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). Obj(), - }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Priority(1). - Request(corev1.ResourceCPU, "3"). - Obj(), - targetCQ: "lend1", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/lend2-mid", kueue.InCohortReclamationReason)), - }, - "preempt from all ClusterQueues in cohort-lend": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("lend1-low", ""). - Priority(-1). - Request(corev1.ResourceCPU, "3"). + *utiltestingapi.MakeWorkload("a2", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("lend1"). + utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3000m"). + Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("lend1-mid", ""). - Request(corev1.ResourceCPU, "2"). + *utiltestingapi.MakeWorkload("a3", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("lend1"). + utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2000m"). + Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("lend2-low", ""). - Priority(-1). - Request(corev1.ResourceCPU, "3"). + *utiltestingapi.MakeWorkload("b1", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("lend2"). + utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3000m"). + Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). Obj(), - *utiltestingapi.MakeWorkload("lend2-mid", ""). - Request(corev1.ResourceCPU, "4"). + *utiltestingapi.MakeWorkload("b2", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("lend2"). + utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4000m"). + Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). Obj(), - }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Request(corev1.ResourceCPU, "4"). - Obj(), - targetCQ: "lend1", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/lend1-low", kueue.InClusterQueueReason), targetKeyReason("/lend2-low", kueue.InCohortReclamationReason)), - }, - "cannot preempt from other ClusterQueues if exceeds requestable quota including lending limit": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ - *utiltestingapi.MakeWorkload("lend2-low", ""). - Priority(-1). - Request(corev1.ResourceCPU, "10"). + *utiltestingapi.MakeWorkload("b3", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( - utiltestingapi.MakeAdmission("lend2"). + utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "10"). + Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Request(corev1.ResourceCPU, "9"). + incoming: baseIncomingWl.Clone(). + Request(corev1.ResourceCPU, "2"). + Request(corev1.ResourceMemory, "2"). + Priority(0). Obj(), - targetCQ: "lend1", + targetCQ: "a", assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ Name: "default", Mode: flavorassigner.Preempt, }, + corev1.ResourceMemory: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, }), - wantPreempted: nil, - }, - "preemptions from cq when target queue is exhausted for the single requested resource": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("a1", ""). Priority(-2). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a2", ""). Priority(-2). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a3", ""). Priority(-1). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, @@ -1842,10 +3590,12 @@ func TestPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("b1", ""). Priority(0). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, @@ -1854,10 +3604,12 @@ func TestPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("b2", ""). Priority(0). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, @@ -1866,41 +3618,29 @@ func TestPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("b3", ""). Priority(0). Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Request(corev1.ResourceCPU, "2"). - Priority(0). - Obj(), - targetCQ: "a", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/a1", kueue.InClusterQueueReason), targetKeyReason("/a2", kueue.InClusterQueueReason)), }, - "preemptions from cq when target queue is exhausted for two requested resources": { + "preemptions from cq when target queue is exhausted for one requested resource, but not the other": { clusterQueues: defaultClusterQueues, admitted: []kueue.Workload{ *utiltestingapi.MakeWorkload("a1", ""). Priority(-2). Request(corev1.ResourceCPU, "1"). - Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). - Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, @@ -1909,12 +3649,10 @@ func TestPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("a2", ""). Priority(-2). Request(corev1.ResourceCPU, "1"). - Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). - Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, @@ -1923,12 +3661,10 @@ func TestPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("a3", ""). Priority(-1). Request(corev1.ResourceCPU, "1"). - Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("a"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). - Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, @@ -1937,12 +3673,10 @@ func TestPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("b1", ""). Priority(0). Request(corev1.ResourceCPU, "1"). - Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). - Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, @@ -1951,12 +3685,10 @@ func TestPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("b2", ""). Priority(0). Request(corev1.ResourceCPU, "1"). - Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). - Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, @@ -1965,19 +3697,17 @@ func TestPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("b3", ""). Priority(0). Request(corev1.ResourceCPU, "1"). - Request(corev1.ResourceMemory, "1"). ReserveQuotaAt( utiltestingapi.MakeAdmission("b"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). - Assignment(corev1.ResourceMemory, "default", "1"). Obj()). Obj(), now, ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Request(corev1.ResourceCPU, "2"). Request(corev1.ResourceMemory, "2"). Priority(0). @@ -1993,11 +3723,8 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/a1", kueue.InClusterQueueReason), targetKeyReason("/a2", kueue.InClusterQueueReason)), - }, - "preemptions from cq when target queue is exhausted for one requested resource, but not the other": { - clusterQueues: defaultClusterQueues, - admitted: []kueue.Workload{ + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("a1", ""). Priority(-2). Request(corev1.ResourceCPU, "1"). @@ -2009,6 +3736,21 @@ func TestPreemption(t *testing.T) { Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a2", ""). Priority(-2). @@ -2021,6 +3763,21 @@ func TestPreemption(t *testing.T) { Obj(), now, ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a3", ""). Priority(-1). @@ -2071,23 +3828,6 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). - Request(corev1.ResourceCPU, "2"). - Request(corev1.ResourceMemory, "2"). - Priority(0). - Obj(), - targetCQ: "a", - assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ - corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - corev1.ResourceMemory: &flavorassigner.FlavorAssignment{ - Name: "default", - Mode: flavorassigner.Preempt, - }, - }), - wantPreempted: sets.New(targetKeyReason("/a1", kueue.InClusterQueueReason), targetKeyReason("/a2", kueue.InClusterQueueReason)), }, "allow preemption from other cluster queues if target cq is not exhausted for the requested resource": { clusterQueues: defaultClusterQueues, @@ -2165,7 +3905,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("in", ""). + incoming: baseIncomingWl.Clone(). Request(corev1.ResourceCPU, "2"). Obj(), targetCQ: "a", @@ -2175,7 +3915,111 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/a1", kueue.InClusterQueueReason), targetKeyReason("/b5", kueue.InCohortReclamationReason)), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("a"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("b1", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b2", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b3", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b4", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + Obj(), + *utiltestingapi.MakeWorkload("b5", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("b"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, "long range preemption": { clusterQueues: []*kueue.ClusterQueue{ @@ -2213,7 +4057,7 @@ func TestPreemption(t *testing.T) { ). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Request(corev1.ResourceCPU, "8"). Obj(), targetCQ: "cq-left", @@ -2223,7 +4067,35 @@ func TestPreemption(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/to-be-preempted", kueue.InCohortReclamationReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("to-be-preempted", ""). + Request(corev1.ResourceCPU, "5"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("cq-right"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "5"). + Obj()). + Obj(), + now, + ). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, } for name, tc := range cases { @@ -2234,6 +4106,8 @@ func TestPreemption(t *testing.T) { ctx, log := utiltesting.ContextWithLog(t) cl := utiltesting.NewClientBuilder(). WithLists(&kueue.WorkloadList{Items: tc.admitted}). + WithStatusSubresource(&kueue.Workload{}). + WithInterceptorFuncs(interceptor.Funcs{SubResourcePatch: utiltesting.TreatSSAAsStrategicMerge}). Build() cqCache := schdcache.New(cl) @@ -2251,8 +4125,6 @@ func TestPreemption(t *testing.T) { } } - var lock sync.Mutex - gotPreempted := sets.New[string]() broadcaster := record.NewBroadcaster() scheme := runtime.NewScheme() if err := kueue.AddToScheme(scheme); err != nil { @@ -2260,12 +4132,6 @@ func TestPreemption(t *testing.T) { } recorder := broadcaster.NewRecorder(scheme, corev1.EventSource{Component: constants.AdmissionName}) preemptor := New(cl, workload.Ordering{}, recorder, config.FairSharing{}, false, clocktesting.NewFakeClock(now)) - preemptor.applyPreemption = func(ctx context.Context, w *kueue.Workload, reason, _ string) error { - lock.Lock() - gotPreempted.Insert(targetKeyReason(workload.Key(w), reason)) - lock.Unlock() - return nil - } beforeSnapshot, err := cqCache.Snapshot(ctx) if err != nil { @@ -2283,11 +4149,22 @@ func TestPreemption(t *testing.T) { if err != nil { t.Fatalf("Failed doing preemption") } - if diff := cmp.Diff(tc.wantPreempted, gotPreempted, cmpopts.EquateEmpty()); diff != "" { - t.Errorf("Issued preemptions (-want,+got):\n%s", diff) + if preempted != tc.wantPreempted { + t.Errorf("Reported %d preemptions, want %d", preempted, tc.wantPreempted) + } + + workloads := &kueue.WorkloadList{} + err = cl.List(ctx, workloads) + if err != nil { + t.Fatalf("Failed to List workloads: %v", err) + } + + defaultCmpOpts := cmp.Options{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion"), } - if preempted != tc.wantPreempted.Len() { - t.Errorf("Reported %d preemptions, want %d", preempted, tc.wantPreempted.Len()) + if diff := cmp.Diff(tc.wantWorkloads, workloads.Items, defaultCmpOpts); diff != "" { + t.Errorf("Unexpected workloads (-want/+got)\n%s", diff) } if diff := cmp.Diff(beforeSnapshot, snapshotWorkingCopy, snapCmpOpts); diff != "" { From 1de8dba67cc7edf29b24ec0ac8cb9e7d2ada89dd Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Tue, 28 Oct 2025 13:37:42 +0530 Subject: [PATCH 040/119] Bump the all group in /cmd/kueueviz/frontend with 2 updates (#7405) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- cmd/kueueviz/frontend/package-lock.json | 32 ++++++++++++------------- cmd/kueueviz/frontend/package.json | 4 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/kueueviz/frontend/package-lock.json b/cmd/kueueviz/frontend/package-lock.json index adc29e4805f..98fdb4078cc 100644 --- a/cmd/kueueviz/frontend/package-lock.json +++ b/cmd/kueueviz/frontend/package-lock.json @@ -21,9 +21,9 @@ "react-toastify": "^11.0.5" }, "devDependencies": { - "@vitejs/plugin-react": "^5.0.4", + "@vitejs/plugin-react": "^5.1.0", "depcheck": "^1.4.7", - "vite": "^7.1.11" + "vite": "^7.1.12" } }, "node_modules/@babel/code-frame": { @@ -1167,9 +1167,9 @@ } }, "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-beta.38", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.38.tgz", - "integrity": "sha512-N/ICGKleNhA5nc9XXQG/kkKHJ7S55u0x0XUJbbkmdCnFuoRkM1Il12q9q0eX19+M7KKUEPw/daUPIRnxhcxAIw==", + "version": "1.0.0-beta.43", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.43.tgz", + "integrity": "sha512-5Uxg7fQUCmfhax7FJke2+8B6cqgeUJUD9o2uXIKXhD+mG0mL6NObmVoi9wXEU1tY89mZKgAYA6fTbftx3q2ZPQ==", "dev": true, "license": "MIT" }, @@ -1556,18 +1556,18 @@ } }, "node_modules/@vitejs/plugin-react": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.0.4.tgz", - "integrity": "sha512-La0KD0vGkVkSk6K+piWDKRUyg8Rl5iAIKRMH0vMJI0Eg47bq1eOxmoObAaQG37WMW9MSyk7Cs8EIWwJC1PtzKA==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-5.1.0.tgz", + "integrity": "sha512-4LuWrg7EKWgQaMJfnN+wcmbAW+VSsCmqGohftWjuct47bv8uE4n/nPpq4XjJPsxgq00GGG5J8dvBczp8uxScew==", "dev": true, "license": "MIT", "dependencies": { "@babel/core": "^7.28.4", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", - "@rolldown/pluginutils": "1.0.0-beta.38", + "@rolldown/pluginutils": "1.0.0-beta.43", "@types/babel__core": "^7.20.5", - "react-refresh": "^0.17.0" + "react-refresh": "^0.18.0" }, "engines": { "node": "^20.19.0 || >=22.12.0" @@ -2853,9 +2853,9 @@ "license": "MIT" }, "node_modules/react-refresh": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.17.0.tgz", - "integrity": "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.18.0.tgz", + "integrity": "sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==", "dev": true, "license": "MIT", "engines": { @@ -3245,9 +3245,9 @@ } }, "node_modules/vite": { - "version": "7.1.11", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.1.11.tgz", - "integrity": "sha512-uzcxnSDVjAopEUjljkWh8EIrg6tlzrjFUfMcR1EVsRDGwf/ccef0qQPRyOrROwhrTDaApueq+ja+KLPlzR/zdg==", + "version": "7.1.12", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.1.12.tgz", + "integrity": "sha512-ZWyE8YXEXqJrrSLvYgrRP7p62OziLW7xI5HYGWFzOvupfAlrLvURSzv/FyGyy0eidogEM3ujU+kUG1zuHgb6Ug==", "dev": true, "license": "MIT", "dependencies": { diff --git a/cmd/kueueviz/frontend/package.json b/cmd/kueueviz/frontend/package.json index e60f3e40fc2..6ef96a1cc03 100644 --- a/cmd/kueueviz/frontend/package.json +++ b/cmd/kueueviz/frontend/package.json @@ -25,9 +25,9 @@ "react-toastify": "^11.0.5" }, "devDependencies": { - "@vitejs/plugin-react": "^5.0.4", + "@vitejs/plugin-react": "^5.1.0", "depcheck": "^1.4.7", - "vite": "^7.1.11" + "vite": "^7.1.12" }, "browserslist": { "production": [ From 54091d15e6ba3bd14d1e10f3e71fcc4fe23e086f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Oct 2025 02:11:37 -0700 Subject: [PATCH 041/119] Bump github.com/onsi/ginkgo/v2 from 2.26.0 to 2.27.1 (#7397) Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.26.0 to 2.27.1. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.26.0...v2.27.1) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-version: 2.27.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 +- vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md | 28 +++++ vendor/github.com/onsi/ginkgo/v2/core_dsl.go | 74 +++++++---- .../onsi/ginkgo/v2/decorator_dsl.go | 7 ++ .../github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go | 6 + .../onsi/ginkgo/v2/internal/group.go | 30 +++++ .../onsi/ginkgo/v2/internal/node.go | 117 +++++++++++++++--- .../onsi/ginkgo/v2/internal/ordering.go | 35 ++++-- .../onsi/ginkgo/v2/internal/suite.go | 20 +++ .../ginkgo/v2/reporters/default_reporter.go | 19 ++- .../onsi/ginkgo/v2/reporting_dsl.go | 45 +++++-- vendor/github.com/onsi/ginkgo/v2/table_dsl.go | 4 +- .../github.com/onsi/ginkgo/v2/types/types.go | 54 ++++++++ .../onsi/ginkgo/v2/types/version.go | 2 +- vendor/modules.txt | 2 +- 16 files changed, 383 insertions(+), 70 deletions(-) diff --git a/go.mod b/go.mod index a02321e904e..cc6d47e30c7 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/kubeflow/mpi-operator v0.6.0 github.com/kubeflow/trainer/v2 v2.1.0-rc.0 github.com/kubeflow/training-operator v1.9.3 - github.com/onsi/ginkgo/v2 v2.26.0 + github.com/onsi/ginkgo/v2 v2.27.1 github.com/onsi/gomega v1.38.2 github.com/open-policy-agent/cert-controller v0.14.0 github.com/project-codeflare/appwrapper v1.1.2 diff --git a/go.sum b/go.sum index c0830049844..196bc9055b5 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BN github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= -github.com/gkampitakis/go-snaps v0.5.14 h1:3fAqdB6BCPKHDMHAKRwtPUwYexKtGrNuw8HX/T/4neo= -github.com/gkampitakis/go-snaps v0.5.14/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -168,8 +168,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/onsi/ginkgo/v2 v2.26.0 h1:1J4Wut1IlYZNEAWIV3ALrT9NfiaGW2cDCJQSFQMs/gE= -github.com/onsi/ginkgo/v2 v2.26.0/go.mod h1:qhEywmzWTBUY88kfO0BRvX4py7scov9yR+Az2oavUzw= +github.com/onsi/ginkgo/v2 v2.27.1 h1:0LJC8MpUSQnfnp4n/3W3GdlmJP3ENGF0ZPzjQGLPP7s= +github.com/onsi/ginkgo/v2 v2.27.1/go.mod h1:wmy3vCqiBjirARfVhAqFpYt8uvX0yaFe+GudAqqcCqA= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/open-policy-agent/cert-controller v0.14.0 h1:TPc19BOHOs4tARruTT5o4bzir7Ed6FF+j3EXP/nmZBs= diff --git a/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md b/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md index 9db388356cc..f3ed259c1e8 100644 --- a/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md +++ b/vendor/github.com/onsi/ginkgo/v2/CHANGELOG.md @@ -1,3 +1,31 @@ +## 2.27.1 + +### Fixes +- Fix Ginkgo Reporter slice-bounds panic [606c1cb] +- Bug Fix: Add GinkoTBWrapper.Attr() and GinkoTBWrapper.Output() [a6463b3] + +## 2.27.0 + +### Features + +#### Transforming Nodes during Tree Construction + +This release adds support for `NodeArgsTransformer`s that can be registered with `AddTreeConstructionNodeArgsTransformer`. + +These are called during the tree construction phase as nodes are constructed and can modify the node strings and decorators. This enables frameworks built on top of Ginkgo to modify Ginkgo nodes and enforce conventions. + +Learn more [here](https://onsi.github.io/ginkgo/#advanced-transforming-node-arguments-during-tree-construction). + +#### Spec Prioritization + +A new `SpecPriority(int)` decorator has been added. Ginkgo will honor priority when ordering specs, ensuring that higher priority specs start running before lower priority specs + +Learn more [here](https://onsi.github.io/ginkgo/#prioritizing-specs). + +### Maintenance +- Bump rexml from 3.4.0 to 3.4.2 in /docs (#1595) [1333dae] +- Bump github.com/gkampitakis/go-snaps from 0.5.14 to 0.5.15 (#1600) [17ae63e] + ## 2.26.0 ### Features diff --git a/vendor/github.com/onsi/ginkgo/v2/core_dsl.go b/vendor/github.com/onsi/ginkgo/v2/core_dsl.go index ec41e8837c8..7e165e47382 100644 --- a/vendor/github.com/onsi/ginkgo/v2/core_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/core_dsl.go @@ -501,6 +501,38 @@ func pushNode(node internal.Node, errors []error) bool { return true } +// NodeArgsTransformer is a hook which is called by the test construction DSL methods +// before creating the new node. If it returns any error, the test suite +// prints those errors and exits. The text and arguments can be modified, +// which includes directly changing the args slice that is passed in. +// Arguments have been flattened already, i.e. none of the entries in args is another []any. +// The result may be nested. +// +// The node type is provided for information and remains the same. +// +// The offset is valid for calling NewLocation directly in the +// implementation of TransformNodeArgs to find the location where +// the Ginkgo DSL function is called. An additional offset supplied +// by the caller via args is already included. +// +// A NodeArgsTransformer can be registered with AddTreeConstructionNodeArgsTransformer. +type NodeArgsTransformer func(nodeType types.NodeType, offset Offset, text string, args []any) (string, []any, []error) + +// AddTreeConstructionNodeArgsTransformer registers a NodeArgsTransformer. +// Only nodes which get created after registering a NodeArgsTransformer +// are transformed by it. The returned function can be called to +// unregister the transformer. +// +// Both may only be called during the construction phase. +// +// If there is more than one registered transformer, then the most +// recently added ones get called first. +func AddTreeConstructionNodeArgsTransformer(transformer NodeArgsTransformer) func() { + // This conversion could be avoided with a type alias, but type aliases make + // developer documentation less useful. + return internal.AddTreeConstructionNodeArgsTransformer(internal.NodeArgsTransformer(transformer)) +} + /* Describe nodes are Container nodes that allow you to organize your specs. A Describe node's closure can contain any number of Setup nodes (e.g. BeforeEach, AfterEach, JustBeforeEach), and Subject nodes (i.e. It). @@ -512,7 +544,7 @@ You can learn more at https://onsi.github.io/ginkgo/#organizing-specs-with-conta In addition, container nodes can be decorated with a variety of decorators. You can learn more here: https://onsi.github.io/ginkgo/#decorator-reference */ func Describe(text string, args ...any) bool { - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeContainer, text, args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeContainer, text, args...))) } /* @@ -520,7 +552,7 @@ FDescribe focuses specs within the Describe block. */ func FDescribe(text string, args ...any) bool { args = append(args, internal.Focus) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeContainer, text, args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeContainer, text, args...))) } /* @@ -528,7 +560,7 @@ PDescribe marks specs within the Describe block as pending. */ func PDescribe(text string, args ...any) bool { args = append(args, internal.Pending) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeContainer, text, args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeContainer, text, args...))) } /* @@ -541,21 +573,21 @@ var XDescribe = PDescribe /* Context is an alias for Describe - it generates the exact same kind of Container node */ var Context, FContext, PContext, XContext = Describe, FDescribe, PDescribe, XDescribe -/* When is an alias for Describe - it generates the exact same kind of Container node */ +/* When is an alias for Describe - it generates the exact same kind of Container node with "when " as prefix for the text. */ func When(text string, args ...any) bool { - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeContainer, "when "+text, args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeContainer, "when "+text, args...))) } -/* When is an alias for Describe - it generates the exact same kind of Container node */ +/* When is an alias for Describe - it generates the exact same kind of Container node with "when " as prefix for the text. */ func FWhen(text string, args ...any) bool { args = append(args, internal.Focus) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeContainer, "when "+text, args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeContainer, "when "+text, args...))) } /* When is an alias for Describe - it generates the exact same kind of Container node */ func PWhen(text string, args ...any) bool { args = append(args, internal.Pending) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeContainer, "when "+text, args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeContainer, "when "+text, args...))) } var XWhen = PWhen @@ -571,7 +603,7 @@ You can learn more at https://onsi.github.io/ginkgo/#spec-subjects-it In addition, subject nodes can be decorated with a variety of decorators. You can learn more here: https://onsi.github.io/ginkgo/#decorator-reference */ func It(text string, args ...any) bool { - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeIt, text, args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeIt, text, args...))) } /* @@ -579,7 +611,7 @@ FIt allows you to focus an individual It. */ func FIt(text string, args ...any) bool { args = append(args, internal.Focus) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeIt, text, args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeIt, text, args...))) } /* @@ -587,7 +619,7 @@ PIt allows you to mark an individual It as pending. */ func PIt(text string, args ...any) bool { args = append(args, internal.Pending) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeIt, text, args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeIt, text, args...))) } /* @@ -634,7 +666,7 @@ You can learn more here: https://onsi.github.io/ginkgo/#suite-setup-and-cleanup- func BeforeSuite(body any, args ...any) bool { combinedArgs := []any{body} combinedArgs = append(combinedArgs, args...) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeBeforeSuite, "", combinedArgs...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeBeforeSuite, "", combinedArgs...))) } /* @@ -653,7 +685,7 @@ You can learn more here: https://onsi.github.io/ginkgo/#suite-setup-and-cleanup- func AfterSuite(body any, args ...any) bool { combinedArgs := []any{body} combinedArgs = append(combinedArgs, args...) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeAfterSuite, "", combinedArgs...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeAfterSuite, "", combinedArgs...))) } /* @@ -691,7 +723,7 @@ func SynchronizedBeforeSuite(process1Body any, allProcessBody any, args ...any) combinedArgs := []any{process1Body, allProcessBody} combinedArgs = append(combinedArgs, args...) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeSynchronizedBeforeSuite, "", combinedArgs...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeSynchronizedBeforeSuite, "", combinedArgs...))) } /* @@ -711,7 +743,7 @@ func SynchronizedAfterSuite(allProcessBody any, process1Body any, args ...any) b combinedArgs := []any{allProcessBody, process1Body} combinedArgs = append(combinedArgs, args...) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeSynchronizedAfterSuite, "", combinedArgs...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeSynchronizedAfterSuite, "", combinedArgs...))) } /* @@ -724,7 +756,7 @@ You cannot nest any other Ginkgo nodes within a BeforeEach node's closure. You can learn more here: https://onsi.github.io/ginkgo/#extracting-common-setup-beforeeach */ func BeforeEach(args ...any) bool { - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeBeforeEach, "", args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeBeforeEach, "", args...))) } /* @@ -737,7 +769,7 @@ You cannot nest any other Ginkgo nodes within a JustBeforeEach node's closure. You can learn more and see some examples here: https://onsi.github.io/ginkgo/#separating-creation-and-configuration-justbeforeeach */ func JustBeforeEach(args ...any) bool { - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeJustBeforeEach, "", args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeJustBeforeEach, "", args...))) } /* @@ -752,7 +784,7 @@ You cannot nest any other Ginkgo nodes within an AfterEach node's closure. You can learn more here: https://onsi.github.io/ginkgo/#spec-cleanup-aftereach-and-defercleanup */ func AfterEach(args ...any) bool { - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeAfterEach, "", args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeAfterEach, "", args...))) } /* @@ -764,7 +796,7 @@ You cannot nest any other Ginkgo nodes within a JustAfterEach node's closure. You can learn more and see some examples here: https://onsi.github.io/ginkgo/#separating-diagnostics-collection-and-teardown-justaftereach */ func JustAfterEach(args ...any) bool { - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeJustAfterEach, "", args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeJustAfterEach, "", args...))) } /* @@ -779,7 +811,7 @@ You can learn more about Ordered Containers at: https://onsi.github.io/ginkgo/#o And you can learn more about BeforeAll at: https://onsi.github.io/ginkgo/#setup-in-ordered-containers-beforeall-and-afterall */ func BeforeAll(args ...any) bool { - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeBeforeAll, "", args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeBeforeAll, "", args...))) } /* @@ -796,7 +828,7 @@ You can learn more about Ordered Containers at: https://onsi.github.io/ginkgo/#o And you can learn more about AfterAll at: https://onsi.github.io/ginkgo/#setup-in-ordered-containers-beforeall-and-afterall */ func AfterAll(args ...any) bool { - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeAfterAll, "", args...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeAfterAll, "", args...))) } /* diff --git a/vendor/github.com/onsi/ginkgo/v2/decorator_dsl.go b/vendor/github.com/onsi/ginkgo/v2/decorator_dsl.go index 8bee5acebdb..e331d7cf8cd 100644 --- a/vendor/github.com/onsi/ginkgo/v2/decorator_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/decorator_dsl.go @@ -154,6 +154,13 @@ Nodes that do not finish within a GracePeriod will be leaked and Ginkgo will pro */ type GracePeriod = internal.GracePeriod +/* +SpecPriority allows you to assign a priority to a spec or container. + +Specs with higher priority will be scheduled to run before specs with lower priority. The default priority is 0 and negative priorities are allowed. +*/ +type SpecPriority = internal.SpecPriority + /* SuppressProgressReporting is a decorator that allows you to disable progress reporting of a particular node. This is useful if `ginkgo -v -progress` is generating too much noise; particularly if you have a `ReportAfterEach` node that is running for every skipped spec and is generating lots of progress reports. diff --git a/vendor/github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go b/vendor/github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go index cabf2814571..40d1e1ab5ce 100644 --- a/vendor/github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/ginkgo_t_dsl.go @@ -190,3 +190,9 @@ func (g *GinkgoTBWrapper) Skipped() bool { func (g *GinkgoTBWrapper) TempDir() string { return g.GinkgoT.TempDir() } +func (g *GinkgoTBWrapper) Attr(key, value string) { + g.GinkgoT.Attr(key, value) +} +func (g *GinkgoTBWrapper) Output() io.Writer { + return g.GinkgoT.Output() +} diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/group.go b/vendor/github.com/onsi/ginkgo/v2/internal/group.go index b88fe2060a5..cc794903e7e 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/group.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/group.go @@ -110,6 +110,7 @@ func newGroup(suite *Suite) *group { } } +// initialReportForSpec constructs a new SpecReport right before running the spec. func (g *group) initialReportForSpec(spec Spec) types.SpecReport { return types.SpecReport{ ContainerHierarchyTexts: spec.Nodes.WithType(types.NodeTypeContainer).Texts(), @@ -127,6 +128,35 @@ func (g *group) initialReportForSpec(spec Spec) types.SpecReport { IsInOrderedContainer: !spec.Nodes.FirstNodeMarkedOrdered().IsZero(), MaxFlakeAttempts: spec.Nodes.GetMaxFlakeAttempts(), MaxMustPassRepeatedly: spec.Nodes.GetMaxMustPassRepeatedly(), + SpecPriority: spec.Nodes.GetSpecPriority(), + } +} + +// constructionNodeReportForTreeNode constructs a new SpecReport right before invoking the body +// of a container node during construction of the full tree. +func constructionNodeReportForTreeNode(node *TreeNode) *types.ConstructionNodeReport { + var report types.ConstructionNodeReport + // Walk up the tree and set attributes accordingly. + addNodeToReportForNode(&report, node) + return &report +} + +// addNodeToReportForNode is conceptually similar to initialReportForSpec and therefore placed here +// although it doesn't do anything with a group. +func addNodeToReportForNode(report *types.ConstructionNodeReport, node *TreeNode) { + if node.Parent != nil { + // First add the parent node, then the current one. + addNodeToReportForNode(report, node.Parent) + } + report.ContainerHierarchyTexts = append(report.ContainerHierarchyTexts, node.Node.Text) + report.ContainerHierarchyLocations = append(report.ContainerHierarchyLocations, node.Node.CodeLocation) + report.ContainerHierarchyLabels = append(report.ContainerHierarchyLabels, node.Node.Labels) + report.ContainerHierarchySemVerConstraints = append(report.ContainerHierarchySemVerConstraints, node.Node.SemVerConstraints) + if node.Node.MarkedSerial { + report.IsSerial = true + } + if node.Node.MarkedOrdered { + report.IsInOrderedContainer = true } } diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/node.go b/vendor/github.com/onsi/ginkgo/v2/internal/node.go index 647368feacb..2bccec2dbf0 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/node.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/node.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "reflect" + "slices" "sort" "sync" "time" @@ -46,22 +47,24 @@ type Node struct { ReportEachBody func(SpecContext, types.SpecReport) ReportSuiteBody func(SpecContext, types.Report) - MarkedFocus bool - MarkedPending bool - MarkedSerial bool - MarkedOrdered bool - MarkedContinueOnFailure bool - MarkedOncePerOrdered bool - FlakeAttempts int - MustPassRepeatedly int - Labels Labels - SemVerConstraints SemVerConstraints - PollProgressAfter time.Duration - PollProgressInterval time.Duration - NodeTimeout time.Duration - SpecTimeout time.Duration - GracePeriod time.Duration - AroundNodes types.AroundNodes + MarkedFocus bool + MarkedPending bool + MarkedSerial bool + MarkedOrdered bool + MarkedContinueOnFailure bool + MarkedOncePerOrdered bool + FlakeAttempts int + MustPassRepeatedly int + Labels Labels + SemVerConstraints SemVerConstraints + PollProgressAfter time.Duration + PollProgressInterval time.Duration + NodeTimeout time.Duration + SpecTimeout time.Duration + GracePeriod time.Duration + AroundNodes types.AroundNodes + HasExplicitlySetSpecPriority bool + SpecPriority int NodeIDWhereCleanupWasGenerated uint } @@ -92,6 +95,7 @@ type PollProgressAfter time.Duration type NodeTimeout time.Duration type SpecTimeout time.Duration type GracePeriod time.Duration +type SpecPriority int type Labels []string @@ -182,6 +186,8 @@ func isDecoration(arg any) bool { return true case t == reflect.TypeOf(types.AroundNodeDecorator{}): return true + case t == reflect.TypeOf(SpecPriority(0)): + return true case t.Kind() == reflect.Slice && isSliceOfDecorations(arg): return true default: @@ -227,7 +233,7 @@ func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeTy } } - args = unrollInterfaceSlice(args) + args = UnrollInterfaceSlice(args) remainingArgs := []any{} // First get the CodeLocation up-to-date @@ -322,6 +328,12 @@ func NewNode(deprecationTracker *types.DeprecationTracker, nodeType types.NodeTy if nodeType.Is(types.NodeTypeContainer) { appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "GracePeriod")) } + case t == reflect.TypeOf(SpecPriority(0)): + if !nodeType.Is(types.NodeTypesForContainerAndIt) { + appendError(types.GinkgoErrors.InvalidDecoratorForNodeType(node.CodeLocation, nodeType, "SpecPriority")) + } + node.SpecPriority = int(arg.(SpecPriority)) + node.HasExplicitlySetSpecPriority = true case t == reflect.TypeOf(types.AroundNodeDecorator{}): node.AroundNodes = append(node.AroundNodes, arg.(types.AroundNodeDecorator)) case t == reflect.TypeOf(Labels{}): @@ -636,7 +648,7 @@ func NewCleanupNode(deprecationTracker *types.DeprecationTracker, fail func(stri }) } - return NewNode(deprecationTracker, types.NodeTypeCleanupInvalid, "", finalArgs...) + return NewNode(deprecationTracker, types.NodeTypeCleanupInvalid, "", finalArgs) } func (n Node) IsZero() bool { @@ -983,7 +995,16 @@ func (n Nodes) GetMaxMustPassRepeatedly() int { return maxMustPassRepeatedly } -func unrollInterfaceSlice(args any) []any { +func (n Nodes) GetSpecPriority() int { + for i := len(n) - 1; i >= 0; i-- { + if n[i].HasExplicitlySetSpecPriority { + return n[i].SpecPriority + } + } + return 0 +} + +func UnrollInterfaceSlice(args any) []any { v := reflect.ValueOf(args) if v.Kind() != reflect.Slice { return []any{args} @@ -992,10 +1013,66 @@ func unrollInterfaceSlice(args any) []any { for i := 0; i < v.Len(); i++ { el := reflect.ValueOf(v.Index(i).Interface()) if el.Kind() == reflect.Slice && el.Type() != reflect.TypeOf(Labels{}) && el.Type() != reflect.TypeOf(SemVerConstraints{}) { - out = append(out, unrollInterfaceSlice(el.Interface())...) + out = append(out, UnrollInterfaceSlice(el.Interface())...) } else { out = append(out, v.Index(i).Interface()) } } return out } + +type NodeArgsTransformer func(nodeType types.NodeType, offset Offset, text string, args []any) (string, []any, []error) + +func AddTreeConstructionNodeArgsTransformer(transformer NodeArgsTransformer) func() { + id := nodeArgsTransformerCounter + nodeArgsTransformerCounter++ + nodeArgsTransformers = append(nodeArgsTransformers, registeredNodeArgsTransformer{id, transformer}) + return func() { + nodeArgsTransformers = slices.DeleteFunc(nodeArgsTransformers, func(transformer registeredNodeArgsTransformer) bool { + return transformer.id == id + }) + } +} + +var ( + nodeArgsTransformerCounter int64 + nodeArgsTransformers []registeredNodeArgsTransformer +) + +type registeredNodeArgsTransformer struct { + id int64 + transformer NodeArgsTransformer +} + +// TransformNewNodeArgs is the helper for DSL functions which handles NodeArgsTransformers. +// +// Its return valus are intentionally the same as the internal.NewNode parameters, +// which makes it possible to chain the invocations: +// +// NewNode(transformNewNodeArgs(...)) +func TransformNewNodeArgs(exitIfErrors func([]error), deprecationTracker *types.DeprecationTracker, nodeType types.NodeType, text string, args ...any) (*types.DeprecationTracker, types.NodeType, string, []any) { + var errs []error + + // Most recent first... + // + // This intentionally doesn't use slices.Backward because + // using iterators influences stack unwinding. + for i := len(nodeArgsTransformers) - 1; i >= 0; i-- { + transformer := nodeArgsTransformers[i].transformer + args = UnrollInterfaceSlice(args) + + // We do not really need to recompute this on additional loop iterations, + // but its fast and simpler this way. + var offset Offset + for _, arg := range args { + if o, ok := arg.(Offset); ok { + offset = o + } + } + offset += 3 // The DSL function, this helper, and the TransformNodeArgs implementation. + + text, args, errs = transformer(nodeType, offset, text, args) + exitIfErrors(errs) + } + return deprecationTracker, nodeType, text, args +} diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/ordering.go b/vendor/github.com/onsi/ginkgo/v2/internal/ordering.go index 84eea0a59e6..da58d54f957 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/ordering.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/ordering.go @@ -125,7 +125,7 @@ func OrderSpecs(specs Specs, suiteConfig types.SuiteConfig) (GroupedSpecIndices, // pick out a representative spec representativeSpec := specs[executionGroups[groupID][0]] - // and grab the node on the spec that will represent which shufflable group this execution group belongs tu + // and grab the node on the spec that will represent which shufflable group this execution group belongs to shufflableGroupingNode := representativeSpec.Nodes.FirstNodeWithType(nodeTypesToShuffle) //add the execution group to its shufflable group @@ -138,14 +138,35 @@ func OrderSpecs(specs Specs, suiteConfig types.SuiteConfig) (GroupedSpecIndices, } } + // now, for each shuffleable group, we compute the priority + shufflableGroupingIDPriorities := map[uint]int{} + for shufflableGroupingID, groupIDs := range shufflableGroupingIDToGroupIDs { + // the priority of a shufflable grouping is the max priority of any spec in any execution group in the shufflable grouping + maxPriority := -1 << 31 // min int + for _, groupID := range groupIDs { + for _, specIdx := range executionGroups[groupID] { + specPriority := specs[specIdx].Nodes.GetSpecPriority() + maxPriority = max(specPriority, maxPriority) + } + } + shufflableGroupingIDPriorities[shufflableGroupingID] = maxPriority + } + // now we permute the sorted shufflable grouping IDs and build the ordered Groups - orderedGroups := GroupedSpecIndices{} permutation := r.Perm(len(shufflableGroupingIDs)) - for _, j := range permutation { - //let's get the execution group IDs for this shufflable group: - executionGroupIDsForJ := shufflableGroupingIDToGroupIDs[shufflableGroupingIDs[j]] - // and we'll add their associated specindices to the orderedGroups slice: - for _, executionGroupID := range executionGroupIDsForJ { + shuffledGroupingIds := make([]uint, len(shufflableGroupingIDs)) + for i, j := range permutation { + shuffledGroupingIds[i] = shufflableGroupingIDs[j] + } + // now, we need to stable sort the shuffledGroupingIds by priority (higher priority first) + sort.SliceStable(shuffledGroupingIds, func(i, j int) bool { + return shufflableGroupingIDPriorities[shuffledGroupingIds[i]] > shufflableGroupingIDPriorities[shuffledGroupingIds[j]] + }) + + // we can now take these prioritized, shuffled, groupings and form the final set of ordered spec groups + orderedGroups := GroupedSpecIndices{} + for _, id := range shuffledGroupingIds { + for _, executionGroupID := range shufflableGroupingIDToGroupIDs[id] { orderedGroups = append(orderedGroups, executionGroups[executionGroupID]) } } diff --git a/vendor/github.com/onsi/ginkgo/v2/internal/suite.go b/vendor/github.com/onsi/ginkgo/v2/internal/suite.go index 14a0688f89e..ef76cd099e6 100644 --- a/vendor/github.com/onsi/ginkgo/v2/internal/suite.go +++ b/vendor/github.com/onsi/ginkgo/v2/internal/suite.go @@ -42,6 +42,8 @@ type Suite struct { config types.SuiteConfig deadline time.Time + currentConstructionNodeReport *types.ConstructionNodeReport + skipAll bool report types.Report currentSpecReport types.SpecReport @@ -203,6 +205,14 @@ func (suite *Suite) PushNode(node Node) error { err = types.GinkgoErrors.CaughtPanicDuringABuildPhase(e, node.CodeLocation) } }() + + // Ensure that code running in the body of the container node + // has access to information about the current container node(s). + suite.currentConstructionNodeReport = constructionNodeReportForTreeNode(suite.tree) + defer func() { + suite.currentConstructionNodeReport = nil + }() + node.Body(nil) return err }() @@ -332,6 +342,16 @@ func (suite *Suite) By(text string, callback ...func()) error { return nil } +func (suite *Suite) CurrentConstructionNodeReport() types.ConstructionNodeReport { + suite.selectiveLock.Lock() + defer suite.selectiveLock.Unlock() + report := suite.currentConstructionNodeReport + if report == nil { + panic("CurrentConstructionNodeReport may only be called during construction of the spec tree") + } + return *report +} + /* Spec Running methods - used during PhaseRun */ diff --git a/vendor/github.com/onsi/ginkgo/v2/reporters/default_reporter.go b/vendor/github.com/onsi/ginkgo/v2/reporters/default_reporter.go index 8c3714b8ae6..026d9cf9b37 100644 --- a/vendor/github.com/onsi/ginkgo/v2/reporters/default_reporter.go +++ b/vendor/github.com/onsi/ginkgo/v2/reporters/default_reporter.go @@ -381,13 +381,22 @@ func (r *DefaultReporter) emitTimeline(indent uint, report types.SpecReport, tim cursor := 0 for _, entry := range timeline { tl := entry.GetTimelineLocation() - if tl.Offset < len(gw) { - r.emit(r.fi(indent, "%s", gw[cursor:tl.Offset])) - cursor = tl.Offset - } else if cursor < len(gw) { + + end := tl.Offset + if end > len(gw) { + end = len(gw) + } + if end < cursor { + end = cursor + } + if cursor < end && cursor <= len(gw) && end <= len(gw) { + r.emit(r.fi(indent, "%s", gw[cursor:end])) + cursor = end + } else if cursor < len(gw) && end == len(gw) { r.emit(r.fi(indent, "%s", gw[cursor:])) cursor = len(gw) } + switch x := entry.(type) { case types.Failure: if isVeryVerbose { @@ -404,7 +413,7 @@ func (r *DefaultReporter) emitTimeline(indent uint, report types.SpecReport, tim case types.ReportEntry: r.emitReportEntry(indent, x) case types.ProgressReport: - r.emitProgressReport(indent, false, false, x) + r.emitProgressReport(indent, false, isVeryVerbose, x) case types.SpecEvent: if isVeryVerbose || !x.IsOnlyVisibleAtVeryVerbose() || r.conf.ShowNodeEvents { r.emitSpecEvent(indent, x, isVeryVerbose) diff --git a/vendor/github.com/onsi/ginkgo/v2/reporting_dsl.go b/vendor/github.com/onsi/ginkgo/v2/reporting_dsl.go index dd135054d4a..4e86dba84d8 100644 --- a/vendor/github.com/onsi/ginkgo/v2/reporting_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/reporting_dsl.go @@ -27,6 +27,8 @@ CurrentSpecReport returns information about the current running spec. The returned object is a types.SpecReport which includes helper methods to make extracting information about the spec easier. +During construction of the test tree the result is empty. + You can learn more about SpecReport here: https://pkg.go.dev/github.com/onsi/ginkgo/types#SpecReport You can learn more about CurrentSpecReport() here: https://onsi.github.io/ginkgo/#getting-a-report-for-the-current-spec */ @@ -34,6 +36,31 @@ func CurrentSpecReport() SpecReport { return global.Suite.CurrentSpecReport() } +/* +ConstructionNodeReport describes the container nodes during construction of +the spec tree. It provides a subset of the information that is provided +by SpecReport at runtime. + +It is documented here: [types.ConstructionNodeReport] +*/ +type ConstructionNodeReport = types.ConstructionNodeReport + +/* +CurrentConstructionNodeReport returns information about the current container nodes +that are leading to the current path in the spec tree. +The returned object is a types.ConstructionNodeReport which includes helper methods +to make extracting information about the spec easier. + +May only be called during construction of the spec tree. It panics when +called while tests are running. Use CurrentSpecReport instead in that +phase. + +You can learn more about ConstructionNodeReport here: [types.ConstructionNodeReport] +*/ +func CurrentTreeConstructionNodeReport() ConstructionNodeReport { + return global.Suite.CurrentConstructionNodeReport() +} + /* ReportEntryVisibility governs the visibility of ReportEntries in Ginkgo's console reporter @@ -92,7 +119,7 @@ func ReportBeforeEach(body any, args ...any) bool { combinedArgs := []any{body} combinedArgs = append(combinedArgs, args...) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeReportBeforeEach, "", combinedArgs...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeReportBeforeEach, "", combinedArgs...))) } /* @@ -116,7 +143,7 @@ func ReportAfterEach(body any, args ...any) bool { combinedArgs := []any{body} combinedArgs = append(combinedArgs, args...) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeReportAfterEach, "", combinedArgs...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeReportAfterEach, "", combinedArgs...))) } /* @@ -145,7 +172,7 @@ You can learn about interruptible nodes here: https://onsi.github.io/ginkgo/#spe func ReportBeforeSuite(body any, args ...any) bool { combinedArgs := []any{body} combinedArgs = append(combinedArgs, args...) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeReportBeforeSuite, "", combinedArgs...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeReportBeforeSuite, "", combinedArgs...))) } /* @@ -177,7 +204,7 @@ You can learn about interruptible nodes here: https://onsi.github.io/ginkgo/#spe func ReportAfterSuite(text string, body any, args ...any) bool { combinedArgs := []any{body} combinedArgs = append(combinedArgs, args...) - return pushNode(internal.NewNode(deprecationTracker, types.NodeTypeReportAfterSuite, text, combinedArgs...)) + return pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeReportAfterSuite, text, combinedArgs...))) } func registerReportAfterSuiteNodeForAutogeneratedReports(reporterConfig types.ReporterConfig) { @@ -222,9 +249,11 @@ func registerReportAfterSuiteNodeForAutogeneratedReports(reporterConfig types.Re flags = append(flags, "--teamcity-report") } pushNode(internal.NewNode( - deprecationTracker, types.NodeTypeReportAfterSuite, - fmt.Sprintf("Autogenerated ReportAfterSuite for %s", strings.Join(flags, " ")), - body, - types.NewCustomCodeLocation("autogenerated by Ginkgo"), + internal.TransformNewNodeArgs( + exitIfErrors, deprecationTracker, types.NodeTypeReportAfterSuite, + fmt.Sprintf("Autogenerated ReportAfterSuite for %s", strings.Join(flags, " ")), + body, + types.NewCustomCodeLocation("autogenerated by Ginkgo"), + ), )) } diff --git a/vendor/github.com/onsi/ginkgo/v2/table_dsl.go b/vendor/github.com/onsi/ginkgo/v2/table_dsl.go index b9e0ca9ef76..1031aa85545 100644 --- a/vendor/github.com/onsi/ginkgo/v2/table_dsl.go +++ b/vendor/github.com/onsi/ginkgo/v2/table_dsl.go @@ -309,11 +309,11 @@ func generateTable(description string, isSubtree bool, args ...any) { internalNodeType = types.NodeTypeContainer } - pushNode(internal.NewNode(deprecationTracker, internalNodeType, description, internalNodeArgs...)) + pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, internalNodeType, description, internalNodeArgs...))) } }) - pushNode(internal.NewNode(deprecationTracker, types.NodeTypeContainer, description, containerNodeArgs...)) + pushNode(internal.NewNode(internal.TransformNewNodeArgs(exitIfErrors, deprecationTracker, types.NodeTypeContainer, description, containerNodeArgs...))) } func invokeFunction(function any, parameters []any) []reflect.Value { diff --git a/vendor/github.com/onsi/ginkgo/v2/types/types.go b/vendor/github.com/onsi/ginkgo/v2/types/types.go index b8e864a5d26..9981a0dd686 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/types.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/types.go @@ -20,6 +20,57 @@ func init() { } } +// ConstructionNodeReport captures information about a Ginkgo spec. +type ConstructionNodeReport struct { + // ContainerHierarchyTexts is a slice containing the text strings of + // all Describe/Context/When containers in this spec's hierarchy. + ContainerHierarchyTexts []string + + // ContainerHierarchyLocations is a slice containing the CodeLocations of + // all Describe/Context/When containers in this spec's hierarchy. + ContainerHierarchyLocations []CodeLocation + + // ContainerHierarchyLabels is a slice containing the labels of + // all Describe/Context/When containers in this spec's hierarchy + ContainerHierarchyLabels [][]string + + // ContainerHierarchySemVerConstraints is a slice containing the semVerConstraints of + // all Describe/Context/When containers in this spec's hierarchy + ContainerHierarchySemVerConstraints [][]string + + // IsSerial captures whether the any container has the Serial decorator + IsSerial bool + + // IsInOrderedContainer captures whether any container is an Ordered container + IsInOrderedContainer bool +} + +// FullText returns a concatenation of all the report.ContainerHierarchyTexts and report.LeafNodeText +func (report ConstructionNodeReport) FullText() string { + texts := []string{} + texts = append(texts, report.ContainerHierarchyTexts...) + texts = slices.DeleteFunc(texts, func(t string) bool { + return t == "" + }) + return strings.Join(texts, " ") +} + +// Labels returns a deduped set of all the spec's Labels. +func (report ConstructionNodeReport) Labels() []string { + out := []string{} + seen := map[string]bool{} + for _, labels := range report.ContainerHierarchyLabels { + for _, label := range labels { + if !seen[label] { + seen[label] = true + out = append(out, label) + } + } + } + + return out +} + // Report captures information about a Ginkgo test run type Report struct { //SuitePath captures the absolute path to the test suite @@ -146,6 +197,9 @@ type SpecReport struct { LeafNodeSemVerConstraints []string LeafNodeText string + // Captures the Spec Priority + SpecPriority int + // State captures whether the spec has passed, failed, etc. State SpecState diff --git a/vendor/github.com/onsi/ginkgo/v2/types/version.go b/vendor/github.com/onsi/ginkgo/v2/types/version.go index a30e68be1ad..855e7cbd977 100644 --- a/vendor/github.com/onsi/ginkgo/v2/types/version.go +++ b/vendor/github.com/onsi/ginkgo/v2/types/version.go @@ -1,3 +1,3 @@ package types -const VERSION = "2.26.0" +const VERSION = "2.27.1" diff --git a/vendor/modules.txt b/vendor/modules.txt index 0844ea8db26..bd3d4c848b3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -253,7 +253,7 @@ github.com/munnerz/goautoneg # github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f ## explicit github.com/mxk/go-flowrate/flowrate -# github.com/onsi/ginkgo/v2 v2.26.0 +# github.com/onsi/ginkgo/v2 v2.27.1 ## explicit; go 1.23.0 github.com/onsi/ginkgo/v2 github.com/onsi/ginkgo/v2/config From 87d985b9443860c0a37918668e5d65c703d95c6a Mon Sep 17 00:00:00 2001 From: bob Date: Tue, 28 Oct 2025 17:39:37 +0800 Subject: [PATCH 042/119] Deprecate QueueVisibility for v1beta2 (#7319) Signed-off-by: song --- .../v1beta1/configuration_conversion.go | 41 ++++++++++ .../config/v1beta1/zz_generated.conversion.go | 80 ++----------------- apis/config/v1beta2/configuration_types.go | 28 ------- apis/config/v1beta2/groupversion_info.go | 2 + apis/config/v1beta2/zz_generated.deepcopy.go | 40 ---------- 5 files changed, 49 insertions(+), 142 deletions(-) create mode 100644 apis/config/v1beta1/configuration_conversion.go diff --git a/apis/config/v1beta1/configuration_conversion.go b/apis/config/v1beta1/configuration_conversion.go new file mode 100644 index 00000000000..249baf6ab7c --- /dev/null +++ b/apis/config/v1beta1/configuration_conversion.go @@ -0,0 +1,41 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + conversionapi "k8s.io/apimachinery/pkg/conversion" + "sigs.k8s.io/controller-runtime/pkg/conversion" + + "sigs.k8s.io/kueue/apis/config/v1beta2" +) + +//lint:file-ignore ST1003 "generated Convert_* calls below use underscores" +//revive:disable:var-naming + +func (src *Configuration) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1beta2.Configuration) + return Convert_v1beta1_Configuration_To_v1beta2_Configuration(src, dst, nil) +} + +func (dst *Configuration) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1beta2.Configuration) + return Convert_v1beta2_Configuration_To_v1beta1_Configuration(src, dst, nil) +} + +func Convert_v1beta1_Configuration_To_v1beta2_Configuration(in *Configuration, out *v1beta2.Configuration, s conversionapi.Scope) error { + return autoConvert_v1beta1_Configuration_To_v1beta2_Configuration(in, out, s) +} diff --git a/apis/config/v1beta1/zz_generated.conversion.go b/apis/config/v1beta1/zz_generated.conversion.go index 3e4c8f2e6bb..9a64937ceb1 100644 --- a/apis/config/v1beta1/zz_generated.conversion.go +++ b/apis/config/v1beta1/zz_generated.conversion.go @@ -59,21 +59,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*ClusterQueueVisibility)(nil), (*v1beta2.ClusterQueueVisibility)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility(a.(*ClusterQueueVisibility), b.(*v1beta2.ClusterQueueVisibility), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueueVisibility)(nil), (*ClusterQueueVisibility)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility(a.(*v1beta2.ClusterQueueVisibility), b.(*ClusterQueueVisibility), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*Configuration)(nil), (*v1beta2.Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_Configuration_To_v1beta2_Configuration(a.(*Configuration), b.(*v1beta2.Configuration), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*v1beta2.Configuration)(nil), (*Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_Configuration_To_v1beta1_Configuration(a.(*v1beta2.Configuration), b.(*Configuration), scope) }); err != nil { @@ -209,16 +194,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*QueueVisibility)(nil), (*v1beta2.QueueVisibility)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility(a.(*QueueVisibility), b.(*v1beta2.QueueVisibility), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta2.QueueVisibility)(nil), (*QueueVisibility)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility(a.(*v1beta2.QueueVisibility), b.(*QueueVisibility), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*RequeuingStrategy)(nil), (*v1beta2.RequeuingStrategy)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_RequeuingStrategy_To_v1beta2_RequeuingStrategy(a.(*RequeuingStrategy), b.(*v1beta2.RequeuingStrategy), scope) }); err != nil { @@ -269,6 +244,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*Configuration)(nil), (*v1beta2.Configuration)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Configuration_To_v1beta2_Configuration(a.(*Configuration), b.(*v1beta2.Configuration), scope) + }); err != nil { + return err + } return nil } @@ -318,26 +298,6 @@ func Convert_v1beta2_ClientConnection_To_v1beta1_ClientConnection(in *v1beta2.Cl return autoConvert_v1beta2_ClientConnection_To_v1beta1_ClientConnection(in, out, s) } -func autoConvert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility(in *ClusterQueueVisibility, out *v1beta2.ClusterQueueVisibility, s conversion.Scope) error { - out.MaxCount = in.MaxCount - return nil -} - -// Convert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility is an autogenerated conversion function. -func Convert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility(in *ClusterQueueVisibility, out *v1beta2.ClusterQueueVisibility, s conversion.Scope) error { - return autoConvert_v1beta1_ClusterQueueVisibility_To_v1beta2_ClusterQueueVisibility(in, out, s) -} - -func autoConvert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility(in *v1beta2.ClusterQueueVisibility, out *ClusterQueueVisibility, s conversion.Scope) error { - out.MaxCount = in.MaxCount - return nil -} - -// Convert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility is an autogenerated conversion function. -func Convert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility(in *v1beta2.ClusterQueueVisibility, out *ClusterQueueVisibility, s conversion.Scope) error { - return autoConvert_v1beta2_ClusterQueueVisibility_To_v1beta1_ClusterQueueVisibility(in, out, s) -} - func autoConvert_v1beta1_Configuration_To_v1beta2_Configuration(in *Configuration, out *v1beta2.Configuration, s conversion.Scope) error { out.Namespace = (*string)(unsafe.Pointer(in.Namespace)) if err := Convert_v1beta1_ControllerManager_To_v1beta2_ControllerManager(&in.ControllerManager, &out.ControllerManager, s); err != nil { @@ -349,7 +309,7 @@ func autoConvert_v1beta1_Configuration_To_v1beta2_Configuration(in *Configuratio out.WaitForPodsReady = (*v1beta2.WaitForPodsReady)(unsafe.Pointer(in.WaitForPodsReady)) out.ClientConnection = (*v1beta2.ClientConnection)(unsafe.Pointer(in.ClientConnection)) out.Integrations = (*v1beta2.Integrations)(unsafe.Pointer(in.Integrations)) - out.QueueVisibility = (*v1beta2.QueueVisibility)(unsafe.Pointer(in.QueueVisibility)) + // WARNING: in.QueueVisibility requires manual conversion: does not exist in peer-type out.MultiKueue = (*v1beta2.MultiKueue)(unsafe.Pointer(in.MultiKueue)) out.FairSharing = (*v1beta2.FairSharing)(unsafe.Pointer(in.FairSharing)) out.AdmissionFairSharing = (*v1beta2.AdmissionFairSharing)(unsafe.Pointer(in.AdmissionFairSharing)) @@ -359,11 +319,6 @@ func autoConvert_v1beta1_Configuration_To_v1beta2_Configuration(in *Configuratio return nil } -// Convert_v1beta1_Configuration_To_v1beta2_Configuration is an autogenerated conversion function. -func Convert_v1beta1_Configuration_To_v1beta2_Configuration(in *Configuration, out *v1beta2.Configuration, s conversion.Scope) error { - return autoConvert_v1beta1_Configuration_To_v1beta2_Configuration(in, out, s) -} - func autoConvert_v1beta2_Configuration_To_v1beta1_Configuration(in *v1beta2.Configuration, out *Configuration, s conversion.Scope) error { out.Namespace = (*string)(unsafe.Pointer(in.Namespace)) if err := Convert_v1beta2_ControllerManager_To_v1beta1_ControllerManager(&in.ControllerManager, &out.ControllerManager, s); err != nil { @@ -375,7 +330,6 @@ func autoConvert_v1beta2_Configuration_To_v1beta1_Configuration(in *v1beta2.Conf out.WaitForPodsReady = (*WaitForPodsReady)(unsafe.Pointer(in.WaitForPodsReady)) out.ClientConnection = (*ClientConnection)(unsafe.Pointer(in.ClientConnection)) out.Integrations = (*Integrations)(unsafe.Pointer(in.Integrations)) - out.QueueVisibility = (*QueueVisibility)(unsafe.Pointer(in.QueueVisibility)) out.MultiKueue = (*MultiKueue)(unsafe.Pointer(in.MultiKueue)) out.FairSharing = (*FairSharing)(unsafe.Pointer(in.FairSharing)) out.AdmissionFairSharing = (*AdmissionFairSharing)(unsafe.Pointer(in.AdmissionFairSharing)) @@ -708,28 +662,6 @@ func Convert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(in * return autoConvert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(in, out, s) } -func autoConvert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility(in *QueueVisibility, out *v1beta2.QueueVisibility, s conversion.Scope) error { - out.ClusterQueues = (*v1beta2.ClusterQueueVisibility)(unsafe.Pointer(in.ClusterQueues)) - out.UpdateIntervalSeconds = in.UpdateIntervalSeconds - return nil -} - -// Convert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility is an autogenerated conversion function. -func Convert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility(in *QueueVisibility, out *v1beta2.QueueVisibility, s conversion.Scope) error { - return autoConvert_v1beta1_QueueVisibility_To_v1beta2_QueueVisibility(in, out, s) -} - -func autoConvert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility(in *v1beta2.QueueVisibility, out *QueueVisibility, s conversion.Scope) error { - out.ClusterQueues = (*ClusterQueueVisibility)(unsafe.Pointer(in.ClusterQueues)) - out.UpdateIntervalSeconds = in.UpdateIntervalSeconds - return nil -} - -// Convert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility is an autogenerated conversion function. -func Convert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility(in *v1beta2.QueueVisibility, out *QueueVisibility, s conversion.Scope) error { - return autoConvert_v1beta2_QueueVisibility_To_v1beta1_QueueVisibility(in, out, s) -} - func autoConvert_v1beta1_RequeuingStrategy_To_v1beta2_RequeuingStrategy(in *RequeuingStrategy, out *v1beta2.RequeuingStrategy, s conversion.Scope) error { out.Timestamp = (*v1beta2.RequeuingTimestamp)(unsafe.Pointer(in.Timestamp)) out.BackoffLimitCount = (*int32)(unsafe.Pointer(in.BackoffLimitCount)) diff --git a/apis/config/v1beta2/configuration_types.go b/apis/config/v1beta2/configuration_types.go index bad26216229..841d02e38a9 100644 --- a/apis/config/v1beta2/configuration_types.go +++ b/apis/config/v1beta2/configuration_types.go @@ -82,13 +82,6 @@ type Configuration struct { // integrations (including K8S job). Integrations *Integrations `json:"integrations,omitempty"` - // QueueVisibility is configuration to expose the information about the top - // pending workloads. - // Deprecated: This field will be removed on v1beta2, use VisibilityOnDemand - // (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) - // instead. - QueueVisibility *QueueVisibility `json:"queueVisibility,omitempty"` - // MultiKueue controls the behaviour of the MultiKueue AdmissionCheck Controller. MultiKueue *MultiKueue `json:"multiKueue,omitempty"` @@ -437,27 +430,6 @@ type PodIntegrationOptions struct { PodSelector *metav1.LabelSelector `json:"podSelector,omitempty"` } -type QueueVisibility struct { - // ClusterQueues is configuration to expose the information - // about the top pending workloads in the cluster queue. - ClusterQueues *ClusterQueueVisibility `json:"clusterQueues,omitempty"` - - // UpdateIntervalSeconds specifies the time interval for updates to the structure - // of the top pending workloads in the queues. - // The minimum value is 1. - // Defaults to 5. - UpdateIntervalSeconds int32 `json:"updateIntervalSeconds,omitempty"` -} - -type ClusterQueueVisibility struct { - // MaxCount indicates the maximal number of pending workloads exposed in the - // cluster queue status. When the value is set to 0, then ClusterQueue - // visibility updates are disabled. - // The maximal value is 4000. - // Defaults to 10. - MaxCount int32 `json:"maxCount,omitempty"` -} - type Resources struct { // ExcludedResourcePrefixes defines which resources should be ignored by Kueue ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"` diff --git a/apis/config/v1beta2/groupversion_info.go b/apis/config/v1beta2/groupversion_info.go index 12d8c6e9c15..0afdf8acbed 100644 --- a/apis/config/v1beta2/groupversion_info.go +++ b/apis/config/v1beta2/groupversion_info.go @@ -46,3 +46,5 @@ func init() { // makes the code compile even when the generated files are missing. localSchemeBuilder.Register(RegisterDefaults) } + +func (*Configuration) Hub() {} diff --git a/apis/config/v1beta2/zz_generated.deepcopy.go b/apis/config/v1beta2/zz_generated.deepcopy.go index 859b944e305..ec28f195985 100644 --- a/apis/config/v1beta2/zz_generated.deepcopy.go +++ b/apis/config/v1beta2/zz_generated.deepcopy.go @@ -77,21 +77,6 @@ func (in *ClientConnection) DeepCopy() *ClientConnection { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterQueueVisibility) DeepCopyInto(out *ClusterQueueVisibility) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterQueueVisibility. -func (in *ClusterQueueVisibility) DeepCopy() *ClusterQueueVisibility { - if in == nil { - return nil - } - out := new(ClusterQueueVisibility) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Configuration) DeepCopyInto(out *Configuration) { *out = *in @@ -127,11 +112,6 @@ func (in *Configuration) DeepCopyInto(out *Configuration) { *out = new(Integrations) (*in).DeepCopyInto(*out) } - if in.QueueVisibility != nil { - in, out := &in.QueueVisibility, &out.QueueVisibility - *out = new(QueueVisibility) - (*in).DeepCopyInto(*out) - } if in.MultiKueue != nil { in, out := &in.MultiKueue, &out.MultiKueue *out = new(MultiKueue) @@ -494,26 +474,6 @@ func (in *PodIntegrationOptions) DeepCopy() *PodIntegrationOptions { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *QueueVisibility) DeepCopyInto(out *QueueVisibility) { - *out = *in - if in.ClusterQueues != nil { - in, out := &in.ClusterQueues, &out.ClusterQueues - *out = new(ClusterQueueVisibility) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueVisibility. -func (in *QueueVisibility) DeepCopy() *QueueVisibility { - if in == nil { - return nil - } - out := new(QueueVisibility) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RequeuingStrategy) DeepCopyInto(out *RequeuingStrategy) { *out = *in From 5062544787213ae93ace933d90ef9000b1a4ccba Mon Sep 17 00:00:00 2001 From: bob Date: Tue, 28 Oct 2025 19:35:36 +0800 Subject: [PATCH 043/119] doc: add Kueue configuration v1beta2 API document (#7417) Signed-off-by: song --- hack/genref/config.yaml | 5 + .../en/docs/reference/kueue-config.v1beta2.md | 974 ++++++++++++++++++ 2 files changed, 979 insertions(+) create mode 100644 site/content/en/docs/reference/kueue-config.v1beta2.md diff --git a/hack/genref/config.yaml b/hack/genref/config.yaml index 3a008a8c8e1..5ba468db54f 100644 --- a/hack/genref/config.yaml +++ b/hack/genref/config.yaml @@ -18,6 +18,11 @@ apis: package: sigs.k8s.io/kueue path: apis/config/v1beta1 + - name: kueue-config + title: Kueue Configuration API + package: sigs.k8s.io/kueue + path: apis/config/v1beta2 + # code fails if there are no types in alpha apis # leave in in case alpha apis are added #- name: kueue-alpha diff --git a/site/content/en/docs/reference/kueue-config.v1beta2.md b/site/content/en/docs/reference/kueue-config.v1beta2.md new file mode 100644 index 00000000000..e7a26995fb8 --- /dev/null +++ b/site/content/en/docs/reference/kueue-config.v1beta2.md @@ -0,0 +1,974 @@ +--- +title: Kueue Configuration API +content_type: tool-reference +package: config.kueue.x-k8s.io/v1beta2 +auto_generated: true +description: Generated API reference documentation for config.kueue.x-k8s.io/v1beta2. +--- + + +## Resource Types + + + + +## `AdmissionFairSharing` {#config-kueue-x-k8s-io-v1beta2-AdmissionFairSharing} + + +**Appears in:** + + + + + + + + + + + + + + + + + + + +
FieldDescription
usageHalfLifeTime [Required]
+k8s.io/apimachinery/pkg/apis/meta/v1.Duration +
+

usageHalfLifeTime indicates the time after which the current usage will decay by a half +If set to 0, usage will be reset to 0 immediately.

+
usageSamplingInterval [Required]
+k8s.io/apimachinery/pkg/apis/meta/v1.Duration +
+

usageSamplingInterval indicates how often Kueue updates consumedResources in FairSharingStatus +Defaults to 5min.

+
resourceWeights [Required]
+map[ResourceName]float64 +
+

resourceWeights assigns weights to resources which then are used to calculate LocalQueue's +resource usage and order Workloads. +Defaults to 1.

+
+ +## `ClientConnection` {#config-kueue-x-k8s-io-v1beta2-ClientConnection} + + +**Appears in:** + + + + + + + + + + + + + + + + +
FieldDescription
qps [Required]
+float32 +
+

QPS controls the number of queries per second allowed for K8S api server +connection.

+

Setting this to a negative value will disable client-side ratelimiting.

+
burst [Required]
+int32 +
+

Burst allows extra queries to accumulate when a client is exceeding its rate.

+
+ +## `ControllerConfigurationSpec` {#config-kueue-x-k8s-io-v1beta2-ControllerConfigurationSpec} + + +**Appears in:** + +- [ControllerManager](#config-kueue-x-k8s-io-v1beta2-ControllerManager) + + +

ControllerConfigurationSpec defines the global configuration for +controllers registered with the manager.

+ + + + + + + + + + + + + + +
FieldDescription
groupKindConcurrency
+map[string]int +
+

GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation +allowed for that controller.

+

When a controller is registered within this manager using the builder utilities, +users have to specify the type the controller reconciles in the For(...) call. +If the object's kind passed matches one of the keys in this map, the concurrency +for that controller is set to the number specified.

+

The key is expected to be consistent in form with GroupKind.String(), +e.g. ReplicaSet in apps group (regardless of version) would be ReplicaSet.apps.

+
cacheSyncTimeout
+time.Duration +
+

CacheSyncTimeout refers to the time limit set to wait for syncing caches. +Defaults to 2 minutes if not set.

+
+ +## `ControllerHealth` {#config-kueue-x-k8s-io-v1beta2-ControllerHealth} + + +**Appears in:** + +- [ControllerManager](#config-kueue-x-k8s-io-v1beta2-ControllerManager) + + +

ControllerHealth defines the health configs.

+ + + + + + + + + + + + + + + + + +
FieldDescription
healthProbeBindAddress
+string +
+

HealthProbeBindAddress is the TCP address that the controller should bind to +for serving health probes +It can be set to "0" or "" to disable serving the health probe.

+
readinessEndpointName
+string +
+

ReadinessEndpointName, defaults to "readyz"

+
livenessEndpointName
+string +
+

LivenessEndpointName, defaults to "healthz"

+
+ +## `ControllerManager` {#config-kueue-x-k8s-io-v1beta2-ControllerManager} + + +**Appears in:** + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
webhook
+ControllerWebhook +
+

Webhook contains the controllers webhook configuration

+
leaderElection
+k8s.io/component-base/config/v1alpha1.LeaderElectionConfiguration +
+

LeaderElection is the LeaderElection config to be used when configuring +the manager.Manager leader election

+
metrics
+ControllerMetrics +
+

Metrics contains the controller metrics configuration

+
health
+ControllerHealth +
+

Health contains the controller health configuration

+
pprofBindAddress
+string +
+

PprofBindAddress is the TCP address that the controller should bind to +for serving pprof. +It can be set to "" or "0" to disable the pprof serving. +Since pprof may contain sensitive information, make sure to protect it +before exposing it to public.

+
controller
+ControllerConfigurationSpec +
+

Controller contains global configuration options for controllers +registered within this manager.

+
+ +## `ControllerMetrics` {#config-kueue-x-k8s-io-v1beta2-ControllerMetrics} + + +**Appears in:** + +- [ControllerManager](#config-kueue-x-k8s-io-v1beta2-ControllerManager) + + +

ControllerMetrics defines the metrics configs.

+ + + + + + + + + + + + + + +
FieldDescription
bindAddress
+string +
+

BindAddress is the TCP address that the controller should bind to +for serving prometheus metrics. +It can be set to "0" to disable the metrics serving.

+
enableClusterQueueResources
+bool +
+

EnableClusterQueueResources, if true the cluster queue resource usage and quotas +metrics will be reported.

+
+ +## `ControllerWebhook` {#config-kueue-x-k8s-io-v1beta2-ControllerWebhook} + + +**Appears in:** + +- [ControllerManager](#config-kueue-x-k8s-io-v1beta2-ControllerManager) + + +

ControllerWebhook defines the webhook server for the controller.

+ + + + + + + + + + + + + + + + + +
FieldDescription
port
+int +
+

Port is the port that the webhook server serves at. +It is used to set webhook.Server.Port.

+
host
+string +
+

Host is the hostname that the webhook server binds to. +It is used to set webhook.Server.Host.

+
certDir
+string +
+

CertDir is the directory that contains the server key and certificate. +if not set, webhook server would look up the server key and certificate in +{TempDir}/k8s-webhook-server/serving-certs. The server key and certificate +must be named tls.key and tls.crt, respectively.

+
+ +## `DeviceClassMapping` {#config-kueue-x-k8s-io-v1beta2-DeviceClassMapping} + + +**Appears in:** + +- [Resources](#config-kueue-x-k8s-io-v1beta2-Resources) + + +

DeviceClassMapping holds device class to logical resource mappings +for Dynamic Resource Allocation support.

+ + + + + + + + + + + + + + +
FieldDescription
name [Required]
+k8s.io/api/core/v1.ResourceName +
+

Name is referenced in ClusterQueue.nominalQuota and Workload status. +Must be a valid fully qualified name consisting of an optional DNS subdomain prefix +followed by a slash and a DNS label, or just a DNS label. +DNS labels consist of lower-case alphanumeric characters or hyphens, +and must start and end with an alphanumeric character. +DNS subdomain prefixes follow the same rules as DNS labels but can contain periods. +The total length must not exceed 253 characters.

+
deviceClassNames [Required]
+[]k8s.io/api/core/v1.ResourceName +
+

DeviceClassNames enumerates the DeviceClasses represented by this resource name. +Each device class name must be a valid qualified name consisting of an optional DNS subdomain prefix +followed by a slash and a DNS label, or just a DNS label. +DNS labels consist of lower-case alphanumeric characters or hyphens, +and must start and end with an alphanumeric character. +DNS subdomain prefixes follow the same rules as DNS labels but can contain periods. +The total length of each name must not exceed 253 characters.

+
+ +## `FairSharing` {#config-kueue-x-k8s-io-v1beta2-FairSharing} + + +**Appears in:** + + + + + + + + + + + + + + + + +
FieldDescription
enable [Required]
+bool +
+

enable indicates whether to enable Fair Sharing for all cohorts. +Defaults to false.

+
preemptionStrategies [Required]
+[]PreemptionStrategy +
+

preemptionStrategies indicates which constraints should a preemption satisfy. +The preemption algorithm will only use the next strategy in the list if the +incoming workload (preemptor) doesn't fit after using the previous strategies. +Possible values are:

+
    +
  • LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ +with the preemptor workload is less than or equal to the share of the preemptee CQ +without the workload to be preempted. +This strategy might favor preemption of smaller workloads in the preemptee CQ, +regardless of priority or start time, in an effort to keep the share of the CQ +as high as possible.
  • +
  • LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ +with the incoming workload is strictly less than the share of the preemptee CQ. +This strategy doesn't depend on the share usage of the workload being preempted. +As a result, the strategy chooses to preempt workloads with the lowest priority and +newest start time first. +The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"].
  • +
+
+ +## `Integrations` {#config-kueue-x-k8s-io-v1beta2-Integrations} + + +**Appears in:** + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
frameworks [Required]
+[]string +
+

List of framework names to be enabled. +Possible options:

+
    +
  • "batch/job"
  • +
  • "kubeflow.org/mpijob"
  • +
  • "ray.io/rayjob"
  • +
  • "ray.io/raycluster"
  • +
  • "jobset.x-k8s.io/jobset"
  • +
  • "kubeflow.org/paddlejob"
  • +
  • "kubeflow.org/pytorchjob"
  • +
  • "kubeflow.org/tfjob"
  • +
  • "kubeflow.org/xgboostjob"
  • +
  • "kubeflow.org/jaxjob"
  • +
  • "trainer.kubeflow.org/trainjob"
  • +
  • "workload.codeflare.dev/appwrapper"
  • +
  • "pod"
  • +
  • "deployment" (requires enabling pod integration)
  • +
  • "statefulset" (requires enabling pod integration)
  • +
  • "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration)
  • +
+
externalFrameworks [Required]
+[]string +
+

List of GroupVersionKinds that are managed for Kueue by external controllers; +the expected format is Kind.version.group.com.

+
podOptions [Required]
+PodIntegrationOptions +
+

PodOptions defines kueue controller behaviour for pod objects +Deprecated: This field will be removed on v1beta2, use ManagedJobsNamespaceSelector +(https://kueue.sigs.k8s.io/docs/tasks/run/plain_pods/) +instead.

+
labelKeysToCopy [Required]
+[]string +
+

labelKeysToCopy is a list of label keys that should be copied from the job into the +workload object. It is not required for the job to have all the labels from this +list. If a job does not have some label with the given key from this list, the +constructed workload object will be created without this label. In the case +of creating a workload from a composable job (pod group), if multiple objects +have labels with some key from the list, the values of these labels must +match or otherwise the workload creation would fail. The labels are copied only +during the workload creation and are not updated even if the labels of the +underlying job are changed.

+
+ +## `InternalCertManagement` {#config-kueue-x-k8s-io-v1beta2-InternalCertManagement} + + +**Appears in:** + + + + + + + + + + + + + + + + + + + +
FieldDescription
enable [Required]
+bool +
+

Enable controls the use of internal cert management for the webhook, +metrics and visibility endpoints. +When enabled Kueue is using libraries to generate and +self-sign the certificates. +When disabled, you need to provide the certificates for +the webhooks, metrics and visibility through a third party certificate +This secret is mounted to the kueue controller manager pod. The mount +path for webhooks is /tmp/k8s-webhook-server/serving-certs, for +metrics endpoint the expected path is /etc/kueue/metrics/certs and for +visibility endpoint the expected path is /visibility. +The keys and certs are named tls.key and tls.crt.

+
webhookServiceName [Required]
+string +
+

WebhookServiceName is the name of the Service used as part of the DNSName. +Defaults to kueue-webhook-service.

+
webhookSecretName [Required]
+string +
+

WebhookSecretName is the name of the Secret used to store CA and server certs. +Defaults to kueue-webhook-server-cert.

+
+ +## `MultiKueue` {#config-kueue-x-k8s-io-v1beta2-MultiKueue} + + +**Appears in:** + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
gcInterval
+k8s.io/apimachinery/pkg/apis/meta/v1.Duration +
+

GCInterval defines the time interval between two consecutive garbage collection runs. +Defaults to 1min. If 0, the garbage collection is disabled.

+
origin
+string +
+

Origin defines a label value used to track the creator of workloads in the worker +clusters. +This is used by multikueue in components like its garbage collector to identify +remote objects that ware created by this multikueue manager cluster and delete +them if their local counterpart no longer exists.

+
workerLostTimeout
+k8s.io/apimachinery/pkg/apis/meta/v1.Duration +
+

WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready +if the connection with its reserving worker cluster is lost.

+

Defaults to 15 minutes.

+
dispatcherName
+string +
+

DispatcherName defines the dispatcher responsible for selecting worker clusters to handle the workload.

+
    +
  • If specified, the workload will be handled by the named dispatcher.
  • +
  • If not specified, the workload will be handled by the default ("kueue.x-k8s.io/multikueue-dispatcher-all-at-once") dispatcher.
  • +
+
externalFrameworks
+[]MultiKueueExternalFramework +
+

ExternalFrameworks defines a list of external frameworks that should be supported +by the generic MultiKueue adapter. Each entry defines how to handle a specific +GroupVersionKind (GVK) for MultiKueue operations.

+
+ +## `MultiKueueExternalFramework` {#config-kueue-x-k8s-io-v1beta2-MultiKueueExternalFramework} + + +**Appears in:** + +- [MultiKueue](#config-kueue-x-k8s-io-v1beta2-MultiKueue) + + +

MultiKueueExternalFramework defines a framework that is not built-in.

+ + + + + + + + + + + +
FieldDescription
name [Required]
+string +
+

Name is the GVK of the resource that are +managed by external controllers +the expected format is kind.version.group.

+
+ +## `ObjectRetentionPolicies` {#config-kueue-x-k8s-io-v1beta2-ObjectRetentionPolicies} + + +**Appears in:** + + + +

ObjectRetentionPolicies holds retention settings for different object types.

+ + + + + + + + + + + +
FieldDescription
workloads
+WorkloadRetentionPolicy +
+

Workloads configures retention for Workloads. +A nil value disables automatic deletion of Workloads.

+
+ +## `PodIntegrationOptions` {#config-kueue-x-k8s-io-v1beta2-PodIntegrationOptions} + + +**Appears in:** + +- [Integrations](#config-kueue-x-k8s-io-v1beta2-Integrations) + + + + + + + + + + + + + + + +
FieldDescription
namespaceSelector [Required]
+k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector +
+

NamespaceSelector can be used to omit some namespaces from pod reconciliation

+
podSelector [Required]
+k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector +
+

PodSelector can be used to choose what pods to reconcile

+
+ +## `PreemptionStrategy` {#config-kueue-x-k8s-io-v1beta2-PreemptionStrategy} + +(Alias of `string`) + +**Appears in:** + +- [FairSharing](#config-kueue-x-k8s-io-v1beta2-FairSharing) + + + + + +## `RequeuingStrategy` {#config-kueue-x-k8s-io-v1beta2-RequeuingStrategy} + + +**Appears in:** + +- [WaitForPodsReady](#config-kueue-x-k8s-io-v1beta2-WaitForPodsReady) + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
timestamp
+RequeuingTimestamp +
+

Timestamp defines the timestamp used for re-queuing a Workload +that was evicted due to Pod readiness. The possible values are:

+
    +
  • Eviction (default) indicates from Workload Evicted condition with PodsReadyTimeout reason.
  • +
  • Creation indicates from Workload .metadata.creationTimestamp.
  • +
+
backoffLimitCount
+int32 +
+

BackoffLimitCount defines the maximum number of re-queuing retries. +Once the number is reached, the workload is deactivated (.spec.activate=false). +When it is null, the workloads will repeatedly and endless re-queueing.

+

Every backoff duration is about "b*2^(n-1)+Rand" where:

+
    +
  • "b" represents the base set by "BackoffBaseSeconds" parameter,
  • +
  • "n" represents the "workloadStatus.requeueState.count",
  • +
  • "Rand" represents the random jitter. +During this time, the workload is taken as an inadmissible and +other workloads will have a chance to be admitted. +By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).
  • +
+

Defaults to null.

+
backoffBaseSeconds
+int32 +
+

BackoffBaseSeconds defines the base for the exponential backoff for +re-queuing an evicted workload.

+

Defaults to 60.

+
backoffMaxSeconds
+int32 +
+

BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.

+

Defaults to 3600.

+
+ +## `RequeuingTimestamp` {#config-kueue-x-k8s-io-v1beta2-RequeuingTimestamp} + +(Alias of `string`) + +**Appears in:** + +- [RequeuingStrategy](#config-kueue-x-k8s-io-v1beta2-RequeuingStrategy) + + + + + +## `ResourceTransformation` {#config-kueue-x-k8s-io-v1beta2-ResourceTransformation} + + +**Appears in:** + +- [Resources](#config-kueue-x-k8s-io-v1beta2-Resources) + + + + + + + + + + + + + + + + + + +
FieldDescription
input [Required]
+k8s.io/api/core/v1.ResourceName +
+

Input is the name of the input resource.

+
strategy [Required]
+ResourceTransformationStrategy +
+

Strategy specifies if the input resource should be replaced or retained. +Defaults to Retain

+
outputs [Required]
+k8s.io/api/core/v1.ResourceList +
+

Outputs specifies the output resources and quantities per unit of input resource. +An empty Outputs combined with a Replace Strategy causes the Input resource to be ignored by Kueue.

+
+ +## `ResourceTransformationStrategy` {#config-kueue-x-k8s-io-v1beta2-ResourceTransformationStrategy} + +(Alias of `string`) + +**Appears in:** + +- [ResourceTransformation](#config-kueue-x-k8s-io-v1beta2-ResourceTransformation) + + + + + +## `Resources` {#config-kueue-x-k8s-io-v1beta2-Resources} + + +**Appears in:** + + + + + + + + + + + + + + + + + + + +
FieldDescription
excludeResourcePrefixes [Required]
+[]string +
+

ExcludedResourcePrefixes defines which resources should be ignored by Kueue

+
transformations [Required]
+[]ResourceTransformation +
+

Transformations defines how to transform PodSpec resources into Workload resource requests. +This is intended to be a map with Input as the key (enforced by validation code)

+
deviceClassMappings
+[]DeviceClassMapping +
+

DeviceClassMappings defines mappings from device classes to logical resources +for Dynamic Resource Allocation support.

+
+ +## `WaitForPodsReady` {#config-kueue-x-k8s-io-v1beta2-WaitForPodsReady} + + +**Appears in:** + + + +

WaitForPodsReady defines configuration for the Wait For Pods Ready feature, +which is used to ensure that all Pods are ready within the specified time.

+ + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
enable [Required]
+bool +
+

Enable indicates whether to enable wait for pods ready feature. +Defaults to false.

+
timeout
+k8s.io/apimachinery/pkg/apis/meta/v1.Duration +
+

Timeout defines the time for an admitted workload to reach the +PodsReady=true condition. When the timeout is exceeded, the workload +evicted and requeued in the same cluster queue. +Defaults to 5min.

+
blockAdmission [Required]
+bool +
+

BlockAdmission when true, cluster queue will block admissions for all +subsequent jobs until the jobs reach the PodsReady=true condition. +This setting is only honored when Enable is set to true.

+
requeuingStrategy
+RequeuingStrategy +
+

RequeuingStrategy defines the strategy for requeuing a Workload.

+
recoveryTimeout
+k8s.io/apimachinery/pkg/apis/meta/v1.Duration +
+

RecoveryTimeout defines an opt-in timeout, measured since the +last transition to the PodsReady=false condition after a Workload is Admitted and running. +Such a transition may happen when a Pod failed and the replacement Pod +is awaited to be scheduled. +After exceeding the timeout the corresponding job gets suspended again +and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. +If not set, there is no timeout.

+
+ +## `WorkloadRetentionPolicy` {#config-kueue-x-k8s-io-v1beta2-WorkloadRetentionPolicy} + + +**Appears in:** + +- [ObjectRetentionPolicies](#config-kueue-x-k8s-io-v1beta2-ObjectRetentionPolicies) + + +

WorkloadRetentionPolicy defines the policies for when Workloads should be deleted.

+ + + + + + + + + + + + + + +
FieldDescription
afterFinished
+k8s.io/apimachinery/pkg/apis/meta/v1.Duration +
+

AfterFinished is the duration to wait after a Workload finishes +before deleting it. +A duration of 0 will delete immediately. +A nil value disables automatic deletion. +Represented using metav1.Duration (e.g. "10m", "1h30m").

+
afterDeactivatedByKueue
+k8s.io/apimachinery/pkg/apis/meta/v1.Duration +
+

AfterDeactivatedByKueue is the duration to wait after any Kueue-managed Workload +(such as a Job, JobSet, or other custom workload types) has been marked +as deactivated by Kueue before automatically deleting it. +Deletion of deactivated workloads may cascade to objects not created by +Kueue, since deleting the parent Workload owner (e.g. JobSet) can trigger +garbage-collection of dependent resources. +A duration of 0 will delete immediately. +A nil value disables automatic deletion. +Represented using metav1.Duration (e.g. "10m", "1h30m").

+
+ \ No newline at end of file From 7491d3575de1198881aaf1435f035e7156893299 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Tue, 28 Oct 2025 22:02:02 +0530 Subject: [PATCH 044/119] Support mutating workload priority class. (#7289) * Support mutating workload priority class. * Skip set and delete. --- apis/kueue/v1beta1/workload_types.go | 2 +- apis/kueue/v1beta2/workload_types.go | 2 +- .../crd/kueue.x-k8s.io_workloads.yaml | 8 +- .../crd/bases/kueue.x-k8s.io_workloads.yaml | 12 +- pkg/controller/core/workload_controller.go | 11 +- pkg/controller/jobframework/reconciler.go | 70 +++-- pkg/controller/jobframework/validation.go | 14 +- .../jobs/deployment/deployment_webhook.go | 8 +- .../deployment/deployment_webhook_test.go | 65 +++++ .../jobs/job/job_controller_test.go | 2 +- pkg/controller/jobs/job/job_webhook_test.go | 38 ++- .../leaderworkerset_reconciler.go | 33 ++- .../leaderworkerset_webhook.go | 11 +- .../leaderworkerset_webhook_test.go | 77 ++++++ .../jobs/statefulset/statefulset_webhook.go | 11 +- .../statefulset/statefulset_webhook_test.go | 113 ++++++++ .../jobs/job/job_controller_test.go | 257 ++++++++++++++---- .../webhook/core/workload_test.go | 172 ++++++++++++ 18 files changed, 777 insertions(+), 129 deletions(-) diff --git a/apis/kueue/v1beta1/workload_types.go b/apis/kueue/v1beta1/workload_types.go index 7d297f96f15..0a6cb4e10a6 100644 --- a/apis/kueue/v1beta1/workload_types.go +++ b/apis/kueue/v1beta1/workload_types.go @@ -766,7 +766,7 @@ const ( // Workload is the Schema for the workloads API // +kubebuilder:validation:XValidation:rule="has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true", message="podSetAssignments must have the same number of podSets as the spec" // +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? (oldSelf.spec.priorityClassSource == self.spec.priorityClassSource) : true", message="priorityClassSource is immutable while workload quota reserved" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true", message="priorityClassName is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && (self.spec.priorityClassSource != 'kueue.x-k8s.io/workloadpriorityclass') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true", message="priorityClassName is immutable while workload quota reserved and priorityClassSource is not equal to kueue.x-k8s.io/workloadpriorityclass" // +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true", message="queueName is immutable while workload quota reserved" // +kubebuilder:validation:XValidation:rule="((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true", message="maximumExecutionTimeSeconds is immutable while workload quota reserved" type Workload struct { diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index 5010fa0e051..623ed4fe849 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -765,7 +765,7 @@ const ( // Workload is the Schema for the workloads API // +kubebuilder:validation:XValidation:rule="has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true", message="podSetAssignments must have the same number of podSets as the spec" // +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true", message="priorityClassSource is immutable while workload quota reserved" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true",message="priorityClassName is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && (self.spec.priorityClassSource != 'kueue.x-k8s.io/workloadpriorityclass') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true", message="priorityClassName is immutable while workload quota reserved and priorityClassSource is not equal to kueue.x-k8s.io/workloadpriorityclass" // +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true", message="queueName is immutable while workload quota reserved" // +kubebuilder:validation:XValidation:rule="((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true", message="maximumExecutionTimeSeconds is immutable while workload quota reserved" type Workload struct { diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index da71f3d5f1b..fd32cadfa6d 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -8846,8 +8846,8 @@ spec: rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true' - message: priorityClassSource is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource == self.spec.priorityClassSource) : true' - - message: priorityClassName is immutable while workload quota reserved - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' + - message: priorityClassName is immutable while workload quota reserved and priorityClassSource is not equal to kueue.x-k8s.io/workloadpriorityclass + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && (self.spec.priorityClassSource != ''kueue.x-k8s.io/workloadpriorityclass'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' - message: queueName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true' - message: maximumExecutionTimeSeconds is immutable while workload quota reserved @@ -17670,8 +17670,8 @@ spec: rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true' - message: priorityClassSource is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true' - - message: priorityClassName is immutable while workload quota reserved - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' + - message: priorityClassName is immutable while workload quota reserved and priorityClassSource is not equal to kueue.x-k8s.io/workloadpriorityclass + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && (self.spec.priorityClassSource != ''kueue.x-k8s.io/workloadpriorityclass'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' - message: queueName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true' - message: maximumExecutionTimeSeconds is immutable while workload quota reserved diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index e837266742a..4ed717e2103 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -9345,9 +9345,11 @@ spec: rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? (oldSelf.spec.priorityClassSource == self.spec.priorityClassSource) : true' - - message: priorityClassName is immutable while workload quota reserved + - message: priorityClassName is immutable while workload quota reserved and + priorityClassSource is not equal to kueue.x-k8s.io/workloadpriorityclass rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) + c.type == ''QuotaReserved'' && c.status == ''True'') && (self.spec.priorityClassSource + != ''kueue.x-k8s.io/workloadpriorityclass'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' - message: queueName is immutable while workload quota reserved @@ -18701,9 +18703,11 @@ spec: && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true' - - message: priorityClassName is immutable while workload quota reserved + - message: priorityClassName is immutable while workload quota reserved and + priorityClassSource is not equal to kueue.x-k8s.io/workloadpriorityclass rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassName) + c.type == ''QuotaReserved'' && c.status == ''True'') && (self.spec.priorityClassSource + != ''kueue.x-k8s.io/workloadpriorityclass'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' - message: queueName is immutable while workload quota reserved diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index 874744f116b..5d32a16b886 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -54,6 +54,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" + "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/dra" "sigs.k8s.io/kueue/pkg/features" @@ -969,7 +970,8 @@ func (r *WorkloadReconciler) Update(e event.TypedUpdateEvent[*kueue.Workload]) b } } case prevStatus == workload.StatusAdmitted && status == workload.StatusAdmitted && !equality.Semantic.DeepEqual(e.ObjectOld.Status.ReclaimablePods, e.ObjectNew.Status.ReclaimablePods), - features.Enabled(features.ElasticJobsViaWorkloadSlices) && workloadslicing.ScaledDown(workload.ExtractPodSetCountsFromWorkload(e.ObjectOld), workload.ExtractPodSetCountsFromWorkload(e.ObjectNew)): + features.Enabled(features.ElasticJobsViaWorkloadSlices) && workloadslicing.ScaledDown(workload.ExtractPodSetCountsFromWorkload(e.ObjectOld), workload.ExtractPodSetCountsFromWorkload(e.ObjectNew)), + workloadPriorityClassChanged(e.ObjectOld, e.ObjectNew): // trigger the move of associated inadmissibleWorkloads, if there are any. r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, e.ObjectNew, func() { // Update the workload from cache while holding the queues lock @@ -991,6 +993,13 @@ func (r *WorkloadReconciler) Update(e event.TypedUpdateEvent[*kueue.Workload]) b return true } +func workloadPriorityClassChanged(old, new *kueue.Workload) bool { + return old.Spec.PriorityClassSource == constants.WorkloadPriorityClassSource && + new.Spec.PriorityClassSource == constants.WorkloadPriorityClassSource && + old.Spec.PriorityClassName != "" && new.Spec.PriorityClassName != "" && + old.Spec.PriorityClassName != new.Spec.PriorityClassName +} + func (r *WorkloadReconciler) Generic(e event.TypedGenericEvent[*kueue.Workload]) bool { r.log.V(3).Info("Ignore Workload generic event", "workload", klog.KObj(e.Object)) return false diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index 19b9935a983..e1bbd1cb46e 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -598,15 +598,6 @@ func (r *JobReconciler) ReconcileGenericJob(ctx context.Context, req ctrl.Reques } return ctrl.Result{}, err } - // Update workload priority if job's label changed. - if wl.Spec.PriorityClassSource == constants.WorkloadPriorityClassSource && WorkloadPriorityClassName(object) != wl.Spec.PriorityClassName { - log.V(2).Info("Job changed priority, updating workload", "oldPriority", wl.Spec.PriorityClassName, "newPriority", WorkloadPriorityClassName(object)) - if _, err = r.updateWorkloadToMatchJob(ctx, job, object, wl); err != nil { - log.Error(err, "Updating workload priority") - return ctrl.Result{}, err - } - return ctrl.Result{}, nil - } log.V(3).Info("Job is suspended and workload not yet admitted by a clusterQueue, nothing to do") return ctrl.Result{}, nil } @@ -934,9 +925,32 @@ func (r *JobReconciler) ensureOneWorkload(ctx context.Context, job GenericJob, o return r.updateWorkloadToMatchJob(ctx, job, object, toUpdate) } + if match != nil { + if err := UpdateWorkloadPriority(ctx, r.client, r.record, job.Object(), match, getCustomPriorityClassFuncFromJob(job)); err != nil { + return nil, err + } + } + return match, nil } +// UpdateWorkloadPriority updates workload priority if object's kueue.x-k8s.io/priority-class label changed. +func UpdateWorkloadPriority(ctx context.Context, c client.Client, r record.EventRecorder, obj client.Object, wl *kueue.Workload, customPriorityClassFunc func() string) error { + if wl.Spec.PriorityClassSource == constants.WorkloadPriorityClassSource && WorkloadPriorityClassName(obj) != wl.Spec.PriorityClassName { + if err := PrepareWorkloadPriority(ctx, c, obj, wl, customPriorityClassFunc); err != nil { + return fmt.Errorf("prepare workload priority: %w", err) + } + if err := c.Update(ctx, wl); err != nil { + return fmt.Errorf("updating existing workload: %w", err) + } + r.Eventf(obj, + corev1.EventTypeNormal, ReasonUpdatedWorkload, + "Updated workload priority class: %v", klog.KObj(wl), + ) + } + return nil +} + func FindMatchingWorkloads(ctx context.Context, c client.Client, job GenericJob) (match *kueue.Workload, toDelete []*kueue.Workload, err error) { object := job.Object() @@ -1090,7 +1104,7 @@ func (r *JobReconciler) updateWorkloadToMatchJob(ctx context.Context, job Generi r.record.Eventf(object, corev1.EventTypeNormal, ReasonUpdatedWorkload, "Updated not matching Workload for suspended job: %v", klog.KObj(wl)) - return newWl, nil + return wl, nil } // startJob will unsuspend the job, and also inject the node affinity. @@ -1263,9 +1277,15 @@ func prepareWorkloadSlice(ctx context.Context, clnt client.Client, job GenericJo } } -// prepareWorkload adds the priority information for the constructed workload -func (r *JobReconciler) prepareWorkload(ctx context.Context, job GenericJob, wl *kueue.Workload) error { - priorityClassName, source, p, err := r.extractPriority(ctx, wl.Spec.PodSets, job) +func getCustomPriorityClassFuncFromJob(job GenericJob) func() string { + if jobWithPriorityClass, isImplemented := job.(JobWithPriorityClass); isImplemented { + return jobWithPriorityClass.PriorityClass + } + return nil +} + +func PrepareWorkloadPriority(ctx context.Context, c client.Client, obj client.Object, wl *kueue.Workload, customPriorityClassFunc func() string) error { + priorityClassName, source, p, err := ExtractPriority(ctx, c, obj, wl.Spec.PodSets, customPriorityClassFunc) if err != nil { return err } @@ -1273,6 +1293,16 @@ func (r *JobReconciler) prepareWorkload(ctx context.Context, job GenericJob, wl wl.Spec.PriorityClassName = priorityClassName wl.Spec.Priority = &p wl.Spec.PriorityClassSource = source + + return nil +} + +// prepareWorkload adds the priority information for the constructed workload +func (r *JobReconciler) prepareWorkload(ctx context.Context, job GenericJob, wl *kueue.Workload) error { + if err := PrepareWorkloadPriority(ctx, r.client, job.Object(), wl, getCustomPriorityClassFuncFromJob(job)); err != nil { + return err + } + wl.Spec.PodSets = clearMinCountsIfFeatureDisabled(wl.Spec.PodSets) if workloadSliceEnabled(job) { @@ -1281,20 +1311,12 @@ func (r *JobReconciler) prepareWorkload(ctx context.Context, job GenericJob, wl return nil } -func (r *JobReconciler) extractPriority(ctx context.Context, podSets []kueue.PodSet, job GenericJob) (string, string, int32, error) { - var customPriorityFunc func() string - if jobWithPriorityClass, isImplemented := job.(JobWithPriorityClass); isImplemented { - customPriorityFunc = jobWithPriorityClass.PriorityClass - } - return ExtractPriority(ctx, r.client, job.Object(), podSets, customPriorityFunc) -} - -func ExtractPriority(ctx context.Context, c client.Client, obj client.Object, podSets []kueue.PodSet, customPriorityFunc func() string) (string, string, int32, error) { +func ExtractPriority(ctx context.Context, c client.Client, obj client.Object, podSets []kueue.PodSet, customPriorityClassFunc func() string) (string, string, int32, error) { if workloadPriorityClass := WorkloadPriorityClassName(obj); len(workloadPriorityClass) > 0 { return utilpriority.GetPriorityFromWorkloadPriorityClass(ctx, c, workloadPriorityClass) } - if customPriorityFunc != nil { - return utilpriority.GetPriorityFromPriorityClass(ctx, c, customPriorityFunc()) + if customPriorityClassFunc != nil { + return utilpriority.GetPriorityFromPriorityClass(ctx, c, customPriorityClassFunc()) } return utilpriority.GetPriorityFromPriorityClass(ctx, c, extractPriorityFromPodSets(podSets)) } diff --git a/pkg/controller/jobframework/validation.go b/pkg/controller/jobframework/validation.go index a3a94cb7c72..07b30bbbb44 100644 --- a/pkg/controller/jobframework/validation.go +++ b/pkg/controller/jobframework/validation.go @@ -148,11 +148,7 @@ func validateUpdateForPrebuiltWorkload(oldJob, newJob GenericJob) field.ErrorLis } func validateJobUpdateForWorkloadPriorityClassName(oldJob, newJob GenericJob) field.ErrorList { - var allErrs field.ErrorList - if !newJob.IsSuspended() || IsWorkloadPriorityClassNameEmpty(newJob.Object()) { - allErrs = append(allErrs, ValidateUpdateForWorkloadPriorityClassName(oldJob.Object(), newJob.Object())...) - } - return allErrs + return ValidateUpdateForWorkloadPriorityClassName(newJob.IsSuspended(), oldJob.Object(), newJob.Object()) } // validatedUpdateForEnabledWorkloadSlice validates that the workload-slicing toggle remains immutable on update. @@ -168,9 +164,11 @@ func validatedUpdateForEnabledWorkloadSlice(oldJob, newJob GenericJob) field.Err return nil } -func ValidateUpdateForWorkloadPriorityClassName(oldObj, newObj client.Object) field.ErrorList { - allErrs := apivalidation.ValidateImmutableField(WorkloadPriorityClassName(newObj), WorkloadPriorityClassName(oldObj), workloadPriorityClassNamePath) - return allErrs +func ValidateUpdateForWorkloadPriorityClassName(isSuspended bool, oldObj, newObj client.Object) field.ErrorList { + if !isSuspended && IsWorkloadPriorityClassNameEmpty(oldObj) || IsWorkloadPriorityClassNameEmpty(newObj) { + return apivalidation.ValidateImmutableField(WorkloadPriorityClassName(newObj), WorkloadPriorityClassName(oldObj), workloadPriorityClassNamePath) + } + return nil } func validateCreateForMaxExecTime(job GenericJob) field.ErrorList { diff --git a/pkg/controller/jobs/deployment/deployment_webhook.go b/pkg/controller/jobs/deployment/deployment_webhook.go index 5a2b71b5964..f47abad4166 100644 --- a/pkg/controller/jobs/deployment/deployment_webhook.go +++ b/pkg/controller/jobs/deployment/deployment_webhook.go @@ -133,9 +133,11 @@ func (wh *Webhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Ob if !isSuspended || newQueueName == "" { allErrs = append(allErrs, apivalidation.ValidateImmutableField(newQueueName, oldQueueName, queueNameLabelPath)...) } - if !isSuspended || jobframework.IsWorkloadPriorityClassNameEmpty(newDeployment.Object()) { - allErrs = append(allErrs, jobframework.ValidateUpdateForWorkloadPriorityClassName(oldDeployment.Object(), newDeployment.Object())...) - } + allErrs = append(allErrs, jobframework.ValidateUpdateForWorkloadPriorityClassName( + isSuspended, + oldDeployment.Object(), + newDeployment.Object(), + )...) return warnings, allErrs.ToAggregate() } diff --git a/pkg/controller/jobs/deployment/deployment_webhook_test.go b/pkg/controller/jobs/deployment/deployment_webhook_test.go index 75cc8dc330d..7b658f62914 100644 --- a/pkg/controller/jobs/deployment/deployment_webhook_test.go +++ b/pkg/controller/jobs/deployment/deployment_webhook_test.go @@ -302,6 +302,23 @@ func TestValidateUpdate(t *testing.T) { Label(constants.WorkloadPriorityClassLabel, "new-test"). Obj(), }, + "set priority-class when replicas ready": { + oldDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + ReadyReplicas(int32(1)). + Obj(), + newDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "test"). + ReadyReplicas(int32(1)). + Obj(), + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "metadata.labels[kueue.x-k8s.io/priority-class]", + }, + }.ToAggregate(), + }, "update priority-class when replicas ready": { oldDeployment: testingdeployment.MakeDeployment("test-pod", ""). Queue("test-queue"). @@ -313,6 +330,54 @@ func TestValidateUpdate(t *testing.T) { Label(constants.WorkloadPriorityClassLabel, "new-test"). ReadyReplicas(int32(1)). Obj(), + wantErr: field.ErrorList{}.ToAggregate(), + }, + "delete priority-class when replicas ready": { + oldDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + ReadyReplicas(int32(1)). + Label(constants.WorkloadPriorityClassLabel, "test"). + Obj(), + newDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + ReadyReplicas(int32(1)). + Obj(), + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "metadata.labels[kueue.x-k8s.io/priority-class]", + }, + }.ToAggregate(), + }, + "set priority-class when replicas not ready": { + oldDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + Obj(), + newDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "test"). + Obj(), + wantErr: field.ErrorList{}.ToAggregate(), + }, + "update priority-class when replicas not ready": { + oldDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "test"). + Obj(), + newDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "new-test"). + Obj(), + wantErr: field.ErrorList{}.ToAggregate(), + }, + "delete priority-class when replicas not ready": { + oldDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "test"). + Obj(), + newDeployment: testingdeployment.MakeDeployment("test-pod", ""). + Queue("test-queue"). + Obj(), wantErr: field.ErrorList{ &field.Error{ Type: field.ErrorTypeInvalid, diff --git a/pkg/controller/jobs/job/job_controller_test.go b/pkg/controller/jobs/job/job_controller_test.go index b5879f272ad..c734bcf3484 100644 --- a/pkg/controller/jobs/job/job_controller_test.go +++ b/pkg/controller/jobs/job/job_controller_test.go @@ -2714,7 +2714,7 @@ func TestReconciler(t *testing.T) { Key: types.NamespacedName{Name: "job", Namespace: "ns"}, EventType: "Normal", Reason: "UpdatedWorkload", - Message: "Updated not matching Workload for suspended job: ns/job", + Message: "Updated workload priority class: ns/job", }, }, }, diff --git a/pkg/controller/jobs/job/job_webhook_test.go b/pkg/controller/jobs/job/job_webhook_test.go index 5c4e7c32cf7..3be566a02e0 100644 --- a/pkg/controller/jobs/job/job_webhook_test.go +++ b/pkg/controller/jobs/job/job_webhook_test.go @@ -518,17 +518,37 @@ func TestValidateUpdate(t *testing.T) { wantValidationErrs: nil, }, { - name: "workloadPriorityClassName is mutable when job is suspended", - oldJob: testingutil.MakeJob("job", "default").WorkloadPriorityClass("test-1").Obj(), - newJob: testingutil.MakeJob("job", "default").WorkloadPriorityClass("test-2").Obj(), + name: "set priority-class when job not suspend", + oldJob: testingutil.MakeJob("job", "default").Suspend(false).Obj(), + newJob: testingutil.MakeJob("job", "default").Suspend(false).WorkloadPriorityClass("test").Obj(), + wantValidationErrs: apivalidation.ValidateImmutableField("test", "", workloadPriorityClassNamePath), }, { - name: "workloadPriorityClassName is immutable when job is running", - oldJob: testingutil.MakeJob("job", "default").WorkloadPriorityClass("test-1").Suspend(false).Obj(), - newJob: testingutil.MakeJob("job", "default").WorkloadPriorityClass("test-2").Suspend(false).Obj(), - wantValidationErrs: field.ErrorList{ - field.Invalid(workloadPriorityClassNamePath, "test-2", apivalidation.FieldImmutableErrorMsg), - }, + name: "update priority-class when job not suspend", + oldJob: testingutil.MakeJob("job", "default").Suspend(false).WorkloadPriorityClass("test").Obj(), + newJob: testingutil.MakeJob("job", "default").Suspend(false).WorkloadPriorityClass("new-test").Obj(), + }, + { + name: "delete priority-class when job not suspend", + oldJob: testingutil.MakeJob("job", "default").Suspend(false).WorkloadPriorityClass("test").Obj(), + newJob: testingutil.MakeJob("job", "default").Suspend(false).Obj(), + wantValidationErrs: apivalidation.ValidateImmutableField("", "test", workloadPriorityClassNamePath), + }, + { + name: "set priority-class when job suspend", + oldJob: testingutil.MakeJob("job", "default").Suspend(true).Obj(), + newJob: testingutil.MakeJob("job", "default").Suspend(true).WorkloadPriorityClass("test").Obj(), + }, + { + name: "update priority-class when job suspend", + oldJob: testingutil.MakeJob("job", "default").Suspend(true).WorkloadPriorityClass("test").Obj(), + newJob: testingutil.MakeJob("job", "default").Suspend(true).WorkloadPriorityClass("new-test").Obj(), + }, + { + name: "delete priority-class when job suspend", + oldJob: testingutil.MakeJob("job", "default").Suspend(true).WorkloadPriorityClass("test").Obj(), + newJob: testingutil.MakeJob("job", "default").Suspend(true).Obj(), + wantValidationErrs: apivalidation.ValidateImmutableField("", "test", workloadPriorityClassNamePath), }, { name: "immutable prebuilt workload ", diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go index 70ad6b64a99..615dff3317d 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go @@ -106,7 +106,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco return ctrl.Result{}, err } - toCreate, toFinalize := r.filterWorkloads(lws, wlList.Items) + toCreate, toUpdate, toFinalize := r.filterWorkloads(lws, wlList.Items) eg, ctx := errgroup.WithContext(ctx) @@ -116,6 +116,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco }) }) + eg.Go(func() error { + return parallelize.Until(ctx, len(toUpdate), func(i int) error { + return jobframework.UpdateWorkloadPriority(ctx, r.client, r.record, lws, toUpdate[i], nil) + }) + }) + eg.Go(func() error { return parallelize.Until(ctx, len(toFinalize), func(i int) error { return r.removeOwnerReference(ctx, lws, toFinalize[i]) @@ -130,15 +136,17 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco return ctrl.Result{}, nil } -// filterWorkloads compares the desired state in a LeaderWorkerSet with existing workloads, -// identifying workloads to create and those to finalize. +// filterWorkloads compares the desired state of a LeaderWorkerSet with existing workloads, +// determining which workloads need to be created, updated, or finalized. // -// It takes a LeaderWorkerSet and a slice of existing Workload objects as input and returns: -// 1. A slice of workload names that need to be created -// 2. A slice of Workload pointers that need to be finalized -func (r *Reconciler) filterWorkloads(lws *leaderworkersetv1.LeaderWorkerSet, existingWorkloads []kueue.Workload) ([]string, []*kueue.Workload) { +// It accepts a LeaderWorkerSet and a slice of existing Workload objects as input and returns: +// 1. A slice of workload names to be created +// 2. A slice of workloads that may require updates +// 3. A slice of Workload pointers to be finalized +func (r *Reconciler) filterWorkloads(lws *leaderworkersetv1.LeaderWorkerSet, existingWorkloads []kueue.Workload) ([]string, []*kueue.Workload, []*kueue.Workload) { var ( toCreate []string + toUpdate []*kueue.Workload toFinalize = utilslices.ToRefMap(existingWorkloads, func(e *kueue.Workload) string { return e.Name }) @@ -147,14 +155,15 @@ func (r *Reconciler) filterWorkloads(lws *leaderworkersetv1.LeaderWorkerSet, exi for i := range replicas { workloadName := GetWorkloadName(lws.UID, lws.Name, fmt.Sprint(i)) - if _, ok := toFinalize[workloadName]; ok { + if wl, ok := toFinalize[workloadName]; ok { + toUpdate = append(toUpdate, wl) delete(toFinalize, workloadName) } else { toCreate = append(toCreate, workloadName) } } - return toCreate, slices.Collect(maps.Values(toFinalize)) + return toCreate, toUpdate, slices.Collect(maps.Values(toFinalize)) } func (r *Reconciler) createPrebuiltWorkload(ctx context.Context, lws *leaderworkersetv1.LeaderWorkerSet, workloadName string) error { @@ -163,15 +172,11 @@ func (r *Reconciler) createPrebuiltWorkload(ctx context.Context, lws *leaderwork return err } - priorityClassName, source, p, err := jobframework.ExtractPriority(ctx, r.client, lws, createdWorkload.Spec.PodSets, nil) + err = jobframework.PrepareWorkloadPriority(ctx, r.client, lws, createdWorkload, nil) if err != nil { return err } - createdWorkload.Spec.PriorityClassName = priorityClassName - createdWorkload.Spec.Priority = &p - createdWorkload.Spec.PriorityClassSource = source - err = r.client.Create(ctx, createdWorkload) if err != nil { return err diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go index 4a2a51c7781..917404b0a54 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook.go @@ -156,12 +156,11 @@ func (wh *Webhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Ob )...) isSuspended := oldLeaderWorkerSet.Status.ReadyReplicas == 0 - if !isSuspended || jobframework.IsWorkloadPriorityClassNameEmpty(newLeaderWorkerSet.Object()) { - allErrs = append(allErrs, jobframework.ValidateUpdateForWorkloadPriorityClassName( - newLeaderWorkerSet.Object(), - oldLeaderWorkerSet.Object(), - )...) - } + allErrs = append(allErrs, jobframework.ValidateUpdateForWorkloadPriorityClassName( + isSuspended, + oldLeaderWorkerSet.Object(), + newLeaderWorkerSet.Object(), + )...) suspend, err := jobframework.WorkloadShouldBeSuspended(ctx, newLeaderWorkerSet.Object(), wh.client, wh.manageJobsWithoutQueueName, wh.managedJobsNamespaceSelector) if err != nil { diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook_test.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook_test.go index ea4e28c8f1e..9ec4fadb264 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook_test.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_webhook_test.go @@ -674,6 +674,25 @@ func TestValidateUpdate(t *testing.T) { Label(constants.WorkloadPriorityClassLabel, "new-test"). Obj(), }, + "set priority class when replicas ready": { + oldObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + ReadyReplicas(int32(1)). + Obj(), + newObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "test"). + ReadyReplicas(int32(1)). + Obj(), + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "metadata.labels[kueue.x-k8s.io/priority-class]", + }, + }.ToAggregate(), + }, "change priority class when replicas ready": { oldObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). LeaderTemplate(corev1.PodTemplateSpec{}). @@ -687,6 +706,64 @@ func TestValidateUpdate(t *testing.T) { Label(constants.WorkloadPriorityClassLabel, "new-test"). ReadyReplicas(int32(1)). Obj(), + wantErr: field.ErrorList{}.ToAggregate(), + }, + "delete priority class when replicas ready": { + oldObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "test"). + ReadyReplicas(int32(1)). + Obj(), + newObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + ReadyReplicas(int32(1)). + Obj(), + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: "metadata.labels[kueue.x-k8s.io/priority-class]", + }, + }.ToAggregate(), + }, + "set priority class when replicas not ready": { + oldObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + Obj(), + newObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "test"). + Obj(), + wantErr: field.ErrorList{}.ToAggregate(), + }, + "change priority class when replicas not ready": { + oldObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "test"). + Obj(), + newObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "new-test"). + ReadyReplicas(int32(1)). + Obj(), + wantErr: field.ErrorList{}.ToAggregate(), + }, + "delete priority class when replicas not ready": { + oldObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + Label(constants.WorkloadPriorityClassLabel, "test"). + Obj(), + newObj: testingleaderworkerset.MakeLeaderWorkerSet("test-lws", ""). + LeaderTemplate(corev1.PodTemplateSpec{}). + Queue("test-queue"). + ReadyReplicas(int32(1)). + Obj(), wantErr: field.ErrorList{ &field.Error{ Type: field.ErrorTypeInvalid, diff --git a/pkg/controller/jobs/statefulset/statefulset_webhook.go b/pkg/controller/jobs/statefulset/statefulset_webhook.go index 6a7aa4ee08c..0b9e36c8fc1 100644 --- a/pkg/controller/jobs/statefulset/statefulset_webhook.go +++ b/pkg/controller/jobs/statefulset/statefulset_webhook.go @@ -144,12 +144,11 @@ func (wh *Webhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Ob if !isSuspended || newQueueName == "" { allErrs = append(allErrs, apivalidation.ValidateImmutableField(newQueueName, oldQueueName, queueNameLabelPath)...) } - if !isSuspended || jobframework.IsWorkloadPriorityClassNameEmpty(newStatefulSet.Object()) { - allErrs = append(allErrs, jobframework.ValidateUpdateForWorkloadPriorityClassName( - oldStatefulSet.Object(), - newStatefulSet.Object(), - )...) - } + allErrs = append(allErrs, jobframework.ValidateUpdateForWorkloadPriorityClassName( + isSuspended, + oldStatefulSet.Object(), + newStatefulSet.Object(), + )...) suspend, err := jobframework.WorkloadShouldBeSuspended(ctx, newStatefulSet.Object(), wh.client, wh.manageJobsWithoutQueueName, wh.managedJobsNamespaceSelector) if err != nil { diff --git a/pkg/controller/jobs/statefulset/statefulset_webhook_test.go b/pkg/controller/jobs/statefulset/statefulset_webhook_test.go index c8d58b6e0ed..a1136351898 100644 --- a/pkg/controller/jobs/statefulset/statefulset_webhook_test.go +++ b/pkg/controller/jobs/statefulset/statefulset_webhook_test.go @@ -376,6 +376,35 @@ func TestValidateUpdate(t *testing.T) { }, }, }, + "set in priority class label when replicas ready": { + oldObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + }, + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: int32(1), + }, + }, + newObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + constants.WorkloadPriorityClassLabel: "priority2", + }, + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: int32(1), + }, + }, + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: priorityClassNameLabelPath.String(), + }, + }.ToAggregate(), + }, "change in priority class label when replicas ready": { oldObj: &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -399,6 +428,90 @@ func TestValidateUpdate(t *testing.T) { ReadyReplicas: int32(1), }, }, + wantErr: field.ErrorList{}.ToAggregate(), + }, + "delete in priority class label when replicas ready": { + oldObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + constants.WorkloadPriorityClassLabel: "priority1", + }, + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: int32(1), + }, + }, + newObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + }, + }, + Status: appsv1.StatefulSetStatus{ + ReadyReplicas: int32(1), + }, + }, + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeInvalid, + Field: priorityClassNameLabelPath.String(), + }, + }.ToAggregate(), + }, + "set in priority class label when replicas not ready": { + oldObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + }, + }, + }, + newObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + constants.WorkloadPriorityClassLabel: "priority2", + }, + }, + }, + wantErr: field.ErrorList{}.ToAggregate(), + }, + "change in priority class label when replicas not ready": { + oldObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + constants.WorkloadPriorityClassLabel: "priority1", + }, + }, + }, + newObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + constants.WorkloadPriorityClassLabel: "priority2", + }, + }, + }, + wantErr: field.ErrorList{}.ToAggregate(), + }, + "delete in priority class label when replicas not ready": { + oldObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + constants.WorkloadPriorityClassLabel: "priority1", + }, + }, + }, + newObj: &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constants.QueueLabel: "queue1", + }, + }, + }, wantErr: field.ErrorList{ &field.Error{ Type: field.ErrorTypeInvalid, diff --git a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go index fcb78a34a0f..d50563d4041 100644 --- a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go @@ -296,53 +296,6 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) - ginkgo.It("Should sync workload priority when job priority label changes", func() { - priorityClass := utiltestingapi.MakeWorkloadPriorityClass(priorityClassName). - PriorityValue(int32(priorityValue)).Obj() - util.MustCreate(ctx, k8sClient, priorityClass) - ginkgo.DeferCleanup(func() { - gomega.Expect(k8sClient.Delete(ctx, priorityClass)).To(gomega.Succeed()) - }) - - highPriorityClass := utiltestingapi.MakeWorkloadPriorityClass(highPriorityClassName). - PriorityValue(int32(highPriorityValue)).Obj() - util.MustCreate(ctx, k8sClient, highPriorityClass) - ginkgo.DeferCleanup(func() { - gomega.Expect(k8sClient.Delete(ctx, highPriorityClass)).To(gomega.Succeed()) - }) - - ginkgo.By("creating job with priority") - job := testingjob.MakeJob(jobName, ns.Name). - WorkloadPriorityClass(priorityClassName). - Obj() - util.MustCreate(ctx, k8sClient, job) - lookupKey := types.NamespacedName{Name: jobName, Namespace: ns.Name} - createdJob := &batchv1.Job{} - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, lookupKey, createdJob)).Should(gomega.Succeed()) - g.Expect(createdJob.Spec.Suspend).Should(gomega.Equal(ptr.To(true))) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) - - ginkgo.By("checking the workload is created with priority, priorityName") - createdWorkload := &kueue.Workload{} - wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job.Name, job.UID), Namespace: ns.Name} - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) - gomega.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) - gomega.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(int32(priorityValue))) - - ginkgo.By("checking the workload priority is updated when the job priority label changes") - gomega.Expect(k8sClient.Get(ctx, lookupKey, createdJob)).Should(gomega.Succeed()) - createdJob.Labels[constants.WorkloadPriorityClassLabel] = highPriorityClassName - gomega.Expect(k8sClient.Update(ctx, createdJob)).Should(gomega.Succeed()) - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(highPriorityClassName)) - g.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(int32(highPriorityValue))) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) - }) - ginkgo.It("Should reconcile job when queueName set by label", func() { ginkgo.By("checking the workload is created with correct queue name assigned") var jobQueueName kueue.LocalQueueName = "test-queue" @@ -1380,6 +1333,216 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con util.ExpectObjectToBeDeleted(ctx, k8sClient, spotUntaintedFlavor, true) }) + ginkgo.When("Use a WorkloadPriorityClass", func() { + var ( + lowWorkloadPriorityClass *kueue.WorkloadPriorityClass + highWorkloadPriorityClass *kueue.WorkloadPriorityClass + ) + + ginkgo.BeforeEach(func() { + lowWorkloadPriorityClass = utiltestingapi.MakeWorkloadPriorityClass("low").PriorityValue(100).Obj() + util.MustCreate(ctx, k8sClient, lowWorkloadPriorityClass) + + highWorkloadPriorityClass = utiltestingapi.MakeWorkloadPriorityClass("high").PriorityValue(1000).Obj() + util.MustCreate(ctx, k8sClient, highWorkloadPriorityClass) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(k8sClient.Delete(ctx, lowWorkloadPriorityClass)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Delete(ctx, highWorkloadPriorityClass)).To(gomega.Succeed()) + }) + + ginkgo.It("running workload priority is decreased causing a pending workload to be scheduled and preempt the running one", func() { + job1 := testingjob.MakeJob(jobName+"1", ns.Name). + WorkloadPriorityClass(highWorkloadPriorityClass.Name). + Queue(kueue.LocalQueueName(devLocalQ.Name)). + Request(corev1.ResourceCPU, "5"). + Obj() + ginkgo.By("creating the first Job with high priority", func() { + util.MustCreate(ctx, k8sClient, job1) + }) + + createdWorkload1 := &kueue.Workload{} + wlLookupKey1 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job1.Name, job1.UID), Namespace: ns.Name} + + ginkgo.By("checking that the first workload is created with a priority and admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey1, createdWorkload1)).Should(gomega.Succeed()) + g.Expect(createdWorkload1.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) + g.Expect(*createdWorkload1.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) + g.Expect(createdWorkload1.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + job2 := testingjob.MakeJob(jobName+"2", ns.Name). + WorkloadPriorityClass(highWorkloadPriorityClass.Name). + Queue(kueue.LocalQueueName(devLocalQ.Name)). + Request(corev1.ResourceCPU, "5"). + Obj() + ginkgo.By("creating the second Job with high priority", func() { + util.MustCreate(ctx, k8sClient, job2) + }) + + createdWorkload2 := &kueue.Workload{} + wlLookupKey2 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job2.Name, job2.UID), Namespace: ns.Name} + + ginkgo.By("checking that the second workload is created with a priority and admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) + g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) + g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) + g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + job3 := testingjob.MakeJob(jobName+"3", ns.Name). + WorkloadPriorityClass(highWorkloadPriorityClass.Name). + Queue(kueue.LocalQueueName(devLocalQ.Name)). + Request(corev1.ResourceCPU, "5"). + Obj() + ginkgo.By("creating the third Job with high priority", func() { + util.MustCreate(ctx, k8sClient, job3) + }) + + createdWorkload3 := &kueue.Workload{} + wlLookupKey3 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job3.Name, job3.UID), Namespace: ns.Name} + + ginkgo.By("checking that the third workload is created with a priority and not admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) + g.Expect(createdWorkload3.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) + g.Expect(*createdWorkload3.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) + g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("updating the second workload’s WorkloadPriorityClass to low", func() { + gomega.Eventually(func(g gomega.Gomega) { + createdJob := &batchv1.Job{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(job2), createdJob)).Should(gomega.Succeed()) + createdJob.Labels[constants.WorkloadPriorityClassLabel] = lowWorkloadPriorityClass.Name + g.Expect(k8sClient.Update(ctx, createdJob)).Should(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("checking that the second workload’s WorkloadPriorityClass is updated when the job’s priority label changes", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) + g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(lowWorkloadPriorityClass.Name)) + g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(lowWorkloadPriorityClass.Value)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("checking that the third workload is admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) + g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("checking that the second workload is evicted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) + g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadEvicted)) + g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadPreempted)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + + ginkgo.It("running workload priority is decreased, but still above the pending workload, and so the workload continues running undistrurbed", func() { + job1 := testingjob.MakeJob(jobName+"1", ns.Name). + WorkloadPriorityClass(highWorkloadPriorityClass.Name). + Queue(kueue.LocalQueueName(devLocalQ.Name)). + Request(corev1.ResourceCPU, "5"). + Obj() + ginkgo.By("creating the first Job with high priority", func() { + util.MustCreate(ctx, k8sClient, job1) + }) + + createdWorkload1 := &kueue.Workload{} + wlLookupKey1 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job1.Name, job1.UID), Namespace: ns.Name} + + ginkgo.By("checking that the first workload is created with a priority and admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey1, createdWorkload1)).Should(gomega.Succeed()) + g.Expect(createdWorkload1.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) + g.Expect(*createdWorkload1.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) + g.Expect(createdWorkload1.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + job2 := testingjob.MakeJob(jobName+"2", ns.Name). + WorkloadPriorityClass(highWorkloadPriorityClass.Name). + Queue(kueue.LocalQueueName(devLocalQ.Name)). + Request(corev1.ResourceCPU, "5"). + Obj() + ginkgo.By("creating the second Job with high priority", func() { + util.MustCreate(ctx, k8sClient, job2) + }) + + createdWorkload2 := &kueue.Workload{} + wlLookupKey2 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job2.Name, job2.UID), Namespace: ns.Name} + + ginkgo.By("checking that the second workload is created with a priority and admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) + g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) + g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) + g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + job3 := testingjob.MakeJob(jobName+"3", ns.Name). + WorkloadPriorityClass(lowWorkloadPriorityClass.Name). + Queue(kueue.LocalQueueName(devLocalQ.Name)). + Request(corev1.ResourceCPU, "5"). + Obj() + ginkgo.By("creating the third Job with low priority", func() { + util.MustCreate(ctx, k8sClient, job3) + }) + + createdWorkload3 := &kueue.Workload{} + wlLookupKey3 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job3.Name, job3.UID), Namespace: ns.Name} + + ginkgo.By("checking that the third workload is created with a priority and not admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) + g.Expect(createdWorkload3.Spec.PriorityClassName).Should(gomega.Equal(lowWorkloadPriorityClass.Name)) + g.Expect(*createdWorkload3.Spec.Priority).Should(gomega.Equal(lowWorkloadPriorityClass.Value)) + g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("updating the second workload’s WorkloadPriorityClass to low", func() { + gomega.Eventually(func(g gomega.Gomega) { + createdJob := &batchv1.Job{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(job2), createdJob)).Should(gomega.Succeed()) + createdJob.Labels[constants.WorkloadPriorityClassLabel] = lowWorkloadPriorityClass.Name + g.Expect(k8sClient.Update(ctx, createdJob)).Should(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("checking that the second workload’s WorkloadPriorityClass is updated when the job’s priority label changes", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) + g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(lowWorkloadPriorityClass.Name)) + g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(lowWorkloadPriorityClass.Value)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("checking that the third workload is not admitted and the second workload is not evicted", func() { + gomega.Consistently(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) + g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload2.Status.Conditions).ShouldNot(testing.HaveConditionStatusTrue(kueue.WorkloadEvicted)) + g.Expect(createdWorkload2.Status.Conditions).ShouldNot(testing.HaveConditionStatusTrue(kueue.WorkloadPreempted)) + g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) + g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) + }, util.ConsistentDuration, util.ConsistentDuration).Should(gomega.Succeed()) + }) + }) + }) + ginkgo.It("Should schedule jobs as they fit in their ClusterQueue", framework.SlowSpec, func() { ginkgo.By("checking the first prod job starts") prodJob1 := testingjob.MakeJob("prod-job1", ns.Name).Queue(kueue.LocalQueueName(prodLocalQ.Name)).Request(corev1.ResourceCPU, "2").Obj() diff --git a/test/integration/singlecluster/webhook/core/workload_test.go b/test/integration/singlecluster/webhook/core/workload_test.go index 6ba2543e964..ac439501706 100644 --- a/test/integration/singlecluster/webhook/core/workload_test.go +++ b/test/integration/singlecluster/webhook/core/workload_test.go @@ -802,6 +802,178 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }, testing.BeInvalidError(), ), + ginkgo.Entry("can set workload priority class when QuotaReserved=false", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassSource = constants.WorkloadPriorityClassSource + newWL.Spec.PriorityClassName = "low" + newWL.Spec.Priority = ptr.To[int32](100) + }, + gomega.Succeed(), + ), + ginkgo.Entry("can't set workload priority class when QuotaReserved=true", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() + }, + true, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassSource = constants.WorkloadPriorityClassSource + newWL.Spec.PriorityClassName = "low" + newWL.Spec.Priority = ptr.To[int32](100) + }, + testing.BeInvalidError(), + ), + ginkgo.Entry("can update workload priority class when QuotaReserved=false", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassSource(constants.WorkloadPriorityClassSource). + PriorityClass("high"). + Priority(1000). + Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassName = "low" + newWL.Spec.Priority = ptr.To[int32](100) + }, + gomega.Succeed(), + ), + ginkgo.Entry("can update workload priority class when QuotaReserved=true", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassSource(constants.WorkloadPriorityClassSource). + PriorityClass("high"). + Priority(1000). + Obj() + }, + true, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassName = "low" + newWL.Spec.Priority = ptr.To[int32](100) + }, + gomega.Succeed(), + ), + ginkgo.Entry("can delete workload priority class when QuotaReserved=false", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassSource(constants.WorkloadPriorityClassSource). + PriorityClass("high"). + Priority(1000). + Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassSource = "" + newWL.Spec.PriorityClassName = "" + newWL.Spec.Priority = nil + }, + gomega.Succeed(), + ), + ginkgo.Entry("can't delete workload priority class when quota reserved when QuotaReserved=true", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassSource(constants.WorkloadPriorityClassSource). + PriorityClass("high"). + Priority(1000). + Obj() + }, + true, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassSource = "" + newWL.Spec.PriorityClassName = "" + newWL.Spec.Priority = nil + }, + testing.BeInvalidError(), + ), + ginkgo.Entry("can set pod priority class QuotaReserved=false", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassSource = constants.PodPriorityClassSource + newWL.Spec.PriorityClassName = "low" + newWL.Spec.Priority = ptr.To[int32](100) + }, + gomega.Succeed(), + ), + ginkgo.Entry("can't set pod priority class when QuotaReserved=true", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() + }, + true, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassSource = constants.PodPriorityClassSource + newWL.Spec.PriorityClassName = "low" + newWL.Spec.Priority = ptr.To[int32](100) + }, + testing.BeInvalidError(), + ), + ginkgo.Entry("can update pod priority class when QuotaReserved=false", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassSource(constants.PodPriorityClassSource). + PriorityClass("high"). + Priority(1000). + Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassName = "low" + newWL.Spec.Priority = ptr.To[int32](100) + }, + gomega.Succeed(), + ), + ginkgo.Entry("can't update pod priority class when QuotaReserved=true", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassSource(constants.PodPriorityClassSource). + PriorityClass("high"). + Priority(1000). + Obj() + }, + true, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassName = "low" + newWL.Spec.Priority = ptr.To[int32](100) + }, + testing.BeInvalidError(), + ), + ginkgo.Entry("can delete pod priority class when QuotaReserved=false", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassSource(constants.PodPriorityClassSource). + PriorityClass("high"). + Priority(1000). + Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassSource = "" + newWL.Spec.PriorityClassName = "" + newWL.Spec.Priority = nil + }, + gomega.Succeed(), + ), + ginkgo.Entry("can't delete pod priority class when QuotaReserved=true", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassSource(constants.PodPriorityClassSource). + PriorityClass("high"). + Priority(1000). + Obj() + }, + true, + func(newWL *kueue.Workload) { + newWL.Spec.PriorityClassSource = "" + newWL.Spec.PriorityClassName = "" + newWL.Spec.Priority = nil + }, + testing.BeInvalidError(), + ), ) ginkgo.It("Should forbid the change of spec.queueName of an admitted workload", func() { From 430f1db96ebf4690afa5166963fed67f86f6189e Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Tue, 28 Oct 2025 13:00:04 -0400 Subject: [PATCH 045/119] enable ssa tags for kubernetes api linter (#7339) --- .golangci-kal.yml | 6 ++++-- apis/kueue/v1beta2/clusterqueue_types.go | 6 +++++- apis/kueue/v1beta2/provisioningrequestconfig_types.go | 2 ++ apis/kueue/v1beta2/workload_types.go | 4 ++++ .../kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml | 5 +++++ charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml | 1 + .../crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml | 3 +++ charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml | 5 +++++ .../components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml | 5 +++++ config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml | 1 + .../bases/kueue.x-k8s.io_provisioningrequestconfigs.yaml | 3 +++ config/components/crd/bases/kueue.x-k8s.io_workloads.yaml | 5 +++++ 12 files changed, 43 insertions(+), 3 deletions(-) diff --git a/.golangci-kal.yml b/.golangci-kal.yml index 81513166644..085be48f50a 100644 --- a/.golangci-kal.yml +++ b/.golangci-kal.yml @@ -30,7 +30,7 @@ linters: # having the `omitempty` value in their `json` tag where appropriate. # - "optionalorrequired" # Every field should be marked as `+optional` or `+required`. # - "requiredfields" # Required fields should not be pointers, and should not have `omitempty`. - # - "ssatags" # Ensure array fields have the appropriate listType markers + - "ssatags" # Ensure array fields have the appropriate listType markers - "statusoptional" # Ensure all first children within status should be optional. - "statussubresource" # All root objects that have a `status` field should have a status subresource. - "uniquemarkers" # Ensure that types and fields do not contain more than a single definition of a marker that should only be present once. @@ -57,12 +57,14 @@ linters: # policy: SuggestFix # SuggestFix | Warn # The policy for pointers in optional fields. Defaults to `SuggestFix`. # omitempty: # policy: SuggestFix # SuggestFix | Warn | Ignore # The policy for omitempty in optional fields. Defaults to `SuggestFix`. + ssatags: + listTypeSetUsage: Ignore # The policy for listType=set usage on object arrays. Defaults to `Warn`. exclusions: generated: strict rules: ## KAL should only run on API folders. - - path-except: "apis/kueue/*" + - path-except: "apis/kueue/v1beta2/*" linters: - kubeapilinter diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index 9f749757bf6..01c067c75e6 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -112,7 +112,7 @@ type ClusterQueueSpec struct { // admissionChecks lists the AdmissionChecks required by this ClusterQueue. // Cannot be used along with AdmissionCheckStrategy. // +optional - AdmissionChecks []AdmissionCheckReference `json:"admissionChecks,omitempty"` + AdmissionChecks []AdmissionCheckReference `json:"admissionChecks,omitempty"` //nolint:kubeapilinter // field is being removed // admissionChecksStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. // This property cannot be used in conjunction with the 'admissionChecks' property. @@ -147,6 +147,8 @@ type ClusterQueueSpec struct { // AdmissionChecksStrategy defines a strategy for a AdmissionCheck. type AdmissionChecksStrategy struct { // admissionChecks is a list of strategies for AdmissionChecks + // +listType=map + // +listMapKey=name AdmissionChecks []AdmissionCheckStrategyRule `json:"admissionChecks,omitempty"` } @@ -158,6 +160,7 @@ type AdmissionCheckStrategyRule struct { // onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. // If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. // +optional + // +listType=set OnFlavors []ResourceFlavorReference `json:"onFlavors,omitempty"` } @@ -184,6 +187,7 @@ type ResourceGroup struct { // of up to 256 covered resources across all resource groups in the ClusterQueue. // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=64 + // +listType=atomic CoveredResources []corev1.ResourceName `json:"coveredResources"` // flavors is the list of flavors that provide the resources of this group. diff --git a/apis/kueue/v1beta2/provisioningrequestconfig_types.go b/apis/kueue/v1beta2/provisioningrequestconfig_types.go index 55fd2ff4336..ff29f7efd4f 100644 --- a/apis/kueue/v1beta2/provisioningrequestconfig_types.go +++ b/apis/kueue/v1beta2/provisioningrequestconfig_types.go @@ -95,6 +95,8 @@ type ProvisioningRequestPodSetUpdates struct { // // +optional // +kubebuilder:validation:MaxItems=8 + // +listType=map + // +listMapKey=key NodeSelector []ProvisioningRequestPodSetUpdatesNodeSelector `json:"nodeSelector,omitempty"` } diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index 623ed4fe849..faa33722168 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -276,6 +276,7 @@ type TopologyAssignment struct { // the lowest level of the topology. // // +required + // +listType=atomic Domains []TopologyDomainAssignment `json:"domains"` } @@ -442,6 +443,8 @@ type WorkloadStatus struct { // Requires enabling the TASFailedNodeReplacement feature gate. // // +optional + // +listType=map + // +listMapKey=name UnhealthyNodes []UnhealthyNode `json:"unhealthyNodes,omitempty"` } @@ -570,6 +573,7 @@ type PodSetUpdate struct { // tolerations of the PodSet to modify. // +optional // +kubebuilder:validation:MaxItems=8 + // +listType=atomic // +kubebuilder:validation:XValidation:rule="self.all(x, !has(x.key) ? x.operator == 'Exists' : true)", message="operator must be Exists when 'key' is empty, which means 'match all values and all keys'" // +kubebuilder:validation:XValidation:rule="self.all(x, has(x.tolerationSeconds) ? x.effect == 'NoExecute' : true)", message="effect must be 'NoExecute' when 'tolerationSeconds' is set" // +kubebuilder:validation:XValidation:rule="self.all(x, !has(x.operator) || x.operator in ['Equal', 'Exists'])", message="supported toleration values: 'Equal'(default), 'Exists'" diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml index 749a5aa0439..5913bca16d5 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml @@ -855,10 +855,14 @@ spec: pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string type: array + x-kubernetes-list-type: set required: - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object admissionScope: description: admissionScope indicates whether ClusterQueue uses the Admission Fair Sharing @@ -1122,6 +1126,7 @@ spec: maxItems: 64 minItems: 1 type: array + x-kubernetes-list-type: atomic flavors: description: |- flavors is the list of flavors that provide the resources of this group. diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml index 8c0f6241669..4324ad3b58e 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml @@ -405,6 +405,7 @@ spec: maxItems: 64 minItems: 1 type: array + x-kubernetes-list-type: atomic flavors: description: |- flavors is the list of flavors that provide the resources of this group. diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml index 3cb8e02c602..90e40babd15 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml @@ -258,6 +258,9 @@ spec: type: object maxItems: 8 type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map type: object provisioningClassName: description: |- diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index fd32cadfa6d..fc58fa143a6 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -17288,6 +17288,7 @@ spec: - values type: object type: array + x-kubernetes-list-type: atomic levels: description: |- levels is an ordered list of keys denoting the levels of the assigned @@ -17404,6 +17405,7 @@ spec: type: object maxItems: 8 type: array + x-kubernetes-list-type: atomic x-kubernetes-validations: - message: operator must be Exists when 'key' is empty, which means 'match all values and all keys' rule: 'self.all(x, !has(x.key) ? x.operator == ''Exists'' : true)' @@ -17658,6 +17660,9 @@ spec: - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object x-kubernetes-validations: - message: clusterName is immutable once set diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml index e42809e8f55..390941107c7 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml @@ -858,10 +858,14 @@ spec: pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string type: array + x-kubernetes-list-type: set required: - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object admissionScope: description: admissionScope indicates whether ClusterQueue uses the @@ -1130,6 +1134,7 @@ spec: maxItems: 64 minItems: 1 type: array + x-kubernetes-list-type: atomic flavors: description: |- flavors is the list of flavors that provide the resources of this group. diff --git a/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml b/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml index 877dd60f14f..add75e420f9 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml @@ -404,6 +404,7 @@ spec: maxItems: 64 minItems: 1 type: array + x-kubernetes-list-type: atomic flavors: description: |- flavors is the list of flavors that provide the resources of this group. diff --git a/config/components/crd/bases/kueue.x-k8s.io_provisioningrequestconfigs.yaml b/config/components/crd/bases/kueue.x-k8s.io_provisioningrequestconfigs.yaml index 08e598f41c5..822a4aa1423 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_provisioningrequestconfigs.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_provisioningrequestconfigs.yaml @@ -260,6 +260,9 @@ spec: type: object maxItems: 8 type: array + x-kubernetes-list-map-keys: + - key + x-kubernetes-list-type: map type: object provisioningClassName: description: |- diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index 4ed717e2103..a45a2ecbdf6 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -18289,6 +18289,7 @@ spec: - values type: object type: array + x-kubernetes-list-type: atomic levels: description: |- levels is an ordered list of keys denoting the levels of the assigned @@ -18408,6 +18409,7 @@ spec: type: object maxItems: 8 type: array + x-kubernetes-list-type: atomic x-kubernetes-validations: - message: operator must be Exists when 'key' is empty, which means 'match all values and all keys' @@ -18678,6 +18680,9 @@ spec: - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object x-kubernetes-validations: - message: clusterName is immutable once set From 17a7ec37eb71a726e729ba611d8dd21bab8291a5 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 29 Oct 2025 02:22:02 -0400 Subject: [PATCH 046/119] increase topology limits to 16 to match topology updates (#7423) --- apis/kueue/v1beta1/localqueue_types.go | 2 +- apis/kueue/v1beta1/workload_types.go | 4 ++-- apis/kueue/v1beta2/localqueue_types.go | 2 +- apis/kueue/v1beta2/workload_types.go | 4 ++-- .../kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml | 4 ++-- charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml | 8 ++++---- .../components/crd/bases/kueue.x-k8s.io_localqueues.yaml | 4 ++-- config/components/crd/bases/kueue.x-k8s.io_workloads.yaml | 8 ++++---- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/apis/kueue/v1beta1/localqueue_types.go b/apis/kueue/v1beta1/localqueue_types.go index c46604d9cab..45ea97a5f3f 100644 --- a/apis/kueue/v1beta1/localqueue_types.go +++ b/apis/kueue/v1beta1/localqueue_types.go @@ -105,7 +105,7 @@ type TopologyInfo struct { // +listType=atomic // +kubebuilder:validation:Required // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:MaxItems=8 + // +kubebuilder:validation:MaxItems=16 Levels []string `json:"levels"` } diff --git a/apis/kueue/v1beta1/workload_types.go b/apis/kueue/v1beta1/workload_types.go index 0a6cb4e10a6..1c1d44696a6 100644 --- a/apis/kueue/v1beta1/workload_types.go +++ b/apis/kueue/v1beta1/workload_types.go @@ -269,7 +269,7 @@ type TopologyAssignment struct { // +required // +listType=atomic // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:MaxItems=8 + // +kubebuilder:validation:MaxItems=16 Levels []string `json:"levels"` // domains is a list of topology assignments split by topology domains at @@ -287,7 +287,7 @@ type TopologyDomainAssignment struct { // +required // +listType=atomic // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:MaxItems=8 + // +kubebuilder:validation:MaxItems=16 Values []string `json:"values"` // count indicates the number of Pods to be scheduled in the topology diff --git a/apis/kueue/v1beta2/localqueue_types.go b/apis/kueue/v1beta2/localqueue_types.go index a19f2d900ad..d0fe9bd0cc7 100644 --- a/apis/kueue/v1beta2/localqueue_types.go +++ b/apis/kueue/v1beta2/localqueue_types.go @@ -105,7 +105,7 @@ type TopologyInfo struct { // +listType=atomic // +kubebuilder:validation:Required // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:MaxItems=8 + // +kubebuilder:validation:MaxItems=16 Levels []string `json:"levels"` } diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index faa33722168..a04782f7412 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -269,7 +269,7 @@ type TopologyAssignment struct { // +required // +listType=atomic // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:MaxItems=8 + // +kubebuilder:validation:MaxItems=16 Levels []string `json:"levels"` // domains is a list of topology assignments split by topology domains at @@ -288,7 +288,7 @@ type TopologyDomainAssignment struct { // +required // +listType=atomic // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:MaxItems=8 + // +kubebuilder:validation:MaxItems=16 Values []string `json:"values"` // count indicates the number of Pods to be scheduled in the topology diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml index b6cf04c6f3c..ef8da08bd05 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml @@ -347,7 +347,7 @@ spec: description: levels define the levels of topology. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -697,7 +697,7 @@ spec: description: levels define the levels of topology. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index fc58fa143a6..d6c90c436ab 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -8455,7 +8455,7 @@ spec: the highest to the lowest. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -8471,7 +8471,7 @@ spec: the topology. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -17279,7 +17279,7 @@ spec: the highest to the lowest. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -17296,7 +17296,7 @@ spec: the topology. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic diff --git a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml index e8c95ba7e79..10233d7cf17 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml @@ -342,7 +342,7 @@ spec: description: levels define the levels of topology. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -704,7 +704,7 @@ spec: description: levels define the levels of topology. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index a45a2ecbdf6..f500d797619 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -8927,7 +8927,7 @@ spec: the highest to the lowest. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -8943,7 +8943,7 @@ spec: the topology. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -18280,7 +18280,7 @@ spec: the highest to the lowest. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic @@ -18297,7 +18297,7 @@ spec: the topology. items: type: string - maxItems: 8 + maxItems: 16 minItems: 1 type: array x-kubernetes-list-type: atomic From fddf1b7f96da7899ae4944aa03ac80203cd4f12c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Oct 2025 01:48:03 -0700 Subject: [PATCH 047/119] Bump github.com/onsi/ginkgo/v2 in /hack/internal/tools (#7401) Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.26.0 to 2.27.1. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.26.0...v2.27.1) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-version: 2.27.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- hack/internal/tools/go.mod | 2 +- hack/internal/tools/go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hack/internal/tools/go.mod b/hack/internal/tools/go.mod index a8a8619fd76..c9289d1dcae 100644 --- a/hack/internal/tools/go.mod +++ b/hack/internal/tools/go.mod @@ -27,7 +27,7 @@ require ( github.com/kubernetes-sigs/reference-docs/genref v0.28.0 github.com/mikefarah/yq/v4 v4.48.1 github.com/norwoodj/helm-docs v1.14.2 - github.com/onsi/ginkgo/v2 v2.26.0 + github.com/onsi/ginkgo/v2 v2.27.2 go.uber.org/mock v0.6.0 go.uber.org/zap v1.27.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 diff --git a/hack/internal/tools/go.sum b/hack/internal/tools/go.sum index d8008f92b0a..445ff09bdcb 100644 --- a/hack/internal/tools/go.sum +++ b/hack/internal/tools/go.sum @@ -442,8 +442,8 @@ github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BN github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= -github.com/gkampitakis/go-snaps v0.5.14 h1:3fAqdB6BCPKHDMHAKRwtPUwYexKtGrNuw8HX/T/4neo= -github.com/gkampitakis/go-snaps v0.5.14/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-critic/go-critic v0.13.0 h1:kJzM7wzltQasSUXtYyTl6UaPVySO6GkaR1thFnJ6afY= github.com/go-critic/go-critic v0.13.0/go.mod h1:M/YeuJ3vOCQDnP2SU+ZhjgRzwzcBW87JqLpMJLrZDLI= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -905,8 +905,8 @@ github.com/olekukonko/tablewriter v1.0.9 h1:XGwRsYLC2bY7bNd93Dk51bcPZksWZmLYuaTH github.com/olekukonko/tablewriter v1.0.9/go.mod h1:5c+EBPeSqvXnLLgkm9isDdzR3wjfBkHR9Nhfp3NWrzo= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.26.0 h1:1J4Wut1IlYZNEAWIV3ALrT9NfiaGW2cDCJQSFQMs/gE= -github.com/onsi/ginkgo/v2 v2.26.0/go.mod h1:qhEywmzWTBUY88kfO0BRvX4py7scov9yR+Az2oavUzw= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= From b177dcc17d426f8029d9f49d428316b845fea583 Mon Sep 17 00:00:00 2001 From: Patryk Bundyra Date: Wed, 29 Oct 2025 16:04:04 +0100 Subject: [PATCH 048/119] Simplify JobSet ReclaimablePods integration (#7420) --- pkg/controller/jobs/jobset/jobset_controller.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/controller/jobs/jobset/jobset_controller.go b/pkg/controller/jobs/jobset/jobset_controller.go index c98cb00cfa4..c3a5a41d972 100644 --- a/pkg/controller/jobs/jobset/jobset_controller.go +++ b/pkg/controller/jobs/jobset/jobset_controller.go @@ -197,13 +197,11 @@ func (j *JobSet) ReclaimablePods(ctx context.Context) ([]kueue.ReclaimablePod, e for i := range j.Spec.ReplicatedJobs { spec := &j.Spec.ReplicatedJobs[i] - if status, found := statuses[spec.Name]; found && status.Succeeded > 0 { - if status.Succeeded > 0 && status.Succeeded <= spec.Replicas { - ret = append(ret, kueue.ReclaimablePod{ - Name: kueue.NewPodSetReference(spec.Name), - Count: status.Succeeded * PodsCountPerReplica(spec), - }) - } + if status, found := statuses[spec.Name]; found && status.Succeeded > 0 && status.Succeeded <= spec.Replicas { + ret = append(ret, kueue.ReclaimablePod{ + Name: kueue.NewPodSetReference(spec.Name), + Count: status.Succeeded * PodsCountPerReplica(spec), + }) } } return ret, nil From cfe148af893eec36c1a125e7785d39b742a50eb8 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 29 Oct 2025 14:22:04 -0400 Subject: [PATCH 049/119] promote MultiKueueBatchJobWithManagedBy to beta (#7341) --- .../multikueue/workload_test.go | 5 + .../jobs/job/job_multikueue_adapter_test.go | 181 +++++++++++++++++- pkg/features/kube_features.go | 1 + site/content/en/docs/installation/_index.md | 3 +- .../multikueue/enabled_integration_test.go | 2 +- test/integration/multikueue/jobs_test.go | 12 +- 6 files changed, 186 insertions(+), 18 deletions(-) diff --git a/pkg/controller/admissionchecks/multikueue/workload_test.go b/pkg/controller/admissionchecks/multikueue/workload_test.go index e389aaaf1ae..d23208c85db 100644 --- a/pkg/controller/admissionchecks/multikueue/workload_test.go +++ b/pkg/controller/admissionchecks/multikueue/workload_test.go @@ -299,6 +299,7 @@ func TestWlReconcile(t *testing.T) { }, }, "wl without reservation, clears the workload objects (withoutJobManagedBy)": { + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: false}, reconcileFor: "wl1", managersJobs: []batchv1.Job{*baseJobBuilder.Clone().Obj()}, managersWorkloads: []kueue.Workload{ @@ -501,6 +502,7 @@ func TestWlReconcile(t *testing.T) { }, }, "remote wl with reservation (withoutJobManagedBy)": { + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: false}, reconcileFor: "wl1", managersWorkloads: []kueue.Workload{ *baseWorkloadBuilder.Clone(). @@ -565,6 +567,7 @@ func TestWlReconcile(t *testing.T) { }, }, "remote wl with reservation (withoutJobManagedBy, MultiKueueDispatcherModeIncremental)": { + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: false}, reconcileFor: "wl1", dispatcherName: ptr.To(config.MultiKueueDispatcherModeIncremental), managersWorkloads: []kueue.Workload{ @@ -702,6 +705,7 @@ func TestWlReconcile(t *testing.T) { }, }, "remote job is changing status, the local job is not updated (withoutJobManagedBy)": { + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: false}, reconcileFor: "wl1", managersWorkloads: []kueue.Workload{ *baseWorkloadBuilder.Clone(). @@ -835,6 +839,7 @@ func TestWlReconcile(t *testing.T) { }, }, "remote wl is finished, the local workload and Job are marked completed (withoutJobManagedBy)": { + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: false}, reconcileFor: "wl1", managersWorkloads: []kueue.Workload{ *baseWorkloadBuilder.Clone(). diff --git a/pkg/controller/jobs/job/job_multikueue_adapter_test.go b/pkg/controller/jobs/job/job_multikueue_adapter_test.go index 01e44675813..2b2953648e4 100644 --- a/pkg/controller/jobs/job/job_multikueue_adapter_test.go +++ b/pkg/controller/jobs/job/job_multikueue_adapter_test.go @@ -394,6 +394,9 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { }, }, "RemoteJobNotFound": { + fields: fields{ + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: false}, + }, args: args{ localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). ManagedBy("parent").Obj()).Build(), @@ -480,9 +483,12 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { Obj(), }, }, - "RemoteJobFinished_Completed": { + "RemoteJobFinished_Completed_MultiKueueBatchJobWithManagedBy_Disabled": { + fields: fields{ + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: false}, + }, args: args{ - localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob().Obj()).Build(), + localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob().Suspend(false).Obj()).Build(), remoteClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). Condition(batchv1.JobCondition{ Type: batchv1.JobComplete, @@ -493,6 +499,7 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { }, want: want{ localJob: newJob(). + Suspend(false). ResourceVersion("2"). Condition( batchv1.JobCondition{ @@ -502,9 +509,38 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { Obj(), }, }, - "RemoteJobFinished_Failed": { + "RemoteJobFinished_Completed_MultiKueueBatchJobWithManagedBy_Enabled": { + fields: fields{ + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: true}, + }, args: args{ - localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob().Obj()).Build(), + localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob().Suspend(false).Obj()).Build(), + remoteClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). + Condition(batchv1.JobCondition{ + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }). + Obj()).Build(), + key: client.ObjectKeyFromObject(newJob().Obj()), + }, + want: want{ + localJob: newJob(). + Condition(batchv1.JobCondition{ + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }). + Suspend(false). + ResourceVersion("2"). + Obj(), + }, + }, + "RemoteJobFinished_Failed_MultiKueueBatchJobManagedBy_Disabled": { + fields: fields{ + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: false}, + }, + + args: args{ + localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob().Suspend(false).Obj()).Build(), remoteClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). Condition(batchv1.JobCondition{ Type: batchv1.JobFailed, @@ -515,6 +551,7 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { }, want: want{ localJob: newJob(). + Suspend(false). ResourceVersion("2"). Condition( batchv1.JobCondition{ @@ -524,6 +561,33 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { Obj(), }, }, + "RemoteJobFinished_Failed_MultiKueueBatchJobManagedBy_Enabled": { + fields: fields{ + features: map[featuregate.Feature]bool{features.MultiKueueBatchJobWithManagedBy: true}, + }, + + args: args{ + localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob().Suspend(false).Obj()).Build(), + remoteClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). + Suspend(false). + Condition(batchv1.JobCondition{ + Type: batchv1.JobFailed, + Status: corev1.ConditionTrue, + }). + Obj()).Build(), + key: client.ObjectKeyFromObject(newJob().Obj()), + }, + want: want{ + localJob: newJob(). + Suspend(false). + Condition(batchv1.JobCondition{ + Type: batchv1.JobFailed, + Status: corev1.ConditionTrue, + }). + ResourceVersion("2"). + Obj(), + }, + }, "ElasticJob_RemoteInSync": { fields: fields{ features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true}, @@ -584,9 +648,9 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { Obj(), }, }, - "ElasticJob_WorkloadNameOnlyChange_EdgeCase": { + "ElasticJob_WorkloadNameOnlyChange_EdgeCase_MultiKueueBatchJobWithManagedBy_Disabled": { fields: fields{ - features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true}, + features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true, features.MultiKueueBatchJobWithManagedBy: false}, }, args: args{ localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). @@ -613,9 +677,37 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { Obj(), }, }, - "ElasticJob_RemoteOutOfSync": { + "ElasticJob_WorkloadNameOnlyChange_EdgeCase_MultiKueueBatchJobWithManagedBy_Enabled": { fields: fields{ - features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true}, + features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true, features.MultiKueueBatchJobWithManagedBy: true}, + }, + args: args{ + localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Condition(runningJobCondition). + Obj()).Build(), + remoteClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Condition(runningJobCondition). + Obj()).Build(), + key: client.ObjectKeyFromObject(newJob().Obj()), + workloadName: "test-workload-new", + }, + want: want{ + localJob: newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Condition(runningJobCondition). + Obj(), + remoteJob: newJob(). + ResourceVersion("1"). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Condition(runningJobCondition). + Obj(), + }, + }, + "ElasticJob_RemoteOutOfSync_MultiKueueBatchJobWithManagedBy_Disabled": { + fields: fields{ + features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true, features.MultiKueueBatchJobWithManagedBy: false}, }, args: args{ localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). @@ -646,9 +738,42 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { Obj(), }, }, - "ElasticJob_RemoteOutOfSync_PatchFailure": { + "ElasticJob_RemoteOutOfSync_MultiKueueBatchJobWithManagedBy_Enabled": { fields: fields{ - features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true}, + features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true, features.MultiKueueBatchJobWithManagedBy: true}, + }, + args: args{ + localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Parallelism(22). + Condition(runningJobCondition). + Obj()).Build(), + remoteClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Label(constants.PrebuiltWorkloadLabel, "test-workload"). + Condition(runningJobCondition). + Obj()).Build(), + key: client.ObjectKeyFromObject(newJob().Obj()), + workloadName: jobframework.GetWorkloadNameForOwnerWithGVKAndGeneration("test", "", gvk, 0), + }, + want: want{ + localJob: newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Parallelism(22). + Condition(runningJobCondition). + Obj(), + remoteJob: newJob(). + ResourceVersion("1"). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Label(constants.PrebuiltWorkloadLabel, "test-workload"). + Parallelism(1). + Condition(runningJobCondition). + Obj(), + }, + }, + "ElasticJob_RemoteOutOfSync_PatchFailure_MultiKueueBatchJobManagedBy_Disabled": { + fields: fields{ + features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true, features.MultiKueueBatchJobWithManagedBy: false}, }, args: args{ localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). @@ -683,6 +808,42 @@ func Test_multiKueueAdapter_SyncJob(t *testing.T) { Obj(), }, }, + "ElasticJob_RemoteOutOfSync_PatchFailure_MultiKueueBatchJobManagedBy_Enabled": { + fields: fields{ + features: map[featuregate.Feature]bool{features.ElasticJobsViaWorkloadSlices: true, features.MultiKueueBatchJobWithManagedBy: true}, + }, + args: args{ + localClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Parallelism(22). + Condition(runningJobCondition). + Obj()).Build(), + remoteClient: fake.NewClientBuilder().WithScheme(schema).WithObjects(newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Label(constants.PrebuiltWorkloadLabel, "test-workload"). + Condition(runningJobCondition). + Obj()). + WithInterceptorFuncs(interceptor.Funcs{ + Patch: func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return errors.New("test-patch-error") + }, + }).Build(), + key: client.ObjectKeyFromObject(newJob().Obj()), + workloadName: jobframework.GetWorkloadNameForOwnerWithGVKAndGeneration("test", "", gvk, 0), + }, + want: want{ + localJob: newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Parallelism(22). + Condition(runningJobCondition). + Obj(), + remoteJob: newJob(). + SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). + Label(constants.PrebuiltWorkloadLabel, "test-workload"). + Condition(runningJobCondition). + Obj(), + }, + }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 230badd3277..6056a64e545 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -240,6 +240,7 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned }, MultiKueueBatchJobWithManagedBy: { {Version: version.MustParse("0.8"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("0.15"), Default: true, PreRelease: featuregate.Beta}, }, TopologyAwareScheduling: { {Version: version.MustParse("0.9"), Default: false, PreRelease: featuregate.Alpha}, diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 03c2da36dd8..52966289160 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -268,7 +268,8 @@ spec: | `FlavorFungibility` | `true` | Beta | 0.5 | | | `MultiKueue` | `false` | Alpha | 0.6 | 0.8 | | `MultiKueue` | `true` | Beta | 0.9 | | -| `MultiKueueBatchJobWithManagedBy` | `false` | Alpha | 0.8 | | +| `MultiKueueBatchJobWithManagedBy` | `false` | Alpha | 0.8 | 0.15 | +| `MultiKueueBatchJobWithManagedBy` | `true` | Beta | 0.15 | | | `PartialAdmission` | `false` | Alpha | 0.4 | 0.4 | | `PartialAdmission` | `true` | Beta | 0.5 | | | `VisibilityOnDemand` | `false` | Alpha | 0.6 | 0.8 | diff --git a/test/integration/multikueue/enabled_integration_test.go b/test/integration/multikueue/enabled_integration_test.go index 1a94471fa11..2678d6b2a50 100644 --- a/test/integration/multikueue/enabled_integration_test.go +++ b/test/integration/multikueue/enabled_integration_test.go @@ -168,7 +168,7 @@ var _ = ginkgo.Describe("MultiKueue when not all integrations are enabled", gink g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) acs := admissioncheck.FindAdmissionCheck(createdWorkload.Status.AdmissionChecks, kueue.AdmissionCheckReference(multiKueueAC.Name)) g.Expect(acs).NotTo(gomega.BeNil()) - g.Expect(acs.State).To(gomega.Equal(kueue.CheckStatePending)) + g.Expect(acs.State).To(gomega.Equal(kueue.CheckStateReady)) g.Expect(acs.Message).To(gomega.Equal(`The workload got reservation on "worker1"`)) ok, err := utiltesting.HasEventAppeared(managerTestCluster.ctx, managerTestCluster.client, corev1.Event{ Reason: "MultiKueue", diff --git a/test/integration/multikueue/jobs_test.go b/test/integration/multikueue/jobs_test.go index 35a6891b816..cf1923732bd 100644 --- a/test/integration/multikueue/jobs_test.go +++ b/test/integration/multikueue/jobs_test.go @@ -245,7 +245,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) acs := admissioncheck.FindAdmissionCheck(createdWorkload.Status.AdmissionChecks, kueue.AdmissionCheckReference(multiKueueAC.Name)) g.Expect(acs).NotTo(gomega.BeNil()) - g.Expect(acs.State).To(gomega.Equal(kueue.CheckStatePending)) + g.Expect(acs.State).To(gomega.Equal(kueue.CheckStateReady)) g.Expect(acs.Message).To(gomega.Equal(`The workload got reservation on "worker1"`)) ok, err := utiltesting.HasEventAppeared(managerTestCluster.ctx, managerTestCluster.client, corev1.Event{ Reason: "MultiKueue", @@ -1694,7 +1694,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, localWorkload := getWorkload(g, manager.ctx, manager.client, workloadKey) acs := admissioncheck.FindAdmissionCheck(localWorkload.Status.AdmissionChecks, kueue.AdmissionCheckReference(multiKueueAC.Name)) g.Expect(acs).NotTo(gomega.BeNil()) - g.Expect(acs.State).To(gomega.Equal(kueue.CheckStatePending)) + g.Expect(acs.State).To(gomega.Equal(kueue.CheckStateReady)) g.Expect(acs.Message).To(gomega.Equal(`The workload got reservation on "worker1"`)) ok, err := utiltesting.HasEventAppeared(manager.ctx, manager.client, corev1.Event{ Reason: "MultiKueue", @@ -1724,9 +1724,9 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, gomega.Expect(list.Items).To(gomega.BeEmpty()) }) - ginkgo.By("observe: job is still suspended in the manager cluster", func() { + ginkgo.By("observe: job is no longer suspended in the manager cluster", func() { getJob(manager.ctx, manager.client, job) - gomega.Expect(job.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(true))) + gomega.Expect(job.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(false))) }) /* @@ -1809,7 +1809,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, workload := getWorkload(g, manager.ctx, manager.client, newWorkloadKey) acs := admissioncheck.FindAdmissionCheck(workload.Status.AdmissionChecks, kueue.AdmissionCheckReference(multiKueueAC.Name)) g.Expect(acs).NotTo(gomega.BeNil()) - g.Expect(acs.State).To(gomega.Equal(kueue.CheckStatePending)) + g.Expect(acs.State).To(gomega.Equal(kueue.CheckStateReady)) g.Expect(acs.Message).To(gomega.Equal(`The workload got reservation on "worker1"`)) ok, err := utiltesting.HasEventAppeared(manager.ctx, manager.client, corev1.Event{ Reason: "MultiKueue", @@ -1825,7 +1825,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, remoteJob := job.DeepCopy() getJob(worker1.ctx, worker1.client, remoteJob) g.Expect(remoteJob.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(false))) - g.Expect(remoteJob.Spec.Parallelism).To(gomega.BeEquivalentTo(ptr.To(int32(2)))) + g.Expect(remoteJob.Spec.Parallelism).To(gomega.BeEquivalentTo(ptr.To(int32(1)))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) From 9bba9917dd4695adad093630d6f2199fe2fee2cf Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 30 Oct 2025 13:28:02 +0530 Subject: [PATCH 050/119] Remove duplicate env variables in podSet template. (#7425) * Remove duplicate env variables in podSet template. * Test EquivalentToWorkload on e2e using events. --- pkg/controller/jobframework/reconciler.go | 6 +- pkg/controller/jobframework/utils.go | 66 +++++++++++++ pkg/controller/jobs/job/job_webhook.go | 2 +- pkg/controller/jobs/jobset/jobset_webhook.go | 2 +- .../kubeflowjob/kubeflowjob_controller.go | 2 +- .../leaderworkerset_reconciler.go | 29 +++--- pkg/controller/jobs/mpijob/mpijob_webhook.go | 2 +- pkg/controller/jobs/pod/pod_controller.go | 2 +- .../jobs/raycluster/raycluster_webhook.go | 2 +- pkg/controller/jobs/rayjob/rayjob_webhook.go | 2 +- pkg/features/kube_features.go | 12 +++ pkg/util/testing/wrappers.go | 12 +++ site/content/en/docs/installation/_index.md | 1 + .../content/zh-CN/docs/installation/_index.md | 1 + test/e2e/singlecluster/e2e_test.go | 99 +++++++++++++++++++ 15 files changed, 215 insertions(+), 25 deletions(-) create mode 100644 pkg/controller/jobframework/utils.go diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index e1bbd1cb46e..dd417e7b7ee 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -827,7 +827,7 @@ func (r *JobReconciler) ensureOneWorkload(ctx context.Context, job GenericJob, o // If workload slicing is enabled for this job, use the slice-based processing path. if workloadSliceEnabled(job) { - podSets, err := job.PodSets(ctx) + podSets, err := JobPodSets(ctx, job) if err != nil { return nil, fmt.Errorf("failed to retrieve pod sets from job: %w", err) } @@ -1068,7 +1068,7 @@ func EquivalentToWorkload(ctx context.Context, c client.Client, job GenericJob, return false, nil } - getPodSets, err := job.PodSets(ctx) + getPodSets, err := JobPodSets(ctx, job) if err != nil { return false, err } @@ -1215,7 +1215,7 @@ func ConstructWorkload(ctx context.Context, c client.Client, job GenericJob, lab log := ctrl.LoggerFrom(ctx) object := job.Object() - podSets, err := job.PodSets(ctx) + podSets, err := JobPodSets(ctx, job) if err != nil { return nil, err } diff --git a/pkg/controller/jobframework/utils.go b/pkg/controller/jobframework/utils.go new file mode 100644 index 00000000000..91497066d5d --- /dev/null +++ b/pkg/controller/jobframework/utils.go @@ -0,0 +1,66 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jobframework + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/pkg/features" + "sigs.k8s.io/kueue/pkg/util/orderedgroups" +) + +// JobPodSets retrieves the pod sets from a GenericJob and applies environment variable +// deduplication if the SanitizePodSets feature gate is enabled. +func JobPodSets(ctx context.Context, job GenericJob) ([]kueue.PodSet, error) { + podSets, err := job.PodSets(ctx) + if err != nil { + return nil, err + } + SanitizePodSets(podSets) + return podSets, nil +} + +// SanitizePodSets sanitizes all PodSets in the given slice by removing duplicate +// environment variables from each container when the SanitizePodSets +// feature is enabled. This function modifies the podSets slice in place. +func SanitizePodSets(podSets []kueue.PodSet) { + for podSetIndex := range podSets { + SanitizePodSet(&podSets[podSetIndex]) + } +} + +// SanitizePodSet sanitizes a single PodSet by removing duplicate environment +// variables from all containers in its pod template, but only if the +// SanitizePodSets feature gate is enabled. +func SanitizePodSet(podSet *kueue.PodSet) { + if features.Enabled(features.SanitizePodSets) { + for containerIndex := range podSet.Template.Spec.Containers { + container := &podSet.Template.Spec.Containers[containerIndex] + envVarGroups := orderedgroups.NewOrderedGroups[string, corev1.EnvVar]() + for _, envVar := range container.Env { + envVarGroups.Insert(envVar.Name, envVar) + } + container.Env = make([]corev1.EnvVar, 0, len(container.Env)) + for _, envVars := range envVarGroups.InOrder { + container.Env = append(container.Env, envVars[len(envVars)-1]) + } + } + } +} diff --git a/pkg/controller/jobs/job/job_webhook.go b/pkg/controller/jobs/job/job_webhook.go index b437ca3c107..0b9bdee2423 100644 --- a/pkg/controller/jobs/job/job_webhook.go +++ b/pkg/controller/jobs/job/job_webhook.go @@ -227,7 +227,7 @@ func (w *JobWebhook) validateTopologyRequest(ctx context.Context, job *Job) (fie return validationErrs, nil } - podSets, err := job.PodSets(ctx) + podSets, err := jobframework.JobPodSets(ctx, job) if err != nil { return nil, err } diff --git a/pkg/controller/jobs/jobset/jobset_webhook.go b/pkg/controller/jobs/jobset/jobset_webhook.go index 0c20b90e959..483b6624464 100644 --- a/pkg/controller/jobs/jobset/jobset_webhook.go +++ b/pkg/controller/jobs/jobset/jobset_webhook.go @@ -142,7 +142,7 @@ func (w *JobSetWebhook) validateCreate(ctx context.Context, jobSet *JobSet) (fie func (w *JobSetWebhook) validateTopologyRequest(ctx context.Context, jobSet *JobSet) (field.ErrorList, error) { var allErrs field.ErrorList - podSets, podSetsErr := jobSet.PodSets(ctx) + podSets, podSetsErr := jobframework.JobPodSets(ctx, jobSet) if podSetsErr == nil { allErrs = append(allErrs, jobframework.ValidatePodSetGroupingTopology(podSets, buildPodSetAnnotationsPathByNameMap(jobSet))...) diff --git a/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_controller.go b/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_controller.go index bb7e3795dd8..52d3b922e33 100644 --- a/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_controller.go +++ b/pkg/controller/jobs/kubeflow/kubeflowjob/kubeflowjob_controller.go @@ -187,7 +187,7 @@ func (j *KubeflowJob) ValidateOnCreate(ctx context.Context) (field.ErrorList, er return nil, nil } - podSets, podSetsErr := j.PodSets(ctx) + podSets, podSetsErr := jobframework.JobPodSets(ctx, j) var allErrs field.ErrorList replicaTypes := j.OrderedReplicaTypes() diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go index 615dff3317d..8646588fa3e 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler.go @@ -200,17 +200,23 @@ func (r *Reconciler) constructWorkload(lws *leaderworkersetv1.LeaderWorkerSet, w return createdWorkload, nil } +func newPodSet(name kueue.PodSetReference, count int32, template *corev1.PodTemplateSpec) kueue.PodSet { + podSet := kueue.PodSet{ + Name: name, + Count: count, + Template: corev1.PodTemplateSpec{ + Spec: *template.Spec.DeepCopy(), + }, + } + jobframework.SanitizePodSet(&podSet) + return podSet +} + func podSets(lws *leaderworkersetv1.LeaderWorkerSet) ([]kueue.PodSet, error) { podSets := make([]kueue.PodSet, 0, 2) if lws.Spec.LeaderWorkerTemplate.LeaderTemplate != nil { - podSet := kueue.PodSet{ - Name: leaderPodSetName, - Count: 1, - Template: corev1.PodTemplateSpec{ - Spec: *lws.Spec.LeaderWorkerTemplate.LeaderTemplate.Spec.DeepCopy(), - }, - } + podSet := newPodSet(leaderPodSetName, 1, lws.Spec.LeaderWorkerTemplate.LeaderTemplate) if features.Enabled(features.TopologyAwareScheduling) { topologyRequest, err := jobframework.NewPodSetTopologyRequest( &lws.Spec.LeaderWorkerTemplate.LeaderTemplate.ObjectMeta).Build() @@ -232,14 +238,7 @@ func podSets(lws *leaderworkersetv1.LeaderWorkerSet) ([]kueue.PodSet, error) { defaultPodSetCount-- } - podSet := kueue.PodSet{ - Name: defaultPodSetName, - Count: defaultPodSetCount, - Template: corev1.PodTemplateSpec{ - Spec: *lws.Spec.LeaderWorkerTemplate.WorkerTemplate.Spec.DeepCopy(), - }, - } - + podSet := newPodSet(defaultPodSetName, defaultPodSetCount, &lws.Spec.LeaderWorkerTemplate.WorkerTemplate) if features.Enabled(features.TopologyAwareScheduling) { topologyRequest, err := jobframework.NewPodSetTopologyRequest( &lws.Spec.LeaderWorkerTemplate.WorkerTemplate.ObjectMeta).PodIndexLabel( diff --git a/pkg/controller/jobs/mpijob/mpijob_webhook.go b/pkg/controller/jobs/mpijob/mpijob_webhook.go index bfac8731e47..03a09d6d0ce 100644 --- a/pkg/controller/jobs/mpijob/mpijob_webhook.go +++ b/pkg/controller/jobs/mpijob/mpijob_webhook.go @@ -155,7 +155,7 @@ func (w *MpiJobWebhook) validateCommon(ctx context.Context, mpiJob *MPIJob) (fie func (w *MpiJobWebhook) validateTopologyRequest(ctx context.Context, mpiJob *MPIJob) (field.ErrorList, error) { var allErrs field.ErrorList - podSets, podSetsErr := mpiJob.PodSets(ctx) + podSets, podSetsErr := jobframework.JobPodSets(ctx, mpiJob) if podSetsErr == nil { allErrs = append(allErrs, jobframework.ValidatePodSetGroupingTopology(podSets, podSetAnnotationsPathByName)...) diff --git a/pkg/controller/jobs/pod/pod_controller.go b/pkg/controller/jobs/pod/pod_controller.go index 693d29610e0..62274891752 100644 --- a/pkg/controller/jobs/pod/pod_controller.go +++ b/pkg/controller/jobs/pod/pod_controller.go @@ -1063,7 +1063,7 @@ func (p *Pod) ConstructComposableWorkload(ctx context.Context, c client.Client, p.list.Items = activePods[:len(activePods)-excessPodsCount] } - podSets, err := p.PodSets(ctx) + podSets, err := jobframework.JobPodSets(ctx, p) if err != nil { if jobframework.IsUnretryableError(err) { r.Eventf(p.Object(), corev1.EventTypeWarning, jobframework.ReasonErrWorkloadCompose, err.Error()) diff --git a/pkg/controller/jobs/raycluster/raycluster_webhook.go b/pkg/controller/jobs/raycluster/raycluster_webhook.go index e31ac1e20bb..607c4d776d9 100644 --- a/pkg/controller/jobs/raycluster/raycluster_webhook.go +++ b/pkg/controller/jobs/raycluster/raycluster_webhook.go @@ -196,7 +196,7 @@ func validateElasticJob(job *rayv1.RayCluster) field.ErrorList { func (w *RayClusterWebhook) validateTopologyRequest(ctx context.Context, rayJob *RayCluster) (field.ErrorList, error) { var allErrs field.ErrorList - podSets, podSetsErr := rayJob.PodSets(ctx) + podSets, podSetsErr := jobframework.JobPodSets(ctx, rayJob) allErrs = append(allErrs, jobframework.ValidateTASPodSetRequest(headGroupMetaPath, &rayJob.Spec.HeadGroupSpec.Template.ObjectMeta)...) diff --git a/pkg/controller/jobs/rayjob/rayjob_webhook.go b/pkg/controller/jobs/rayjob/rayjob_webhook.go index 98cbcbf195c..dc87d0bca2c 100644 --- a/pkg/controller/jobs/rayjob/rayjob_webhook.go +++ b/pkg/controller/jobs/rayjob/rayjob_webhook.go @@ -177,7 +177,7 @@ func (w *RayJobWebhook) validateTopologyRequest(ctx context.Context, rayJob *ray return allErrs, nil } - podSets, podSetsErr := (*RayJob)(rayJob).PodSets(ctx) + podSets, podSetsErr := jobframework.JobPodSets(ctx, (*RayJob)(rayJob)) allErrs = append(allErrs, jobframework.ValidateTASPodSetRequest(headGroupMetaPath, &rayJob.Spec.RayClusterSpec.HeadGroupSpec.Template.ObjectMeta)...) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 6056a64e545..d36f55d6bf6 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -198,6 +198,15 @@ const ( // // Enable all updates to Workload objects to use Patch Merge instead of Patch Apply. WorkloadRequestUseMergePatch featuregate.Feature = "WorkloadRequestUseMergePatch" + + // owner: @mbobrovskyi + // + // SanitizePodSets enables automatic sanitization of PodSets when creating the Workload object. + // The main use case it deduplication of environment variables + // in PodSet templates within Workload objects. When enabled, duplicate env var entries + // are resolved by keeping only the last occurrence, allowing workload creation to succeed + // even when duplicates are present in the spec. + SanitizePodSets featuregate.Feature = "SanitizePodSets" ) func init() { @@ -313,6 +322,9 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned WorkloadRequestUseMergePatch: { {Version: version.MustParse("0.14"), Default: false, PreRelease: featuregate.Alpha}, }, + SanitizePodSets: { + {Version: version.MustParse("0.15"), Default: true, PreRelease: featuregate.Beta}, + }, } func SetFeatureGateDuringTest(tb testing.TB, f featuregate.Feature, value bool) { diff --git a/pkg/util/testing/wrappers.go b/pkg/util/testing/wrappers.go index 97fb3721792..ef342b46853 100644 --- a/pkg/util/testing/wrappers.go +++ b/pkg/util/testing/wrappers.go @@ -162,6 +162,12 @@ func (c *ContainerWrapper) Name(name string) *ContainerWrapper { return c } +// Image sets the image of the container. +func (c *ContainerWrapper) Image(image string) *ContainerWrapper { + c.Container.Image = image + return c +} + // WithResourceReq appends a resource request to the container. func (c *ContainerWrapper) WithResourceReq(resourceName corev1.ResourceName, quantity string) *ContainerWrapper { requests := utilResource.MergeResourceListKeepFirst(c.Resources.Requests, corev1.ResourceList{ @@ -182,6 +188,12 @@ func (c *ContainerWrapper) WithResourceLimit(resourceName corev1.ResourceName, q return c } +// WithEnvVar appends a env variable to the container. +func (c *ContainerWrapper) WithEnvVar(envVar corev1.EnvVar) *ContainerWrapper { + c.Env = append(c.Env, envVar) + return c +} + // AsSidecar makes the container a sidecar when used as an Init Container. func (c *ContainerWrapper) AsSidecar() *ContainerWrapper { c.RestartPolicy = ptr.To(corev1.ContainerRestartPolicyAlways) diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 52966289160..a7b4966c707 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -296,6 +296,7 @@ spec: | `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | | `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | +| `SanitizePodSets` | `true` | Beta | 0.15 | | ### Feature gates for graduated or deprecated features diff --git a/site/content/zh-CN/docs/installation/_index.md b/site/content/zh-CN/docs/installation/_index.md index 8ad673b0312..ffeb3341f2e 100644 --- a/site/content/zh-CN/docs/installation/_index.md +++ b/site/content/zh-CN/docs/installation/_index.md @@ -291,6 +291,7 @@ spec: | `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | | `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | +| `SanitizePodSets` | `true` | Beta | 0.15 | | ### 已毕业或已弃用特性的特性门控 {#feature-gates-for-graduated-or-deprecated-features} diff --git a/test/e2e/singlecluster/e2e_test.go b/test/e2e/singlecluster/e2e_test.go index 9dd1deab927..563e34e8faa 100644 --- a/test/e2e/singlecluster/e2e_test.go +++ b/test/e2e/singlecluster/e2e_test.go @@ -31,6 +31,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" + "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/slices" "sigs.k8s.io/kueue/pkg/util/testing" @@ -495,6 +496,104 @@ var _ = ginkgo.Describe("Kueue", func() { }) }) + ginkgo.It("Should deduplicate env variables", func() { + highPriorityClass := testing.MakePriorityClass("high").PriorityValue(100).Obj() + util.MustCreate(ctx, k8sClient, highPriorityClass) + ginkgo.DeferCleanup(func() { + gomega.Expect(k8sClient.Delete(ctx, highPriorityClass)).To(gomega.Succeed()) + }) + + lowJob := testingjob.MakeJob("low", ns.Name). + Queue(kueue.LocalQueueName(localQueue.Name)). + Parallelism(1). + NodeSelector("instance-type", "on-demand"). + Containers( + *testing.MakeContainer(). + Name("c"). + Image("sleep"). + WithResourceReq(corev1.ResourceCPU, "1"). + WithEnvVar(corev1.EnvVar{Name: "TEST_ENV", Value: "test1"}). + WithEnvVar(corev1.EnvVar{Name: "TEST_ENV", Value: "test2"}). + Obj(), + ). + Obj() + + ginkgo.By("Creating a low-priority job with duplicated environment variables", func() { + util.MustCreate(ctx, k8sClient, lowJob) + }) + + lowCreatedWorkload := &kueue.Workload{} + lowWlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(lowJob.Name, lowJob.UID), Namespace: ns.Name} + + ginkgo.By("Checking that the low-priority workload is created with deduplicated environment variables", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, lowWlLookupKey, lowCreatedWorkload)).Should(gomega.Succeed()) + g.Expect(lowCreatedWorkload.Spec.PodSets).Should(gomega.HaveLen(1)) + g.Expect(lowCreatedWorkload.Spec.PodSets[0].Template.Spec.Containers).Should(gomega.HaveLen(1)) + g.Expect(lowCreatedWorkload.Spec.PodSets[0].Template.Spec.Containers[0].Env).Should(gomega.BeComparableTo( + []corev1.EnvVar{{Name: "TEST_ENV", Value: "test2"}}, + )) + g.Expect(workload.IsAdmitted(lowCreatedWorkload)).Should(gomega.BeTrue()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + highJob := testingjob.MakeJob("high", ns.Name). + Queue(kueue.LocalQueueName(localQueue.Name)). + Parallelism(1). + PriorityClass(highPriorityClass.Name). + Request(corev1.ResourceCPU, "1"). + NodeSelector("instance-type", "on-demand"). + Obj() + + ginkgo.By("Creating a high-priority job", func() { + util.MustCreate(ctx, k8sClient, highJob) + }) + + highCreatedWorkload := &kueue.Workload{} + highWlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(lowJob.Name, lowJob.UID), Namespace: ns.Name} + + ginkgo.By("Checking that the high-priority workload is created and admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, highWlLookupKey, highCreatedWorkload)).Should(gomega.Succeed()) + g.Expect(workload.IsAdmitted(highCreatedWorkload)).Should(gomega.BeTrue()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Checking that the low-priority workload is successfully preempted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, lowWlLookupKey, lowCreatedWorkload)).Should(gomega.Succeed()) + g.Expect(workload.IsEvicted(lowCreatedWorkload)).Should(gomega.BeTrue()) + g.Expect(lowCreatedWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadPreempted)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Checking that the low-priority job still has duplication of environment variables", func() { + createdLowJob := &batchv1.Job{} + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lowJob), createdLowJob)).Should(gomega.Succeed()) + g.Expect(createdLowJob.Spec.Template.Spec.Containers).Should(gomega.HaveLen(1)) + g.Expect(createdLowJob.Spec.Template.Spec.Containers[0].Env).Should(gomega.BeComparableTo( + []corev1.EnvVar{ + {Name: "TEST_ENV", Value: "test1"}, + {Name: "TEST_ENV", Value: "test2"}, + }, + )) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + // Verify that the workload is not being continuously reconciled and updated + // due to a misbehaving EquivalentToWorkload implementation. + ginkgo.By("Use events to observe the workload is not updated", func() { + eventList := &corev1.EventList{} + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, eventList, &client.ListOptions{Namespace: ns.Name})).To(gomega.Succeed()) + for _, event := range eventList.Items { + g.Expect(event.Reason).ShouldNot(gomega.Equal(jobframework.ReasonUpdatedWorkload)) + } + }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) + }) + }) + ginkgo.It("Should not allow removing the workload's priority through the job", func() { samplePriority := "sample-priority" samplePriorityClass := utiltestingapi.MakeWorkloadPriorityClass(samplePriority).PriorityValue(100).Obj() From a24556e533e70d98f0c62872d8a4e1b813e16dc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Irving=20Mondrag=C3=B3n?= Date: Thu, 30 Oct 2025 09:28:04 +0100 Subject: [PATCH 051/119] Fix multikueue/provisioning indexer conflict setup (#7432) * Fix multikueue/provisioning indexer conflict setup * Update unit tests --- .../admissionchecks/multikueue/indexer.go | 14 -------------- .../admissionchecks/multikueue/indexer_test.go | 2 ++ .../admissionchecks/provisioning/indexer.go | 11 ----------- .../admissionchecks/provisioning/indexer_test.go | 2 ++ pkg/controller/core/indexer/indexer.go | 14 ++++++++++++++ 5 files changed, 18 insertions(+), 25 deletions(-) diff --git a/pkg/controller/admissionchecks/multikueue/indexer.go b/pkg/controller/admissionchecks/multikueue/indexer.go index 4a41079e7e8..03c2370eb61 100644 --- a/pkg/controller/admissionchecks/multikueue/indexer.go +++ b/pkg/controller/admissionchecks/multikueue/indexer.go @@ -25,7 +25,6 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/admissioncheck" - "sigs.k8s.io/kueue/pkg/util/slices" ) const ( @@ -57,16 +56,6 @@ func indexUsingMultiKueueClusters(obj client.Object) []string { return config.Spec.Clusters } -func indexWorkloadsAdmissionChecks(obj client.Object) []string { - workload, isWorkload := obj.(*kueue.Workload) - if !isWorkload || len(workload.Status.AdmissionChecks) == 0 { - return nil - } - return slices.Map(workload.Status.AdmissionChecks, func(checkState *kueue.AdmissionCheckState) string { - return string(checkState.Name) - }) -} - func SetupIndexer(ctx context.Context, indexer client.FieldIndexer, configNamespace string) error { if err := indexer.IndexField(ctx, &kueue.MultiKueueCluster{}, UsingKubeConfigs, getIndexUsingKubeConfigs(configNamespace)); err != nil { return fmt.Errorf("setting index on clusters using kubeconfig: %w", err) @@ -77,8 +66,5 @@ func SetupIndexer(ctx context.Context, indexer client.FieldIndexer, configNamesp if err := indexer.IndexField(ctx, &kueue.AdmissionCheck{}, AdmissionCheckUsingConfigKey, admissioncheck.IndexerByConfigFunction(kueue.MultiKueueControllerName, configGVK)); err != nil { return fmt.Errorf("setting index on admission checks config: %w", err) } - if err := indexer.IndexField(ctx, &kueue.Workload{}, WorkloadsWithAdmissionCheckKey, indexWorkloadsAdmissionChecks); err != nil { - return fmt.Errorf("setting index on workloads admission checks: %w", err) - } return nil } diff --git a/pkg/controller/admissionchecks/multikueue/indexer_test.go b/pkg/controller/admissionchecks/multikueue/indexer_test.go index 3f746c59318..de13218d76c 100644 --- a/pkg/controller/admissionchecks/multikueue/indexer_test.go +++ b/pkg/controller/admissionchecks/multikueue/indexer_test.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" @@ -53,6 +54,7 @@ func getClientBuilder(ctx context.Context) *fake.ClientBuilder { })) builder := fake.NewClientBuilder().WithScheme(scheme).WithObjects(utiltesting.MakeNamespace(TestNamespace)) + _ = indexer.Setup(ctx, utiltesting.AsIndexer(builder)) _ = SetupIndexer(ctx, utiltesting.AsIndexer(builder), TestNamespace) return builder } diff --git a/pkg/controller/admissionchecks/provisioning/indexer.go b/pkg/controller/admissionchecks/provisioning/indexer.go index 060b7fe787e..9e7c525ac23 100644 --- a/pkg/controller/admissionchecks/provisioning/indexer.go +++ b/pkg/controller/admissionchecks/provisioning/indexer.go @@ -49,21 +49,10 @@ func indexRequestsOwner(obj client.Object) []string { return slices.Map(refs, func(r *metav1.OwnerReference) string { return r.Name }) } -func indexWorkloadsChecks(obj client.Object) []string { - wl, isWl := obj.(*kueue.Workload) - if !isWl || len(wl.Status.AdmissionChecks) == 0 { - return nil - } - return slices.Map(wl.Status.AdmissionChecks, func(c *kueue.AdmissionCheckState) string { return string(c.Name) }) -} - func SetupIndexer(ctx context.Context, indexer client.FieldIndexer) error { if err := indexer.IndexField(ctx, &autoscaling.ProvisioningRequest{}, RequestsOwnedByWorkloadKey, indexRequestsOwner); err != nil { return fmt.Errorf("setting index on provisionRequest owner: %w", err) } - if err := indexer.IndexField(ctx, &kueue.Workload{}, WorkloadsWithAdmissionCheckKey, indexWorkloadsChecks); err != nil { - return fmt.Errorf("setting index on workloads checks: %w", err) - } if err := indexer.IndexField(ctx, &kueue.AdmissionCheck{}, AdmissionCheckUsingConfigKey, admissioncheck.IndexerByConfigFunction(kueue.ProvisioningRequestControllerName, configGVK)); err != nil { return fmt.Errorf("setting index on admission checks config: %w", err) diff --git a/pkg/controller/admissionchecks/provisioning/indexer_test.go b/pkg/controller/admissionchecks/provisioning/indexer_test.go index 02107bf3d2d..90a3bf817f8 100644 --- a/pkg/controller/admissionchecks/provisioning/indexer_test.go +++ b/pkg/controller/admissionchecks/provisioning/indexer_test.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" @@ -47,6 +48,7 @@ func getClientBuilder(ctx context.Context) (*fake.ClientBuilder, context.Context utilruntime.Must(autoscaling.AddToScheme(scheme)) builder := fake.NewClientBuilder().WithScheme(scheme).WithObjects(utiltesting.MakeNamespace(TestNamespace)) + _ = indexer.Setup(ctx, utiltesting.AsIndexer(builder)) _ = SetupIndexer(ctx, utiltesting.AsIndexer(builder)) return builder, ctx } diff --git a/pkg/controller/core/indexer/indexer.go b/pkg/controller/core/indexer/indexer.go index 8c5aeff1cea..aebbd6540a9 100644 --- a/pkg/controller/core/indexer/indexer.go +++ b/pkg/controller/core/indexer/indexer.go @@ -40,6 +40,7 @@ const ( WorkloadQuotaReservedKey = "status.quotaReserved" WorkloadRuntimeClassKey = "spec.runtimeClass" OwnerReferenceUID = "metadata.ownerReferences.uid" + WorkloadAdmissionCheckKey = "status.admissionChecks" // OwnerReferenceGroupKindFmt defines the format string used to construct a field path // for indexing or matching against a specific owner Group and Kind in a Kubernetes object's metadata. @@ -166,6 +167,16 @@ func IndexOwnerUID(obj client.Object) []string { return slices.Map(obj.GetOwnerReferences(), func(o *metav1.OwnerReference) string { return string(o.UID) }) } +func IndexWorkloadAdmissionCheck(obj client.Object) []string { + wl, ok := obj.(*kueue.Workload) + if !ok || len(wl.Status.AdmissionChecks) == 0 { + return nil + } + return slices.Map(wl.Status.AdmissionChecks, func(checkState *kueue.AdmissionCheckState) string { + return string(checkState.Name) + }) +} + // Setup sets the index with the given fields for core apis. func Setup(ctx context.Context, indexer client.FieldIndexer) error { if err := indexer.IndexField(ctx, &kueue.Workload{}, WorkloadQueueKey, IndexWorkloadQueue); err != nil { @@ -180,6 +191,9 @@ func Setup(ctx context.Context, indexer client.FieldIndexer) error { if err := indexer.IndexField(ctx, &kueue.Workload{}, WorkloadRuntimeClassKey, IndexWorkloadRuntimeClass); err != nil { return fmt.Errorf("setting index on runtimeClass for Workload: %w", err) } + if err := indexer.IndexField(ctx, &kueue.Workload{}, WorkloadAdmissionCheckKey, IndexWorkloadAdmissionCheck); err != nil { + return fmt.Errorf("setting index on admissionCheck for Workload: %w", err) + } if err := indexer.IndexField(ctx, &kueue.LocalQueue{}, QueueClusterQueueKey, IndexQueueClusterQueue); err != nil { return fmt.Errorf("setting index on clusterQueue for localQueue: %w", err) } From 5891828904448042464098489097fed5adc71710 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 30 Oct 2025 17:28:04 +0530 Subject: [PATCH 052/119] Fix SanitizePodSets feature gate version. (#7444) --- pkg/features/kube_features.go | 2 +- site/content/en/docs/installation/_index.md | 8 ++++++-- site/content/zh-CN/docs/installation/_index.md | 6 +++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index d36f55d6bf6..96121a272bb 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -323,7 +323,7 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned {Version: version.MustParse("0.14"), Default: false, PreRelease: featuregate.Alpha}, }, SanitizePodSets: { - {Version: version.MustParse("0.15"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("0.13"), Default: true, PreRelease: featuregate.Beta}, }, } diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index a7b4966c707..4405f1fe36e 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -269,7 +269,7 @@ spec: | `MultiKueue` | `false` | Alpha | 0.6 | 0.8 | | `MultiKueue` | `true` | Beta | 0.9 | | | `MultiKueueBatchJobWithManagedBy` | `false` | Alpha | 0.8 | 0.15 | -| `MultiKueueBatchJobWithManagedBy` | `true` | Beta | 0.15 | | +| `MultiKueueBatchJobWithManagedBy` | `true` | Beta | 0.15 | | | `PartialAdmission` | `false` | Alpha | 0.4 | 0.4 | | `PartialAdmission` | `true` | Beta | 0.5 | | | `VisibilityOnDemand` | `false` | Alpha | 0.6 | 0.8 | @@ -296,7 +296,11 @@ spec: | `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | | `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | -| `SanitizePodSets` | `true` | Beta | 0.15 | | +| `SanitizePodSets` | `true` | Beta | 0.13 | | + +{{% alert title="Note" color="primary" %}} +The SanitizePodSets feature is available starting from versions 0.13.8 and 0.14.3. +{{% /alert %}} ### Feature gates for graduated or deprecated features diff --git a/site/content/zh-CN/docs/installation/_index.md b/site/content/zh-CN/docs/installation/_index.md index ffeb3341f2e..635ce5ecb59 100644 --- a/site/content/zh-CN/docs/installation/_index.md +++ b/site/content/zh-CN/docs/installation/_index.md @@ -263,8 +263,8 @@ spec: ### Alpha 和 Beta 级别特性的特性门控 {#feature-gates-for-alpha-and-beta-features} -| 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | -| --------------------------------------------- | ------- | ----- | -------- | -------- | +| 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | +| --------------------------------------------- | ------- | ----- |----------| -------- | | `FlavorFungibility` | `true` | Beta | 0.5 | | | `MultiKueue` | `false` | Alpha | 0.6 | 0.8 | | `MultiKueue` | `true` | Beta | 0.9 | | @@ -291,7 +291,7 @@ spec: | `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | | `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | -| `SanitizePodSets` | `true` | Beta | 0.15 | | +| `SanitizePodSets` | `true` | Beta | 0.13 | | ### 已毕业或已弃用特性的特性门控 {#feature-gates-for-graduated-or-deprecated-features} From d0338b210ce9cf9e3af8c1ce332f5898d532648a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Thu, 30 Oct 2025 13:56:05 +0100 Subject: [PATCH 053/119] MultiKueue remote client kubeconfig validation (#7439) * Detect invalid/insecure MultiKueue kubeconfigs * Update MultiKueue setup tests * Add skip validation flag * Update after review --- .../multikueue/multikueuecluster.go | 97 +++++++++ .../multikueue/multikueuecluster_test.go | 203 +++++++++++++++--- .../multikueue/testdata/worker1KubeConfig | 1 - pkg/features/kube_features.go | 10 + pkg/util/testing/kubeconfig.go | 103 +++++++-- site/content/en/docs/installation/_index.md | 1 + .../en/docs/tasks/manage/setup_multikueue.md | 8 + .../content/zh-CN/docs/installation/_index.md | 1 + test/integration/multikueue/setup_test.go | 203 ++++++++++++++++++ 9 files changed, 575 insertions(+), 52 deletions(-) delete mode 100644 pkg/controller/admissionchecks/multikueue/testdata/worker1KubeConfig diff --git a/pkg/controller/admissionchecks/multikueue/multikueuecluster.go b/pkg/controller/admissionchecks/multikueue/multikueuecluster.go index 278a585f0e7..ba9b1c0e4f2 100644 --- a/pkg/controller/admissionchecks/multikueue/multikueuecluster.go +++ b/pkg/controller/admissionchecks/multikueue/multikueuecluster.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" "maps" + "net" + "net/url" "os" "slices" "strings" @@ -36,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/workqueue" @@ -52,6 +55,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" + "sigs.k8s.io/kueue/pkg/features" ) const ( @@ -414,6 +418,19 @@ func (c *clustersReconciler) Reconcile(ctx context.Context, req reconcile.Reques return reconcile.Result{}, c.updateStatus(ctx, cluster, false, "BadConfig", err.Error()) } + if features.Enabled(features.MultiKueueAllowInsecureKubeconfigs) { + log.V(3).Info("Feature MultiKueueAllowInsecureKubeconfigs is enabled, skipping kubeconfig validation") + } else { + err = validateKubeconfig(kubeConfig) + if err != nil { + log.Error(err, "validating kubeconfig failed") + if updateErr := c.updateStatus(ctx, cluster, false, "InsecureKubeConfig", fmt.Sprintf("insecure kubeconfig: %v", err)); updateErr != nil { + return reconcile.Result{}, fmt.Errorf("failed to update MultiKueueCluster status: %w after detecting insecure kubeconfig: %w", updateErr, err) + } + return reconcile.Result{}, fmt.Errorf("validating kubeconfig failed: %w", err) + } + } + if retryAfter, err := c.setRemoteClientConfig(ctx, cluster.Name, kubeConfig, c.origin); err != nil { log.Error(err, "setting kubeconfig", "retryAfter", retryAfter) if err := c.updateStatus(ctx, cluster, false, "ClientConnectionFailed", err.Error()); err != nil { @@ -426,6 +443,86 @@ func (c *clustersReconciler) Reconcile(ctx context.Context, req reconcile.Reques return reconcile.Result{}, client.IgnoreNotFound(c.updateStatus(ctx, cluster, true, "Active", "Connected")) } +// validateKubeconfig checks that the provided kubeconfig content is safe to use +// in a MultiKueueCluster context. +func validateKubeconfig(kubeconfig []byte) error { + // Parse the raw kubeconfig + config, err := clientcmd.Load(kubeconfig) + if err != nil { + return fmt.Errorf("failed to parse kubeconfig(apiconfig): %v", err) + } + + if config.AuthInfos != nil { + for _, authInfo := range config.AuthInfos { + // Token is allowed due to service account tokens usage + if authInfo.TokenFile != "" { + return errors.New("tokenFile is not allowed") + } + } + } + + // Validate each cluster config + for name, cluster := range config.Clusters { + // Require TLS + if cluster.InsecureSkipTLSVerify { + return fmt.Errorf("insecure TLS verification is not allowed for cluster %s", name) + } + // Require CA cert data (not file paths) + if cluster.CertificateAuthority != "" { + return fmt.Errorf("certificate-authority file paths are not allowed, use certificate-authority-data for cluster %s", name) + } + // Validate server URL + if cluster.Server == "" { + return fmt.Errorf("server URL is required for cluster %s", name) + } + } + + // Get the restconfig to validate the final settings + restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig) + if err != nil { + return err + } + + // Block dangerous auth mechanisms + // BearerToken is allowed due to service account tokens usage + if restConfig.BearerTokenFile != "" { + return errors.New("bearerTokenFile is not allowed") + } + // lowercase the host to allow FQDN with uppercase letters + if !isValidHostname(strings.ToLower(restConfig.Host)) { + return errors.New("untrusted server endpoint") + } + if restConfig.Username != "" || restConfig.Password != "" { + return errors.New("basic auth is not allowed") + } + if restConfig.ExecProvider != nil { + return errors.New("exec plugins are not allowed") + } + if restConfig.AuthProvider != nil { + return errors.New("auth providers are not allowed") + } + + return nil +} + +func isValidHostname(server string) bool { + u, err := url.Parse(server) + if err != nil { + return false + } + host, _, err := net.SplitHostPort(u.Host) + if err != nil { + host = u.Host // If no port is present + } + // Check if host is a valid IP + if net.ParseIP(host) != nil { + return true + } + // Check if host is a valid DNS hostname + errs := validation.IsDNS1123Subdomain(host) + return len(errs) == 0 +} + func (c *clustersReconciler) getKubeConfig(ctx context.Context, ref *kueue.KubeConfig) ([]byte, bool, error) { if ref.LocationType == kueue.SecretLocationType { return c.getKubeConfigFromSecret(ctx, ref.Location) diff --git a/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go b/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go index 594cdbb476e..a78c1c9a631 100644 --- a/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go +++ b/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go @@ -19,6 +19,10 @@ package multikueue import ( "context" "errors" + "fmt" + "os" + "path/filepath" + "strings" "testing" "time" @@ -30,12 +34,16 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobframework" + "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/slices" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" @@ -51,13 +59,13 @@ var ( func fakeClientBuilder(ctx context.Context) func([]byte, client.Options) (client.WithWatch, error) { return func(kubeconfig []byte, _ client.Options) (client.WithWatch, error) { - if string(kubeconfig) == "invalid" { + if strings.Contains(string(kubeconfig), "invalid") { return nil, errInvalidConfig } b := getClientBuilder(ctx) b = b.WithInterceptorFuncs(interceptor.Funcs{ Watch: func(ctx context.Context, client client.WithWatch, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { - if string(kubeconfig) == "nowatch" { + if strings.Contains(string(kubeconfig), "nowatch") { return nil, errCannotWatch } return client.Watch(ctx, obj, opts...) @@ -98,10 +106,33 @@ func makeTestSecret(name string, kubeconfig string) corev1.Secret { } } +func kubeconfigBase(user string) *utiltesting.TestKubeconfigWrapper { + return utiltesting.NewTestKubeConfigWrapper(). + Cluster("test", "https://10.10.10.10", []byte{'-', '-', '-', '-', '-'}). + User(user, nil, nil). + Context("test-context", "test", user). + CurrentContext("test-context") +} + +func testKubeconfig(user string) string { + kubeconfig, _ := kubeconfigBase(user). + TokenAuthInfo(user, "FAKE-TOKEN-123456"). + Build() + return string(kubeconfig) +} + +func testKubeconfigInsecure(user string, tokenFile *string) string { + kubeconfig, _ := kubeconfigBase(user). + TokenFileAuthInfo(user, *tokenFile). + Build() + return string(kubeconfig) +} + func TestUpdateConfig(t *testing.T) { ctx, _ := utiltesting.ContextWithLog(t) cancelCalledCount := 0 cancelCalled := func() { cancelCalledCount++ } + validKubeconfigLocation := filepath.Join(t.TempDir(), "worker1KubeConfig") cases := map[string]struct { reconcileFor string @@ -109,10 +140,12 @@ func TestUpdateConfig(t *testing.T) { clusters []kueue.MultiKueueCluster secrets []corev1.Secret - wantRemoteClients map[string]*remoteClient - wantClusters []kueue.MultiKueueCluster - wantRequeueAfter time.Duration - wantCancelCalled int + wantRemoteClients map[string]*remoteClient + wantClusters []kueue.MultiKueueCluster + wantRequeueAfter time.Duration + wantCancelCalled int + wantErr error + skipInsecureKubeconfig bool }{ "new valid client is added": { reconcileFor: "worker1", @@ -123,7 +156,7 @@ func TestUpdateConfig(t *testing.T) { Obj(), }, secrets: []corev1.Secret{ - makeTestSecret("worker1", "worker1 kubeconfig"), + makeTestSecret("worker1", testKubeconfig("worker1")), }, wantClusters: []kueue.MultiKueueCluster{ *utiltestingapi.MakeMultiKueueCluster("worker1"). @@ -134,7 +167,7 @@ func TestUpdateConfig(t *testing.T) { }, wantRemoteClients: map[string]*remoteClient{ "worker1": { - kubeconfig: []byte("worker1 kubeconfig"), + kubeconfig: []byte(testKubeconfig("worker1")), }, }, }, @@ -147,7 +180,7 @@ func TestUpdateConfig(t *testing.T) { Obj(), }, secrets: []corev1.Secret{ - makeTestSecret("worker1", "worker1 kubeconfig"), + makeTestSecret("worker1", testKubeconfig("worker1")), }, remoteClients: map[string]*remoteClient{ "worker1": newTestClient(ctx, "worker1 old kubeconfig", cancelCalled), @@ -161,7 +194,7 @@ func TestUpdateConfig(t *testing.T) { }, wantRemoteClients: map[string]*remoteClient{ "worker1": { - kubeconfig: []byte("worker1 kubeconfig"), + kubeconfig: []byte(testKubeconfig("worker1")), }, }, wantCancelCalled: 1, @@ -170,23 +203,23 @@ func TestUpdateConfig(t *testing.T) { reconcileFor: "worker1", clusters: []kueue.MultiKueueCluster{ *utiltestingapi.MakeMultiKueueCluster("worker1"). - KubeConfig(kueue.PathLocationType, "testdata/worker1KubeConfig"). + KubeConfig(kueue.PathLocationType, validKubeconfigLocation). Generation(1). Obj(), }, remoteClients: map[string]*remoteClient{ - "worker1": newTestClient(ctx, "worker1 old kubeconfig", cancelCalled), + "worker1": newTestClient(ctx, testKubeconfig("worker1_old"), cancelCalled), }, wantClusters: []kueue.MultiKueueCluster{ *utiltestingapi.MakeMultiKueueCluster("worker1"). - KubeConfig(kueue.PathLocationType, "testdata/worker1KubeConfig"). + KubeConfig(kueue.PathLocationType, validKubeconfigLocation). Active(metav1.ConditionTrue, "Active", "Connected", 1). Generation(1). Obj(), }, wantRemoteClients: map[string]*remoteClient{ "worker1": { - kubeconfig: []byte("worker1 kubeconfig"), + kubeconfig: []byte(testKubeconfig("worker1")), }, }, wantCancelCalled: 1, @@ -200,13 +233,13 @@ func TestUpdateConfig(t *testing.T) { Obj(), }, secrets: []corev1.Secret{ - makeTestSecret("worker1", "invalid"), + makeTestSecret("worker1", testKubeconfig("invalid")), }, remoteClients: map[string]*remoteClient{ "worker1": newTestClient(ctx, "worker1 old kubeconfig", cancelCalled), }, wantRemoteClients: map[string]*remoteClient{ - "worker1": newTestClient(ctx, "invalid", nil), + "worker1": newTestClient(ctx, testKubeconfig("invalid"), nil), }, wantClusters: []kueue.MultiKueueCluster{ *utiltestingapi.MakeMultiKueueCluster("worker1"). @@ -271,13 +304,13 @@ func TestUpdateConfig(t *testing.T) { Obj(), }, secrets: []corev1.Secret{ - makeTestSecret("worker1", "nowatch"), + makeTestSecret("worker1", testKubeconfig("nowatch")), }, remoteClients: map[string]*remoteClient{ "worker1": newTestClient(ctx, "worker1 old kubeconfig", cancelCalled), }, wantRemoteClients: map[string]*remoteClient{ - "worker1": setReconnectState(newTestClient(ctx, "nowatch", nil), 1), + "worker1": setReconnectState(newTestClient(ctx, testKubeconfig("nowatch"), nil), 1), }, wantClusters: []kueue.MultiKueueCluster{ *utiltestingapi.MakeMultiKueueCluster("worker1"). @@ -299,13 +332,13 @@ func TestUpdateConfig(t *testing.T) { Obj(), }, secrets: []corev1.Secret{ - makeTestSecret("worker1", "nowatch"), + makeTestSecret("worker1", testKubeconfig("nowatch")), }, remoteClients: map[string]*remoteClient{ - "worker1": setReconnectState(newTestClient(ctx, "nowatch", cancelCalled), 2), + "worker1": setReconnectState(newTestClient(ctx, testKubeconfig("nowatch"), cancelCalled), 2), }, wantRemoteClients: map[string]*remoteClient{ - "worker1": setReconnectState(newTestClient(ctx, "nowatch", nil), 3), + "worker1": setReconnectState(newTestClient(ctx, testKubeconfig("nowatch"), nil), 3), }, wantClusters: []kueue.MultiKueueCluster{ *utiltestingapi.MakeMultiKueueCluster("worker1"). @@ -327,13 +360,13 @@ func TestUpdateConfig(t *testing.T) { Obj(), }, secrets: []corev1.Secret{ - makeTestSecret("worker1", "good config"), + makeTestSecret("worker1", testKubeconfig("good_user")), }, remoteClients: map[string]*remoteClient{ - "worker1": setReconnectState(newTestClient(ctx, "nowatch", cancelCalled), 5), + "worker1": setReconnectState(newTestClient(ctx, testKubeconfig("nowatch"), cancelCalled), 5), }, wantRemoteClients: map[string]*remoteClient{ - "worker1": newTestClient(ctx, "good config", nil), + "worker1": newTestClient(ctx, testKubeconfig("good_user"), nil), }, wantClusters: []kueue.MultiKueueCluster{ *utiltestingapi.MakeMultiKueueCluster("worker1"). @@ -354,13 +387,13 @@ func TestUpdateConfig(t *testing.T) { Obj(), }, secrets: []corev1.Secret{ - makeTestSecret("worker1", "invalid"), + makeTestSecret("worker1", testKubeconfig("invalid")), }, remoteClients: map[string]*remoteClient{ "worker1": setReconnectState(newTestClient(ctx, "nowatch", cancelCalled), 5), }, wantRemoteClients: map[string]*remoteClient{ - "worker1": newTestClient(ctx, "invalid", nil), + "worker1": newTestClient(ctx, testKubeconfig("invalid"), nil), }, wantClusters: []kueue.MultiKueueCluster{ *utiltestingapi.MakeMultiKueueCluster("worker1"). @@ -371,6 +404,51 @@ func TestUpdateConfig(t *testing.T) { }, wantCancelCalled: 1, }, + "failed due to insecure kubeconfig": { + reconcileFor: "worker1", + clusters: []kueue.MultiKueueCluster{ + *utiltestingapi.MakeMultiKueueCluster("worker1"). + KubeConfig(kueue.SecretLocationType, "worker1"). + Generation(1). + Obj(), + }, + secrets: []corev1.Secret{ + makeTestSecret("worker1", testKubeconfigInsecure("worker1", ptr.To("/path/to/tokenfile"))), + }, + wantClusters: []kueue.MultiKueueCluster{ + *utiltestingapi.MakeMultiKueueCluster("worker1"). + KubeConfig(kueue.SecretLocationType, "worker1"). + Active(metav1.ConditionFalse, "InsecureKubeConfig", "insecure kubeconfig: tokenFile is not allowed", 1). + Generation(1). + Obj(), + }, + wantErr: fmt.Errorf("validating kubeconfig failed: %w", errors.New("tokenFile is not allowed")), + }, + "skip insecure kubeconfig validation": { + reconcileFor: "worker1", + clusters: []kueue.MultiKueueCluster{ + *utiltestingapi.MakeMultiKueueCluster("worker1"). + KubeConfig(kueue.SecretLocationType, "worker1"). + Generation(1). + Obj(), + }, + secrets: []corev1.Secret{ + makeTestSecret("worker1", testKubeconfigInsecure("worker1", ptr.To("/path/to/tokenfile"))), + }, + wantClusters: []kueue.MultiKueueCluster{ + *utiltestingapi.MakeMultiKueueCluster("worker1"). + KubeConfig(kueue.SecretLocationType, "worker1"). + Active(metav1.ConditionTrue, "Active", "Connected", 1). + Generation(1). + Obj(), + }, + wantRemoteClients: map[string]*remoteClient{ + "worker1": { + kubeconfig: []byte(testKubeconfigInsecure("worker1", ptr.To("/path/to/tokenfile"))), + }, + }, + skipInsecureKubeconfig: true, + }, } for name, tc := range cases { @@ -392,10 +470,27 @@ func TestUpdateConfig(t *testing.T) { } reconciler.builderOverride = fakeClientBuilder(ctx) + if tc.skipInsecureKubeconfig { + features.SetFeatureGateDuringTest(t, features.MultiKueueAllowInsecureKubeconfigs, true) + } + + // Create test kubeconfig file for path location type + if tc.clusters != nil && tc.clusters[0].Spec.KubeConfig.LocationType == kueue.PathLocationType && tc.clusters[0].Spec.KubeConfig.Location != "" { + kubeconfigBytes := testKubeconfig("worker1") + if err := os.WriteFile(tc.clusters[0].Spec.KubeConfig.Location, []byte(kubeconfigBytes), 0666); err != nil { + t.Errorf("Failed to create test file (%s): %v", tc.clusters[0].Spec.KubeConfig.Location, err) + } + } + cancelCalledCount = 0 res, gotErr := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: tc.reconcileFor}}) - if gotErr != nil { - t.Errorf("unexpected reconcile error: %s", gotErr) + if diff := cmp.Diff(gotErr, tc.wantErr, cmp.Comparer(func(a, b error) bool { + if a == nil || b == nil { + return a == b + } + return a.Error() == b.Error() + })); diff != "" { + t.Errorf("unexpected reconcile error: \nwant:\n%v\ngot:%v\n", tc.wantErr, gotErr) } if diff := cmp.Diff(tc.wantRequeueAfter, res.RequeueAfter); diff != "" { @@ -573,3 +668,55 @@ func TestRemoteClientGC(t *testing.T) { }) } } + +func TestValidateKubeconfig(t *testing.T) { + kubeconfigBase := utiltesting.NewTestKubeConfigWrapper().Cluster("test", "https://10.10.10.10", []byte{0x2d, 0x2d, 0x2d, 0x2d, 0x2d}). + User("u", nil, nil). + Context("test-context", "test", "u"). + CurrentContext("test-context") + + cases := map[string]struct { + cfgFn func() *clientcmdapi.Config + wantErr bool + }{ + "tokenFile not allowed": { + cfgFn: func() *clientcmdapi.Config { + c := kubeconfigBase.Clone().TokenFileAuthInfo("u", "/tmp/tokenfile").Obj() + return &c + }, + wantErr: true, + }, + "insecure skip-tls": { + cfgFn: func() *clientcmdapi.Config { + c := kubeconfigBase.Clone().InsecureSkipTLSVerify("test", true).Obj() + return &c + }, + wantErr: true, + }, + "certificate-authority file disallowed": { + cfgFn: func() *clientcmdapi.Config { + c := kubeconfigBase.Clone().CAFileCluster("test", "/tmp/ca").Obj() + return &c + }, + wantErr: true, + }, + "valid config": { + cfgFn: func() *clientcmdapi.Config { + c := kubeconfigBase.Clone().Obj() + return &c + }, + wantErr: false, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + c := tc.cfgFn() + raw, _ := clientcmd.Write(*c) + err := validateKubeconfig(raw) + if (err != nil) != tc.wantErr { + t.Fatalf("wantErr=%v, got err: %v", tc.wantErr, err) + } + }) + } +} diff --git a/pkg/controller/admissionchecks/multikueue/testdata/worker1KubeConfig b/pkg/controller/admissionchecks/multikueue/testdata/worker1KubeConfig deleted file mode 100644 index 8827346240b..00000000000 --- a/pkg/controller/admissionchecks/multikueue/testdata/worker1KubeConfig +++ /dev/null @@ -1 +0,0 @@ -worker1 kubeconfig \ No newline at end of file diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 96121a272bb..fd0d1bf62f1 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -207,6 +207,13 @@ const ( // are resolved by keeping only the last occurrence, allowing workload creation to succeed // even when duplicates are present in the spec. SanitizePodSets featuregate.Feature = "SanitizePodSets" + + // owner: @mszadkow + // + // Allow insecure kubeconfigs in MultiKueue setup. + // Requires careful consideration as it may lead to security issues. + // Deprecated: planned to be removed in 0.17 + MultiKueueAllowInsecureKubeconfigs featuregate.Feature = "MultiKueueAllowInsecureKubeconfigs" ) func init() { @@ -325,6 +332,9 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned SanitizePodSets: { {Version: version.MustParse("0.13"), Default: true, PreRelease: featuregate.Beta}, }, + MultiKueueAllowInsecureKubeconfigs: { + {Version: version.MustParse("0.15"), Default: false, PreRelease: featuregate.Alpha}, + }, } func SetFeatureGateDuringTest(tb testing.TB, f featuregate.Feature, value bool) { diff --git a/pkg/util/testing/kubeconfig.go b/pkg/util/testing/kubeconfig.go index 2cd29de4ef5..2ee0b41e39f 100644 --- a/pkg/util/testing/kubeconfig.go +++ b/pkg/util/testing/kubeconfig.go @@ -22,29 +22,86 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) -func RestConfigToKubeConfig(restConfig *rest.Config) ([]byte, error) { - cfg := clientcmdapi.Config{ - Kind: "config", - APIVersion: "v1", - Clusters: map[string]*clientcmdapi.Cluster{ - "default-cluster": { - Server: restConfig.Host, - CertificateAuthorityData: restConfig.CAData, - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{ - "default-user": { - ClientCertificateData: restConfig.CertData, - ClientKeyData: restConfig.KeyData, - }, - }, - Contexts: map[string]*clientcmdapi.Context{ - "default-context": { - Cluster: "default-cluster", - AuthInfo: "default-user", - }, +type TestKubeconfigWrapper struct { + Config clientcmdapi.Config +} + +func NewTestKubeConfigWrapper() *TestKubeconfigWrapper { + return &TestKubeconfigWrapper{ + Config: clientcmdapi.Config{ + Kind: "config", + APIVersion: "v1", + Clusters: map[string]*clientcmdapi.Cluster{}, + AuthInfos: map[string]*clientcmdapi.AuthInfo{}, + Contexts: map[string]*clientcmdapi.Context{}, }, - CurrentContext: "default-context", } - return clientcmd.Write(cfg) +} + +func (k *TestKubeconfigWrapper) Cluster(name, server string, caData []byte) *TestKubeconfigWrapper { + k.Config.Clusters[name] = &clientcmdapi.Cluster{ + Server: server, + CertificateAuthorityData: caData, + } + return k +} + +func (k *TestKubeconfigWrapper) User(name string, certData, keyData []byte) *TestKubeconfigWrapper { + k.Config.AuthInfos[name] = &clientcmdapi.AuthInfo{ + ClientCertificateData: certData, + ClientKeyData: keyData, + } + return k +} + +func (k *TestKubeconfigWrapper) Context(name, clusterName, userName string) *TestKubeconfigWrapper { + k.Config.Contexts[name] = &clientcmdapi.Context{ + Cluster: clusterName, + AuthInfo: userName, + } + return k +} + +func (k *TestKubeconfigWrapper) CurrentContext(name string) *TestKubeconfigWrapper { + k.Config.CurrentContext = name + return k +} + +func (k *TestKubeconfigWrapper) TokenAuthInfo(name, token string) *TestKubeconfigWrapper { + k.Config.AuthInfos[name].Token = token + return k +} + +func (k *TestKubeconfigWrapper) TokenFileAuthInfo(name, tokenFilePath string) *TestKubeconfigWrapper { + k.Config.AuthInfos[name].TokenFile = tokenFilePath + return k +} + +func (k *TestKubeconfigWrapper) InsecureSkipTLSVerify(clusterName string, skip bool) *TestKubeconfigWrapper { + k.Config.Clusters[clusterName].InsecureSkipTLSVerify = skip + return k +} + +func (k *TestKubeconfigWrapper) CAFileCluster(clusterName, caFilePath string) *TestKubeconfigWrapper { + k.Config.Clusters[clusterName].CertificateAuthority = caFilePath + return k +} + +func (k *TestKubeconfigWrapper) Clone() *TestKubeconfigWrapper { + return &TestKubeconfigWrapper{Config: *k.Config.DeepCopy()} +} + +func (k *TestKubeconfigWrapper) Obj() clientcmdapi.Config { + return k.Config +} + +func (k *TestKubeconfigWrapper) Build() ([]byte, error) { + return clientcmd.Write(k.Config) +} + +func RestConfigToKubeConfig(restConfig *rest.Config) ([]byte, error) { + return NewTestKubeConfigWrapper().Cluster("default-cluster", restConfig.Host, restConfig.CAData). + User("default-user", restConfig.CertData, restConfig.KeyData). + Context("default-context", "default-cluster", "default-user"). + CurrentContext("default-context").Build() } diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 4405f1fe36e..5481a0325df 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -301,6 +301,7 @@ spec: {{% alert title="Note" color="primary" %}} The SanitizePodSets feature is available starting from versions 0.13.8 and 0.14.3. {{% /alert %}} +| `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.15 | | ### Feature gates for graduated or deprecated features diff --git a/site/content/en/docs/tasks/manage/setup_multikueue.md b/site/content/en/docs/tasks/manage/setup_multikueue.md index 1b27a0f6187..abb94c173a9 100644 --- a/site/content/en/docs/tasks/manage/setup_multikueue.md +++ b/site/content/en/docs/tasks/manage/setup_multikueue.md @@ -49,6 +49,14 @@ chmod +x create-multikueue-kubeconfig.sh To create a Kubeconfig that can be used in the manager cluster to delegate Jobs in the current worker. +{{% alert title="Security Notice" color="primary" %}} + +MultiKueue validates kubeconfig files to protect against known arbitrary code execution vulnerabilities. +For your security, it is strongly recommended not to use the MultiKueueAllowInsecureKubeconfigs flag. +This flag was introduced in Kueue v0.15.0 solely for backward compatibility and will be deprecated in Kueue v0.17.0. + +{{% /alert %}} + ### Kubeflow Installation Install Kubeflow Trainer in the Worker cluster (see [Kubeflow Trainer Installation](https://www.kubeflow.org/docs/components/training/installation/) diff --git a/site/content/zh-CN/docs/installation/_index.md b/site/content/zh-CN/docs/installation/_index.md index 635ce5ecb59..c9a30995772 100644 --- a/site/content/zh-CN/docs/installation/_index.md +++ b/site/content/zh-CN/docs/installation/_index.md @@ -292,6 +292,7 @@ spec: | `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | | `SanitizePodSets` | `true` | Beta | 0.13 | | +| `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.15 | | ### 已毕业或已弃用特性的特性门控 {#feature-gates-for-graduated-or-deprecated-features} diff --git a/test/integration/multikueue/setup_test.go b/test/integration/multikueue/setup_test.go index 89e367f2692..85a44cc3e02 100644 --- a/test/integration/multikueue/setup_test.go +++ b/test/integration/multikueue/setup_test.go @@ -27,11 +27,13 @@ import ( "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" @@ -408,4 +410,205 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, }) }) }) + + ginkgo.It("Should properly detect insecure kubeconfig of MultiKueueClusters, kubeconfig provided by secret", func() { + var w1KubeconfigInvalidBytes []byte + ginkgo.By("Create a kubeconfig with an invalid certificate authority path", func() { + cfg, err := worker1TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + w1KubeconfigInvalid, err := clientcmd.Load(cfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(w1KubeconfigInvalid).NotTo(gomega.BeNil()) + + w1KubeconfigInvalid.Clusters["default-cluster"].CertificateAuthority = "/some/random/path" + w1KubeconfigInvalidBytes, err = clientcmd.Write(*w1KubeconfigInvalid) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testing-secret", + Namespace: managersConfigNamespace.Name, + }, + Data: map[string][]byte{ + kueue.MultiKueueConfigSecretKey: w1KubeconfigInvalidBytes, + }, + } + + ginkgo.By("creating the secret, the kubeconfig is insecure", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, secret)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, secret) }) + }) + + cluster := utiltestingapi.MakeMultiKueueCluster("testing-cluster").KubeConfig(kueue.SecretLocationType, "testing-secret").Obj() + ginkgo.By("creating the cluster, its Active state is updated", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, cluster)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, cluster) }) + + ginkgo.By("wait for the cluster's active state update", func() { + updatedCluster := kueue.MultiKueueCluster{} + clusterKey := client.ObjectKeyFromObject(cluster) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, clusterKey, &updatedCluster)).To(gomega.Succeed()) + g.Expect(updatedCluster.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.MultiKueueClusterActive, + Status: metav1.ConditionFalse, + Reason: "InsecureKubeConfig", + Message: "insecure kubeconfig: certificate-authority file paths are not allowed, use certificate-authority-data for cluster default-cluster", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + + w1Kubeconfig, err := worker1TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("updating the secret with a valid kubeconfig", func() { + updatedSecret := &corev1.Secret{} + secretKey := client.ObjectKeyFromObject(secret) + gomega.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, secretKey, updatedSecret)).To(gomega.Succeed()) + updatedSecret.Data[kueue.MultiKueueConfigSecretKey] = w1Kubeconfig + gomega.Expect(managerTestCluster.client.Update(managerTestCluster.ctx, updatedSecret)).To(gomega.Succeed()) + }) + + ginkgo.By("the cluster become active", func() { + ginkgo.By("wait for the cluster's active state update", func() { + updatedCluster := kueue.MultiKueueCluster{} + clusterKey := client.ObjectKeyFromObject(cluster) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, clusterKey, &updatedCluster)).To(gomega.Succeed()) + g.Expect(updatedCluster.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.MultiKueueClusterActive, + Status: metav1.ConditionTrue, + Reason: "Active", + Message: "Connected", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) + + ginkgo.It("Should properly detect insecure kubeconfig of MultiKueueClusters, kubeconfig provided by file", func() { + var w1KubeconfigInvalidBytes []byte + ginkgo.By("Create a kubeconfig with disallowed tokenFile", func() { + cfg, err := worker1TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + w1KubeconfigInvalid, err := clientcmd.Load(cfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(w1KubeconfigInvalid).NotTo(gomega.BeNil()) + + w1KubeconfigInvalid.AuthInfos["default-user"].TokenFile = "/some/random/path" + w1KubeconfigInvalidBytes, err = clientcmd.Write(*w1KubeconfigInvalid) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + tempDir := ginkgo.GinkgoT().TempDir() + fsKubeConfig := filepath.Join(tempDir, "testing.kubeconfig") + + ginkgo.By("creating the kubeconfig file, the kubeconfig is insecure", func() { + gomega.Expect(os.WriteFile(fsKubeConfig, w1KubeconfigInvalidBytes, 0666)).Should(gomega.Succeed()) + }) + + cluster := utiltestingapi.MakeMultiKueueCluster("testing-cluster").KubeConfig(kueue.PathLocationType, fsKubeConfig).Obj() + ginkgo.By("creating the cluster, its Active state is updated", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, cluster)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, cluster) }) + + ginkgo.By("wait for the cluster's active state update", func() { + updatedCluster := kueue.MultiKueueCluster{} + clusterKey := client.ObjectKeyFromObject(cluster) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, clusterKey, &updatedCluster)).To(gomega.Succeed()) + g.Expect(updatedCluster.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.MultiKueueClusterActive, + Status: metav1.ConditionFalse, + Reason: "InsecureKubeConfig", + Message: "insecure kubeconfig: tokenFile is not allowed", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + + w1Kubeconfig, err := worker1TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("creating the kubeconfig file, the cluster become active", func() { + gomega.Expect(os.WriteFile(fsKubeConfig, w1Kubeconfig, 0666)).Should(gomega.Succeed()) + ginkgo.By("wait for the cluster's active state update", func() { + updatedCluster := kueue.MultiKueueCluster{} + clusterKey := client.ObjectKeyFromObject(cluster) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, clusterKey, &updatedCluster)).To(gomega.Succeed()) + g.Expect(updatedCluster.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.MultiKueueClusterActive, + Status: metav1.ConditionTrue, + Reason: "Active", + Message: "Connected", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) + + ginkgo.It("Should allow insecure kubeconfig of MultiKueueClusters, kubeconfig provided by secret, when MultiKueueAllowInsecureKubeconfigs enabled", func() { + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.MultiKueueAllowInsecureKubeconfigs, true) + + tempDir := ginkgo.GinkgoT().TempDir() + tokenFile := filepath.Join(tempDir, "testing.tokenfile") + + ginkgo.By("creating the tokenFile file", func() { + gomega.Expect(os.WriteFile(tokenFile, []byte("FAKE-TOKEN-123456"), 0666)).Should(gomega.Succeed()) + }) + + var w1KubeconfigInvalidBytes []byte + ginkgo.By("Create a kubeconfig with disallowed tokenFile", func() { + cfg, err := worker1TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + w1KubeconfigInvalid, err := clientcmd.Load(cfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(w1KubeconfigInvalid).NotTo(gomega.BeNil()) + + w1KubeconfigInvalid.AuthInfos["default-user"].TokenFile = tokenFile + w1KubeconfigInvalidBytes, err = clientcmd.Write(*w1KubeconfigInvalid) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testing-secret", + Namespace: managersConfigNamespace.Name, + }, + Data: map[string][]byte{ + kueue.MultiKueueConfigSecretKey: w1KubeconfigInvalidBytes, + }, + } + + ginkgo.By("creating the secret, the kubeconfig is insecure", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, secret)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, secret) }) + }) + + cluster := utiltestingapi.MakeMultiKueueCluster("testing-cluster").KubeConfig(kueue.SecretLocationType, "testing-secret").Obj() + ginkgo.By("creating the cluster, its Active state is updated", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, cluster)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, cluster) }) + + ginkgo.By("wait for the cluster's active state update", func() { + updatedCluster := kueue.MultiKueueCluster{} + clusterKey := client.ObjectKeyFromObject(cluster) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, clusterKey, &updatedCluster)).To(gomega.Succeed()) + g.Expect(updatedCluster.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.MultiKueueClusterActive, + Status: metav1.ConditionTrue, + Reason: "Active", + Message: "Connected", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) }) From d81e75b984a798c6f79c773d38be031899f79dc4 Mon Sep 17 00:00:00 2001 From: Ryan Phillips Date: Thu, 30 Oct 2025 08:38:04 -0500 Subject: [PATCH 054/119] services: update app.kuberntes.io/component for services (#7371) --- charts/kueue/templates/_helpers.tpl | 26 +++++++++++++- charts/kueue/templates/manager/manager.yaml | 2 +- .../visibility/apiservice_v1beta1.yaml | 2 +- .../templates/visibility/role_binding.yaml | 2 +- .../kueue/templates/visibility/service.yaml | 2 +- charts/kueue/templates/webhook/service.yaml | 2 +- config/default/kustomization.yaml | 35 ++++++++++++++++++- hack/processing-plan.yaml | 4 +-- 8 files changed, 66 insertions(+), 9 deletions(-) diff --git a/charts/kueue/templates/_helpers.tpl b/charts/kueue/templates/_helpers.tpl index cb9a404cfc6..f734c171234 100644 --- a/charts/kueue/templates/_helpers.tpl +++ b/charts/kueue/templates/_helpers.tpl @@ -56,7 +56,31 @@ Labels for metrics service */}} {{- define "kueue.metricsService.labels" -}} {{ include "kueue.labels" . }} -app.kubernetes.io/component: metrics +app.kubernetes.io/component: metrics-service +{{- end }} + +{{/* +Labels for webhook service +*/}} +{{- define "kueue.webhookService.labels" -}} +{{ include "kueue.labels" . }} +app.kubernetes.io/component: webhook-service +{{- end }} + +{{/* +Labels for visibility service +*/}} +{{- define "kueue.visibilityService.labels" -}} +{{ include "kueue.labels" . }} +app.kubernetes.io/component: visibility-service +{{- end }} + +{{/* +Labels for controller-manager +*/}} +{{- define "kueue.controllerManager.labels" -}} +{{ include "kueue.labels" . }} +app.kubernetes.io/component: controller {{- end }} {{/* diff --git a/charts/kueue/templates/manager/manager.yaml b/charts/kueue/templates/manager/manager.yaml index 3b0f7de04e8..9f85fcbaa32 100644 --- a/charts/kueue/templates/manager/manager.yaml +++ b/charts/kueue/templates/manager/manager.yaml @@ -4,7 +4,7 @@ metadata: name: {{ include "kueue.fullname" . }}-controller-manager namespace: '{{ .Release.Namespace }}' labels: - {{- include "kueue.labels" . | nindent 4 }} + {{- include "kueue.controllerManager.labels" . | nindent 4 }} spec: replicas: {{ .Values.controllerManager.replicas }} selector: diff --git a/charts/kueue/templates/visibility/apiservice_v1beta1.yaml b/charts/kueue/templates/visibility/apiservice_v1beta1.yaml index 5062954e0bf..68eff2d0c41 100644 --- a/charts/kueue/templates/visibility/apiservice_v1beta1.yaml +++ b/charts/kueue/templates/visibility/apiservice_v1beta1.yaml @@ -8,7 +8,7 @@ metadata: cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ include "kueue.fullname" . }}-visibility-server-cert' {{- end }} labels: - {{- include "kueue.labels" . | nindent 4 }} + {{- include "kueue.visibilityService.labels" . | nindent 4 }} name: v1beta1.visibility.kueue.x-k8s.io spec: {{- if not .Values.enableCertManager }} diff --git a/charts/kueue/templates/visibility/role_binding.yaml b/charts/kueue/templates/visibility/role_binding.yaml index 963e5e83e1c..dcb68b6bf24 100644 --- a/charts/kueue/templates/visibility/role_binding.yaml +++ b/charts/kueue/templates/visibility/role_binding.yaml @@ -4,7 +4,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: - {{- include "kueue.labels" . | nindent 4 }} + {{- include "kueue.visibilityService.labels" . | nindent 4 }} name: '{{ include "kueue.fullname" . }}-visibility-server-auth-reader' namespace: kube-system roleRef: diff --git a/charts/kueue/templates/visibility/service.yaml b/charts/kueue/templates/visibility/service.yaml index d1efa4b729b..c4275d7f159 100644 --- a/charts/kueue/templates/visibility/service.yaml +++ b/charts/kueue/templates/visibility/service.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: Service metadata: labels: - {{- include "kueue.labels" . | nindent 4 }} + {{- include "kueue.visibilityService.labels" . | nindent 4 }} name: '{{ include "kueue.fullname" . }}-visibility-server' namespace: '{{ .Release.Namespace }}' spec: diff --git a/charts/kueue/templates/webhook/service.yaml b/charts/kueue/templates/webhook/service.yaml index 03328b06fae..90da33f1157 100644 --- a/charts/kueue/templates/webhook/service.yaml +++ b/charts/kueue/templates/webhook/service.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: Service metadata: labels: - {{- include "kueue.labels" . | nindent 4 }} + {{- include "kueue.webhookService.labels" . | nindent 4 }} name: '{{ include "kueue.fullname" . }}-webhook-service' namespace: '{{ .Release.Namespace }}' spec: diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index a6c683456f6..de1c104b520 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -11,7 +11,6 @@ namePrefix: kueue- labels: - pairs: app.kubernetes.io/name: kueue - app.kubernetes.io/component: controller includeTemplates: true - pairs: control-plane: controller-manager @@ -52,6 +51,40 @@ patches: # Expose port used by the metrics service - path: manager_metrics_patch.yaml +# Add component labels to services +- patch: |- + apiVersion: v1 + kind: Service + metadata: + name: webhook-service + namespace: system + labels: + app.kubernetes.io/component: webhook-service +- patch: |- + apiVersion: v1 + kind: Service + metadata: + name: visibility-server + namespace: kueue-system + labels: + app.kubernetes.io/component: visibility-service +- patch: |- + apiVersion: v1 + kind: Service + metadata: + name: controller-manager-metrics-service + namespace: system + labels: + app.kubernetes.io/component: metrics-service +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: controller-manager + namespace: system + labels: + app.kubernetes.io/component: controller + # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection diff --git a/hack/processing-plan.yaml b/hack/processing-plan.yaml index a1eb3b5223b..dce721b7157 100644 --- a/hack/processing-plan.yaml +++ b/hack/processing-plan.yaml @@ -105,7 +105,7 @@ files: key: .metadata value: | labels: - {{- include "kueue.labels" . | nindent 4 }} + {{- include "kueue.visibilityService.labels" . | nindent 4 }} indentation: 2 - type: INSERT_TEXT key: .spec @@ -192,7 +192,7 @@ files: key: .metadata value: | labels: - {{- include "kueue.labels" . | nindent 4 }} + {{- include "kueue.webhookService.labels" . | nindent 4 }} indentation: 2 - type: INSERT_TEXT key: .spec.type From 7c4de982cad06d1a0fbf7ce4e99600cf26ecd098 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 30 Oct 2025 19:08:11 +0530 Subject: [PATCH 055/119] Add License prefix for helm templates. (#7438) --- charts/kueue/templates/_helpers.tpl | 16 ++++++++++ .../certmanager/certificate-metrics.yaml | 16 ++++++++++ .../certificate-visibility-server.yaml | 16 ++++++++++ .../templates/certmanager/certificate.yaml | 16 ++++++++++ .../crd/kueue.x-k8s.io_admissionchecks.yaml | 16 ++++++++++ .../crd/kueue.x-k8s.io_clusterqueues.yaml | 16 ++++++++++ .../templates/crd/kueue.x-k8s.io_cohorts.yaml | 16 ++++++++++ .../crd/kueue.x-k8s.io_localqueues.yaml | 16 ++++++++++ .../kueue.x-k8s.io_multikueueclusters.yaml | 16 ++++++++++ .../crd/kueue.x-k8s.io_multikueueconfigs.yaml | 16 ++++++++++ ...e.x-k8s.io_provisioningrequestconfigs.yaml | 16 ++++++++++ .../crd/kueue.x-k8s.io_resourceflavors.yaml | 16 ++++++++++ .../crd/kueue.x-k8s.io_topologies.yaml | 16 ++++++++++ ...ueue.x-k8s.io_workloadpriorityclasses.yaml | 16 ++++++++++ .../crd/kueue.x-k8s.io_workloads.yaml | 16 ++++++++++ .../kueue/templates/internalcert/secret.yaml | 16 ++++++++++ .../kueueviz/backend-deployment.yaml | 16 ++++++++++ .../templates/kueueviz/backend-ingress.yaml | 16 ++++++++++ .../templates/kueueviz/backend-service.yaml | 16 ++++++++++ .../kueueviz/cluster-role-binding.yaml | 16 ++++++++++ .../kueue/templates/kueueviz/clusterrole.yaml | 16 ++++++++++ .../kueueviz/frontend-deployment.yaml | 16 ++++++++++ .../templates/kueueviz/frontend-ingress.yaml | 16 ++++++++++ .../templates/kueueviz/frontend-service.yaml | 16 ++++++++++ .../templates/manager/auth_proxy_service.yaml | 16 ++++++++++ .../templates/manager/manager-config.yaml | 16 ++++++++++ .../kueue/templates/manager/manager-pdb.yaml | 16 ++++++++++ charts/kueue/templates/manager/manager.yaml | 16 ++++++++++ .../kueue/templates/prometheus/monitor.yaml | 16 ++++++++++ charts/kueue/templates/prometheus/role.yaml | 16 ++++++++++ .../templates/rbac/batch_admin_role.yaml | 16 ++++++++++ .../kueue/templates/rbac/batch_user_role.yaml | 16 ++++++++++ .../rbac/clusterqueue_editor_role.yaml | 16 ++++++++++ .../rbac/clusterqueue_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/cohort_editor_role.yaml | 16 ++++++++++ .../templates/rbac/cohort_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/jaxjob_editor_role.yaml | 16 ++++++++++ .../templates/rbac/jaxjob_viewer_role.yaml | 16 ++++++++++ .../kueue/templates/rbac/job_editor_role.yaml | 16 ++++++++++ .../kueue/templates/rbac/job_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/jobset_editor_role.yaml | 16 ++++++++++ .../templates/rbac/jobset_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/leader_election_role.yaml | 16 ++++++++++ .../rbac/leader_election_role_binding.yaml | 16 ++++++++++ .../rbac/localqueue_editor_role.yaml | 16 ++++++++++ .../rbac/localqueue_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/manager_secrets_role.yaml | 16 ++++++++++ .../rbac/manager_secrets_role_binding.yaml | 16 ++++++++++ .../templates/rbac/metrics_auth_role.yaml | 16 ++++++++++ .../rbac/metrics_auth_role_binding.yaml | 16 ++++++++++ .../templates/rbac/metrics_reader_role.yaml | 16 ++++++++++ .../templates/rbac/mpijob_editor_role.yaml | 16 ++++++++++ .../templates/rbac/mpijob_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/paddlejob_editor_role.yaml | 16 ++++++++++ .../templates/rbac/paddlejob_viewer_role.yaml | 16 ++++++++++ .../pending_workloads_cq_viewer_role.yaml | 16 ++++++++++ .../pending_workloads_lq_viewer_role.yaml | 16 ++++++++++ .../rbac/pytorchjob_editor_role.yaml | 16 ++++++++++ .../rbac/pytorchjob_viewer_role.yaml | 16 ++++++++++ .../rbac/raycluster_editor_role.yaml | 16 ++++++++++ .../rbac/raycluster_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/rayjob_editor_role.yaml | 16 ++++++++++ .../templates/rbac/rayjob_viewer_role.yaml | 16 ++++++++++ .../rbac/resourceflavor_editor_role.yaml | 16 ++++++++++ .../rbac/resourceflavor_viewer_role.yaml | 16 ++++++++++ charts/kueue/templates/rbac/role.yaml | 16 ++++++++++ charts/kueue/templates/rbac/role_binding.yaml | 16 ++++++++++ .../kueue/templates/rbac/service_account.yaml | 16 ++++++++++ .../templates/rbac/tfjob_editor_role.yaml | 16 ++++++++++ .../templates/rbac/tfjob_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/topology_editor_role.yaml | 16 ++++++++++ .../templates/rbac/topology_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/trainjob_editor_role.yaml | 16 ++++++++++ .../templates/rbac/trainjob_viewer_role.yaml | 16 ++++++++++ .../templates/rbac/workload_editor_role.yaml | 16 ++++++++++ .../templates/rbac/workload_viewer_role.yaml | 16 ++++++++++ .../rbac/xgboostjob_editor_role.yaml | 16 ++++++++++ .../rbac/xgboostjob_viewer_role.yaml | 16 ++++++++++ .../templates/visibility-apf/flowschema.yaml | 16 ++++++++++ .../prioritylevelconfigurations.yaml | 16 ++++++++++ .../visibility/apiservice_v1beta1.yaml | 16 ++++++++++ .../templates/visibility/role_binding.yaml | 16 ++++++++++ .../kueue/templates/visibility/service.yaml | 16 ++++++++++ charts/kueue/templates/webhook/manifests.yaml | 16 ++++++++++ charts/kueue/templates/webhook/service.yaml | 16 ++++++++++ .../yaml-processor/yamlproc/processor.go | 31 +++++++++++++++++-- hack/processing-plan.yaml | 2 ++ 87 files changed, 1390 insertions(+), 3 deletions(-) diff --git a/charts/kueue/templates/_helpers.tpl b/charts/kueue/templates/_helpers.tpl index f734c171234..efa7adbafef 100644 --- a/charts/kueue/templates/_helpers.tpl +++ b/charts/kueue/templates/_helpers.tpl @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Expand the name of the chart. */}} diff --git a/charts/kueue/templates/certmanager/certificate-metrics.yaml b/charts/kueue/templates/certmanager/certificate-metrics.yaml index 9419230da18..2219a305d53 100644 --- a/charts/kueue/templates/certmanager/certificate-metrics.yaml +++ b/charts/kueue/templates/certmanager/certificate-metrics.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableCertManager }} diff --git a/charts/kueue/templates/certmanager/certificate-visibility-server.yaml b/charts/kueue/templates/certmanager/certificate-visibility-server.yaml index 50f6f699a27..20cdc246e60 100644 --- a/charts/kueue/templates/certmanager/certificate-visibility-server.yaml +++ b/charts/kueue/templates/certmanager/certificate-visibility-server.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableCertManager }} diff --git a/charts/kueue/templates/certmanager/certificate.yaml b/charts/kueue/templates/certmanager/certificate.yaml index 0a79a9f7d85..011f47e05b1 100644 --- a/charts/kueue/templates/certmanager/certificate.yaml +++ b/charts/kueue/templates/certmanager/certificate.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableCertManager }} diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml index 40b3a352f13..6bf25e5743d 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml index 5913bca16d5..6066d734808 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml index 4324ad3b58e..e7d0ed710d0 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml index ef8da08bd05..4299eb41c4d 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml index 1a61f31ba5d..5ae9b67d06f 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml index 6041c805dbe..ffbb7296148 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml index 90e40babd15..a7d3a0c63f3 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_resourceflavors.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_resourceflavors.yaml index 90cae4cc6e0..4e936c6847c 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_resourceflavors.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_resourceflavors.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_topologies.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_topologies.yaml index bd9a463bea2..4f8e763fcfb 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_topologies.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_topologies.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml index 4ddb00e4d03..801fe8502fb 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index d6c90c436ab..c607621733e 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kueue/templates/internalcert/secret.yaml b/charts/kueue/templates/internalcert/secret.yaml index 91211b97c45..f1080ad58a4 100644 --- a/charts/kueue/templates/internalcert/secret.yaml +++ b/charts/kueue/templates/internalcert/secret.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if not .Values.enableCertManager }} diff --git a/charts/kueue/templates/kueueviz/backend-deployment.yaml b/charts/kueue/templates/kueueviz/backend-deployment.yaml index 60ed4a5db66..d4a1da53ce9 100644 --- a/charts/kueue/templates/kueueviz/backend-deployment.yaml +++ b/charts/kueue/templates/kueueviz/backend-deployment.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableKueueViz }} diff --git a/charts/kueue/templates/kueueviz/backend-ingress.yaml b/charts/kueue/templates/kueueviz/backend-ingress.yaml index 2d063895061..9af249ed8a0 100644 --- a/charts/kueue/templates/kueueviz/backend-ingress.yaml +++ b/charts/kueue/templates/kueueviz/backend-ingress.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableKueueViz }} diff --git a/charts/kueue/templates/kueueviz/backend-service.yaml b/charts/kueue/templates/kueueviz/backend-service.yaml index 7556dc49031..1fc2e632edd 100644 --- a/charts/kueue/templates/kueueviz/backend-service.yaml +++ b/charts/kueue/templates/kueueviz/backend-service.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableKueueViz }} diff --git a/charts/kueue/templates/kueueviz/cluster-role-binding.yaml b/charts/kueue/templates/kueueviz/cluster-role-binding.yaml index d812a76fa66..2ca4e143ab6 100644 --- a/charts/kueue/templates/kueueviz/cluster-role-binding.yaml +++ b/charts/kueue/templates/kueueviz/cluster-role-binding.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableKueueViz }} diff --git a/charts/kueue/templates/kueueviz/clusterrole.yaml b/charts/kueue/templates/kueueviz/clusterrole.yaml index d4268081f13..878f9d656b5 100644 --- a/charts/kueue/templates/kueueviz/clusterrole.yaml +++ b/charts/kueue/templates/kueueviz/clusterrole.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableKueueViz }} diff --git a/charts/kueue/templates/kueueviz/frontend-deployment.yaml b/charts/kueue/templates/kueueviz/frontend-deployment.yaml index 47d6b6bd559..d592ea3cdd4 100644 --- a/charts/kueue/templates/kueueviz/frontend-deployment.yaml +++ b/charts/kueue/templates/kueueviz/frontend-deployment.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableKueueViz }} diff --git a/charts/kueue/templates/kueueviz/frontend-ingress.yaml b/charts/kueue/templates/kueueviz/frontend-ingress.yaml index ca0270402f3..0b0a1ccc5b7 100644 --- a/charts/kueue/templates/kueueviz/frontend-ingress.yaml +++ b/charts/kueue/templates/kueueviz/frontend-ingress.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableKueueViz }} diff --git a/charts/kueue/templates/kueueviz/frontend-service.yaml b/charts/kueue/templates/kueueviz/frontend-service.yaml index 96cbc458c20..f87ff080141 100644 --- a/charts/kueue/templates/kueueviz/frontend-service.yaml +++ b/charts/kueue/templates/kueueviz/frontend-service.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableKueueViz }} diff --git a/charts/kueue/templates/manager/auth_proxy_service.yaml b/charts/kueue/templates/manager/auth_proxy_service.yaml index e3eb2f72a74..85cedb6dfee 100644 --- a/charts/kueue/templates/manager/auth_proxy_service.yaml +++ b/charts/kueue/templates/manager/auth_proxy_service.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + apiVersion: v1 kind: Service metadata: diff --git a/charts/kueue/templates/manager/manager-config.yaml b/charts/kueue/templates/manager/manager-config.yaml index 39440ea0ed4..6b47aba816d 100644 --- a/charts/kueue/templates/manager/manager-config.yaml +++ b/charts/kueue/templates/manager/manager-config.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kueue/templates/manager/manager-pdb.yaml b/charts/kueue/templates/manager/manager-pdb.yaml index ce41d317b2d..26def36b6f7 100644 --- a/charts/kueue/templates/manager/manager-pdb.yaml +++ b/charts/kueue/templates/manager/manager-pdb.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{- if .Values.controllerManager.podDisruptionBudget.enabled }} apiVersion: policy/v1 kind: PodDisruptionBudget diff --git a/charts/kueue/templates/manager/manager.yaml b/charts/kueue/templates/manager/manager.yaml index 9f85fcbaa32..91130348b2c 100644 --- a/charts/kueue/templates/manager/manager.yaml +++ b/charts/kueue/templates/manager/manager.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + apiVersion: apps/v1 kind: Deployment metadata: diff --git a/charts/kueue/templates/prometheus/monitor.yaml b/charts/kueue/templates/prometheus/monitor.yaml index 5f1444fb5ef..e8b21dba639 100644 --- a/charts/kueue/templates/prometheus/monitor.yaml +++ b/charts/kueue/templates/prometheus/monitor.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enablePrometheus }} diff --git a/charts/kueue/templates/prometheus/role.yaml b/charts/kueue/templates/prometheus/role.yaml index 80d0e1fe54d..7047bc6a7ab 100644 --- a/charts/kueue/templates/prometheus/role.yaml +++ b/charts/kueue/templates/prometheus/role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enablePrometheus }} diff --git a/charts/kueue/templates/rbac/batch_admin_role.yaml b/charts/kueue/templates/rbac/batch_admin_role.yaml index 7efde679e50..c5b68bab5f0 100644 --- a/charts/kueue/templates/rbac/batch_admin_role.yaml +++ b/charts/kueue/templates/rbac/batch_admin_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/batch_user_role.yaml b/charts/kueue/templates/rbac/batch_user_role.yaml index c848d7971a2..530cfd7f0d9 100644 --- a/charts/kueue/templates/rbac/batch_user_role.yaml +++ b/charts/kueue/templates/rbac/batch_user_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/clusterqueue_editor_role.yaml b/charts/kueue/templates/rbac/clusterqueue_editor_role.yaml index c3b6151238c..1cc9ac18a4b 100644 --- a/charts/kueue/templates/rbac/clusterqueue_editor_role.yaml +++ b/charts/kueue/templates/rbac/clusterqueue_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/clusterqueue_viewer_role.yaml b/charts/kueue/templates/rbac/clusterqueue_viewer_role.yaml index f138ba3b5e8..01c85112042 100644 --- a/charts/kueue/templates/rbac/clusterqueue_viewer_role.yaml +++ b/charts/kueue/templates/rbac/clusterqueue_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/cohort_editor_role.yaml b/charts/kueue/templates/rbac/cohort_editor_role.yaml index c621509ee43..a2804295d07 100644 --- a/charts/kueue/templates/rbac/cohort_editor_role.yaml +++ b/charts/kueue/templates/rbac/cohort_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/cohort_viewer_role.yaml b/charts/kueue/templates/rbac/cohort_viewer_role.yaml index 8672da1e563..918daccd4d6 100644 --- a/charts/kueue/templates/rbac/cohort_viewer_role.yaml +++ b/charts/kueue/templates/rbac/cohort_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/jaxjob_editor_role.yaml b/charts/kueue/templates/rbac/jaxjob_editor_role.yaml index 6dd9a57a481..7d27a132b7f 100644 --- a/charts/kueue/templates/rbac/jaxjob_editor_role.yaml +++ b/charts/kueue/templates/rbac/jaxjob_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/jaxjob_viewer_role.yaml b/charts/kueue/templates/rbac/jaxjob_viewer_role.yaml index c2ba453660d..d508fec52f1 100644 --- a/charts/kueue/templates/rbac/jaxjob_viewer_role.yaml +++ b/charts/kueue/templates/rbac/jaxjob_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/job_editor_role.yaml b/charts/kueue/templates/rbac/job_editor_role.yaml index 9d588521e81..c5aac325743 100644 --- a/charts/kueue/templates/rbac/job_editor_role.yaml +++ b/charts/kueue/templates/rbac/job_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/job_viewer_role.yaml b/charts/kueue/templates/rbac/job_viewer_role.yaml index e51be5a8297..613bbad4067 100644 --- a/charts/kueue/templates/rbac/job_viewer_role.yaml +++ b/charts/kueue/templates/rbac/job_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/jobset_editor_role.yaml b/charts/kueue/templates/rbac/jobset_editor_role.yaml index a2adc2de6a1..3472b586e7f 100644 --- a/charts/kueue/templates/rbac/jobset_editor_role.yaml +++ b/charts/kueue/templates/rbac/jobset_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/jobset_viewer_role.yaml b/charts/kueue/templates/rbac/jobset_viewer_role.yaml index bc3ee22cfcd..96a0c76b529 100644 --- a/charts/kueue/templates/rbac/jobset_viewer_role.yaml +++ b/charts/kueue/templates/rbac/jobset_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/leader_election_role.yaml b/charts/kueue/templates/rbac/leader_election_role.yaml index 8fede5ae493..c5212500ca9 100644 --- a/charts/kueue/templates/rbac/leader_election_role.yaml +++ b/charts/kueue/templates/rbac/leader_election_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/leader_election_role_binding.yaml b/charts/kueue/templates/rbac/leader_election_role_binding.yaml index 67637dc43d7..b6cfb45562c 100644 --- a/charts/kueue/templates/rbac/leader_election_role_binding.yaml +++ b/charts/kueue/templates/rbac/leader_election_role_binding.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/localqueue_editor_role.yaml b/charts/kueue/templates/rbac/localqueue_editor_role.yaml index 1223ab70e80..b46b903e814 100644 --- a/charts/kueue/templates/rbac/localqueue_editor_role.yaml +++ b/charts/kueue/templates/rbac/localqueue_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/localqueue_viewer_role.yaml b/charts/kueue/templates/rbac/localqueue_viewer_role.yaml index c340c90e969..a2755e11130 100644 --- a/charts/kueue/templates/rbac/localqueue_viewer_role.yaml +++ b/charts/kueue/templates/rbac/localqueue_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/manager_secrets_role.yaml b/charts/kueue/templates/rbac/manager_secrets_role.yaml index b1869f62fea..72173328d29 100644 --- a/charts/kueue/templates/rbac/manager_secrets_role.yaml +++ b/charts/kueue/templates/rbac/manager_secrets_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/manager_secrets_role_binding.yaml b/charts/kueue/templates/rbac/manager_secrets_role_binding.yaml index c71c0d906e6..9ec5c67e85e 100644 --- a/charts/kueue/templates/rbac/manager_secrets_role_binding.yaml +++ b/charts/kueue/templates/rbac/manager_secrets_role_binding.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/metrics_auth_role.yaml b/charts/kueue/templates/rbac/metrics_auth_role.yaml index 4d17d4cab6a..f7bbefb14a4 100644 --- a/charts/kueue/templates/rbac/metrics_auth_role.yaml +++ b/charts/kueue/templates/rbac/metrics_auth_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/metrics_auth_role_binding.yaml b/charts/kueue/templates/rbac/metrics_auth_role_binding.yaml index 7e873283b4a..82cd7e5bd86 100644 --- a/charts/kueue/templates/rbac/metrics_auth_role_binding.yaml +++ b/charts/kueue/templates/rbac/metrics_auth_role_binding.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/metrics_reader_role.yaml b/charts/kueue/templates/rbac/metrics_reader_role.yaml index c0a57a88aef..8272e545d1f 100644 --- a/charts/kueue/templates/rbac/metrics_reader_role.yaml +++ b/charts/kueue/templates/rbac/metrics_reader_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/mpijob_editor_role.yaml b/charts/kueue/templates/rbac/mpijob_editor_role.yaml index 9da72772db7..6ccde99d9b5 100644 --- a/charts/kueue/templates/rbac/mpijob_editor_role.yaml +++ b/charts/kueue/templates/rbac/mpijob_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/mpijob_viewer_role.yaml b/charts/kueue/templates/rbac/mpijob_viewer_role.yaml index fcba36f1354..68ba7c7f19e 100644 --- a/charts/kueue/templates/rbac/mpijob_viewer_role.yaml +++ b/charts/kueue/templates/rbac/mpijob_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/paddlejob_editor_role.yaml b/charts/kueue/templates/rbac/paddlejob_editor_role.yaml index 95fbd593496..3c29e54b9cb 100644 --- a/charts/kueue/templates/rbac/paddlejob_editor_role.yaml +++ b/charts/kueue/templates/rbac/paddlejob_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/paddlejob_viewer_role.yaml b/charts/kueue/templates/rbac/paddlejob_viewer_role.yaml index e4c11f224df..5cb49d99470 100644 --- a/charts/kueue/templates/rbac/paddlejob_viewer_role.yaml +++ b/charts/kueue/templates/rbac/paddlejob_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/pending_workloads_cq_viewer_role.yaml b/charts/kueue/templates/rbac/pending_workloads_cq_viewer_role.yaml index d7019a23313..fe11ea6f800 100644 --- a/charts/kueue/templates/rbac/pending_workloads_cq_viewer_role.yaml +++ b/charts/kueue/templates/rbac/pending_workloads_cq_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/pending_workloads_lq_viewer_role.yaml b/charts/kueue/templates/rbac/pending_workloads_lq_viewer_role.yaml index 62c9b6551c4..5c32eeb6e21 100644 --- a/charts/kueue/templates/rbac/pending_workloads_lq_viewer_role.yaml +++ b/charts/kueue/templates/rbac/pending_workloads_lq_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/pytorchjob_editor_role.yaml b/charts/kueue/templates/rbac/pytorchjob_editor_role.yaml index bb7f6d40401..a1cf23bfd84 100644 --- a/charts/kueue/templates/rbac/pytorchjob_editor_role.yaml +++ b/charts/kueue/templates/rbac/pytorchjob_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/pytorchjob_viewer_role.yaml b/charts/kueue/templates/rbac/pytorchjob_viewer_role.yaml index f7f0bde5f04..a2614d1916d 100644 --- a/charts/kueue/templates/rbac/pytorchjob_viewer_role.yaml +++ b/charts/kueue/templates/rbac/pytorchjob_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/raycluster_editor_role.yaml b/charts/kueue/templates/rbac/raycluster_editor_role.yaml index 4fdacbea3a1..665aac802a6 100644 --- a/charts/kueue/templates/rbac/raycluster_editor_role.yaml +++ b/charts/kueue/templates/rbac/raycluster_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/raycluster_viewer_role.yaml b/charts/kueue/templates/rbac/raycluster_viewer_role.yaml index f9af5033ece..13d753eff8a 100644 --- a/charts/kueue/templates/rbac/raycluster_viewer_role.yaml +++ b/charts/kueue/templates/rbac/raycluster_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/rayjob_editor_role.yaml b/charts/kueue/templates/rbac/rayjob_editor_role.yaml index 02a44f67475..f2d0a018663 100644 --- a/charts/kueue/templates/rbac/rayjob_editor_role.yaml +++ b/charts/kueue/templates/rbac/rayjob_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/rayjob_viewer_role.yaml b/charts/kueue/templates/rbac/rayjob_viewer_role.yaml index 38ed145793f..5d10494cff2 100644 --- a/charts/kueue/templates/rbac/rayjob_viewer_role.yaml +++ b/charts/kueue/templates/rbac/rayjob_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/resourceflavor_editor_role.yaml b/charts/kueue/templates/rbac/resourceflavor_editor_role.yaml index 87a9028b442..f3a330bc6f9 100644 --- a/charts/kueue/templates/rbac/resourceflavor_editor_role.yaml +++ b/charts/kueue/templates/rbac/resourceflavor_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/resourceflavor_viewer_role.yaml b/charts/kueue/templates/rbac/resourceflavor_viewer_role.yaml index f5a857eedd5..f073bce7cbc 100644 --- a/charts/kueue/templates/rbac/resourceflavor_viewer_role.yaml +++ b/charts/kueue/templates/rbac/resourceflavor_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/role.yaml b/charts/kueue/templates/rbac/role.yaml index 43931821aa8..0b6758e2fa8 100644 --- a/charts/kueue/templates/rbac/role.yaml +++ b/charts/kueue/templates/rbac/role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/role_binding.yaml b/charts/kueue/templates/rbac/role_binding.yaml index 4031e260544..9fba729e80c 100644 --- a/charts/kueue/templates/rbac/role_binding.yaml +++ b/charts/kueue/templates/rbac/role_binding.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/service_account.yaml b/charts/kueue/templates/rbac/service_account.yaml index caa665b9f7d..e28686a1420 100644 --- a/charts/kueue/templates/rbac/service_account.yaml +++ b/charts/kueue/templates/rbac/service_account.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: v1 diff --git a/charts/kueue/templates/rbac/tfjob_editor_role.yaml b/charts/kueue/templates/rbac/tfjob_editor_role.yaml index c6348b24f3d..1204f6b3664 100644 --- a/charts/kueue/templates/rbac/tfjob_editor_role.yaml +++ b/charts/kueue/templates/rbac/tfjob_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/tfjob_viewer_role.yaml b/charts/kueue/templates/rbac/tfjob_viewer_role.yaml index ce3c1647660..58a3b486def 100644 --- a/charts/kueue/templates/rbac/tfjob_viewer_role.yaml +++ b/charts/kueue/templates/rbac/tfjob_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/topology_editor_role.yaml b/charts/kueue/templates/rbac/topology_editor_role.yaml index 81df6ce231c..63572dfb1e9 100644 --- a/charts/kueue/templates/rbac/topology_editor_role.yaml +++ b/charts/kueue/templates/rbac/topology_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/topology_viewer_role.yaml b/charts/kueue/templates/rbac/topology_viewer_role.yaml index cf5d57debd5..8159d9d24a6 100644 --- a/charts/kueue/templates/rbac/topology_viewer_role.yaml +++ b/charts/kueue/templates/rbac/topology_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/trainjob_editor_role.yaml b/charts/kueue/templates/rbac/trainjob_editor_role.yaml index 89800a7197a..c4d821a2254 100644 --- a/charts/kueue/templates/rbac/trainjob_editor_role.yaml +++ b/charts/kueue/templates/rbac/trainjob_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/trainjob_viewer_role.yaml b/charts/kueue/templates/rbac/trainjob_viewer_role.yaml index 8310f6234a1..ff0704e7f2e 100644 --- a/charts/kueue/templates/rbac/trainjob_viewer_role.yaml +++ b/charts/kueue/templates/rbac/trainjob_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/workload_editor_role.yaml b/charts/kueue/templates/rbac/workload_editor_role.yaml index 0e05d6b3dd3..be62b900092 100644 --- a/charts/kueue/templates/rbac/workload_editor_role.yaml +++ b/charts/kueue/templates/rbac/workload_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/workload_viewer_role.yaml b/charts/kueue/templates/rbac/workload_viewer_role.yaml index 00aa14f6c5a..c1508639428 100644 --- a/charts/kueue/templates/rbac/workload_viewer_role.yaml +++ b/charts/kueue/templates/rbac/workload_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/xgboostjob_editor_role.yaml b/charts/kueue/templates/rbac/xgboostjob_editor_role.yaml index 5fe0623876e..edd086c15a8 100644 --- a/charts/kueue/templates/rbac/xgboostjob_editor_role.yaml +++ b/charts/kueue/templates/rbac/xgboostjob_editor_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/rbac/xgboostjob_viewer_role.yaml b/charts/kueue/templates/rbac/xgboostjob_viewer_role.yaml index ff81fcdb249..9dd08b71e62 100644 --- a/charts/kueue/templates/rbac/xgboostjob_viewer_role.yaml +++ b/charts/kueue/templates/rbac/xgboostjob_viewer_role.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/visibility-apf/flowschema.yaml b/charts/kueue/templates/visibility-apf/flowschema.yaml index 6cf7d323b7e..185deb9883f 100644 --- a/charts/kueue/templates/visibility-apf/flowschema.yaml +++ b/charts/kueue/templates/visibility-apf/flowschema.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableVisibilityAPF }} diff --git a/charts/kueue/templates/visibility-apf/prioritylevelconfigurations.yaml b/charts/kueue/templates/visibility-apf/prioritylevelconfigurations.yaml index e78e217192b..d60f6f378cd 100644 --- a/charts/kueue/templates/visibility-apf/prioritylevelconfigurations.yaml +++ b/charts/kueue/templates/visibility-apf/prioritylevelconfigurations.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- if .Values.enableVisibilityAPF }} diff --git a/charts/kueue/templates/visibility/apiservice_v1beta1.yaml b/charts/kueue/templates/visibility/apiservice_v1beta1.yaml index 68eff2d0c41..d823e119ad4 100644 --- a/charts/kueue/templates/visibility/apiservice_v1beta1.yaml +++ b/charts/kueue/templates/visibility/apiservice_v1beta1.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: apiregistration.k8s.io/v1 diff --git a/charts/kueue/templates/visibility/role_binding.yaml b/charts/kueue/templates/visibility/role_binding.yaml index dcb68b6bf24..72dac1025f1 100644 --- a/charts/kueue/templates/visibility/role_binding.yaml +++ b/charts/kueue/templates/visibility/role_binding.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: rbac.authorization.k8s.io/v1 diff --git a/charts/kueue/templates/visibility/service.yaml b/charts/kueue/templates/visibility/service.yaml index c4275d7f159..4fd7f80541d 100644 --- a/charts/kueue/templates/visibility/service.yaml +++ b/charts/kueue/templates/visibility/service.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: v1 diff --git a/charts/kueue/templates/webhook/manifests.yaml b/charts/kueue/templates/webhook/manifests.yaml index 4cfb6b87812..be09161d15b 100644 --- a/charts/kueue/templates/webhook/manifests.yaml +++ b/charts/kueue/templates/webhook/manifests.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} {{- $integrationsConfig := (fromYaml .Values.managerConfig.controllerManagerConfigYaml).integrations }} diff --git a/charts/kueue/templates/webhook/service.yaml b/charts/kueue/templates/webhook/service.yaml index 90da33f1157..024ed6021a7 100644 --- a/charts/kueue/templates/webhook/service.yaml +++ b/charts/kueue/templates/webhook/service.yaml @@ -1,3 +1,19 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + {{/* Code generated by yaml-processor. DO NOT EDIT. */}} apiVersion: v1 diff --git a/hack/internal/tools/yaml-processor/yamlproc/processor.go b/hack/internal/tools/yaml-processor/yamlproc/processor.go index 81b89061412..2e8e180cc61 100644 --- a/hack/internal/tools/yaml-processor/yamlproc/processor.go +++ b/hack/internal/tools/yaml-processor/yamlproc/processor.go @@ -27,7 +27,12 @@ import ( ) type ProcessingPlan struct { - Files []FileOperations `yaml:"files"` + GlobalOptions ProcessingPlanOptions `yaml:"globalOptions"` + Files []FileOperations `yaml:"files"` +} + +type ProcessingPlanOptions struct { + BoilerplatePath string `yaml:"boilerplatePath"` } type FileOperations struct { @@ -100,7 +105,7 @@ func (fp *FileProcessor) ProcessPlan(plan ProcessingPlan) { for _, filePath := range filteredFiles { file.Path = filePath - err := fp.ProcessFile(file) + err := fp.ProcessFile(file, plan.GlobalOptions) if err != nil && !file.ContinueOnError { logger.Fatal("Failed to process file", zap.String("file", file.Path), zap.Error(err)) } @@ -138,7 +143,7 @@ func isExcluded(file string, excludes []string) (bool, error) { return false, nil } -func (fp *FileProcessor) ProcessFile(fileOps FileOperations) error { +func (fp *FileProcessor) ProcessFile(fileOps FileOperations, options ProcessingPlanOptions) error { if fileOps.OutputDir != "" { if err := os.MkdirAll(fileOps.OutputDir, 0755); err != nil { logger.Error("Failed to create output directory", zap.String("outputDir", fileOps.OutputDir), zap.Error(err)) @@ -182,6 +187,14 @@ func (fp *FileProcessor) ProcessFile(fileOps FileOperations) error { prefix := []byte(fmt.Sprintf("{{/* Code generated by %s. DO NOT EDIT. */}}\n\n", filepath.Base(os.Args[0]))) + if options.BoilerplatePath != "" { + boilerplate, err := loadBoilerplate(options.BoilerplatePath) + if err != nil { + logger.Fatal("Failed to load boilerplate", zap.Error(err)) + } + prefix = append(boilerplate, prefix...) + } + // Append prefix to data data = append(prefix, data...) @@ -350,3 +363,15 @@ func LoadProcessingPlan(path string) (*ProcessingPlan, error) { } return &plan, nil } + +func loadBoilerplate(path string) ([]byte, error) { + if path != "" { + boilerplate, err := os.ReadFile(path) + if err != nil { + logger.Fatal("Failed to load boilerplate", zap.Error(err)) + } + boilerplate = append([]byte("{{- "), boilerplate...) + return append(boilerplate, []byte(" -}}\n\n")...), nil + } + return nil, nil +} diff --git a/hack/processing-plan.yaml b/hack/processing-plan.yaml index dce721b7157..dedafe9394a 100644 --- a/hack/processing-plan.yaml +++ b/hack/processing-plan.yaml @@ -1,3 +1,5 @@ +globalOptions: + boilerplatePath: ./hack/boilerplate.go.txt files: - path: ./config/components/crd/bases/*.yaml outputDir: ./charts/kueue/templates/crd/ From cbc270bc8dec2dfc6435e213a5a08c291cfdd12a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Irving=20Mondrag=C3=B3n?= Date: Thu, 30 Oct 2025 16:10:06 +0100 Subject: [PATCH 056/119] Self-nominate IrvingMg as reviewer for internal tool yaml-processor (#7448) --- hack/internal/tools/yaml-processor/OWNERS | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 hack/internal/tools/yaml-processor/OWNERS diff --git a/hack/internal/tools/yaml-processor/OWNERS b/hack/internal/tools/yaml-processor/OWNERS new file mode 100644 index 00000000000..16391b9752e --- /dev/null +++ b/hack/internal/tools/yaml-processor/OWNERS @@ -0,0 +1,4 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +reviewers: +- IrvingMg From 56bdbcc3344413070787b55a9d05f1b0b929ef3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Thu, 30 Oct 2025 17:24:05 +0100 Subject: [PATCH 057/119] Update main with the latest v0.14.3 (#7455) --- CHANGELOG/CHANGELOG-0.14.md | 29 +++++++++++++++++++++++++ Makefile | 2 +- README.md | 2 +- SECURITY-INSIGHTS.yaml | 12 +++++----- charts/kueue/Chart.yaml | 4 ++-- charts/kueue/README.md | 8 +++---- charts/kueue/README.md.gotmpl | 6 ++--- cmd/kueueviz/INSTALL.md | 6 ++--- cmd/kueueviz/frontend/package-lock.json | 4 ++-- cmd/kueueviz/frontend/package.json | 2 +- site/hugo.toml | 4 ++-- test/e2e/kueueviz/package-lock.json | 4 ++-- test/e2e/kueueviz/package.json | 2 +- 13 files changed, 57 insertions(+), 28 deletions(-) diff --git a/CHANGELOG/CHANGELOG-0.14.md b/CHANGELOG/CHANGELOG-0.14.md index 9bbbe272db0..f6e672d5214 100644 --- a/CHANGELOG/CHANGELOG-0.14.md +++ b/CHANGELOG/CHANGELOG-0.14.md @@ -1,3 +1,32 @@ +## v0.14.3 + +Changes since `v0.14.2`: + +## Urgent Upgrade Notes + +### (No, really, you MUST read this before you upgrade) + +- MultiKueue: validate remote client kubeconfigs and reject insecure kubeconfigs by default; add feature gate MultiKueueAllowInsecureKubeconfigs to temporarily allow insecure kubeconfigs until v0.17.0. + + if you are using MultiKueue kubeconfigs which are not passing the new validation please + enable the `MultiKueueAllowInsecureKubeconfigs` feature gate and let us know so that we can re-consider + the deprecation plans for the feature gate. (#7452, @mszadkow) + +## Changes by Kind + +### Bug or Regression + +- Fix a bug where a workload would not get requeued after eviction due to failed hotswap. (#7379, @pajakd) +- Fix the kueue-controller-manager startup failures. + + This fixed the Kueue CrashLoopBackOff due to the log message: "Unable to setup indexes","error":"could not setup multikueue indexer: setting index on workloads admission checks: indexer conflict. (#7440, @IrvingMg) +- Fixed the bug that prevented managing workloads with duplicated environment variable names in containers. This issue manifested when creating the Workload via the API. (#7443, @mbobrovskyi) +- Increase the number of Topology levels limitations for localqueue and workloads to 16 (#7427, @kannon92) +- Services: fix the setting of the `app.kubernetes.io/component` label to discriminate between different service components within Kueue as follows: + - controller-manager-metrics-service for kueue-controller-manager-metrics-service + - visibility-service for kueue-visibility-server + - webhook-service for kueue-webhook-service (#7450, @rphillips) + ## v0.14.2 Changes since `v0.14.1`: diff --git a/Makefile b/Makefile index 7c9b2db58b7..c55aa88fe85 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ LD_FLAGS += -X '$(version_pkg).BuildDate=$(shell date -u +%Y-%m-%dT%H:%M:%SZ)' # Update these variables when preparing a new release or a release branch. # Then run `make prepare-release-branch` -RELEASE_VERSION=v0.14.2 +RELEASE_VERSION=v0.14.3 RELEASE_BRANCH=main # Application version for Helm and npm (strips leading 'v' from RELEASE_VERSION) APP_VERSION := $(shell echo $(RELEASE_VERSION) | cut -c2-) diff --git a/README.md b/README.md index 38603c01b53..3938f3a6277 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Read the [overview](https://kueue.sigs.k8s.io/docs/overview/) and watch the Kueu To install the latest release of Kueue in your cluster, run the following command: ```shell -kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.2/manifests.yaml +kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.3/manifests.yaml ``` The controller runs in the `kueue-system` namespace. diff --git a/SECURITY-INSIGHTS.yaml b/SECURITY-INSIGHTS.yaml index 2384c6af60a..57ef963334c 100644 --- a/SECURITY-INSIGHTS.yaml +++ b/SECURITY-INSIGHTS.yaml @@ -1,11 +1,11 @@ header: schema-version: 1.0.0 expiration-date: '2024-09-28T01:00:00.000Z' - last-updated: '2025-10-23' - last-reviewed: '2025-10-23' - commit-hash: "afb96efdd444eb95e357e6d6b77ce9cd2817fc91" + last-updated: '' + last-reviewed: '' + commit-hash: "" project-url: 'https://github.com/kubernetes-sigs/kueue' - project-release: "0.14.2" + project-release: "0.14.3" changelog: 'https://github.com/kubernetes-sigs/kueue/tree/main/CHANGELOG' license: 'https://github.com/kubernetes-sigs/kueue/blob/main/LICENSE' project-lifecycle: @@ -28,7 +28,7 @@ documentation: - 'https://kueue.sigs.k8s.io/docs/' distribution-points: - >- - https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.2/manifests.yaml + https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.3/manifests.yaml security-artifacts: threat-model: threat-model-created: false @@ -62,5 +62,5 @@ dependencies: dependencies-lists: - 'https://github.com/kubernetes-sigs/kueue/blob/main/go.mod' sbom: - - sbom-file: https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.2/kueue-v0.14.2.spdx.json + - sbom-file: https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.3/kueue-v0.14.3.spdx.json sbom-format: SPDX diff --git a/charts/kueue/Chart.yaml b/charts/kueue/Chart.yaml index 702b56000a7..6c5a9b64983 100644 --- a/charts/kueue/Chart.yaml +++ b/charts/kueue/Chart.yaml @@ -16,9 +16,9 @@ type: application # NOTE: Do not modify manually. In Kueue, the version and appVersion are # overridden to GIT_TAG when building the artifacts, including the helm charts, # via Makefile. -version: 0.14.2 +version: 0.14.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v0.14.2" +appVersion: "v0.14.3" diff --git a/charts/kueue/README.md b/charts/kueue/README.md index 16353fc2cfe..6ae7ae56572 100644 --- a/charts/kueue/README.md +++ b/charts/kueue/README.md @@ -1,6 +1,6 @@ # kueue -![Version: 0.14.2](https://img.shields.io/badge/Version-0.14.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.14.2](https://img.shields.io/badge/AppVersion-v0.14.2-informational?style=flat-square) +![Version: 0.14.3](https://img.shields.io/badge/Version-0.14.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.14.3](https://img.shields.io/badge/AppVersion-v0.14.3-informational?style=flat-square) Kueue is a set of APIs and controllers for job queueing. It is a job-level manager that decides when a job should be admitted to start (as in pods can be created) and when it should stop (as in active pods should be deleted). @@ -28,7 +28,7 @@ $ helm install kueue kueue/ --create-namespace --namespace kueue-system Or use the charts pushed to `oci://registry.k8s.io/kueue/charts/kueue`: ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" --create-namespace --namespace=kueue-system +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" --create-namespace --namespace=kueue-system ``` For more advanced parametrization of Kueue, we recommend using a local overrides file, passed via the `--values` flag. For example: @@ -50,7 +50,7 @@ controllerManager: ``` ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ --create-namespace --namespace=kueue-system \ --values overrides.yaml ``` @@ -58,7 +58,7 @@ helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ You can also use the `--set` flag. For example, to enable a feature gate (e.g., `TopologyAwareScheduling`): ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ --create-namespace --namespace=kueue-system \ --set "controllerManager.featureGates[0].name=TopologyAwareScheduling" \ --set "controllerManager.featureGates[0].enabled=true" diff --git a/charts/kueue/README.md.gotmpl b/charts/kueue/README.md.gotmpl index 6a93180b739..404ad302df0 100644 --- a/charts/kueue/README.md.gotmpl +++ b/charts/kueue/README.md.gotmpl @@ -30,7 +30,7 @@ $ helm install kueue kueue/ --create-namespace --namespace kueue-system Or use the charts pushed to `oci://registry.k8s.io/kueue/charts/kueue`: ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" --create-namespace --namespace=kueue-system +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" --create-namespace --namespace=kueue-system ``` For more advanced parametrization of Kueue, we recommend using a local overrides file, passed via the `--values` flag. For example: @@ -52,7 +52,7 @@ controllerManager: ``` ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ --create-namespace --namespace=kueue-system \ --values overrides.yaml ``` @@ -60,7 +60,7 @@ helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ You can also use the `--set` flag. For example, to enable a feature gate (e.g., `TopologyAwareScheduling`): ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.2" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ --create-namespace --namespace=kueue-system \ --set "controllerManager.featureGates[0].name=TopologyAwareScheduling" \ --set "controllerManager.featureGates[0].enabled=true" diff --git a/cmd/kueueviz/INSTALL.md b/cmd/kueueviz/INSTALL.md index b8c2fd9bc4f..36ac6f55d82 100644 --- a/cmd/kueueviz/INSTALL.md +++ b/cmd/kueueviz/INSTALL.md @@ -3,7 +3,7 @@ KueueViz can be installed using `kubectl` with the following command: ``` -kubectl create -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.2/kueueviz.yaml +kubectl create -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.3/kueueviz.yaml ``` If you are using `kind` and that you don't have an `ingress` controller, you can use `port-forward` to configure and run `KueueViz`: @@ -23,7 +23,7 @@ by ensuring that `enableKueueViz` is set to `true`: ``` helm upgrade --install kueue oci://registry.k8s.io/kueue/charts/kueue \ - --version="0.14.2" + --version="0.14.3" --namespace kueue-system \ --set enableKueueViz=true \ --create-namespace @@ -44,7 +44,7 @@ kind create cluster kind get kubeconfig > kubeconfig export KUBECONFIG=$PWD/kubeconfig helm install kueue oci://us-central1-docker.pkg.dev/k8s-staging-images/charts/kueue \ - --version="0.14.2" --create-namespace --namespace=kueue-system + --version="0.14.3" --create-namespace --namespace=kueue-system ``` ## Build diff --git a/cmd/kueueviz/frontend/package-lock.json b/cmd/kueueviz/frontend/package-lock.json index 98fdb4078cc..6d594e9f4de 100644 --- a/cmd/kueueviz/frontend/package-lock.json +++ b/cmd/kueueviz/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "kueueviz-frontend", - "version": "0.14.2", + "version": "0.14.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "kueueviz-frontend", - "version": "0.14.2", + "version": "0.14.3", "license": "Apache 2.0", "dependencies": { "@emotion/react": "^11.14.0", diff --git a/cmd/kueueviz/frontend/package.json b/cmd/kueueviz/frontend/package.json index 6ef96a1cc03..63ed18fc410 100644 --- a/cmd/kueueviz/frontend/package.json +++ b/cmd/kueueviz/frontend/package.json @@ -1,6 +1,6 @@ { "name": "kueueviz-frontend", - "version": "0.14.2", + "version": "0.14.3", "private": true, "description": "Frontend dashboard for visualizing Kueue status", "main": "src/index.jsx", diff --git a/site/hugo.toml b/site/hugo.toml index b79c676f2a7..9a7f6754706 100644 --- a/site/hugo.toml +++ b/site/hugo.toml @@ -99,10 +99,10 @@ ignoreFiles = [] # The major.minor version tag for the version of the docs represented in this # branch of the repository. Used in the "version-banner" partial to display a # version number for this doc set. - version = "v0.14.2" + version = "v0.14.3" # Version of Kueue without the leading "v", as used for Helm charts. - chart_version = "0.14.2" + chart_version = "0.14.3" # Flag used in the "version-banner" partial to decide whether to display a # banner on every page indicating that this is an archived version of the docs. diff --git a/test/e2e/kueueviz/package-lock.json b/test/e2e/kueueviz/package-lock.json index 6962a8fb3fd..6d2b85090bf 100644 --- a/test/e2e/kueueviz/package-lock.json +++ b/test/e2e/kueueviz/package-lock.json @@ -1,12 +1,12 @@ { "name": "kueueviz-frontend-tests", - "version": "0.14.2", + "version": "0.14.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "kueueviz-frontend-tests", - "version": "0.14.2", + "version": "0.14.3", "devDependencies": { "@testing-library/cypress": "^10.1.0", "cypress": "^15.5.0", diff --git a/test/e2e/kueueviz/package.json b/test/e2e/kueueviz/package.json index 4ec1e1d15da..a3dcded44b3 100644 --- a/test/e2e/kueueviz/package.json +++ b/test/e2e/kueueviz/package.json @@ -1,6 +1,6 @@ { "name": "kueueviz-frontend-tests", - "version": "0.14.2", + "version": "0.14.3", "scripts": { "cypress:run": "cypress run --headless", "check-unused": "depcheck" From b4edcb9eebc2ff5940b1fd0427afa1686ca81745 Mon Sep 17 00:00:00 2001 From: Yuki Iwai Date: Fri, 31 Oct 2025 01:24:12 +0900 Subject: [PATCH 058/119] Add v0.13.8 Release note to CHANGELOG (#7458) Signed-off-by: Yuki Iwai --- CHANGELOG/CHANGELOG-0.13.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CHANGELOG/CHANGELOG-0.13.md b/CHANGELOG/CHANGELOG-0.13.md index 0a374526785..c6c83eba315 100644 --- a/CHANGELOG/CHANGELOG-0.13.md +++ b/CHANGELOG/CHANGELOG-0.13.md @@ -1,3 +1,32 @@ +## v0.13.8 + +Changes since `v0.13.7`: + +## Urgent Upgrade Notes + +### (No, really, you MUST read this before you upgrade) + +- MultiKueue: validate remote client kubeconfigs and reject insecure kubeconfigs by default; add feature gate MultiKueueAllowInsecureKubeconfigs to temporarily allow insecure kubeconfigs until v0.17.0. + + if you are using MultiKueue kubeconfigs which are not passing the new validation please + enable the `MultiKueueAllowInsecureKubeconfigs` feature gate and let us know so that we can re-consider + the deprecation plans for the feature gate. (#7453, @mszadkow) + +## Changes by Kind + +### Bug or Regression + +- Fix a bug where a workload would not get requeued after eviction due to failed hotswap. (#7380, @pajakd) +- Fix the kueue-controller-manager startup failures. + + This fixed the Kueue CrashLoopBackOff due to the log message: "Unable to setup indexes","error":"could not setup multikueue indexer: setting index on workloads admission checks: indexer conflict. (#7441, @IrvingMg) +- Fixed the bug that prevented managing workloads with duplicated environment variable names in containers. This issue manifested when creating the Workload via the API. (#7442, @mbobrovskyi) +- Services: fix the setting of the `app.kubernetes.io/component` label to discriminate between different service components within Kueue as follows: + - controller-manager-metrics-service for kueue-controller-manager-metrics-service + - visibility-service for kueue-visibility-server + - webhook-service for kueue-webhook-service (#7451, @rphillips) +- TAS: Increase the number of Topology levels limitations for localqueue and workloads to 16 (#7428, @kannon92) + ## v0.13.7 Changes since `v0.13.6`: From ab8b367404d78298a826212e96fbad01f13f49c9 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 30 Oct 2025 23:18:01 +0530 Subject: [PATCH 059/119] Replace preemtion stub with interceptor function in TestHierarchicalPreemptions. (#7445) --- .../preemption_hierarchical_test.go | 884 +++++++++++++++--- 1 file changed, 742 insertions(+), 142 deletions(-) diff --git a/pkg/scheduler/preemption/preemption_hierarchical_test.go b/pkg/scheduler/preemption/preemption_hierarchical_test.go index cec87bb50d0..358c3d9ecc2 100644 --- a/pkg/scheduler/preemption/preemption_hierarchical_test.go +++ b/pkg/scheduler/preemption/preemption_hierarchical_test.go @@ -17,8 +17,6 @@ limitations under the License. package preemption import ( - "context" - "sync" "testing" "time" @@ -27,15 +25,16 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" clocktesting "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" + controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/scheduler/flavorassigner" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" @@ -43,10 +42,13 @@ import ( ) func TestHierarchicalPreemptions(t *testing.T) { - now := time.Now() + now := time.Now().Truncate(time.Second) flavors := []*kueue.ResourceFlavor{ utiltestingapi.MakeResourceFlavor("default").Obj(), } + baseIncomingWl := utiltestingapi.MakeWorkload("in", ""). + UID("wl-in"). + Label(controllerconstants.JobUIDLabel, "job-in") cases := map[string]struct { clusterQueues []*kueue.ClusterQueue cohorts []*kueue.Cohort @@ -54,7 +56,8 @@ func TestHierarchicalPreemptions(t *testing.T) { incoming *kueue.Workload targetCQ kueue.ClusterQueueReference assignment flavorassigner.Assignment - wantPreempted sets.Set[string] + wantPreempted int + wantWorkloads []kueue.Workload }{ // // R @@ -91,12 +94,12 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted2", ""). Priority(0). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "2"). Obj(), @@ -107,7 +110,31 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/admitted2", kueue.InCohortReclamationReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted2", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, // // R @@ -152,19 +179,19 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted1", ""). Priority(-10). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_nominal"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_nominal"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted2", ""). Priority(0). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "2"). Obj(), @@ -175,7 +202,38 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/admitted2", kueue.InCohortReclamationReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted1", ""). + Priority(-10). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_nominal"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted2", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, // // R(0) @@ -212,19 +270,19 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted1", ""). Priority(1). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted2", ""). Priority(2). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "2"). Obj(), @@ -235,9 +293,53 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New( - targetKeyReason("/admitted1", kueue.InCohortReclamationReason), - targetKeyReason("/admitted2", kueue.InCohortReclamationReason)), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted1", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted2", ""). + Priority(2). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, // // R(0) @@ -278,26 +380,26 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_not_preemptible", ""). Priority(1). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_preemptible", ""). Priority(0). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_own_queue", ""). Priority(-1). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(1). Request(corev1.ResourceCPU, "2"). Obj(), @@ -308,9 +410,60 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New( - targetKeyReason("/admitted_preemptible", kueue.InCohortReclaimWhileBorrowingReason), - targetKeyReason("/admitted_own_queue", kueue.InClusterQueueReason)), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_not_preemptible", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_own_queue", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted_preemptible", ""). + Priority(0). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclaimWhileBorrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, // // R(0) @@ -347,19 +500,19 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_borrowing", ""). Priority(1). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_same_queue", ""). Priority(-2). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "1"). Obj(), @@ -370,8 +523,38 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_queue", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + Obj(), + }, }, // // R(0) @@ -421,19 +604,19 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_nominal", ""). Priority(-10). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_nominal"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_nominal"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). Priority(-1). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "2"). Obj(), @@ -444,8 +627,38 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New( - targetKeyReason("/admitted_same_cohort", kueue.InCohortReclaimWhileBorrowingReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_nominal", ""). + Priority(-10). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_nominal"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). + Priority(-1). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclaimWhileBorrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, // // R(0) @@ -490,19 +703,19 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_borrowing", ""). Priority(10). Request(corev1.ResourceCPU, "3"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). Priority(10). Request(corev1.ResourceCPU, "3"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "4"). Obj(), @@ -513,9 +726,53 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason), - targetKeyReason("/admitted_same_cohort", kueue.InCohortReclamationReason)), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(10). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). + Priority(10). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, // // R(1) @@ -566,26 +823,26 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_borrowing", ""). Priority(-1). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_same_cohort_preemptible", ""). Priority(-1). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_not_preemptible", ""). Priority(1). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "3"). Obj(), @@ -596,9 +853,60 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason), - targetKeyReason("/admitted_same_cohort_preemptible", kueue.InCohortReclaimWhileBorrowingReason)), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(-1). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_not_preemptible", ""). + Priority(1). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort_preemptible", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclaimWhileBorrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, // // R(1) @@ -649,26 +957,26 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_borrowing", ""). Priority(-1). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_same_queue_preemptible", ""). Priority(-1). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_not_preemptible", ""). Priority(1). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "3"). Obj(), @@ -679,9 +987,60 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason), - targetKeyReason("/admitted_same_queue_preemptible", kueue.InClusterQueueReason)), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(-1). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_not_preemptible", ""). + Priority(1). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_queue_preemptible", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, // // R(0) @@ -726,33 +1085,33 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_borrowing_prio_8", ""). Priority(8). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_prio_9", ""). Priority(9). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_prio_10", ""). Priority(9). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_nominal", ""). Priority(-2). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_nominal"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_nominal"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "1"). Obj(), @@ -763,8 +1122,52 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing_prio_8", kueue.InCohortReclamationReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing_prio_10", ""). + Priority(9). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_prio_8", ""). + Priority(8). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_prio_9", ""). + Priority(9). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_nominal", ""). + Priority(-2). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_nominal"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + Obj(), + }, }, // // R @@ -813,26 +1216,26 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_other_1", ""). Priority(-10). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_other"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_other"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_other_2", ""). Priority(-10). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_other"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_other"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). Priority(0). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "2"). Obj(), @@ -843,7 +1246,29 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New[string](), + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_other_1", ""). + Priority(-10). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_other"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_other_2", ""). + Priority(-10). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_other"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). + Priority(0). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + Obj(), + }, }, // // R(3CPU, 0Gi) @@ -892,22 +1317,22 @@ func TestHierarchicalPreemptions(t *testing.T) { Priority(0). Request(corev1.ResourceCPU, "3"). Request(corev1.ResourceMemory, "1Gi"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "3"). - Assignment(corev1.ResourceMemory, "default", "1Gi").Obj()).Obj()). + Assignment(corev1.ResourceMemory, "default", "1Gi").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). Priority(-2). Request(corev1.ResourceCPU, "1"). Request(corev1.ResourceMemory, "3Gi"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "1"). - Assignment(corev1.ResourceMemory, "default", "3Gi").Obj()).Obj()). + Assignment(corev1.ResourceMemory, "default", "3Gi").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(-2). Request(corev1.ResourceCPU, "2"). Request(corev1.ResourceMemory, "1Gi"). @@ -923,7 +1348,42 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New(targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(0). + Request(corev1.ResourceCPU, "3"). + Request(corev1.ResourceMemory, "1Gi"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3"). + Assignment(corev1.ResourceMemory, "default", "1Gi").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted_same_cohort", ""). + Priority(-2). + Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "3Gi"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1"). + Assignment(corev1.ResourceMemory, "default", "3Gi").Obj()).Obj(), now). + Obj(), + }, }, // // R(0) @@ -972,16 +1432,16 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_borrowing", ""). Priority(-10). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("evicted_same_cohort", ""). Priority(-1). Request(corev1.ResourceCPU, "1"). - ReserveQuota(utiltestingapi.MakeAdmission("q_same_cohort"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, @@ -989,7 +1449,7 @@ func TestHierarchicalPreemptions(t *testing.T) { }). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(0). Request(corev1.ResourceCPU, "1"). Obj(), @@ -1000,7 +1460,28 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New[string](), + wantPreempted: 1, // preemption on going + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(-10). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("evicted_same_cohort", ""). + Priority(-1). + Request(corev1.ResourceCPU, "1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_same_cohort"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "1").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(now), + }). + Obj(), + }, }, // // R(0) @@ -1039,12 +1520,12 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_borrowing", ""). Priority(0). Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q_borrowing"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(-2). Request(corev1.ResourceCPU, "5"). Obj(), @@ -1055,8 +1536,31 @@ func TestHierarchicalPreemptions(t *testing.T) { Mode: flavorassigner.Preempt, }, }), - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing", kueue.InCohortReclamationReason)), + wantPreempted: 1, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing", ""). + Priority(0). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q_borrowing"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + }, }, // r // / \ @@ -1150,61 +1654,61 @@ func TestHierarchicalPreemptions(t *testing.T) { *utiltestingapi.MakeWorkload("admitted_borrowing_1", ""). Priority(-6). Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_2", ""). Priority(-5). Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q1"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q1"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_3", ""). Priority(-9). Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_4", ""). Priority(-10). Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q2"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_5", ""). Priority(-4). Request(corev1.ResourceCPU, "4"). - ReserveQuota(utiltestingapi.MakeAdmission("q3"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q3"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_6", ""). Priority(-3). Request(corev1.ResourceCPU, "3"). - ReserveQuota(utiltestingapi.MakeAdmission("q3"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q3"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_7", ""). Priority(4). Request(corev1.ResourceCPU, "2"). - ReserveQuota(utiltestingapi.MakeAdmission("q4"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q4"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("admitted_borrowing_8", ""). Priority(2). Request(corev1.ResourceCPU, "3"). - ReserveQuota(utiltestingapi.MakeAdmission("q4"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q4"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). - Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj()). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). Obj(), }, - incoming: utiltestingapi.MakeWorkload("incoming", ""). + incoming: baseIncomingWl.Clone(). Priority(-2). Request(corev1.ResourceCPU, "7"). Obj(), @@ -1218,9 +1722,95 @@ func TestHierarchicalPreemptions(t *testing.T) { // only one of workloads from q2 will be preempted because // after preempting the first one, the usage of cohort // c23 will be back within nominal quota - wantPreempted: sets.New( - targetKeyReason("/admitted_borrowing_1", kueue.InCohortReclamationReason), - targetKeyReason("/admitted_borrowing_4", kueue.InCohortReclamationReason)), + wantPreempted: 2, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("admitted_borrowing_1", ""). + Priority(-6). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_2", ""). + Priority(-5). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q1"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_3", ""). + Priority(-9). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_4", ""). + Priority(-10). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q2"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_5", ""). + Priority(-4). + Request(corev1.ResourceCPU, "4"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q3"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "4").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_6", ""). + Priority(-3). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q3"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_7", ""). + Priority(4). + Request(corev1.ResourceCPU, "2"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q4"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "2").Obj()).Obj(), now). + Obj(), + *utiltestingapi.MakeWorkload("admitted_borrowing_8", ""). + Priority(2). + Request(corev1.ResourceCPU, "3"). + ReserveQuotaAt(utiltestingapi.MakeAdmission("q4"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). + Obj(), + }, }, } for name, tc := range cases { @@ -1228,6 +1818,8 @@ func TestHierarchicalPreemptions(t *testing.T) { ctx, log := utiltesting.ContextWithLog(t) cl := utiltesting.NewClientBuilder(). WithLists(&kueue.WorkloadList{Items: tc.admitted}). + WithStatusSubresource(&kueue.Workload{}). + WithInterceptorFuncs(interceptor.Funcs{SubResourcePatch: utiltesting.TreatSSAAsStrategicMerge}). Build() cqCache := schdcache.New(cl) @@ -1245,8 +1837,6 @@ func TestHierarchicalPreemptions(t *testing.T) { } } - var lock sync.Mutex - gotPreempted := sets.New[string]() broadcaster := record.NewBroadcaster() scheme := runtime.NewScheme() if err := kueue.AddToScheme(scheme); err != nil { @@ -1254,12 +1844,7 @@ func TestHierarchicalPreemptions(t *testing.T) { } recorder := broadcaster.NewRecorder(scheme, corev1.EventSource{Component: constants.AdmissionName}) preemptor := New(cl, workload.Ordering{}, recorder, config.FairSharing{}, false, clocktesting.NewFakeClock(now)) - preemptor.applyPreemption = func(ctx context.Context, w *kueue.Workload, reason, _ string) error { - lock.Lock() - gotPreempted.Insert(targetKeyReason(workload.Key(w), reason)) - lock.Unlock() - return nil - } + beforeSnapshot, err := cqCache.Snapshot(ctx) if err != nil { t.Fatalf("unexpected error while building snapshot: %v", err) @@ -1272,13 +1857,28 @@ func TestHierarchicalPreemptions(t *testing.T) { wlInfo := workload.NewInfo(tc.incoming) wlInfo.ClusterQueue = tc.targetCQ targets := preemptor.GetTargets(log, *wlInfo, tc.assignment, snapshotWorkingCopy) - _, err = preemptor.IssuePreemptions(ctx, wlInfo, targets) + preempted, err := preemptor.IssuePreemptions(ctx, wlInfo, targets) if err != nil { t.Fatalf("Failed doing preemption") } - if diff := cmp.Diff(tc.wantPreempted, gotPreempted, cmpopts.EquateEmpty()); diff != "" { - t.Errorf("Issued preemptions (-want,+got):\n%s", diff) + if preempted != tc.wantPreempted { + t.Errorf("Reported %d preemptions, want %d", preempted, tc.wantPreempted) + } + + workloads := &kueue.WorkloadList{} + err = cl.List(ctx, workloads) + if err != nil { + t.Fatalf("Failed to List workloads: %v", err) + } + + defaultCmpOpts := cmp.Options{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion"), + } + if diff := cmp.Diff(tc.wantWorkloads, workloads.Items, defaultCmpOpts); diff != "" { + t.Errorf("Unexpected workloads (-want/+got)\n%s", diff) } + if diff := cmp.Diff(beforeSnapshot, snapshotWorkingCopy, snapCmpOpts); diff != "" { t.Errorf("Snapshot was modified (-initial,+end):\n%s", diff) } From 4f8172d2a84b8dfcd93e8df6fea0d3fb1aec863b Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 31 Oct 2025 11:12:02 +0530 Subject: [PATCH 060/119] Drop graduated ManagedJobsNamespaceSelector feature gate. (#7466) --- keps/3589-manage-jobs-selectively/kep.yaml | 5 +++-- pkg/features/kube_features.go | 10 ---------- site/content/en/docs/installation/_index.md | 2 -- .../reference/components-tools/feature-gate-removed.md | 4 +++- site/content/zh-CN/docs/installation/_index.md | 2 -- 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/keps/3589-manage-jobs-selectively/kep.yaml b/keps/3589-manage-jobs-selectively/kep.yaml index feb27235722..b367a98c514 100644 --- a/keps/3589-manage-jobs-selectively/kep.yaml +++ b/keps/3589-manage-jobs-selectively/kep.yaml @@ -12,16 +12,17 @@ approvers: - "@tenzen-y" # The target maturity stage in the current dev cycle for this KEP. -stage: beta +stage: stable # The most recent milestone for which work toward delivery of this KEP has been # done. This can be the current (upcoming) milestone, if it is being actively # worked on. -latest-milestone: "v0.10" +latest-milestone: "v0.13" # The milestone at which this feature was, or is targeted to be, at each stage. milestone: beta: "v0.10" + stable: "v0.13" feature-gates: - name: ManagedJobsNamespaceSelector diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index fd0d1bf62f1..5e629a3441a 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -96,12 +96,6 @@ const ( // all currently available ResourceFlavors for the LocalQueue. ExposeFlavorsInLocalQueue featuregate.Feature = "ExposeFlavorsInLocalQueue" - // owner: @dgrove-oss - // kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/3589-manage-jobs-selectively - // - // Enable namespace-based control of manageJobsWithoutQueueNames for all Job integrations - ManagedJobsNamespaceSelector featuregate.Feature = "ManagedJobsNamespaceSelector" - // owner: @kpostoffice // kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/1833-metrics-for-local-queue // @@ -270,10 +264,6 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned ExposeFlavorsInLocalQueue: { {Version: version.MustParse("0.9"), Default: true, PreRelease: featuregate.Beta}, }, - ManagedJobsNamespaceSelector: { - {Version: version.MustParse("0.10"), Default: true, PreRelease: featuregate.Beta}, - {Version: version.MustParse("0.13"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 0.15 - }, LocalQueueMetrics: { {Version: version.MustParse("0.10"), Default: false, PreRelease: featuregate.Alpha}, }, diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 5481a0325df..1673f063406 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -307,8 +307,6 @@ The SanitizePodSets feature is available starting from versions 0.13.8 and 0.14. | Feature | Default | Stage | Since | Until | |---------------------------------------| ------- | ---------- | ----- | ----- | -| `ManagedJobsNamespaceSelector` | `true` | Beta | 0.10 | 0.13 | -| `ManagedJobsNamespaceSelector` | `true` | GA | 0.13 | | | `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | | `ProvisioningACC` | `true` | Beta | 0.7 | | | `ProvisioningACC` | `true` | GA | 0.14 | | diff --git a/site/content/en/docs/reference/components-tools/feature-gate-removed.md b/site/content/en/docs/reference/components-tools/feature-gate-removed.md index 1c54db2df7e..d8b87755f78 100644 --- a/site/content/en/docs/reference/components-tools/feature-gate-removed.md +++ b/site/content/en/docs/reference/components-tools/feature-gate-removed.md @@ -33,4 +33,6 @@ In the following table: | `WorkloadResourceRequestsSummary` | `true` | Beta | 0.10 | 0.11 | | `WorkloadResourceRequestsSummary` | `true` | GA | 0.11 | 0.13 | | `QueueVisibility` | `false` | Alpha | 0.5 | 0.9 | -| `QueueVisibility` | `false` | Deprecated | 0.9 | 0.14 | \ No newline at end of file +| `QueueVisibility` | `false` | Deprecated | 0.9 | 0.14 | +| `ManagedJobsNamespaceSelector` | `true` | Beta | 0.10 | 0.13 | +| `ManagedJobsNamespaceSelector` | `true` | GA | 0.13 | 0.15 | \ No newline at end of file diff --git a/site/content/zh-CN/docs/installation/_index.md b/site/content/zh-CN/docs/installation/_index.md index c9a30995772..cc341b66700 100644 --- a/site/content/zh-CN/docs/installation/_index.md +++ b/site/content/zh-CN/docs/installation/_index.md @@ -298,8 +298,6 @@ spec: | 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | |---------------------------------------| ------- | ---------- |---------|---------| -| `ManagedJobsNamespaceSelector` | `true` | Beta | 0.10 | 0.13 | -| `ManagedJobsNamespaceSelector` | `true` | GA | 0.13 | | | `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | | `ProvisioningACC` | `true` | Beta | 0.7 | | | `ProvisioningACC` | `true` | GA | 0.14 | | From 76a89e097f2c982c80d17af987fce6b208399115 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 31 Oct 2025 13:10:03 +0530 Subject: [PATCH 061/119] v1beta2: graduate the visibility API. (#7411) --- .../openapi/zz_generated.openapi.go | 331 ++++++++++++++++++ apis/visibility/v1beta1/doc.go | 2 +- apis/visibility/v1beta1/groupversion_info.go | 1 - .../v1beta1/zz_generated.conversion.go | 242 +++++++++++++ apis/visibility/v1beta2/conversion_test.go | 93 +++++ apis/visibility/v1beta2/defaults.go | 37 ++ apis/visibility/v1beta2/doc.go | 23 ++ apis/visibility/v1beta2/groupversion_info.go | 52 +++ apis/visibility/v1beta2/types.go | 115 ++++++ .../v1beta2/zz_generated.conversion.go | 68 ++++ .../v1beta2/zz_generated.deepcopy.go | 213 +++++++++++ .../v1beta2/zz_generated.defaults.go | 37 ++ .../visibility/apiservice_v1beta2.yaml | 39 +++ client-go/applyconfiguration/utils.go | 12 + .../visibility/v1beta2/clusterqueue.go | 231 ++++++++++++ .../visibility/v1beta2/localqueue.go | 232 ++++++++++++ .../visibility/v1beta2/pendingworkload.go | 227 ++++++++++++ .../v1beta2/pendingworkloadssummary.go | 235 +++++++++++++ client-go/clientset/versioned/clientset.go | 13 + .../versioned/fake/clientset_generated.go | 7 + .../clientset/versioned/fake/register.go | 2 + .../clientset/versioned/scheme/register.go | 2 + .../typed/visibility/v1beta2/clusterqueue.go | 84 +++++ .../versioned/typed/visibility/v1beta2/doc.go | 19 + .../typed/visibility/v1beta2/fake/doc.go | 19 + .../v1beta2/fake/fake_clusterqueue.go | 67 ++++ .../v1beta2/fake/fake_localqueue.go | 66 ++++ .../v1beta2/fake/fake_visibility_client.go | 43 +++ .../visibility/v1beta2/generated_expansion.go | 22 ++ .../typed/visibility/v1beta2/localqueue.go | 85 +++++ .../visibility/v1beta2/visibility_client.go | 105 ++++++ .../informers/externalversions/generic.go | 7 + .../externalversions/visibility/interface.go | 8 + .../visibility/v1beta2/clusterqueue.go | 100 ++++++ .../visibility/v1beta2/interface.go | 51 +++ .../visibility/v1beta2/localqueue.go | 101 ++++++ .../visibility/v1beta2/clusterqueue.go | 47 +++ .../visibility/v1beta2/expansion_generated.go | 30 ++ .../listers/visibility/v1beta2/localqueue.go | 69 ++++ cmd/kueuectl/app/list/list_workload.go | 4 +- .../app/list/list_workload_printer.go | 2 +- cmd/kueuectl/app/list/list_workload_test.go | 2 +- .../visibility/apiservice_v1beta2.yaml | 13 + .../components/visibility/kustomization.yaml | 1 + pkg/visibility/api/install.go | 49 --- pkg/visibility/server.go | 44 ++- .../{api/v1beta1 => storage}/cluster_queue.go | 4 +- .../{api/v1beta1 => storage}/local_queue.go | 4 +- .../pending_workloads_cq.go | 4 +- .../pending_workloads_cq_test.go | 4 +- .../pending_workloads_lq.go | 4 +- .../pending_workloads_lq_test.go | 4 +- .../{api/v1beta1 => storage}/storage.go | 2 +- .../{api/v1beta1 => storage}/utils.go | 7 +- .../test_utils.go => storage/utils_test.go} | 4 +- .../pending_workloads_on_demand.md | 30 +- .../pending_workloads_on_demand.md | 30 +- test/e2e/certmanager/suite_test.go | 7 +- test/e2e/singlecluster/suite_test.go | 11 +- test/e2e/singlecluster/visibility_test.go | 89 +++-- test/util/e2e.go | 27 +- 61 files changed, 3326 insertions(+), 157 deletions(-) create mode 100644 apis/visibility/v1beta2/conversion_test.go create mode 100644 apis/visibility/v1beta2/defaults.go create mode 100644 apis/visibility/v1beta2/doc.go create mode 100644 apis/visibility/v1beta2/groupversion_info.go create mode 100644 apis/visibility/v1beta2/types.go create mode 100644 apis/visibility/v1beta2/zz_generated.conversion.go create mode 100644 apis/visibility/v1beta2/zz_generated.deepcopy.go create mode 100644 apis/visibility/v1beta2/zz_generated.defaults.go create mode 100644 charts/kueue/templates/visibility/apiservice_v1beta2.yaml create mode 100644 client-go/applyconfiguration/visibility/v1beta2/clusterqueue.go create mode 100644 client-go/applyconfiguration/visibility/v1beta2/localqueue.go create mode 100644 client-go/applyconfiguration/visibility/v1beta2/pendingworkload.go create mode 100644 client-go/applyconfiguration/visibility/v1beta2/pendingworkloadssummary.go create mode 100644 client-go/clientset/versioned/typed/visibility/v1beta2/clusterqueue.go create mode 100644 client-go/clientset/versioned/typed/visibility/v1beta2/doc.go create mode 100644 client-go/clientset/versioned/typed/visibility/v1beta2/fake/doc.go create mode 100644 client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_clusterqueue.go create mode 100644 client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_localqueue.go create mode 100644 client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_visibility_client.go create mode 100644 client-go/clientset/versioned/typed/visibility/v1beta2/generated_expansion.go create mode 100644 client-go/clientset/versioned/typed/visibility/v1beta2/localqueue.go create mode 100644 client-go/clientset/versioned/typed/visibility/v1beta2/visibility_client.go create mode 100644 client-go/informers/externalversions/visibility/v1beta2/clusterqueue.go create mode 100644 client-go/informers/externalversions/visibility/v1beta2/interface.go create mode 100644 client-go/informers/externalversions/visibility/v1beta2/localqueue.go create mode 100644 client-go/listers/visibility/v1beta2/clusterqueue.go create mode 100644 client-go/listers/visibility/v1beta2/expansion_generated.go create mode 100644 client-go/listers/visibility/v1beta2/localqueue.go create mode 100644 config/components/visibility/apiservice_v1beta2.yaml delete mode 100644 pkg/visibility/api/install.go rename pkg/visibility/{api/v1beta1 => storage}/cluster_queue.go (95%) rename pkg/visibility/{api/v1beta1 => storage}/local_queue.go (95%) rename pkg/visibility/{api/v1beta1 => storage}/pending_workloads_cq.go (97%) rename pkg/visibility/{api/v1beta1 => storage}/pending_workloads_cq_test.go (99%) rename pkg/visibility/{api/v1beta1 => storage}/pending_workloads_lq.go (97%) rename pkg/visibility/{api/v1beta1 => storage}/pending_workloads_lq_test.go (99%) rename pkg/visibility/{api/v1beta1 => storage}/storage.go (98%) rename pkg/visibility/{api/v1beta1 => storage}/utils.go (87%) rename pkg/visibility/{api/v1beta1/test_utils.go => storage/utils_test.go} (91%) diff --git a/apis/visibility/openapi/zz_generated.openapi.go b/apis/visibility/openapi/zz_generated.openapi.go index d79b55e3fd4..74f4ddaff6c 100644 --- a/apis/visibility/openapi/zz_generated.openapi.go +++ b/apis/visibility/openapi/zz_generated.openapi.go @@ -88,6 +88,13 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "sigs.k8s.io/kueue/apis/visibility/v1beta1.PendingWorkload": schema_kueue_apis_visibility_v1beta1_PendingWorkload(ref), "sigs.k8s.io/kueue/apis/visibility/v1beta1.PendingWorkloadOptions": schema_kueue_apis_visibility_v1beta1_PendingWorkloadOptions(ref), "sigs.k8s.io/kueue/apis/visibility/v1beta1.PendingWorkloadsSummary": schema_kueue_apis_visibility_v1beta1_PendingWorkloadsSummary(ref), + "sigs.k8s.io/kueue/apis/visibility/v1beta2.ClusterQueue": schema_kueue_apis_visibility_v1beta2_ClusterQueue(ref), + "sigs.k8s.io/kueue/apis/visibility/v1beta2.ClusterQueueList": schema_kueue_apis_visibility_v1beta2_ClusterQueueList(ref), + "sigs.k8s.io/kueue/apis/visibility/v1beta2.LocalQueue": schema_kueue_apis_visibility_v1beta2_LocalQueue(ref), + "sigs.k8s.io/kueue/apis/visibility/v1beta2.LocalQueueList": schema_kueue_apis_visibility_v1beta2_LocalQueueList(ref), + "sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkload": schema_kueue_apis_visibility_v1beta2_PendingWorkload(ref), + "sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkloadOptions": schema_kueue_apis_visibility_v1beta2_PendingWorkloadOptions(ref), + "sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkloadsSummary": schema_kueue_apis_visibility_v1beta2_PendingWorkloadsSummary(ref), } } @@ -3040,3 +3047,327 @@ func schema_kueue_apis_visibility_v1beta1_PendingWorkloadsSummary(ref common.Ref "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta", "sigs.k8s.io/kueue/apis/visibility/v1beta1.PendingWorkload"}, } } + +func schema_kueue_apis_visibility_v1beta2_ClusterQueue(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "pendingWorkloadsSummary": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkloadsSummary"), + }, + }, + }, + Required: []string{"pendingWorkloadsSummary"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta", "sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkloadsSummary"}, + } +} + +func schema_kueue_apis_visibility_v1beta2_ClusterQueueList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/kueue/apis/visibility/v1beta2.ClusterQueue"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta", "sigs.k8s.io/kueue/apis/visibility/v1beta2.ClusterQueue"}, + } +} + +func schema_kueue_apis_visibility_v1beta2_LocalQueue(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "pendingWorkloadsSummary": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkloadsSummary"), + }, + }, + }, + Required: []string{"pendingWorkloadsSummary"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta", "sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkloadsSummary"}, + } +} + +func schema_kueue_apis_visibility_v1beta2_LocalQueueList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/kueue/apis/visibility/v1beta2.LocalQueue"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta", "sigs.k8s.io/kueue/apis/visibility/v1beta2.LocalQueue"}, + } +} + +func schema_kueue_apis_visibility_v1beta2_PendingWorkload(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PendingWorkload is a user-facing representation of a pending workload that summarizes the relevant information for position in the cluster queue.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "priority": { + SchemaProps: spec.SchemaProps{ + Description: "Priority indicates the workload's priority", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + "localQueueName": { + SchemaProps: spec.SchemaProps{ + Description: "LocalQueueName indicates the name of the LocalQueue the workload is submitted to", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "positionInClusterQueue": { + SchemaProps: spec.SchemaProps{ + Description: "PositionInClusterQueue indicates the workload's position in the ClusterQueue, starting from 0", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + "positionInLocalQueue": { + SchemaProps: spec.SchemaProps{ + Description: "PositionInLocalQueue indicates the workload's position in the LocalQueue, starting from 0", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + Required: []string{"priority", "localQueueName", "positionInClusterQueue", "positionInLocalQueue"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_kueue_apis_visibility_v1beta2_PendingWorkloadOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PendingWorkloadOptions are query params used in the visibility queries", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "offset": { + SchemaProps: spec.SchemaProps{ + Description: "Offset indicates position of the first pending workload that should be fetched, starting from 0. 0 by default", + Default: 0, + Type: []string{"integer"}, + Format: "int64", + }, + }, + "limit": { + SchemaProps: spec.SchemaProps{ + Description: "Limit indicates max number of pending workloads that should be fetched. 1000 by default", + Type: []string{"integer"}, + Format: "int64", + }, + }, + }, + Required: []string{"offset"}, + }, + }, + } +} + +func schema_kueue_apis_visibility_v1beta2_PendingWorkloadsSummary(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PendingWorkloadsSummary contains a list of pending workloads in the context of the query (within LocalQueue or ClusterQueue).", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkload"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta", "sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkload"}, + } +} diff --git a/apis/visibility/v1beta1/doc.go b/apis/visibility/v1beta1/doc.go index 2cdc7ef5212..0a4c8440255 100644 --- a/apis/visibility/v1beta1/doc.go +++ b/apis/visibility/v1beta1/doc.go @@ -18,6 +18,6 @@ limitations under the License. // +kubebuilder:skip // +groupName=visibility.kueue.x-k8s.io // +k8s:openapi-gen=true -// +k8s:conversion-gen=false +// +k8s:conversion-gen=sigs.k8s.io/kueue/apis/visibility/v1beta2 package v1beta1 diff --git a/apis/visibility/v1beta1/groupversion_info.go b/apis/visibility/v1beta1/groupversion_info.go index f7e97c63b24..31484ab3188 100644 --- a/apis/visibility/v1beta1/groupversion_info.go +++ b/apis/visibility/v1beta1/groupversion_info.go @@ -19,7 +19,6 @@ limitations under the License. // +kubebuilder:skip // +groupName=visibility.kueue.x-k8s.io // +k8s:openapi-gen=true -// +k8s:conversion-gen=false package v1beta1 diff --git a/apis/visibility/v1beta1/zz_generated.conversion.go b/apis/visibility/v1beta1/zz_generated.conversion.go index 396ba67ec10..b863b3df2c7 100644 --- a/apis/visibility/v1beta1/zz_generated.conversion.go +++ b/apis/visibility/v1beta1/zz_generated.conversion.go @@ -22,9 +22,13 @@ package v1beta1 import ( url "net/url" + unsafe "unsafe" conversion "k8s.io/apimachinery/pkg/conversion" runtime "k8s.io/apimachinery/pkg/runtime" + kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" + v1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" ) func init() { @@ -34,6 +38,76 @@ func init() { // RegisterConversions adds conversion functions to the given scheme. // Public to allow building arbitrary schemes. func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*ClusterQueue)(nil), (*v1beta2.ClusterQueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(a.(*ClusterQueue), b.(*v1beta2.ClusterQueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueue)(nil), (*ClusterQueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(a.(*v1beta2.ClusterQueue), b.(*ClusterQueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*ClusterQueueList)(nil), (*v1beta2.ClusterQueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList(a.(*ClusterQueueList), b.(*v1beta2.ClusterQueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueueList)(nil), (*ClusterQueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(a.(*v1beta2.ClusterQueueList), b.(*ClusterQueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LocalQueue)(nil), (*v1beta2.LocalQueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(a.(*LocalQueue), b.(*v1beta2.LocalQueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.LocalQueue)(nil), (*LocalQueue)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(a.(*v1beta2.LocalQueue), b.(*LocalQueue), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*LocalQueueList)(nil), (*v1beta2.LocalQueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList(a.(*LocalQueueList), b.(*v1beta2.LocalQueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.LocalQueueList)(nil), (*LocalQueueList)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList(a.(*v1beta2.LocalQueueList), b.(*LocalQueueList), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PendingWorkload)(nil), (*v1beta2.PendingWorkload)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PendingWorkload_To_v1beta2_PendingWorkload(a.(*PendingWorkload), b.(*v1beta2.PendingWorkload), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.PendingWorkload)(nil), (*PendingWorkload)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_PendingWorkload_To_v1beta1_PendingWorkload(a.(*v1beta2.PendingWorkload), b.(*PendingWorkload), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PendingWorkloadOptions)(nil), (*v1beta2.PendingWorkloadOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PendingWorkloadOptions_To_v1beta2_PendingWorkloadOptions(a.(*PendingWorkloadOptions), b.(*v1beta2.PendingWorkloadOptions), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.PendingWorkloadOptions)(nil), (*PendingWorkloadOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_PendingWorkloadOptions_To_v1beta1_PendingWorkloadOptions(a.(*v1beta2.PendingWorkloadOptions), b.(*PendingWorkloadOptions), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*PendingWorkloadsSummary)(nil), (*v1beta2.PendingWorkloadsSummary)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_PendingWorkloadsSummary_To_v1beta2_PendingWorkloadsSummary(a.(*PendingWorkloadsSummary), b.(*v1beta2.PendingWorkloadsSummary), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*v1beta2.PendingWorkloadsSummary)(nil), (*PendingWorkloadsSummary)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_PendingWorkloadsSummary_To_v1beta1_PendingWorkloadsSummary(a.(*v1beta2.PendingWorkloadsSummary), b.(*PendingWorkloadsSummary), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*url.Values)(nil), (*PendingWorkloadOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_url_Values_To_v1beta1_PendingWorkloadOptions(a.(*url.Values), b.(*PendingWorkloadOptions), scope) }); err != nil { @@ -42,6 +116,152 @@ func RegisterConversions(s *runtime.Scheme) error { return nil } +func autoConvert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(in *ClusterQueue, out *v1beta2.ClusterQueue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_PendingWorkloadsSummary_To_v1beta2_PendingWorkloadsSummary(&in.Summary, &out.Summary, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue is an autogenerated conversion function. +func Convert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(in *ClusterQueue, out *v1beta2.ClusterQueue, s conversion.Scope) error { + return autoConvert_v1beta1_ClusterQueue_To_v1beta2_ClusterQueue(in, out, s) +} + +func autoConvert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(in *v1beta2.ClusterQueue, out *ClusterQueue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_PendingWorkloadsSummary_To_v1beta1_PendingWorkloadsSummary(&in.Summary, &out.Summary, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue is an autogenerated conversion function. +func Convert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(in *v1beta2.ClusterQueue, out *ClusterQueue, s conversion.Scope) error { + return autoConvert_v1beta2_ClusterQueue_To_v1beta1_ClusterQueue(in, out, s) +} + +func autoConvert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList(in *ClusterQueueList, out *v1beta2.ClusterQueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.ClusterQueue)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList is an autogenerated conversion function. +func Convert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList(in *ClusterQueueList, out *v1beta2.ClusterQueueList, s conversion.Scope) error { + return autoConvert_v1beta1_ClusterQueueList_To_v1beta2_ClusterQueueList(in, out, s) +} + +func autoConvert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(in *v1beta2.ClusterQueueList, out *ClusterQueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]ClusterQueue)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList is an autogenerated conversion function. +func Convert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(in *v1beta2.ClusterQueueList, out *ClusterQueueList, s conversion.Scope) error { + return autoConvert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(in, out, s) +} + +func autoConvert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(in *LocalQueue, out *v1beta2.LocalQueue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta1_PendingWorkloadsSummary_To_v1beta2_PendingWorkloadsSummary(&in.Summary, &out.Summary, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta1_LocalQueue_To_v1beta2_LocalQueue is an autogenerated conversion function. +func Convert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(in *LocalQueue, out *v1beta2.LocalQueue, s conversion.Scope) error { + return autoConvert_v1beta1_LocalQueue_To_v1beta2_LocalQueue(in, out, s) +} + +func autoConvert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(in *v1beta2.LocalQueue, out *LocalQueue, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + if err := Convert_v1beta2_PendingWorkloadsSummary_To_v1beta1_PendingWorkloadsSummary(&in.Summary, &out.Summary, s); err != nil { + return err + } + return nil +} + +// Convert_v1beta2_LocalQueue_To_v1beta1_LocalQueue is an autogenerated conversion function. +func Convert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(in *v1beta2.LocalQueue, out *LocalQueue, s conversion.Scope) error { + return autoConvert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(in, out, s) +} + +func autoConvert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList(in *LocalQueueList, out *v1beta2.LocalQueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]v1beta2.LocalQueue)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList is an autogenerated conversion function. +func Convert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList(in *LocalQueueList, out *v1beta2.LocalQueueList, s conversion.Scope) error { + return autoConvert_v1beta1_LocalQueueList_To_v1beta2_LocalQueueList(in, out, s) +} + +func autoConvert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList(in *v1beta2.LocalQueueList, out *LocalQueueList, s conversion.Scope) error { + out.ListMeta = in.ListMeta + out.Items = *(*[]LocalQueue)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList is an autogenerated conversion function. +func Convert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList(in *v1beta2.LocalQueueList, out *LocalQueueList, s conversion.Scope) error { + return autoConvert_v1beta2_LocalQueueList_To_v1beta1_LocalQueueList(in, out, s) +} + +func autoConvert_v1beta1_PendingWorkload_To_v1beta2_PendingWorkload(in *PendingWorkload, out *v1beta2.PendingWorkload, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + out.Priority = in.Priority + out.LocalQueueName = kueuev1beta2.LocalQueueName(in.LocalQueueName) + out.PositionInClusterQueue = in.PositionInClusterQueue + out.PositionInLocalQueue = in.PositionInLocalQueue + return nil +} + +// Convert_v1beta1_PendingWorkload_To_v1beta2_PendingWorkload is an autogenerated conversion function. +func Convert_v1beta1_PendingWorkload_To_v1beta2_PendingWorkload(in *PendingWorkload, out *v1beta2.PendingWorkload, s conversion.Scope) error { + return autoConvert_v1beta1_PendingWorkload_To_v1beta2_PendingWorkload(in, out, s) +} + +func autoConvert_v1beta2_PendingWorkload_To_v1beta1_PendingWorkload(in *v1beta2.PendingWorkload, out *PendingWorkload, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + out.Priority = in.Priority + out.LocalQueueName = kueuev1beta1.LocalQueueName(in.LocalQueueName) + out.PositionInClusterQueue = in.PositionInClusterQueue + out.PositionInLocalQueue = in.PositionInLocalQueue + return nil +} + +// Convert_v1beta2_PendingWorkload_To_v1beta1_PendingWorkload is an autogenerated conversion function. +func Convert_v1beta2_PendingWorkload_To_v1beta1_PendingWorkload(in *v1beta2.PendingWorkload, out *PendingWorkload, s conversion.Scope) error { + return autoConvert_v1beta2_PendingWorkload_To_v1beta1_PendingWorkload(in, out, s) +} + +func autoConvert_v1beta1_PendingWorkloadOptions_To_v1beta2_PendingWorkloadOptions(in *PendingWorkloadOptions, out *v1beta2.PendingWorkloadOptions, s conversion.Scope) error { + out.Offset = in.Offset + out.Limit = in.Limit + return nil +} + +// Convert_v1beta1_PendingWorkloadOptions_To_v1beta2_PendingWorkloadOptions is an autogenerated conversion function. +func Convert_v1beta1_PendingWorkloadOptions_To_v1beta2_PendingWorkloadOptions(in *PendingWorkloadOptions, out *v1beta2.PendingWorkloadOptions, s conversion.Scope) error { + return autoConvert_v1beta1_PendingWorkloadOptions_To_v1beta2_PendingWorkloadOptions(in, out, s) +} + +func autoConvert_v1beta2_PendingWorkloadOptions_To_v1beta1_PendingWorkloadOptions(in *v1beta2.PendingWorkloadOptions, out *PendingWorkloadOptions, s conversion.Scope) error { + out.Offset = in.Offset + out.Limit = in.Limit + return nil +} + +// Convert_v1beta2_PendingWorkloadOptions_To_v1beta1_PendingWorkloadOptions is an autogenerated conversion function. +func Convert_v1beta2_PendingWorkloadOptions_To_v1beta1_PendingWorkloadOptions(in *v1beta2.PendingWorkloadOptions, out *PendingWorkloadOptions, s conversion.Scope) error { + return autoConvert_v1beta2_PendingWorkloadOptions_To_v1beta1_PendingWorkloadOptions(in, out, s) +} + func autoConvert_url_Values_To_v1beta1_PendingWorkloadOptions(in *url.Values, out *PendingWorkloadOptions, s conversion.Scope) error { // WARNING: Field TypeMeta does not have json tag, skipping. @@ -66,3 +286,25 @@ func autoConvert_url_Values_To_v1beta1_PendingWorkloadOptions(in *url.Values, ou func Convert_url_Values_To_v1beta1_PendingWorkloadOptions(in *url.Values, out *PendingWorkloadOptions, s conversion.Scope) error { return autoConvert_url_Values_To_v1beta1_PendingWorkloadOptions(in, out, s) } + +func autoConvert_v1beta1_PendingWorkloadsSummary_To_v1beta2_PendingWorkloadsSummary(in *PendingWorkloadsSummary, out *v1beta2.PendingWorkloadsSummary, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + out.Items = *(*[]v1beta2.PendingWorkload)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta1_PendingWorkloadsSummary_To_v1beta2_PendingWorkloadsSummary is an autogenerated conversion function. +func Convert_v1beta1_PendingWorkloadsSummary_To_v1beta2_PendingWorkloadsSummary(in *PendingWorkloadsSummary, out *v1beta2.PendingWorkloadsSummary, s conversion.Scope) error { + return autoConvert_v1beta1_PendingWorkloadsSummary_To_v1beta2_PendingWorkloadsSummary(in, out, s) +} + +func autoConvert_v1beta2_PendingWorkloadsSummary_To_v1beta1_PendingWorkloadsSummary(in *v1beta2.PendingWorkloadsSummary, out *PendingWorkloadsSummary, s conversion.Scope) error { + out.ObjectMeta = in.ObjectMeta + out.Items = *(*[]PendingWorkload)(unsafe.Pointer(&in.Items)) + return nil +} + +// Convert_v1beta2_PendingWorkloadsSummary_To_v1beta1_PendingWorkloadsSummary is an autogenerated conversion function. +func Convert_v1beta2_PendingWorkloadsSummary_To_v1beta1_PendingWorkloadsSummary(in *v1beta2.PendingWorkloadsSummary, out *PendingWorkloadsSummary, s conversion.Scope) error { + return autoConvert_v1beta2_PendingWorkloadsSummary_To_v1beta1_PendingWorkloadsSummary(in, out, s) +} diff --git a/apis/visibility/v1beta2/conversion_test.go b/apis/visibility/v1beta2/conversion_test.go new file mode 100644 index 00000000000..9dc05f834a8 --- /dev/null +++ b/apis/visibility/v1beta2/conversion_test.go @@ -0,0 +1,93 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/diff" +) + +func TestPendingWorkloadsOptions(t *testing.T) { + scheme := runtime.NewScheme() + err := AddToScheme(scheme) + if err != nil { + t.Fatal(err) + } + codec := runtime.NewParameterCodec(scheme) + + cases := map[string]struct { + inputQueryParams url.Values + wantQueryParams url.Values + wantPendingWorkloadOptions PendingWorkloadOptions + }{ + "correct parameters": { + inputQueryParams: url.Values{ + "limit": {"1"}, + "offset": {"2"}, + }, + wantQueryParams: url.Values{ + "limit": {"1"}, + "offset": {"2"}, + }, + wantPendingWorkloadOptions: PendingWorkloadOptions{ + Limit: 1, + Offset: 2, + }, + }, + "default values": { + inputQueryParams: url.Values{ + "limit": {"0"}, + "offset": {"0"}, + }, + wantQueryParams: url.Values{ + "limit": {"1000"}, + "offset": {"0"}, + }, + wantPendingWorkloadOptions: PendingWorkloadOptions{ + Limit: 1000, + Offset: 0, + }, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + // versioned -> query params + actualParameters, err := codec.EncodeParameters(&tc.wantPendingWorkloadOptions, SchemeGroupVersion) + if err != nil { + t.Fatal(err) + } + if d := cmp.Diff(actualParameters, tc.wantQueryParams); d != "" { + t.Fatalf("Unexpected serialization:\n%s", diff.ObjectGoPrintSideBySide(tc.wantQueryParams, actualParameters)) + } + + // query params -> versioned + convertedPendingWorkloadOptions := PendingWorkloadOptions{} + err = codec.DecodeParameters(tc.inputQueryParams, SchemeGroupVersion, &convertedPendingWorkloadOptions) + if err != nil { + t.Fatal(err) + } + if d := cmp.Diff(convertedPendingWorkloadOptions, tc.wantPendingWorkloadOptions); d != "" { + t.Fatalf("Unexpected deserialization:\n%s", diff.ObjectGoPrintSideBySide(tc.wantPendingWorkloadOptions, convertedPendingWorkloadOptions)) + } + }) + } +} diff --git a/apis/visibility/v1beta2/defaults.go b/apis/visibility/v1beta2/defaults.go new file mode 100644 index 00000000000..53697893d35 --- /dev/null +++ b/apis/visibility/v1beta2/defaults.go @@ -0,0 +1,37 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +func init() { + localSchemeBuilder.Register(addDefaultingFuncs) +} + +func addDefaultingFuncs(scheme *runtime.Scheme) error { + return RegisterDefaults(scheme) +} + +//nolint:revive // format required by generated code for defaulting +func SetDefaults_PendingWorkloadOptions(obj *PendingWorkloadOptions) { + defaultPendingWorkloadsLimit := int64(1000) + if obj.Limit == 0 { + obj.Limit = defaultPendingWorkloadsLimit + } +} diff --git a/apis/visibility/v1beta2/doc.go b/apis/visibility/v1beta2/doc.go new file mode 100644 index 00000000000..052e0020047 --- /dev/null +++ b/apis/visibility/v1beta2/doc.go @@ -0,0 +1,23 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +kubebuilder:object:generate=true +// +kubebuilder:skip +// +groupName=visibility.kueue.x-k8s.io +// +k8s:openapi-gen=true +// +k8s:conversion-gen=false + +package v1beta2 diff --git a/apis/visibility/v1beta2/groupversion_info.go b/apis/visibility/v1beta2/groupversion_info.go new file mode 100644 index 00000000000..e89b9721b62 --- /dev/null +++ b/apis/visibility/v1beta2/groupversion_info.go @@ -0,0 +1,52 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta2 contains API Schema definitions for the pending workloads kueue v1beta2 API group +// +kubebuilder:object:generate=true +// +kubebuilder:skip +// +groupName=visibility.kueue.x-k8s.io +// +k8s:openapi-gen=true + +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "visibility.kueue.x-k8s.io", Version: "v1beta2"} + + // SchemeGroupVersion is alias to GroupVersion for client-go libraries. + // It is required by pkg/client/informers/externalversions/... + SchemeGroupVersion = GroupVersion + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // localSchemeBuilder is used to register autogenerated conversion and defaults functions + // It is required by ./zz_generated.conversion.go and ./zz_generated.defaults.go + localSchemeBuilder = &SchemeBuilder.SchemeBuilder + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +// Resource is required by pkg/client/listers/... +func Resource(resource string) schema.GroupResource { + return GroupVersion.WithResource(resource).GroupResource() +} diff --git a/apis/visibility/v1beta2/types.go b/apis/visibility/v1beta2/types.go new file mode 100644 index 00000000000..e83f37e9c50 --- /dev/null +++ b/apis/visibility/v1beta2/types.go @@ -0,0 +1,115 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +// +genclient +// +kubebuilder:object:root=true +// +k8s:openapi-gen=true +// +genclient:nonNamespaced +// +genclient:method=GetPendingWorkloadsSummary,verb=get,subresource=pendingworkloads,result=sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkloadsSummary +type ClusterQueue struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Summary PendingWorkloadsSummary `json:"pendingWorkloadsSummary"` +} + +// +kubebuilder:object:root=true +type ClusterQueueList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []ClusterQueue `json:"items"` +} + +// +genclient +// +kubebuilder:object:root=true +// +k8s:openapi-gen=true +// +genclient:method=GetPendingWorkloadsSummary,verb=get,subresource=pendingworkloads,result=sigs.k8s.io/kueue/apis/visibility/v1beta2.PendingWorkloadsSummary +type LocalQueue struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Summary PendingWorkloadsSummary `json:"pendingWorkloadsSummary"` +} + +// +kubebuilder:object:root=true +type LocalQueueList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []LocalQueue `json:"items"` +} + +// PendingWorkload is a user-facing representation of a pending workload that summarizes the relevant information for +// position in the cluster queue. +type PendingWorkload struct { + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Priority indicates the workload's priority + Priority int32 `json:"priority"` + + // LocalQueueName indicates the name of the LocalQueue the workload is submitted to + LocalQueueName v1beta2.LocalQueueName `json:"localQueueName"` + + // PositionInClusterQueue indicates the workload's position in the ClusterQueue, starting from 0 + PositionInClusterQueue int32 `json:"positionInClusterQueue"` + + // PositionInLocalQueue indicates the workload's position in the LocalQueue, starting from 0 + PositionInLocalQueue int32 `json:"positionInLocalQueue"` +} + +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true + +// PendingWorkloadsSummary contains a list of pending workloads in the context +// of the query (within LocalQueue or ClusterQueue). +type PendingWorkloadsSummary struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Items []PendingWorkload `json:"items"` +} + +// +kubebuilder:object:root=true +// +k8s:openapi-gen=true +// +k8s:conversion-gen:explicit-from=net/url.Values +// +k8s:defaulter-gen=true + +// PendingWorkloadOptions are query params used in the visibility queries +type PendingWorkloadOptions struct { + metav1.TypeMeta `json:",inline"` + + // Offset indicates position of the first pending workload that should be fetched, starting from 0. 0 by default + Offset int64 `json:"offset"` + + // Limit indicates max number of pending workloads that should be fetched. 1000 by default + Limit int64 `json:"limit,omitempty"` +} + +func init() { + SchemeBuilder.Register( + &PendingWorkloadsSummary{}, + &PendingWorkloadOptions{}, + ) +} diff --git a/apis/visibility/v1beta2/zz_generated.conversion.go b/apis/visibility/v1beta2/zz_generated.conversion.go new file mode 100644 index 00000000000..957a4132ce7 --- /dev/null +++ b/apis/visibility/v1beta2/zz_generated.conversion.go @@ -0,0 +1,68 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by conversion-gen. DO NOT EDIT. + +package v1beta2 + +import ( + url "net/url" + + conversion "k8s.io/apimachinery/pkg/conversion" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +func init() { + localSchemeBuilder.Register(RegisterConversions) +} + +// RegisterConversions adds conversion functions to the given scheme. +// Public to allow building arbitrary schemes. +func RegisterConversions(s *runtime.Scheme) error { + if err := s.AddGeneratedConversionFunc((*url.Values)(nil), (*PendingWorkloadOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_url_Values_To_v1beta2_PendingWorkloadOptions(a.(*url.Values), b.(*PendingWorkloadOptions), scope) + }); err != nil { + return err + } + return nil +} + +func autoConvert_url_Values_To_v1beta2_PendingWorkloadOptions(in *url.Values, out *PendingWorkloadOptions, s conversion.Scope) error { + // WARNING: Field TypeMeta does not have json tag, skipping. + + if values, ok := map[string][]string(*in)["offset"]; ok && len(values) > 0 { + if err := runtime.Convert_Slice_string_To_int64(&values, &out.Offset, s); err != nil { + return err + } + } else { + out.Offset = 0 + } + if values, ok := map[string][]string(*in)["limit"]; ok && len(values) > 0 { + if err := runtime.Convert_Slice_string_To_int64(&values, &out.Limit, s); err != nil { + return err + } + } else { + out.Limit = 0 + } + return nil +} + +// Convert_url_Values_To_v1beta2_PendingWorkloadOptions is an autogenerated conversion function. +func Convert_url_Values_To_v1beta2_PendingWorkloadOptions(in *url.Values, out *PendingWorkloadOptions, s conversion.Scope) error { + return autoConvert_url_Values_To_v1beta2_PendingWorkloadOptions(in, out, s) +} diff --git a/apis/visibility/v1beta2/zz_generated.deepcopy.go b/apis/visibility/v1beta2/zz_generated.deepcopy.go new file mode 100644 index 00000000000..572a6a9dd71 --- /dev/null +++ b/apis/visibility/v1beta2/zz_generated.deepcopy.go @@ -0,0 +1,213 @@ +//go:build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta2 + +import ( + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterQueue) DeepCopyInto(out *ClusterQueue) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Summary.DeepCopyInto(&out.Summary) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterQueue. +func (in *ClusterQueue) DeepCopy() *ClusterQueue { + if in == nil { + return nil + } + out := new(ClusterQueue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterQueue) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterQueueList) DeepCopyInto(out *ClusterQueueList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ClusterQueue, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterQueueList. +func (in *ClusterQueueList) DeepCopy() *ClusterQueueList { + if in == nil { + return nil + } + out := new(ClusterQueueList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ClusterQueueList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalQueue) DeepCopyInto(out *LocalQueue) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Summary.DeepCopyInto(&out.Summary) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalQueue. +func (in *LocalQueue) DeepCopy() *LocalQueue { + if in == nil { + return nil + } + out := new(LocalQueue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LocalQueue) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalQueueList) DeepCopyInto(out *LocalQueueList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LocalQueue, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalQueueList. +func (in *LocalQueueList) DeepCopy() *LocalQueueList { + if in == nil { + return nil + } + out := new(LocalQueueList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LocalQueueList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PendingWorkload) DeepCopyInto(out *PendingWorkload) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PendingWorkload. +func (in *PendingWorkload) DeepCopy() *PendingWorkload { + if in == nil { + return nil + } + out := new(PendingWorkload) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PendingWorkloadOptions) DeepCopyInto(out *PendingWorkloadOptions) { + *out = *in + out.TypeMeta = in.TypeMeta +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PendingWorkloadOptions. +func (in *PendingWorkloadOptions) DeepCopy() *PendingWorkloadOptions { + if in == nil { + return nil + } + out := new(PendingWorkloadOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PendingWorkloadOptions) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PendingWorkloadsSummary) DeepCopyInto(out *PendingWorkloadsSummary) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PendingWorkload, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PendingWorkloadsSummary. +func (in *PendingWorkloadsSummary) DeepCopy() *PendingWorkloadsSummary { + if in == nil { + return nil + } + out := new(PendingWorkloadsSummary) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PendingWorkloadsSummary) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/apis/visibility/v1beta2/zz_generated.defaults.go b/apis/visibility/v1beta2/zz_generated.defaults.go new file mode 100644 index 00000000000..66e2b9a2ef6 --- /dev/null +++ b/apis/visibility/v1beta2/zz_generated.defaults.go @@ -0,0 +1,37 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by defaulter-gen. DO NOT EDIT. + +package v1beta2 + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// RegisterDefaults adds defaulters functions to the given scheme. +// Public to allow building arbitrary schemes. +// All generated defaulters are covering - they call all nested defaulters. +func RegisterDefaults(scheme *runtime.Scheme) error { + scheme.AddTypeDefaultingFunc(&PendingWorkloadOptions{}, func(obj interface{}) { SetObjectDefaults_PendingWorkloadOptions(obj.(*PendingWorkloadOptions)) }) + return nil +} + +func SetObjectDefaults_PendingWorkloadOptions(in *PendingWorkloadOptions) { + SetDefaults_PendingWorkloadOptions(in) +} diff --git a/charts/kueue/templates/visibility/apiservice_v1beta2.yaml b/charts/kueue/templates/visibility/apiservice_v1beta2.yaml new file mode 100644 index 00000000000..93eb8639ad1 --- /dev/null +++ b/charts/kueue/templates/visibility/apiservice_v1beta2.yaml @@ -0,0 +1,39 @@ +{{- /* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ -}} + +{{/* Code generated by yaml-processor. DO NOT EDIT. */}} + +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + {{- if .Values.enableCertManager }} + annotations: + cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/{{ include "kueue.fullname" . }}-visibility-server-cert' + {{- end }} + labels: + {{- include "kueue.visibilityService.labels" . | nindent 4 }} + name: v1beta2.visibility.kueue.x-k8s.io +spec: + {{- if not .Values.enableCertManager }} + insecureSkipTLSVerify: true + {{- end }} + group: visibility.kueue.x-k8s.io + groupPriorityMinimum: 100 + service: + name: '{{ include "kueue.fullname" . }}-visibility-server' + namespace: '{{ .Release.Namespace }}' + version: v1beta2 + versionPriority: 100 diff --git a/client-go/applyconfiguration/utils.go b/client-go/applyconfiguration/utils.go index 9b6de000d20..673a24f15a7 100644 --- a/client-go/applyconfiguration/utils.go +++ b/client-go/applyconfiguration/utils.go @@ -24,10 +24,12 @@ import ( v1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" v1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibilityv1beta1 "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" internal "sigs.k8s.io/kueue/client-go/applyconfiguration/internal" kueuev1beta1 "sigs.k8s.io/kueue/client-go/applyconfiguration/kueue/v1beta1" kueuev1beta2 "sigs.k8s.io/kueue/client-go/applyconfiguration/kueue/v1beta2" applyconfigurationvisibilityv1beta1 "sigs.k8s.io/kueue/client-go/applyconfiguration/visibility/v1beta1" + applyconfigurationvisibilityv1beta2 "sigs.k8s.io/kueue/client-go/applyconfiguration/visibility/v1beta2" ) // ForKind returns an apply configuration type for the given GroupVersionKind, or nil if no @@ -316,6 +318,16 @@ func ForKind(kind schema.GroupVersionKind) interface{} { case visibilityv1beta1.SchemeGroupVersion.WithKind("PendingWorkloadsSummary"): return &applyconfigurationvisibilityv1beta1.PendingWorkloadsSummaryApplyConfiguration{} + // Group=visibility.kueue.x-k8s.io, Version=v1beta2 + case visibilityv1beta2.SchemeGroupVersion.WithKind("ClusterQueue"): + return &applyconfigurationvisibilityv1beta2.ClusterQueueApplyConfiguration{} + case visibilityv1beta2.SchemeGroupVersion.WithKind("LocalQueue"): + return &applyconfigurationvisibilityv1beta2.LocalQueueApplyConfiguration{} + case visibilityv1beta2.SchemeGroupVersion.WithKind("PendingWorkload"): + return &applyconfigurationvisibilityv1beta2.PendingWorkloadApplyConfiguration{} + case visibilityv1beta2.SchemeGroupVersion.WithKind("PendingWorkloadsSummary"): + return &applyconfigurationvisibilityv1beta2.PendingWorkloadsSummaryApplyConfiguration{} + } return nil } diff --git a/client-go/applyconfiguration/visibility/v1beta2/clusterqueue.go b/client-go/applyconfiguration/visibility/v1beta2/clusterqueue.go new file mode 100644 index 00000000000..0188dded61b --- /dev/null +++ b/client-go/applyconfiguration/visibility/v1beta2/clusterqueue.go @@ -0,0 +1,231 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +// ClusterQueueApplyConfiguration represents a declarative configuration of the ClusterQueue type for use +// with apply. +type ClusterQueueApplyConfiguration struct { + v1.TypeMetaApplyConfiguration `json:",inline"` + *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Summary *PendingWorkloadsSummaryApplyConfiguration `json:"pendingWorkloadsSummary,omitempty"` +} + +// ClusterQueue constructs a declarative configuration of the ClusterQueue type for use with +// apply. +func ClusterQueue(name string) *ClusterQueueApplyConfiguration { + b := &ClusterQueueApplyConfiguration{} + b.WithName(name) + b.WithKind("ClusterQueue") + b.WithAPIVersion("visibility.kueue.x-k8s.io/v1beta2") + return b +} +func (b ClusterQueueApplyConfiguration) IsApplyConfiguration() {} + +// WithKind sets the Kind field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Kind field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithKind(value string) *ClusterQueueApplyConfiguration { + b.TypeMetaApplyConfiguration.Kind = &value + return b +} + +// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the APIVersion field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithAPIVersion(value string) *ClusterQueueApplyConfiguration { + b.TypeMetaApplyConfiguration.APIVersion = &value + return b +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithName(value string) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Name = &value + return b +} + +// WithGenerateName sets the GenerateName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the GenerateName field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithGenerateName(value string) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.GenerateName = &value + return b +} + +// WithNamespace sets the Namespace field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Namespace field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithNamespace(value string) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Namespace = &value + return b +} + +// WithUID sets the UID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UID field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithUID(value types.UID) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.UID = &value + return b +} + +// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceVersion field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithResourceVersion(value string) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.ResourceVersion = &value + return b +} + +// WithGeneration sets the Generation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Generation field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithGeneration(value int64) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Generation = &value + return b +} + +// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CreationTimestamp field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithCreationTimestamp(value metav1.Time) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.CreationTimestamp = &value + return b +} + +// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionTimestamp field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionTimestamp = &value + return b +} + +// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionGracePeriodSeconds = &value + return b +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *ClusterQueueApplyConfiguration) WithLabels(entries map[string]string) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Labels == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *ClusterQueueApplyConfiguration) WithAnnotations(entries map[string]string) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Annotations == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Annotations[k] = v + } + return b +} + +// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the OwnerReferences field. +func (b *ClusterQueueApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + if values[i] == nil { + panic("nil value passed to WithOwnerReferences") + } + b.ObjectMetaApplyConfiguration.OwnerReferences = append(b.ObjectMetaApplyConfiguration.OwnerReferences, *values[i]) + } + return b +} + +// WithFinalizers adds the given value to the Finalizers field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Finalizers field. +func (b *ClusterQueueApplyConfiguration) WithFinalizers(values ...string) *ClusterQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + b.ObjectMetaApplyConfiguration.Finalizers = append(b.ObjectMetaApplyConfiguration.Finalizers, values[i]) + } + return b +} + +func (b *ClusterQueueApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { + if b.ObjectMetaApplyConfiguration == nil { + b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} + } +} + +// WithSummary sets the Summary field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Summary field is set to the value of the last call. +func (b *ClusterQueueApplyConfiguration) WithSummary(value *PendingWorkloadsSummaryApplyConfiguration) *ClusterQueueApplyConfiguration { + b.Summary = value + return b +} + +// GetKind retrieves the value of the Kind field in the declarative configuration. +func (b *ClusterQueueApplyConfiguration) GetKind() *string { + return b.TypeMetaApplyConfiguration.Kind +} + +// GetAPIVersion retrieves the value of the APIVersion field in the declarative configuration. +func (b *ClusterQueueApplyConfiguration) GetAPIVersion() *string { + return b.TypeMetaApplyConfiguration.APIVersion +} + +// GetName retrieves the value of the Name field in the declarative configuration. +func (b *ClusterQueueApplyConfiguration) GetName() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Name +} + +// GetNamespace retrieves the value of the Namespace field in the declarative configuration. +func (b *ClusterQueueApplyConfiguration) GetNamespace() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Namespace +} diff --git a/client-go/applyconfiguration/visibility/v1beta2/localqueue.go b/client-go/applyconfiguration/visibility/v1beta2/localqueue.go new file mode 100644 index 00000000000..c4eb8ed6ccb --- /dev/null +++ b/client-go/applyconfiguration/visibility/v1beta2/localqueue.go @@ -0,0 +1,232 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +// LocalQueueApplyConfiguration represents a declarative configuration of the LocalQueue type for use +// with apply. +type LocalQueueApplyConfiguration struct { + v1.TypeMetaApplyConfiguration `json:",inline"` + *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Summary *PendingWorkloadsSummaryApplyConfiguration `json:"pendingWorkloadsSummary,omitempty"` +} + +// LocalQueue constructs a declarative configuration of the LocalQueue type for use with +// apply. +func LocalQueue(name, namespace string) *LocalQueueApplyConfiguration { + b := &LocalQueueApplyConfiguration{} + b.WithName(name) + b.WithNamespace(namespace) + b.WithKind("LocalQueue") + b.WithAPIVersion("visibility.kueue.x-k8s.io/v1beta2") + return b +} +func (b LocalQueueApplyConfiguration) IsApplyConfiguration() {} + +// WithKind sets the Kind field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Kind field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithKind(value string) *LocalQueueApplyConfiguration { + b.TypeMetaApplyConfiguration.Kind = &value + return b +} + +// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the APIVersion field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithAPIVersion(value string) *LocalQueueApplyConfiguration { + b.TypeMetaApplyConfiguration.APIVersion = &value + return b +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithName(value string) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Name = &value + return b +} + +// WithGenerateName sets the GenerateName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the GenerateName field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithGenerateName(value string) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.GenerateName = &value + return b +} + +// WithNamespace sets the Namespace field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Namespace field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithNamespace(value string) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Namespace = &value + return b +} + +// WithUID sets the UID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UID field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithUID(value types.UID) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.UID = &value + return b +} + +// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceVersion field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithResourceVersion(value string) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.ResourceVersion = &value + return b +} + +// WithGeneration sets the Generation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Generation field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithGeneration(value int64) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Generation = &value + return b +} + +// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CreationTimestamp field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithCreationTimestamp(value metav1.Time) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.CreationTimestamp = &value + return b +} + +// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionTimestamp field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionTimestamp = &value + return b +} + +// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionGracePeriodSeconds = &value + return b +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *LocalQueueApplyConfiguration) WithLabels(entries map[string]string) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Labels == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *LocalQueueApplyConfiguration) WithAnnotations(entries map[string]string) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Annotations == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Annotations[k] = v + } + return b +} + +// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the OwnerReferences field. +func (b *LocalQueueApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + if values[i] == nil { + panic("nil value passed to WithOwnerReferences") + } + b.ObjectMetaApplyConfiguration.OwnerReferences = append(b.ObjectMetaApplyConfiguration.OwnerReferences, *values[i]) + } + return b +} + +// WithFinalizers adds the given value to the Finalizers field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Finalizers field. +func (b *LocalQueueApplyConfiguration) WithFinalizers(values ...string) *LocalQueueApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + b.ObjectMetaApplyConfiguration.Finalizers = append(b.ObjectMetaApplyConfiguration.Finalizers, values[i]) + } + return b +} + +func (b *LocalQueueApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { + if b.ObjectMetaApplyConfiguration == nil { + b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} + } +} + +// WithSummary sets the Summary field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Summary field is set to the value of the last call. +func (b *LocalQueueApplyConfiguration) WithSummary(value *PendingWorkloadsSummaryApplyConfiguration) *LocalQueueApplyConfiguration { + b.Summary = value + return b +} + +// GetKind retrieves the value of the Kind field in the declarative configuration. +func (b *LocalQueueApplyConfiguration) GetKind() *string { + return b.TypeMetaApplyConfiguration.Kind +} + +// GetAPIVersion retrieves the value of the APIVersion field in the declarative configuration. +func (b *LocalQueueApplyConfiguration) GetAPIVersion() *string { + return b.TypeMetaApplyConfiguration.APIVersion +} + +// GetName retrieves the value of the Name field in the declarative configuration. +func (b *LocalQueueApplyConfiguration) GetName() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Name +} + +// GetNamespace retrieves the value of the Namespace field in the declarative configuration. +func (b *LocalQueueApplyConfiguration) GetNamespace() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Namespace +} diff --git a/client-go/applyconfiguration/visibility/v1beta2/pendingworkload.go b/client-go/applyconfiguration/visibility/v1beta2/pendingworkload.go new file mode 100644 index 00000000000..a10a3efc758 --- /dev/null +++ b/client-go/applyconfiguration/visibility/v1beta2/pendingworkload.go @@ -0,0 +1,227 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" + kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +// PendingWorkloadApplyConfiguration represents a declarative configuration of the PendingWorkload type for use +// with apply. +type PendingWorkloadApplyConfiguration struct { + *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Priority *int32 `json:"priority,omitempty"` + LocalQueueName *kueuev1beta2.LocalQueueName `json:"localQueueName,omitempty"` + PositionInClusterQueue *int32 `json:"positionInClusterQueue,omitempty"` + PositionInLocalQueue *int32 `json:"positionInLocalQueue,omitempty"` +} + +// PendingWorkloadApplyConfiguration constructs a declarative configuration of the PendingWorkload type for use with +// apply. +func PendingWorkload() *PendingWorkloadApplyConfiguration { + return &PendingWorkloadApplyConfiguration{} +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithName(value string) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Name = &value + return b +} + +// WithGenerateName sets the GenerateName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the GenerateName field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithGenerateName(value string) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.GenerateName = &value + return b +} + +// WithNamespace sets the Namespace field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Namespace field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithNamespace(value string) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Namespace = &value + return b +} + +// WithUID sets the UID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UID field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithUID(value types.UID) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.UID = &value + return b +} + +// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceVersion field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithResourceVersion(value string) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.ResourceVersion = &value + return b +} + +// WithGeneration sets the Generation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Generation field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithGeneration(value int64) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Generation = &value + return b +} + +// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CreationTimestamp field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithCreationTimestamp(value metav1.Time) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.CreationTimestamp = &value + return b +} + +// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionTimestamp field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionTimestamp = &value + return b +} + +// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionGracePeriodSeconds = &value + return b +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *PendingWorkloadApplyConfiguration) WithLabels(entries map[string]string) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Labels == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *PendingWorkloadApplyConfiguration) WithAnnotations(entries map[string]string) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Annotations == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Annotations[k] = v + } + return b +} + +// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the OwnerReferences field. +func (b *PendingWorkloadApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + if values[i] == nil { + panic("nil value passed to WithOwnerReferences") + } + b.ObjectMetaApplyConfiguration.OwnerReferences = append(b.ObjectMetaApplyConfiguration.OwnerReferences, *values[i]) + } + return b +} + +// WithFinalizers adds the given value to the Finalizers field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Finalizers field. +func (b *PendingWorkloadApplyConfiguration) WithFinalizers(values ...string) *PendingWorkloadApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + b.ObjectMetaApplyConfiguration.Finalizers = append(b.ObjectMetaApplyConfiguration.Finalizers, values[i]) + } + return b +} + +func (b *PendingWorkloadApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { + if b.ObjectMetaApplyConfiguration == nil { + b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} + } +} + +// WithPriority sets the Priority field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Priority field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithPriority(value int32) *PendingWorkloadApplyConfiguration { + b.Priority = &value + return b +} + +// WithLocalQueueName sets the LocalQueueName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the LocalQueueName field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithLocalQueueName(value kueuev1beta2.LocalQueueName) *PendingWorkloadApplyConfiguration { + b.LocalQueueName = &value + return b +} + +// WithPositionInClusterQueue sets the PositionInClusterQueue field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the PositionInClusterQueue field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithPositionInClusterQueue(value int32) *PendingWorkloadApplyConfiguration { + b.PositionInClusterQueue = &value + return b +} + +// WithPositionInLocalQueue sets the PositionInLocalQueue field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the PositionInLocalQueue field is set to the value of the last call. +func (b *PendingWorkloadApplyConfiguration) WithPositionInLocalQueue(value int32) *PendingWorkloadApplyConfiguration { + b.PositionInLocalQueue = &value + return b +} + +// GetName retrieves the value of the Name field in the declarative configuration. +func (b *PendingWorkloadApplyConfiguration) GetName() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Name +} + +// GetNamespace retrieves the value of the Namespace field in the declarative configuration. +func (b *PendingWorkloadApplyConfiguration) GetNamespace() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Namespace +} diff --git a/client-go/applyconfiguration/visibility/v1beta2/pendingworkloadssummary.go b/client-go/applyconfiguration/visibility/v1beta2/pendingworkloadssummary.go new file mode 100644 index 00000000000..d6cfab21e95 --- /dev/null +++ b/client-go/applyconfiguration/visibility/v1beta2/pendingworkloadssummary.go @@ -0,0 +1,235 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" +) + +// PendingWorkloadsSummaryApplyConfiguration represents a declarative configuration of the PendingWorkloadsSummary type for use +// with apply. +type PendingWorkloadsSummaryApplyConfiguration struct { + v1.TypeMetaApplyConfiguration `json:",inline"` + *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Items []PendingWorkloadApplyConfiguration `json:"items,omitempty"` +} + +// PendingWorkloadsSummaryApplyConfiguration constructs a declarative configuration of the PendingWorkloadsSummary type for use with +// apply. +func PendingWorkloadsSummary() *PendingWorkloadsSummaryApplyConfiguration { + b := &PendingWorkloadsSummaryApplyConfiguration{} + b.WithKind("PendingWorkloadsSummary") + b.WithAPIVersion("visibility.kueue.x-k8s.io/v1beta2") + return b +} +func (b PendingWorkloadsSummaryApplyConfiguration) IsApplyConfiguration() {} + +// WithKind sets the Kind field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Kind field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithKind(value string) *PendingWorkloadsSummaryApplyConfiguration { + b.TypeMetaApplyConfiguration.Kind = &value + return b +} + +// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the APIVersion field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithAPIVersion(value string) *PendingWorkloadsSummaryApplyConfiguration { + b.TypeMetaApplyConfiguration.APIVersion = &value + return b +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithName(value string) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Name = &value + return b +} + +// WithGenerateName sets the GenerateName field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the GenerateName field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithGenerateName(value string) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.GenerateName = &value + return b +} + +// WithNamespace sets the Namespace field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Namespace field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithNamespace(value string) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Namespace = &value + return b +} + +// WithUID sets the UID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the UID field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithUID(value types.UID) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.UID = &value + return b +} + +// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ResourceVersion field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithResourceVersion(value string) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.ResourceVersion = &value + return b +} + +// WithGeneration sets the Generation field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Generation field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithGeneration(value int64) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.Generation = &value + return b +} + +// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the CreationTimestamp field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithCreationTimestamp(value metav1.Time) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.CreationTimestamp = &value + return b +} + +// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionTimestamp field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionTimestamp = &value + return b +} + +// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + b.ObjectMetaApplyConfiguration.DeletionGracePeriodSeconds = &value + return b +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithLabels(entries map[string]string) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Labels == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithAnnotations(entries map[string]string) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + if b.ObjectMetaApplyConfiguration.Annotations == nil && len(entries) > 0 { + b.ObjectMetaApplyConfiguration.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.ObjectMetaApplyConfiguration.Annotations[k] = v + } + return b +} + +// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the OwnerReferences field. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + if values[i] == nil { + panic("nil value passed to WithOwnerReferences") + } + b.ObjectMetaApplyConfiguration.OwnerReferences = append(b.ObjectMetaApplyConfiguration.OwnerReferences, *values[i]) + } + return b +} + +// WithFinalizers adds the given value to the Finalizers field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Finalizers field. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithFinalizers(values ...string) *PendingWorkloadsSummaryApplyConfiguration { + b.ensureObjectMetaApplyConfigurationExists() + for i := range values { + b.ObjectMetaApplyConfiguration.Finalizers = append(b.ObjectMetaApplyConfiguration.Finalizers, values[i]) + } + return b +} + +func (b *PendingWorkloadsSummaryApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { + if b.ObjectMetaApplyConfiguration == nil { + b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} + } +} + +// WithItems adds the given value to the Items field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Items field. +func (b *PendingWorkloadsSummaryApplyConfiguration) WithItems(values ...*PendingWorkloadApplyConfiguration) *PendingWorkloadsSummaryApplyConfiguration { + for i := range values { + if values[i] == nil { + panic("nil value passed to WithItems") + } + b.Items = append(b.Items, *values[i]) + } + return b +} + +// GetKind retrieves the value of the Kind field in the declarative configuration. +func (b *PendingWorkloadsSummaryApplyConfiguration) GetKind() *string { + return b.TypeMetaApplyConfiguration.Kind +} + +// GetAPIVersion retrieves the value of the APIVersion field in the declarative configuration. +func (b *PendingWorkloadsSummaryApplyConfiguration) GetAPIVersion() *string { + return b.TypeMetaApplyConfiguration.APIVersion +} + +// GetName retrieves the value of the Name field in the declarative configuration. +func (b *PendingWorkloadsSummaryApplyConfiguration) GetName() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Name +} + +// GetNamespace retrieves the value of the Namespace field in the declarative configuration. +func (b *PendingWorkloadsSummaryApplyConfiguration) GetNamespace() *string { + b.ensureObjectMetaApplyConfigurationExists() + return b.ObjectMetaApplyConfiguration.Namespace +} diff --git a/client-go/clientset/versioned/clientset.go b/client-go/clientset/versioned/clientset.go index eae8e735f8e..679e26aec4d 100644 --- a/client-go/clientset/versioned/clientset.go +++ b/client-go/clientset/versioned/clientset.go @@ -28,6 +28,7 @@ import ( kueuev1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta1" kueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2" visibilityv1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta1" + visibilityv1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta2" ) type Interface interface { @@ -36,6 +37,7 @@ type Interface interface { KueueV1beta1() kueuev1beta1.KueueV1beta1Interface KueueV1beta2() kueuev1beta2.KueueV1beta2Interface VisibilityV1beta1() visibilityv1beta1.VisibilityV1beta1Interface + VisibilityV1beta2() visibilityv1beta2.VisibilityV1beta2Interface } // Clientset contains the clients for groups. @@ -45,6 +47,7 @@ type Clientset struct { kueueV1beta1 *kueuev1beta1.KueueV1beta1Client kueueV1beta2 *kueuev1beta2.KueueV1beta2Client visibilityV1beta1 *visibilityv1beta1.VisibilityV1beta1Client + visibilityV1beta2 *visibilityv1beta2.VisibilityV1beta2Client } // KueueV1alpha1 retrieves the KueueV1alpha1Client @@ -67,6 +70,11 @@ func (c *Clientset) VisibilityV1beta1() visibilityv1beta1.VisibilityV1beta1Inter return c.visibilityV1beta1 } +// VisibilityV1beta2 retrieves the VisibilityV1beta2Client +func (c *Clientset) VisibilityV1beta2() visibilityv1beta2.VisibilityV1beta2Interface { + return c.visibilityV1beta2 +} + // Discovery retrieves the DiscoveryClient func (c *Clientset) Discovery() discovery.DiscoveryInterface { if c == nil { @@ -127,6 +135,10 @@ func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, if err != nil { return nil, err } + cs.visibilityV1beta2, err = visibilityv1beta2.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) if err != nil { @@ -152,6 +164,7 @@ func New(c rest.Interface) *Clientset { cs.kueueV1beta1 = kueuev1beta1.New(c) cs.kueueV1beta2 = kueuev1beta2.New(c) cs.visibilityV1beta1 = visibilityv1beta1.New(c) + cs.visibilityV1beta2 = visibilityv1beta2.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) return &cs diff --git a/client-go/clientset/versioned/fake/clientset_generated.go b/client-go/clientset/versioned/fake/clientset_generated.go index e3cdaa3a2d4..ec200ab8976 100644 --- a/client-go/clientset/versioned/fake/clientset_generated.go +++ b/client-go/clientset/versioned/fake/clientset_generated.go @@ -34,6 +34,8 @@ import ( fakekueuev1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/kueue/v1beta2/fake" visibilityv1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta1" fakevisibilityv1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta1/fake" + visibilityv1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta2" + fakevisibilityv1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta2/fake" ) // NewSimpleClientset returns a clientset that will respond with the provided objects. @@ -149,3 +151,8 @@ func (c *Clientset) KueueV1beta2() kueuev1beta2.KueueV1beta2Interface { func (c *Clientset) VisibilityV1beta1() visibilityv1beta1.VisibilityV1beta1Interface { return &fakevisibilityv1beta1.FakeVisibilityV1beta1{Fake: &c.Fake} } + +// VisibilityV1beta2 retrieves the VisibilityV1beta2Client +func (c *Clientset) VisibilityV1beta2() visibilityv1beta2.VisibilityV1beta2Interface { + return &fakevisibilityv1beta2.FakeVisibilityV1beta2{Fake: &c.Fake} +} diff --git a/client-go/clientset/versioned/fake/register.go b/client-go/clientset/versioned/fake/register.go index 31fb5ba9d97..eb4b0b962d7 100644 --- a/client-go/clientset/versioned/fake/register.go +++ b/client-go/clientset/versioned/fake/register.go @@ -27,6 +27,7 @@ import ( kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibilityv1beta1 "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" ) var scheme = runtime.NewScheme() @@ -37,6 +38,7 @@ var localSchemeBuilder = runtime.SchemeBuilder{ kueuev1beta1.AddToScheme, kueuev1beta2.AddToScheme, visibilityv1beta1.AddToScheme, + visibilityv1beta2.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/client-go/clientset/versioned/scheme/register.go b/client-go/clientset/versioned/scheme/register.go index 50034e5898d..eb61f49b8d7 100644 --- a/client-go/clientset/versioned/scheme/register.go +++ b/client-go/clientset/versioned/scheme/register.go @@ -27,6 +27,7 @@ import ( kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibilityv1beta1 "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" ) var Scheme = runtime.NewScheme() @@ -37,6 +38,7 @@ var localSchemeBuilder = runtime.SchemeBuilder{ kueuev1beta1.AddToScheme, kueuev1beta2.AddToScheme, visibilityv1beta1.AddToScheme, + visibilityv1beta2.AddToScheme, } // AddToScheme adds all types of this clientset into the given scheme. This allows composition diff --git a/client-go/clientset/versioned/typed/visibility/v1beta2/clusterqueue.go b/client-go/clientset/versioned/typed/visibility/v1beta2/clusterqueue.go new file mode 100644 index 00000000000..354eb69b48e --- /dev/null +++ b/client-go/clientset/versioned/typed/visibility/v1beta2/clusterqueue.go @@ -0,0 +1,84 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1beta2 + +import ( + context "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" + applyconfigurationvisibilityv1beta2 "sigs.k8s.io/kueue/client-go/applyconfiguration/visibility/v1beta2" + scheme "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" +) + +// ClusterQueuesGetter has a method to return a ClusterQueueInterface. +// A group's client should implement this interface. +type ClusterQueuesGetter interface { + ClusterQueues() ClusterQueueInterface +} + +// ClusterQueueInterface has methods to work with ClusterQueue resources. +type ClusterQueueInterface interface { + Create(ctx context.Context, clusterQueue *visibilityv1beta2.ClusterQueue, opts v1.CreateOptions) (*visibilityv1beta2.ClusterQueue, error) + Update(ctx context.Context, clusterQueue *visibilityv1beta2.ClusterQueue, opts v1.UpdateOptions) (*visibilityv1beta2.ClusterQueue, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*visibilityv1beta2.ClusterQueue, error) + List(ctx context.Context, opts v1.ListOptions) (*visibilityv1beta2.ClusterQueueList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *visibilityv1beta2.ClusterQueue, err error) + Apply(ctx context.Context, clusterQueue *applyconfigurationvisibilityv1beta2.ClusterQueueApplyConfiguration, opts v1.ApplyOptions) (result *visibilityv1beta2.ClusterQueue, err error) + GetPendingWorkloadsSummary(ctx context.Context, clusterQueueName string, options v1.GetOptions) (*visibilityv1beta2.PendingWorkloadsSummary, error) + + ClusterQueueExpansion +} + +// clusterQueues implements ClusterQueueInterface +type clusterQueues struct { + *gentype.ClientWithListAndApply[*visibilityv1beta2.ClusterQueue, *visibilityv1beta2.ClusterQueueList, *applyconfigurationvisibilityv1beta2.ClusterQueueApplyConfiguration] +} + +// newClusterQueues returns a ClusterQueues +func newClusterQueues(c *VisibilityV1beta2Client) *clusterQueues { + return &clusterQueues{ + gentype.NewClientWithListAndApply[*visibilityv1beta2.ClusterQueue, *visibilityv1beta2.ClusterQueueList, *applyconfigurationvisibilityv1beta2.ClusterQueueApplyConfiguration]( + "clusterqueues", + c.RESTClient(), + scheme.ParameterCodec, + "", + func() *visibilityv1beta2.ClusterQueue { return &visibilityv1beta2.ClusterQueue{} }, + func() *visibilityv1beta2.ClusterQueueList { return &visibilityv1beta2.ClusterQueueList{} }, + ), + } +} + +// GetPendingWorkloadsSummary takes name of the clusterQueue, and returns the corresponding visibilityv1beta2.PendingWorkloadsSummary object, and an error if there is any. +func (c *clusterQueues) GetPendingWorkloadsSummary(ctx context.Context, clusterQueueName string, options v1.GetOptions) (result *visibilityv1beta2.PendingWorkloadsSummary, err error) { + result = &visibilityv1beta2.PendingWorkloadsSummary{} + err = c.GetClient().Get(). + Resource("clusterqueues"). + Name(clusterQueueName). + SubResource("pendingworkloads"). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} diff --git a/client-go/clientset/versioned/typed/visibility/v1beta2/doc.go b/client-go/clientset/versioned/typed/visibility/v1beta2/doc.go new file mode 100644 index 00000000000..bfa7cbb63b9 --- /dev/null +++ b/client-go/clientset/versioned/typed/visibility/v1beta2/doc.go @@ -0,0 +1,19 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1beta2 diff --git a/client-go/clientset/versioned/typed/visibility/v1beta2/fake/doc.go b/client-go/clientset/versioned/typed/visibility/v1beta2/fake/doc.go new file mode 100644 index 00000000000..3b001592595 --- /dev/null +++ b/client-go/clientset/versioned/typed/visibility/v1beta2/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_clusterqueue.go b/client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_clusterqueue.go new file mode 100644 index 00000000000..61925edfba6 --- /dev/null +++ b/client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_clusterqueue.go @@ -0,0 +1,67 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + context "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gentype "k8s.io/client-go/gentype" + testing "k8s.io/client-go/testing" + v1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" + visibilityv1beta2 "sigs.k8s.io/kueue/client-go/applyconfiguration/visibility/v1beta2" + typedvisibilityv1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta2" +) + +// fakeClusterQueues implements ClusterQueueInterface +type fakeClusterQueues struct { + *gentype.FakeClientWithListAndApply[*v1beta2.ClusterQueue, *v1beta2.ClusterQueueList, *visibilityv1beta2.ClusterQueueApplyConfiguration] + Fake *FakeVisibilityV1beta2 +} + +func newFakeClusterQueues(fake *FakeVisibilityV1beta2) typedvisibilityv1beta2.ClusterQueueInterface { + return &fakeClusterQueues{ + gentype.NewFakeClientWithListAndApply[*v1beta2.ClusterQueue, *v1beta2.ClusterQueueList, *visibilityv1beta2.ClusterQueueApplyConfiguration]( + fake.Fake, + "", + v1beta2.SchemeGroupVersion.WithResource("clusterqueues"), + v1beta2.SchemeGroupVersion.WithKind("ClusterQueue"), + func() *v1beta2.ClusterQueue { return &v1beta2.ClusterQueue{} }, + func() *v1beta2.ClusterQueueList { return &v1beta2.ClusterQueueList{} }, + func(dst, src *v1beta2.ClusterQueueList) { dst.ListMeta = src.ListMeta }, + func(list *v1beta2.ClusterQueueList) []*v1beta2.ClusterQueue { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1beta2.ClusterQueueList, items []*v1beta2.ClusterQueue) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} + +// GetPendingWorkloadsSummary takes name of the clusterQueue, and returns the corresponding pendingWorkloadsSummary object, and an error if there is any. +func (c *fakeClusterQueues) GetPendingWorkloadsSummary(ctx context.Context, clusterQueueName string, options v1.GetOptions) (result *v1beta2.PendingWorkloadsSummary, err error) { + emptyResult := &v1beta2.PendingWorkloadsSummary{} + obj, err := c.Fake. + Invokes(testing.NewRootGetSubresourceActionWithOptions(c.Resource(), "pendingworkloads", clusterQueueName, options), emptyResult) + if obj == nil { + return emptyResult, err + } + return obj.(*v1beta2.PendingWorkloadsSummary), err +} diff --git a/client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_localqueue.go b/client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_localqueue.go new file mode 100644 index 00000000000..5e44b480937 --- /dev/null +++ b/client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_localqueue.go @@ -0,0 +1,66 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + context "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gentype "k8s.io/client-go/gentype" + testing "k8s.io/client-go/testing" + v1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" + visibilityv1beta2 "sigs.k8s.io/kueue/client-go/applyconfiguration/visibility/v1beta2" + typedvisibilityv1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta2" +) + +// fakeLocalQueues implements LocalQueueInterface +type fakeLocalQueues struct { + *gentype.FakeClientWithListAndApply[*v1beta2.LocalQueue, *v1beta2.LocalQueueList, *visibilityv1beta2.LocalQueueApplyConfiguration] + Fake *FakeVisibilityV1beta2 +} + +func newFakeLocalQueues(fake *FakeVisibilityV1beta2, namespace string) typedvisibilityv1beta2.LocalQueueInterface { + return &fakeLocalQueues{ + gentype.NewFakeClientWithListAndApply[*v1beta2.LocalQueue, *v1beta2.LocalQueueList, *visibilityv1beta2.LocalQueueApplyConfiguration]( + fake.Fake, + namespace, + v1beta2.SchemeGroupVersion.WithResource("localqueues"), + v1beta2.SchemeGroupVersion.WithKind("LocalQueue"), + func() *v1beta2.LocalQueue { return &v1beta2.LocalQueue{} }, + func() *v1beta2.LocalQueueList { return &v1beta2.LocalQueueList{} }, + func(dst, src *v1beta2.LocalQueueList) { dst.ListMeta = src.ListMeta }, + func(list *v1beta2.LocalQueueList) []*v1beta2.LocalQueue { return gentype.ToPointerSlice(list.Items) }, + func(list *v1beta2.LocalQueueList, items []*v1beta2.LocalQueue) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} + +// GetPendingWorkloadsSummary takes name of the localQueue, and returns the corresponding pendingWorkloadsSummary object, and an error if there is any. +func (c *fakeLocalQueues) GetPendingWorkloadsSummary(ctx context.Context, localQueueName string, options v1.GetOptions) (result *v1beta2.PendingWorkloadsSummary, err error) { + emptyResult := &v1beta2.PendingWorkloadsSummary{} + obj, err := c.Fake. + Invokes(testing.NewGetSubresourceActionWithOptions(c.Resource(), c.Namespace(), "pendingworkloads", localQueueName, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1beta2.PendingWorkloadsSummary), err +} diff --git a/client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_visibility_client.go b/client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_visibility_client.go new file mode 100644 index 00000000000..19f42d03632 --- /dev/null +++ b/client-go/clientset/versioned/typed/visibility/v1beta2/fake/fake_visibility_client.go @@ -0,0 +1,43 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" + v1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta2" +) + +type FakeVisibilityV1beta2 struct { + *testing.Fake +} + +func (c *FakeVisibilityV1beta2) ClusterQueues() v1beta2.ClusterQueueInterface { + return newFakeClusterQueues(c) +} + +func (c *FakeVisibilityV1beta2) LocalQueues(namespace string) v1beta2.LocalQueueInterface { + return newFakeLocalQueues(c, namespace) +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeVisibilityV1beta2) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/client-go/clientset/versioned/typed/visibility/v1beta2/generated_expansion.go b/client-go/clientset/versioned/typed/visibility/v1beta2/generated_expansion.go new file mode 100644 index 00000000000..ab34ee10b53 --- /dev/null +++ b/client-go/clientset/versioned/typed/visibility/v1beta2/generated_expansion.go @@ -0,0 +1,22 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1beta2 + +type ClusterQueueExpansion interface{} + +type LocalQueueExpansion interface{} diff --git a/client-go/clientset/versioned/typed/visibility/v1beta2/localqueue.go b/client-go/clientset/versioned/typed/visibility/v1beta2/localqueue.go new file mode 100644 index 00000000000..c4b2d520304 --- /dev/null +++ b/client-go/clientset/versioned/typed/visibility/v1beta2/localqueue.go @@ -0,0 +1,85 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1beta2 + +import ( + context "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" + applyconfigurationvisibilityv1beta2 "sigs.k8s.io/kueue/client-go/applyconfiguration/visibility/v1beta2" + scheme "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" +) + +// LocalQueuesGetter has a method to return a LocalQueueInterface. +// A group's client should implement this interface. +type LocalQueuesGetter interface { + LocalQueues(namespace string) LocalQueueInterface +} + +// LocalQueueInterface has methods to work with LocalQueue resources. +type LocalQueueInterface interface { + Create(ctx context.Context, localQueue *visibilityv1beta2.LocalQueue, opts v1.CreateOptions) (*visibilityv1beta2.LocalQueue, error) + Update(ctx context.Context, localQueue *visibilityv1beta2.LocalQueue, opts v1.UpdateOptions) (*visibilityv1beta2.LocalQueue, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*visibilityv1beta2.LocalQueue, error) + List(ctx context.Context, opts v1.ListOptions) (*visibilityv1beta2.LocalQueueList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *visibilityv1beta2.LocalQueue, err error) + Apply(ctx context.Context, localQueue *applyconfigurationvisibilityv1beta2.LocalQueueApplyConfiguration, opts v1.ApplyOptions) (result *visibilityv1beta2.LocalQueue, err error) + GetPendingWorkloadsSummary(ctx context.Context, localQueueName string, options v1.GetOptions) (*visibilityv1beta2.PendingWorkloadsSummary, error) + + LocalQueueExpansion +} + +// localQueues implements LocalQueueInterface +type localQueues struct { + *gentype.ClientWithListAndApply[*visibilityv1beta2.LocalQueue, *visibilityv1beta2.LocalQueueList, *applyconfigurationvisibilityv1beta2.LocalQueueApplyConfiguration] +} + +// newLocalQueues returns a LocalQueues +func newLocalQueues(c *VisibilityV1beta2Client, namespace string) *localQueues { + return &localQueues{ + gentype.NewClientWithListAndApply[*visibilityv1beta2.LocalQueue, *visibilityv1beta2.LocalQueueList, *applyconfigurationvisibilityv1beta2.LocalQueueApplyConfiguration]( + "localqueues", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *visibilityv1beta2.LocalQueue { return &visibilityv1beta2.LocalQueue{} }, + func() *visibilityv1beta2.LocalQueueList { return &visibilityv1beta2.LocalQueueList{} }, + ), + } +} + +// GetPendingWorkloadsSummary takes name of the localQueue, and returns the corresponding visibilityv1beta2.PendingWorkloadsSummary object, and an error if there is any. +func (c *localQueues) GetPendingWorkloadsSummary(ctx context.Context, localQueueName string, options v1.GetOptions) (result *visibilityv1beta2.PendingWorkloadsSummary, err error) { + result = &visibilityv1beta2.PendingWorkloadsSummary{} + err = c.GetClient().Get(). + Namespace(c.GetNamespace()). + Resource("localqueues"). + Name(localQueueName). + SubResource("pendingworkloads"). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} diff --git a/client-go/clientset/versioned/typed/visibility/v1beta2/visibility_client.go b/client-go/clientset/versioned/typed/visibility/v1beta2/visibility_client.go new file mode 100644 index 00000000000..fd1b452b21b --- /dev/null +++ b/client-go/clientset/versioned/typed/visibility/v1beta2/visibility_client.go @@ -0,0 +1,105 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1beta2 + +import ( + http "net/http" + + rest "k8s.io/client-go/rest" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" + scheme "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" +) + +type VisibilityV1beta2Interface interface { + RESTClient() rest.Interface + ClusterQueuesGetter + LocalQueuesGetter +} + +// VisibilityV1beta2Client is used to interact with features provided by the visibility.kueue.x-k8s.io group. +type VisibilityV1beta2Client struct { + restClient rest.Interface +} + +func (c *VisibilityV1beta2Client) ClusterQueues() ClusterQueueInterface { + return newClusterQueues(c) +} + +func (c *VisibilityV1beta2Client) LocalQueues(namespace string) LocalQueueInterface { + return newLocalQueues(c, namespace) +} + +// NewForConfig creates a new VisibilityV1beta2Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*VisibilityV1beta2Client, error) { + config := *c + setConfigDefaults(&config) + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new VisibilityV1beta2Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*VisibilityV1beta2Client, error) { + config := *c + setConfigDefaults(&config) + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &VisibilityV1beta2Client{client}, nil +} + +// NewForConfigOrDie creates a new VisibilityV1beta2Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *VisibilityV1beta2Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new VisibilityV1beta2Client for the given RESTClient. +func New(c rest.Interface) *VisibilityV1beta2Client { + return &VisibilityV1beta2Client{c} +} + +func setConfigDefaults(config *rest.Config) { + gv := visibilityv1beta2.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = rest.CodecFactoryForGeneratedClient(scheme.Scheme, scheme.Codecs).WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *VisibilityV1beta2Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/client-go/informers/externalversions/generic.go b/client-go/informers/externalversions/generic.go index 55e66176f38..1e19dcc75d7 100644 --- a/client-go/informers/externalversions/generic.go +++ b/client-go/informers/externalversions/generic.go @@ -25,6 +25,7 @@ import ( v1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" v1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibilityv1beta1 "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" ) // GenericInformer is type of SharedIndexInformer which will locate and delegate to other @@ -107,6 +108,12 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource case visibilityv1beta1.SchemeGroupVersion.WithResource("localqueues"): return &genericInformer{resource: resource.GroupResource(), informer: f.Visibility().V1beta1().LocalQueues().Informer()}, nil + // Group=visibility.kueue.x-k8s.io, Version=v1beta2 + case visibilityv1beta2.SchemeGroupVersion.WithResource("clusterqueues"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Visibility().V1beta2().ClusterQueues().Informer()}, nil + case visibilityv1beta2.SchemeGroupVersion.WithResource("localqueues"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Visibility().V1beta2().LocalQueues().Informer()}, nil + } return nil, fmt.Errorf("no informer found for %v", resource) diff --git a/client-go/informers/externalversions/visibility/interface.go b/client-go/informers/externalversions/visibility/interface.go index 90d318603e5..0166df6164c 100644 --- a/client-go/informers/externalversions/visibility/interface.go +++ b/client-go/informers/externalversions/visibility/interface.go @@ -20,12 +20,15 @@ package visibility import ( internalinterfaces "sigs.k8s.io/kueue/client-go/informers/externalversions/internalinterfaces" v1beta1 "sigs.k8s.io/kueue/client-go/informers/externalversions/visibility/v1beta1" + v1beta2 "sigs.k8s.io/kueue/client-go/informers/externalversions/visibility/v1beta2" ) // Interface provides access to each of this group's versions. type Interface interface { // V1beta1 provides access to shared informers for resources in V1beta1. V1beta1() v1beta1.Interface + // V1beta2 provides access to shared informers for resources in V1beta2. + V1beta2() v1beta2.Interface } type group struct { @@ -43,3 +46,8 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList func (g *group) V1beta1() v1beta1.Interface { return v1beta1.New(g.factory, g.namespace, g.tweakListOptions) } + +// V1beta2 returns a new v1beta2.Interface. +func (g *group) V1beta2() v1beta2.Interface { + return v1beta2.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/client-go/informers/externalversions/visibility/v1beta2/clusterqueue.go b/client-go/informers/externalversions/visibility/v1beta2/clusterqueue.go new file mode 100644 index 00000000000..ea0dda2b2f1 --- /dev/null +++ b/client-go/informers/externalversions/visibility/v1beta2/clusterqueue.go @@ -0,0 +1,100 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1beta2 + +import ( + context "context" + time "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" + apisvisibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" + versioned "sigs.k8s.io/kueue/client-go/clientset/versioned" + internalinterfaces "sigs.k8s.io/kueue/client-go/informers/externalversions/internalinterfaces" + visibilityv1beta2 "sigs.k8s.io/kueue/client-go/listers/visibility/v1beta2" +) + +// ClusterQueueInformer provides access to a shared informer and lister for +// ClusterQueues. +type ClusterQueueInformer interface { + Informer() cache.SharedIndexInformer + Lister() visibilityv1beta2.ClusterQueueLister +} + +type clusterQueueInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewClusterQueueInformer constructs a new informer for ClusterQueue type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewClusterQueueInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredClusterQueueInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredClusterQueueInformer constructs a new informer for ClusterQueue type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredClusterQueueInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VisibilityV1beta2().ClusterQueues().List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VisibilityV1beta2().ClusterQueues().Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VisibilityV1beta2().ClusterQueues().List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VisibilityV1beta2().ClusterQueues().Watch(ctx, options) + }, + }, + &apisvisibilityv1beta2.ClusterQueue{}, + resyncPeriod, + indexers, + ) +} + +func (f *clusterQueueInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredClusterQueueInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *clusterQueueInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisvisibilityv1beta2.ClusterQueue{}, f.defaultInformer) +} + +func (f *clusterQueueInformer) Lister() visibilityv1beta2.ClusterQueueLister { + return visibilityv1beta2.NewClusterQueueLister(f.Informer().GetIndexer()) +} diff --git a/client-go/informers/externalversions/visibility/v1beta2/interface.go b/client-go/informers/externalversions/visibility/v1beta2/interface.go new file mode 100644 index 00000000000..b13c09b7362 --- /dev/null +++ b/client-go/informers/externalversions/visibility/v1beta2/interface.go @@ -0,0 +1,51 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1beta2 + +import ( + internalinterfaces "sigs.k8s.io/kueue/client-go/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // ClusterQueues returns a ClusterQueueInformer. + ClusterQueues() ClusterQueueInformer + // LocalQueues returns a LocalQueueInformer. + LocalQueues() LocalQueueInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// ClusterQueues returns a ClusterQueueInformer. +func (v *version) ClusterQueues() ClusterQueueInformer { + return &clusterQueueInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} + +// LocalQueues returns a LocalQueueInformer. +func (v *version) LocalQueues() LocalQueueInformer { + return &localQueueInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/client-go/informers/externalversions/visibility/v1beta2/localqueue.go b/client-go/informers/externalversions/visibility/v1beta2/localqueue.go new file mode 100644 index 00000000000..0380eb1fdda --- /dev/null +++ b/client-go/informers/externalversions/visibility/v1beta2/localqueue.go @@ -0,0 +1,101 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1beta2 + +import ( + context "context" + time "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" + apisvisibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" + versioned "sigs.k8s.io/kueue/client-go/clientset/versioned" + internalinterfaces "sigs.k8s.io/kueue/client-go/informers/externalversions/internalinterfaces" + visibilityv1beta2 "sigs.k8s.io/kueue/client-go/listers/visibility/v1beta2" +) + +// LocalQueueInformer provides access to a shared informer and lister for +// LocalQueues. +type LocalQueueInformer interface { + Informer() cache.SharedIndexInformer + Lister() visibilityv1beta2.LocalQueueLister +} + +type localQueueInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewLocalQueueInformer constructs a new informer for LocalQueue type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewLocalQueueInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredLocalQueueInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredLocalQueueInformer constructs a new informer for LocalQueue type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredLocalQueueInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VisibilityV1beta2().LocalQueues(namespace).List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VisibilityV1beta2().LocalQueues(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VisibilityV1beta2().LocalQueues(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.VisibilityV1beta2().LocalQueues(namespace).Watch(ctx, options) + }, + }, + &apisvisibilityv1beta2.LocalQueue{}, + resyncPeriod, + indexers, + ) +} + +func (f *localQueueInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredLocalQueueInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *localQueueInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisvisibilityv1beta2.LocalQueue{}, f.defaultInformer) +} + +func (f *localQueueInformer) Lister() visibilityv1beta2.LocalQueueLister { + return visibilityv1beta2.NewLocalQueueLister(f.Informer().GetIndexer()) +} diff --git a/client-go/listers/visibility/v1beta2/clusterqueue.go b/client-go/listers/visibility/v1beta2/clusterqueue.go new file mode 100644 index 00000000000..a9f88fb7dcf --- /dev/null +++ b/client-go/listers/visibility/v1beta2/clusterqueue.go @@ -0,0 +1,47 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1beta2 + +import ( + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" +) + +// ClusterQueueLister helps list ClusterQueues. +// All objects returned here must be treated as read-only. +type ClusterQueueLister interface { + // List lists all ClusterQueues in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*visibilityv1beta2.ClusterQueue, err error) + // Get retrieves the ClusterQueue from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*visibilityv1beta2.ClusterQueue, error) + ClusterQueueListerExpansion +} + +// clusterQueueLister implements the ClusterQueueLister interface. +type clusterQueueLister struct { + listers.ResourceIndexer[*visibilityv1beta2.ClusterQueue] +} + +// NewClusterQueueLister returns a new ClusterQueueLister. +func NewClusterQueueLister(indexer cache.Indexer) ClusterQueueLister { + return &clusterQueueLister{listers.New[*visibilityv1beta2.ClusterQueue](indexer, visibilityv1beta2.Resource("clusterqueue"))} +} diff --git a/client-go/listers/visibility/v1beta2/expansion_generated.go b/client-go/listers/visibility/v1beta2/expansion_generated.go new file mode 100644 index 00000000000..4100065a3a3 --- /dev/null +++ b/client-go/listers/visibility/v1beta2/expansion_generated.go @@ -0,0 +1,30 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1beta2 + +// ClusterQueueListerExpansion allows custom methods to be added to +// ClusterQueueLister. +type ClusterQueueListerExpansion interface{} + +// LocalQueueListerExpansion allows custom methods to be added to +// LocalQueueLister. +type LocalQueueListerExpansion interface{} + +// LocalQueueNamespaceListerExpansion allows custom methods to be added to +// LocalQueueNamespaceLister. +type LocalQueueNamespaceListerExpansion interface{} diff --git a/client-go/listers/visibility/v1beta2/localqueue.go b/client-go/listers/visibility/v1beta2/localqueue.go new file mode 100644 index 00000000000..2c5e46a2a24 --- /dev/null +++ b/client-go/listers/visibility/v1beta2/localqueue.go @@ -0,0 +1,69 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1beta2 + +import ( + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" +) + +// LocalQueueLister helps list LocalQueues. +// All objects returned here must be treated as read-only. +type LocalQueueLister interface { + // List lists all LocalQueues in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*visibilityv1beta2.LocalQueue, err error) + // LocalQueues returns an object that can list and get LocalQueues. + LocalQueues(namespace string) LocalQueueNamespaceLister + LocalQueueListerExpansion +} + +// localQueueLister implements the LocalQueueLister interface. +type localQueueLister struct { + listers.ResourceIndexer[*visibilityv1beta2.LocalQueue] +} + +// NewLocalQueueLister returns a new LocalQueueLister. +func NewLocalQueueLister(indexer cache.Indexer) LocalQueueLister { + return &localQueueLister{listers.New[*visibilityv1beta2.LocalQueue](indexer, visibilityv1beta2.Resource("localqueue"))} +} + +// LocalQueues returns an object that can list and get LocalQueues. +func (s *localQueueLister) LocalQueues(namespace string) LocalQueueNamespaceLister { + return localQueueNamespaceLister{listers.NewNamespaced[*visibilityv1beta2.LocalQueue](s.ResourceIndexer, namespace)} +} + +// LocalQueueNamespaceLister helps list and get LocalQueues. +// All objects returned here must be treated as read-only. +type LocalQueueNamespaceLister interface { + // List lists all LocalQueues in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*visibilityv1beta2.LocalQueue, err error) + // Get retrieves the LocalQueue from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*visibilityv1beta2.LocalQueue, error) + LocalQueueNamespaceListerExpansion +} + +// localQueueNamespaceLister implements the LocalQueueNamespaceLister +// interface. +type localQueueNamespaceLister struct { + listers.ResourceIndexer[*visibilityv1beta2.LocalQueue] +} diff --git a/cmd/kueuectl/app/list/list_workload.go b/cmd/kueuectl/app/list/list_workload.go index 6d7933319de..e790e6430a0 100644 --- a/cmd/kueuectl/app/list/list_workload.go +++ b/cmd/kueuectl/app/list/list_workload.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" clientset "sigs.k8s.io/kueue/client-go/clientset/versioned" "sigs.k8s.io/kueue/client-go/clientset/versioned/scheme" "sigs.k8s.io/kueue/cmd/kueuectl/app/completion" @@ -451,7 +451,7 @@ func (o *WorkloadOptions) pendingWorkloads(ctx context.Context, list *kueue.Work } pendingWorkloadsSummary, ok := pendingWorkloadsSummaries[clusterQueueName] if !ok { - pendingWorkloadsSummary, err = o.ClientSet.VisibilityV1beta1().ClusterQueues(). + pendingWorkloadsSummary, err = o.ClientSet.VisibilityV1beta2().ClusterQueues(). GetPendingWorkloadsSummary(ctx, string(clusterQueueName), metav1.GetOptions{}) if client.IgnoreNotFound(err) != nil { return nil, err diff --git a/cmd/kueuectl/app/list/list_workload_printer.go b/cmd/kueuectl/app/list/list_workload_printer.go index 0bec93d1c18..8a6d05506b8 100644 --- a/cmd/kueuectl/app/list/list_workload_printer.go +++ b/cmd/kueuectl/app/list/list_workload_printer.go @@ -31,7 +31,7 @@ import ( "k8s.io/utils/clock" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) diff --git a/cmd/kueuectl/app/list/list_workload_test.go b/cmd/kueuectl/app/list/list_workload_test.go index 066fed37982..52f52c4e52e 100644 --- a/cmd/kueuectl/app/list/list_workload_test.go +++ b/cmd/kueuectl/app/list/list_workload_test.go @@ -44,7 +44,7 @@ import ( testingclock "k8s.io/utils/clock/testing" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" cmdtesting "sigs.k8s.io/kueue/cmd/kueuectl/app/testing" "sigs.k8s.io/kueue/pkg/controller/constants" diff --git a/config/components/visibility/apiservice_v1beta2.yaml b/config/components/visibility/apiservice_v1beta2.yaml new file mode 100644 index 00000000000..011aba59691 --- /dev/null +++ b/config/components/visibility/apiservice_v1beta2.yaml @@ -0,0 +1,13 @@ +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta2.visibility.kueue.x-k8s.io +spec: + group: visibility.kueue.x-k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: visibility-server + namespace: kueue-system + version: v1beta2 + versionPriority: 100 \ No newline at end of file diff --git a/config/components/visibility/kustomization.yaml b/config/components/visibility/kustomization.yaml index 18bd0173ce1..96162046c7f 100644 --- a/config/components/visibility/kustomization.yaml +++ b/config/components/visibility/kustomization.yaml @@ -1,5 +1,6 @@ resources: - apiservice_v1beta1.yaml +- apiservice_v1beta2.yaml - role_binding.yaml - service.yaml diff --git a/pkg/visibility/api/install.go b/pkg/visibility/api/install.go deleted file mode 100644 index 4d75ae40f49..00000000000 --- a/pkg/visibility/api/install.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package api - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - genericapiserver "k8s.io/apiserver/pkg/server" - - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" - qcache "sigs.k8s.io/kueue/pkg/cache/queue" - visibilityapi "sigs.k8s.io/kueue/pkg/visibility/api/v1beta1" -) - -var ( - Scheme = runtime.NewScheme() - Codecs = serializer.NewCodecFactory(Scheme) - ParameterCodec = runtime.NewParameterCodec(Scheme) -) - -func init() { - utilruntime.Must(visibility.AddToScheme(Scheme)) - utilruntime.Must(Scheme.SetVersionPriority(visibility.GroupVersion)) - metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) -} - -// Install installs API scheme and registers storages -func Install(server *genericapiserver.GenericAPIServer, kueueMgr *qcache.Manager) error { - apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(visibility.GroupVersion.Group, Scheme, ParameterCodec, Codecs) - apiGroupInfo.VersionedResourcesStorageMap[visibility.GroupVersion.Version] = visibilityapi.NewStorage(kueueMgr) - return server.InstallAPIGroups(&apiGroupInfo) -} diff --git a/pkg/visibility/server.go b/pkg/visibility/server.go index 4fe37c1a421..9bf97ffae6e 100644 --- a/pkg/visibility/server.go +++ b/pkg/visibility/server.go @@ -22,6 +22,11 @@ import ( "net" "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" validatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/validating" "k8s.io/apiserver/pkg/admission/plugin/resourcequota" mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating" @@ -34,14 +39,18 @@ import ( "k8s.io/component-base/version" generatedopenapi "sigs.k8s.io/kueue/apis/visibility/openapi" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibilityv1beta1 "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibilityv1beta2 "sigs.k8s.io/kueue/apis/visibility/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" - "sigs.k8s.io/kueue/pkg/visibility/api" + "sigs.k8s.io/kueue/pkg/visibility/storage" _ "k8s.io/component-base/metrics/prometheus/restclient" // for client-go metrics registration ) var ( + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) + parameterCodec = runtime.NewParameterCodec(scheme) // Admission plugins that are enabled by default in the kubeapi server // but are not required for the visibility server. disabledPlugins = []string{ @@ -53,11 +62,17 @@ var ( certDir = "/visibility" ) +func init() { + utilruntime.Must(visibilityv1beta2.AddToScheme(scheme)) + utilruntime.Must(visibilityv1beta1.AddToScheme(scheme)) + metav1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) +} + // +kubebuilder:rbac:groups=flowcontrol.apiserver.k8s.io,resources=prioritylevelconfigurations,verbs=list;watch // +kubebuilder:rbac:groups=flowcontrol.apiserver.k8s.io,resources=flowschemas,verbs=list;watch // +kubebuilder:rbac:groups=flowcontrol.apiserver.k8s.io,resources=flowschemas/status,verbs=patch -// CreateAndStartVisibilityServer creates visibility server injecting KueueManager and starts it +// CreateAndStartVisibilityServer creates a visibility server injecting KueueManager and starts it func CreateAndStartVisibilityServer(ctx context.Context, kueueMgr *qcache.Manager, enableInternalCertManagement bool, kubeConfig *rest.Config) error { config := newVisibilityServerConfig(kubeConfig) if err := applyVisibilityServerOptions(config, enableInternalCertManagement); err != nil { @@ -69,7 +84,7 @@ func CreateAndStartVisibilityServer(ctx context.Context, kueueMgr *qcache.Manage return fmt.Errorf("unable to create visibility server: %w", err) } - if err := api.Install(visibilityServer, kueueMgr); err != nil { + if err := install(visibilityServer, kueueMgr); err != nil { return fmt.Errorf("unable to install visibility.kueue.x-k8s.io API: %w", err) } @@ -81,7 +96,10 @@ func CreateAndStartVisibilityServer(ctx context.Context, kueueMgr *qcache.Manage } func applyVisibilityServerOptions(config *genericapiserver.RecommendedConfig, enableInternalCertManagement bool) error { - o := genericoptions.NewRecommendedOptions("", api.Codecs.LegacyCodec(visibility.SchemeGroupVersion)) + o := genericoptions.NewRecommendedOptions("", codecs.LegacyCodec( + visibilityv1beta2.SchemeGroupVersion, + visibilityv1beta1.SchemeGroupVersion, + )) o.Etcd = nil o.SecureServing.BindPort = 8082 if enableInternalCertManagement { @@ -99,13 +117,13 @@ func applyVisibilityServerOptions(config *genericapiserver.RecommendedConfig, en } func newVisibilityServerConfig(kubeConfig *rest.Config) *genericapiserver.RecommendedConfig { - c := genericapiserver.NewRecommendedConfig(api.Codecs) + c := genericapiserver.NewRecommendedConfig(codecs) versionInfo := version.Get() version := strings.Split(versionInfo.String(), "-")[0] // enable OpenAPI schemas c.EffectiveVersion = compatibility.NewEffectiveVersionFromString(version, "", "") - c.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(api.Scheme)) - c.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(api.Scheme)) + c.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(scheme)) + c.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(scheme)) c.OpenAPIConfig.Info.Title = "Kueue visibility-server" c.OpenAPIV3Config.Info.Title = "Kueue visibility-server" c.OpenAPIConfig.Info.Version = version @@ -116,3 +134,13 @@ func newVisibilityServerConfig(kubeConfig *rest.Config) *genericapiserver.Recomm return c } + +// install installs API scheme and registers storages +func install(server *genericapiserver.GenericAPIServer, kueueMgr *qcache.Manager) error { + apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(visibilityv1beta2.GroupVersion.Group, scheme, parameterCodec, codecs) + storage := storage.NewStorage(kueueMgr) + apiGroupInfo.VersionedResourcesStorageMap[visibilityv1beta2.GroupVersion.Version] = storage + apiGroupInfo.VersionedResourcesStorageMap[visibilityv1beta1.GroupVersion.Version] = storage + apiGroupInfo.PrioritizedVersions = []schema.GroupVersion{visibilityv1beta2.GroupVersion, visibilityv1beta1.GroupVersion} + return server.InstallAPIGroups(&apiGroupInfo) +} diff --git a/pkg/visibility/api/v1beta1/cluster_queue.go b/pkg/visibility/storage/cluster_queue.go similarity index 95% rename from pkg/visibility/api/v1beta1/cluster_queue.go rename to pkg/visibility/storage/cluster_queue.go index fa19d5e4ac9..e380a20f294 100644 --- a/pkg/visibility/api/v1beta1/cluster_queue.go +++ b/pkg/visibility/storage/cluster_queue.go @@ -14,13 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package storage import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/rest" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" ) // CqREST type is used only to install clusterqueues/ resource, so we can install clusterqueues/pending_workloads subresource. diff --git a/pkg/visibility/api/v1beta1/local_queue.go b/pkg/visibility/storage/local_queue.go similarity index 95% rename from pkg/visibility/api/v1beta1/local_queue.go rename to pkg/visibility/storage/local_queue.go index 43748edaf2a..ef352a7d8e7 100644 --- a/pkg/visibility/api/v1beta1/local_queue.go +++ b/pkg/visibility/storage/local_queue.go @@ -14,13 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package storage import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/rest" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" ) // LqREST type is used only to install localqueues/ resource, so we can install localqueues/pending_workloads subresource. diff --git a/pkg/visibility/api/v1beta1/pending_workloads_cq.go b/pkg/visibility/storage/pending_workloads_cq.go similarity index 97% rename from pkg/visibility/api/v1beta1/pending_workloads_cq.go rename to pkg/visibility/storage/pending_workloads_cq.go index 12fc8e0286a..8378e522fd4 100644 --- a/pkg/visibility/api/v1beta1/pending_workloads_cq.go +++ b/pkg/visibility/storage/pending_workloads_cq.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package storage import ( "context" @@ -27,7 +27,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" diff --git a/pkg/visibility/api/v1beta1/pending_workloads_cq_test.go b/pkg/visibility/storage/pending_workloads_cq_test.go similarity index 99% rename from pkg/visibility/api/v1beta1/pending_workloads_cq_test.go rename to pkg/visibility/storage/pending_workloads_cq_test.go index 26f623666eb..00b92e71d6d 100644 --- a/pkg/visibility/api/v1beta1/pending_workloads_cq_test.go +++ b/pkg/visibility/storage/pending_workloads_cq_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package storage import ( "context" @@ -28,7 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/visibility/api/v1beta1/pending_workloads_lq.go b/pkg/visibility/storage/pending_workloads_lq.go similarity index 97% rename from pkg/visibility/api/v1beta1/pending_workloads_lq.go rename to pkg/visibility/storage/pending_workloads_lq.go index 227a4bcb515..8a0b4e2009f 100644 --- a/pkg/visibility/api/v1beta1/pending_workloads_lq.go +++ b/pkg/visibility/storage/pending_workloads_lq.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package storage import ( "context" @@ -28,7 +28,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" utilqueue "sigs.k8s.io/kueue/pkg/util/queue" diff --git a/pkg/visibility/api/v1beta1/pending_workloads_lq_test.go b/pkg/visibility/storage/pending_workloads_lq_test.go similarity index 99% rename from pkg/visibility/api/v1beta1/pending_workloads_lq_test.go rename to pkg/visibility/storage/pending_workloads_lq_test.go index 367464d1b38..4165eb172b2 100644 --- a/pkg/visibility/api/v1beta1/pending_workloads_lq_test.go +++ b/pkg/visibility/storage/pending_workloads_lq_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package storage import ( "context" @@ -29,7 +29,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/request" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" diff --git a/pkg/visibility/api/v1beta1/storage.go b/pkg/visibility/storage/storage.go similarity index 98% rename from pkg/visibility/api/v1beta1/storage.go rename to pkg/visibility/storage/storage.go index f641d2e8b10..6ac13e382b2 100644 --- a/pkg/visibility/api/v1beta1/storage.go +++ b/pkg/visibility/storage/storage.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package storage import ( "k8s.io/apiserver/pkg/registry/rest" diff --git a/pkg/visibility/api/v1beta1/utils.go b/pkg/visibility/storage/utils.go similarity index 87% rename from pkg/visibility/api/v1beta1/utils.go rename to pkg/visibility/storage/utils.go index f9f7a455f46..f9f66c01693 100644 --- a/pkg/visibility/api/v1beta1/utils.go +++ b/pkg/visibility/storage/utils.go @@ -14,13 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package storage import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" "sigs.k8s.io/kueue/pkg/workload" ) @@ -43,7 +42,7 @@ func newPendingWorkload(wlInfo *workload.Info, positionInLq int32, positionInCq }, PositionInClusterQueue: int32(positionInCq), Priority: *wlInfo.Obj.Spec.Priority, - LocalQueueName: kueuev1beta1.LocalQueueName(wlInfo.Obj.Spec.QueueName), + LocalQueueName: wlInfo.Obj.Spec.QueueName, PositionInLocalQueue: positionInLq, } } diff --git a/pkg/visibility/api/v1beta1/test_utils.go b/pkg/visibility/storage/utils_test.go similarity index 91% rename from pkg/visibility/api/v1beta1/test_utils.go rename to pkg/visibility/storage/utils_test.go index 31a613f5ea3..e3d0dab6f2a 100644 --- a/pkg/visibility/api/v1beta1/test_utils.go +++ b/pkg/visibility/storage/utils_test.go @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1beta1 +package storage import ( - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" ) type req struct { diff --git a/site/content/en/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand.md b/site/content/en/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand.md index 584c957c757..8d5b5857027 100644 --- a/site/content/en/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand.md +++ b/site/content/en/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand.md @@ -72,7 +72,7 @@ kubectl apply -f https://kueue.sigs.k8s.io/examples/visibility/cluster-role-and- You can disable it by setting the `VisibilityOnDemand` feature gate. Check the [Installation](/docs/installation/#change-the-feature-gates-configuration) guide for details on feature gate configuration. -If you disable the feature, you also need to remove the associated `APIService` from your cluster by doing `kubectl delete APIService v1beta1.visibility.kueue.x-k8s.io`. +If you disable the feature, you also need to remove the associated `APIService` from your cluster by doing `kubectl delete APIService v1beta2.visibility.kueue.x-k8s.io`. {{% /alert %}} @@ -103,7 +103,7 @@ for i in {1..6}; do kubectl create -f https://kueue.sigs.k8s.io/examples/jobs/sa To view pending workloads in ClusterQueue `cluster-queue` run the following command: ```shell -kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta1/clusterqueues/cluster-queue/pendingworkloads" +kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta2/clusterqueues/cluster-queue/pendingworkloads" ``` You should get results similar to: @@ -111,7 +111,7 @@ You should get results similar to: ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -184,7 +184,7 @@ You can pass optional query parameters: To view only 1 pending workloads use, starting from position 1 in ClusterQueue run: ```shell -kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta1/clusterqueues/cluster-queue/pendingworkloads?limit=1&offset=1" +kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta2/clusterqueues/cluster-queue/pendingworkloads?limit=1&offset=1" ``` You should get results similar to @@ -192,7 +192,7 @@ You should get results similar to ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -226,8 +226,8 @@ If you followed steps described in [Directly accessing the Visibility API](#dire you can use curl to view pending workloads in ClusterQueue using following commands: {{< tabpane lang="shell" persist=disabled >}} -{{< tab header="Using kubectl proxy" >}} curl http://localhost:8080/apis/visibility.kueue.x-k8s.io/v1beta1/clusterqueues/cluster-queue/pendingworkloads {{< /tab >}} -{{< tab header="Without kubectl proxy" >}} curl -X GET $APISERVER/apis/visibility.kueue.x-k8s.io/v1beta1/clusterqueues/cluster-queue/pendingworkloads --header "Authorization: Bearer $TOKEN" --insecure {{< /tab >}} +{{< tab header="Using kubectl proxy" >}} curl http://localhost:8080/apis/visibility.kueue.x-k8s.io/v1beta2/clusterqueues/cluster-queue/pendingworkloads {{< /tab >}} +{{< tab header="Without kubectl proxy" >}} curl -X GET $APISERVER/apis/visibility.kueue.x-k8s.io/v1beta2/clusterqueues/cluster-queue/pendingworkloads --header "Authorization: Bearer $TOKEN" --insecure {{< /tab >}} {{< /tabpane >}} You should get results similar to: @@ -235,7 +235,7 @@ You should get results similar to: ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -306,7 +306,7 @@ You should get results similar to: Similarly to ClusterQueue, to view pending workloads in LocalQueue `user-queue` run the following command: ```shell -kubectl get --raw /apis/visibility.kueue.x-k8s.io/v1beta1/namespaces/default/localqueues/user-queue/pendingworkloads +kubectl get --raw /apis/visibility.kueue.x-k8s.io/v1beta2/namespaces/default/localqueues/user-queue/pendingworkloads ``` You should get results similar to: @@ -314,7 +314,7 @@ You should get results similar to: ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -387,7 +387,7 @@ You can pass optional query parameters: To view only 1 pending workloads use, starting from position 1 in LocalQueue run: ```shell -kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta1/namespaces/default/localqueues/user-queue/pendingworkloads?limit=1&offset=1" +kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta2/namespaces/default/localqueues/user-queue/pendingworkloads?limit=1&offset=1" ``` You should get results similar to @@ -395,7 +395,7 @@ You should get results similar to ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -429,8 +429,8 @@ If you followed steps described in [Directly accessing the Visibility API](#dire above, you can use curl to view pending workloads in LocalQueue using following commands: {{< tabpane lang="shell" persist=disabled >}} -{{< tab header="Using kubectl proxy" >}} curl http://localhost:8080/apis/visibility.kueue.x-k8s.io/v1beta1/namespaces/default/localqueues/user-queue/pendingworkloads {{< /tab >}} -{{< tab header="Without kubectl proxy" >}} curl -X GET $APISERVER/apis/visibility.kueue.x-k8s.io/v1beta1/namespaces/default/localqueues/user-queue/pendingworkloads --header "Authorization: Bearer $TOKEN" --insecure {{< /tab >}} +{{< tab header="Using kubectl proxy" >}} curl http://localhost:8080/apis/visibility.kueue.x-k8s.io/v1beta2/namespaces/default/localqueues/user-queue/pendingworkloads {{< /tab >}} +{{< tab header="Without kubectl proxy" >}} curl -X GET $APISERVER/apis/visibility.kueue.x-k8s.io/v1beta2/namespaces/default/localqueues/user-queue/pendingworkloads --header "Authorization: Bearer $TOKEN" --insecure {{< /tab >}} {{< /tabpane >}} You should get results similar to: @@ -438,7 +438,7 @@ You should get results similar to: ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, diff --git a/site/content/zh-CN/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand.md b/site/content/zh-CN/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand.md index 584c957c757..8d5b5857027 100644 --- a/site/content/zh-CN/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand.md +++ b/site/content/zh-CN/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand.md @@ -72,7 +72,7 @@ kubectl apply -f https://kueue.sigs.k8s.io/examples/visibility/cluster-role-and- You can disable it by setting the `VisibilityOnDemand` feature gate. Check the [Installation](/docs/installation/#change-the-feature-gates-configuration) guide for details on feature gate configuration. -If you disable the feature, you also need to remove the associated `APIService` from your cluster by doing `kubectl delete APIService v1beta1.visibility.kueue.x-k8s.io`. +If you disable the feature, you also need to remove the associated `APIService` from your cluster by doing `kubectl delete APIService v1beta2.visibility.kueue.x-k8s.io`. {{% /alert %}} @@ -103,7 +103,7 @@ for i in {1..6}; do kubectl create -f https://kueue.sigs.k8s.io/examples/jobs/sa To view pending workloads in ClusterQueue `cluster-queue` run the following command: ```shell -kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta1/clusterqueues/cluster-queue/pendingworkloads" +kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta2/clusterqueues/cluster-queue/pendingworkloads" ``` You should get results similar to: @@ -111,7 +111,7 @@ You should get results similar to: ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -184,7 +184,7 @@ You can pass optional query parameters: To view only 1 pending workloads use, starting from position 1 in ClusterQueue run: ```shell -kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta1/clusterqueues/cluster-queue/pendingworkloads?limit=1&offset=1" +kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta2/clusterqueues/cluster-queue/pendingworkloads?limit=1&offset=1" ``` You should get results similar to @@ -192,7 +192,7 @@ You should get results similar to ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -226,8 +226,8 @@ If you followed steps described in [Directly accessing the Visibility API](#dire you can use curl to view pending workloads in ClusterQueue using following commands: {{< tabpane lang="shell" persist=disabled >}} -{{< tab header="Using kubectl proxy" >}} curl http://localhost:8080/apis/visibility.kueue.x-k8s.io/v1beta1/clusterqueues/cluster-queue/pendingworkloads {{< /tab >}} -{{< tab header="Without kubectl proxy" >}} curl -X GET $APISERVER/apis/visibility.kueue.x-k8s.io/v1beta1/clusterqueues/cluster-queue/pendingworkloads --header "Authorization: Bearer $TOKEN" --insecure {{< /tab >}} +{{< tab header="Using kubectl proxy" >}} curl http://localhost:8080/apis/visibility.kueue.x-k8s.io/v1beta2/clusterqueues/cluster-queue/pendingworkloads {{< /tab >}} +{{< tab header="Without kubectl proxy" >}} curl -X GET $APISERVER/apis/visibility.kueue.x-k8s.io/v1beta2/clusterqueues/cluster-queue/pendingworkloads --header "Authorization: Bearer $TOKEN" --insecure {{< /tab >}} {{< /tabpane >}} You should get results similar to: @@ -235,7 +235,7 @@ You should get results similar to: ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -306,7 +306,7 @@ You should get results similar to: Similarly to ClusterQueue, to view pending workloads in LocalQueue `user-queue` run the following command: ```shell -kubectl get --raw /apis/visibility.kueue.x-k8s.io/v1beta1/namespaces/default/localqueues/user-queue/pendingworkloads +kubectl get --raw /apis/visibility.kueue.x-k8s.io/v1beta2/namespaces/default/localqueues/user-queue/pendingworkloads ``` You should get results similar to: @@ -314,7 +314,7 @@ You should get results similar to: ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -387,7 +387,7 @@ You can pass optional query parameters: To view only 1 pending workloads use, starting from position 1 in LocalQueue run: ```shell -kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta1/namespaces/default/localqueues/user-queue/pendingworkloads?limit=1&offset=1" +kubectl get --raw "/apis/visibility.kueue.x-k8s.io/v1beta2/namespaces/default/localqueues/user-queue/pendingworkloads?limit=1&offset=1" ``` You should get results similar to @@ -395,7 +395,7 @@ You should get results similar to ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, @@ -429,8 +429,8 @@ If you followed steps described in [Directly accessing the Visibility API](#dire above, you can use curl to view pending workloads in LocalQueue using following commands: {{< tabpane lang="shell" persist=disabled >}} -{{< tab header="Using kubectl proxy" >}} curl http://localhost:8080/apis/visibility.kueue.x-k8s.io/v1beta1/namespaces/default/localqueues/user-queue/pendingworkloads {{< /tab >}} -{{< tab header="Without kubectl proxy" >}} curl -X GET $APISERVER/apis/visibility.kueue.x-k8s.io/v1beta1/namespaces/default/localqueues/user-queue/pendingworkloads --header "Authorization: Bearer $TOKEN" --insecure {{< /tab >}} +{{< tab header="Using kubectl proxy" >}} curl http://localhost:8080/apis/visibility.kueue.x-k8s.io/v1beta2/namespaces/default/localqueues/user-queue/pendingworkloads {{< /tab >}} +{{< tab header="Without kubectl proxy" >}} curl -X GET $APISERVER/apis/visibility.kueue.x-k8s.io/v1beta2/namespaces/default/localqueues/user-queue/pendingworkloads --header "Authorization: Bearer $TOKEN" --insecure {{< /tab >}} {{< /tabpane >}} You should get results similar to: @@ -438,7 +438,7 @@ You should get results similar to: ```json { "kind": "PendingWorkloadsSummary", - "apiVersion": "visibility.kueue.x-k8s.io/v1beta1", + "apiVersion": "visibility.kueue.x-k8s.io/v1beta2", "metadata": { "creationTimestamp": null }, diff --git a/test/e2e/certmanager/suite_test.go b/test/e2e/certmanager/suite_test.go index 17cf6f365c1..edb294274ec 100644 --- a/test/e2e/certmanager/suite_test.go +++ b/test/e2e/certmanager/suite_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - visibilityv1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -38,7 +38,7 @@ var ( cfg *rest.Config restClient *rest.RESTClient kueueNS = util.GetKueueNamespace() - visibilityClient visibilityv1beta1.VisibilityV1beta1Interface + visibilityClient visibility.VisibilityV1beta2Interface ) func TestAPIs(t *testing.T) { @@ -59,8 +59,7 @@ var _ = ginkgo.BeforeSuite(func() { k8sClient, cfg, err = util.CreateClientUsingCluster("") gomega.Expect(err).NotTo(gomega.HaveOccurred()) restClient = util.CreateRestClient(cfg) - visibilityClient, err = util.CreateVisibilityClient("") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) + visibilityClient = util.CreateVisibilityClient("") ctx = ginkgo.GinkgoT().Context() waitForAvailableStart := time.Now() diff --git a/test/e2e/singlecluster/suite_test.go b/test/e2e/singlecluster/suite_test.go index 3154cdd1d55..afe32b742f8 100644 --- a/test/e2e/singlecluster/suite_test.go +++ b/test/e2e/singlecluster/suite_test.go @@ -30,7 +30,8 @@ import ( "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" - visibilityv1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta1" + kueueclientset "sigs.k8s.io/kueue/client-go/clientset/versioned" + visibility "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -41,8 +42,8 @@ var ( cfg *rest.Config restClient *rest.RESTClient ctx context.Context - visibilityClient visibilityv1beta1.VisibilityV1beta1Interface - impersonatedVisibilityClient visibilityv1beta1.VisibilityV1beta1Interface + kueueClientset kueueclientset.Interface + impersonatedVisibilityClient visibility.VisibilityV1beta2Interface kueueNS = util.GetKueueNamespace() ) @@ -64,9 +65,9 @@ var _ = ginkgo.BeforeSuite(func() { k8sClient, cfg, err = util.CreateClientUsingCluster("") gomega.Expect(err).NotTo(gomega.HaveOccurred()) restClient = util.CreateRestClient(cfg) - visibilityClient, err = util.CreateVisibilityClient("") + kueueClientset = util.CreateKueueClientset("") gomega.Expect(err).NotTo(gomega.HaveOccurred()) - impersonatedVisibilityClient, err = util.CreateVisibilityClient(fmt.Sprintf("system:serviceaccount:%s:default", kueueNS)) + impersonatedVisibilityClient = util.CreateVisibilityClient(fmt.Sprintf("system:serviceaccount:%s:default", kueueNS)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ctx = ginkgo.GinkgoT().Context() diff --git a/test/e2e/singlecluster/visibility_test.go b/test/e2e/singlecluster/visibility_test.go index d803939834e..60912474e50 100644 --- a/test/e2e/singlecluster/visibility_test.go +++ b/test/e2e/singlecluster/visibility_test.go @@ -29,9 +29,8 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" @@ -135,9 +134,55 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { util.ExpectObjectToBeDeleted(ctx, k8sClient, defaultRF, true) }) + ginkgo.It("Should allow fetching information about pending workloads in ClusterQueue (v1beta1)", func() { + ginkgo.By("Verify there are zero pending workloads", func() { + info, err := kueueClientset.VisibilityV1beta1().ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(info.Items).Should(gomega.BeEmpty()) + }) + + ginkgo.By("Schedule a job which is pending due to lower priority", func() { + sampleJob2 = testingjob.MakeJob("test-job-2", nsA.Name). + Queue(kueue.LocalQueueName(localQueueA.Name)). + Image(util.GetAgnHostImage(), util.BehaviorExitFast). + RequestAndLimit(corev1.ResourceCPU, "1"). + PriorityClass(lowPriorityClass.Name). + Obj() + util.MustCreate(ctx, k8sClient, sampleJob2) + }) + + ginkgo.By("Verify there is one pending workload", func() { + gomega.Eventually(func(g gomega.Gomega) { + info, err := kueueClientset.VisibilityV1beta1().ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(info.Items).Should(gomega.HaveLen(1)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Await for pods to be running", func() { + gomega.Eventually(func(g gomega.Gomega) { + createdJob := &batchv1.Job{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(blockingJob), createdJob)).Should(gomega.Succeed()) + g.Expect(createdJob.Status.Ready).Should(gomega.Equal(ptr.To[int32](1))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Terminate execution of the first workload to release the quota", func() { + gomega.Expect(util.DeleteAllPodsInNamespace(ctx, k8sClient, nsA)).Should(gomega.Succeed()) + }) + + ginkgo.By("Verify there are zero pending workloads, after the second workload is admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + info, err := kueueClientset.VisibilityV1beta1().ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(info.Items).Should(gomega.BeEmpty()) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + }) + ginkgo.It("Should allow fetching information about pending workloads in ClusterQueue", func() { ginkgo.By("Verify there are zero pending workloads", func() { - info, err := visibilityClient.ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(info.Items).Should(gomega.BeEmpty()) }) @@ -154,7 +199,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.By("Verify there is one pending workload", func() { gomega.Eventually(func(g gomega.Gomega) { - info, err := visibilityClient.ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(info.Items).Should(gomega.HaveLen(1)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -174,7 +219,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.By("Verify there are zero pending workloads, after the second workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { - info, err := visibilityClient.ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(info.Items).Should(gomega.BeEmpty()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) @@ -225,7 +270,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: highPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 0, - LocalQueueName: kueuev1beta1.LocalQueueName(localQueueA.Name), + LocalQueueName: kueue.LocalQueueName(localQueueA.Name), }, { ObjectMeta: metav1.ObjectMeta{ @@ -235,7 +280,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: midPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 1, - LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), + LocalQueueName: kueue.LocalQueueName(localQueueB.Name), }, { ObjectMeta: metav1.ObjectMeta{ @@ -245,11 +290,11 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: lowPriorityClass.Value, PositionInLocalQueue: 1, PositionInClusterQueue: 2, - LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), + LocalQueueName: kueue.LocalQueueName(localQueueB.Name), }, } gomega.Eventually(func(g gomega.Gomega) { - info, err := visibilityClient.ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().ClusterQueues().GetPendingWorkloadsSummary(ctx, clusterQueue.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(info.Items).Should(gomega.BeComparableTo(wantPendingWorkloads, pendingWorkloadsCmpOpts...)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -258,7 +303,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.It("Should allow fetching information about pending workloads in LocalQueue", func() { ginkgo.By("Verify there are zero pending workloads", func() { - info, err := visibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(info.Items).Should(gomega.BeEmpty()) }) @@ -275,7 +320,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.By("Verify there is one pending workload", func() { gomega.Eventually(func(g gomega.Gomega) { - info, err := visibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(info.Items).Should(gomega.HaveLen(1)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -295,7 +340,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.By("Verify there are zero pending workloads, after the second workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { - info, err := visibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(info.Items).Should(gomega.BeEmpty()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) @@ -346,11 +391,11 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: highPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 0, - LocalQueueName: kueuev1beta1.LocalQueueName(localQueueA.Name), + LocalQueueName: kueue.LocalQueueName(localQueueA.Name), }, } gomega.Eventually(func(g gomega.Gomega) { - info, err := visibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(info.Items).Should(gomega.BeComparableTo(wantPendingWorkloads, pendingWorkloadsCmpOpts...)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -366,7 +411,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: midPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 1, - LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), + LocalQueueName: kueue.LocalQueueName(localQueueB.Name), }, { ObjectMeta: metav1.ObjectMeta{ @@ -376,11 +421,11 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: lowPriorityClass.Value, PositionInLocalQueue: 1, PositionInClusterQueue: 2, - LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), + LocalQueueName: kueue.LocalQueueName(localQueueB.Name), }, } gomega.Eventually(func(g gomega.Gomega) { - info, err := visibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueB.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueB.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(info.Items).Should(gomega.BeComparableTo(wantPendingWorkloads, pendingWorkloadsCmpOpts...)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -439,11 +484,11 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: highPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 0, - LocalQueueName: kueuev1beta1.LocalQueueName(localQueueA.Name), + LocalQueueName: kueue.LocalQueueName(localQueueA.Name), }, } gomega.Eventually(func(g gomega.Gomega) { - info, err := visibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueA.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(info.Items).Should(gomega.BeComparableTo(wantPendingWorkloads, pendingWorkloadsCmpOpts...)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -459,7 +504,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: midPriorityClass.Value, PositionInLocalQueue: 0, PositionInClusterQueue: 1, - LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), + LocalQueueName: kueue.LocalQueueName(localQueueB.Name), }, { ObjectMeta: metav1.ObjectMeta{ @@ -469,11 +514,11 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { Priority: lowPriorityClass.Value, PositionInLocalQueue: 1, PositionInClusterQueue: 2, - LocalQueueName: kueuev1beta1.LocalQueueName(localQueueB.Name), + LocalQueueName: kueue.LocalQueueName(localQueueB.Name), }, } gomega.Eventually(func(g gomega.Gomega) { - info, err := visibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueB.Name, metav1.GetOptions{}) + info, err := kueueClientset.VisibilityV1beta2().LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, localQueueB.Name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(info.Items).Should(gomega.BeComparableTo(wantPendingWorkloads, pendingWorkloadsCmpOpts...)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) diff --git a/test/util/e2e.go b/test/util/e2e.go index 07025aec8c5..96b423db0d2 100644 --- a/test/util/e2e.go +++ b/test/util/e2e.go @@ -52,9 +52,9 @@ import ( configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueuev1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - visibility "sigs.k8s.io/kueue/apis/visibility/v1beta1" + visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" kueueclientset "sigs.k8s.io/kueue/client-go/clientset/versioned" - visibilityv1beta1 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta1" + visibilityv1beta2 "sigs.k8s.io/kueue/client-go/clientset/versioned/typed/visibility/v1beta2" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" "sigs.k8s.io/kueue/pkg/workload" ) @@ -188,23 +188,22 @@ func CreateRestClient(cfg *rest.Config) *rest.RESTClient { return restClient } -func CreateVisibilityClient(user string) (visibilityv1beta1.VisibilityV1beta1Interface, error) { +func CreateKueueClientset(user string) kueueclientset.Interface { + ginkgo.GinkgoHelper() cfg, err := config.GetConfigWithContext("") - if err != nil { - return nil, fmt.Errorf("unable to get kubeconfig: %w", err) - } - gomega.ExpectWithOffset(1, cfg).NotTo(gomega.BeNil()) - + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cfg).NotTo(gomega.BeNil()) if user != "" { cfg.Impersonate.UserName = user } - kueueClient, err := kueueclientset.NewForConfig(cfg) - if err != nil { - return nil, fmt.Errorf("unable to create kueue clientset: %w", err) - } - visibilityClient := kueueClient.VisibilityV1beta1() - return visibilityClient, nil + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return kueueClient +} + +func CreateVisibilityClient(user string) visibilityv1beta2.VisibilityV1beta2Interface { + kueueClientset := CreateKueueClientset(user) + return kueueClientset.VisibilityV1beta2() } func rolloutOperatorDeployment(ctx context.Context, k8sClient client.Client, key types.NamespacedName, kindClusterName string) { From 8e4abe2c39dfafed558ffe0347a5bb5c9a92cf1e Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 31 Oct 2025 04:10:01 -0400 Subject: [PATCH 062/119] add support for maxlength linter command for kubernetes-api-linter (#7283) --- .golangci-kal.yml | 4 +- apis/kueue/v1beta2/admissioncheck_types.go | 4 ++ apis/kueue/v1beta2/clusterqueue_types.go | 10 ++++- apis/kueue/v1beta2/fairsharing_types.go | 1 + apis/kueue/v1beta2/localqueue_types.go | 3 ++ apis/kueue/v1beta2/multikueue_types.go | 4 ++ apis/kueue/v1beta2/workload_types.go | 23 +++++++++++ .../v1beta2/workloadpriorityclass_types.go | 2 + .../crd/kueue.x-k8s.io_admissionchecks.yaml | 4 ++ .../crd/kueue.x-k8s.io_clusterqueues.yaml | 11 +++++ .../crd/kueue.x-k8s.io_localqueues.yaml | 3 ++ .../kueue.x-k8s.io_multikueueclusters.yaml | 3 ++ .../crd/kueue.x-k8s.io_multikueueconfigs.yaml | 1 + ...ueue.x-k8s.io_workloadpriorityclasses.yaml | 2 + .../crd/kueue.x-k8s.io_workloads.yaml | 32 +++++++++++---- .../bases/kueue.x-k8s.io_admissionchecks.yaml | 4 ++ .../bases/kueue.x-k8s.io_clusterqueues.yaml | 11 +++++ .../crd/bases/kueue.x-k8s.io_localqueues.yaml | 3 ++ .../kueue.x-k8s.io_multikueueclusters.yaml | 3 ++ .../kueue.x-k8s.io_multikueueconfigs.yaml | 1 + ...ueue.x-k8s.io_workloadpriorityclasses.yaml | 2 + .../crd/bases/kueue.x-k8s.io_workloads.yaml | 41 +++++++++++++++---- .../en/docs/reference/kueue.v1beta2.md | 37 +++++++++++------ 23 files changed, 176 insertions(+), 33 deletions(-) diff --git a/.golangci-kal.yml b/.golangci-kal.yml index 085be48f50a..a7cf9268690 100644 --- a/.golangci-kal.yml +++ b/.golangci-kal.yml @@ -19,7 +19,7 @@ linters: - "duplicatemarkers" # Ensure there are no exact duplicate markers. for types and fields. - "integers" # Ensure only int32 and int64 are used for integers. - "jsontags" # Ensure every field has a json tag. - # - "maxlength" # Ensure all strings and arrays have maximum lengths/maximum items. + - "maxlength" # Ensure all strings and arrays have maximum lengths/maximum items. # - "nobools" # Bools do not evolve over time, should use enums instead. - "nodurations" # Prevents usage of `Duration` types. - "nofloats" # Ensure floats are not used. @@ -64,7 +64,7 @@ linters: generated: strict rules: ## KAL should only run on API folders. - - path-except: "apis/kueue/v1beta2/*" + - path-except: "apis/kueue/v1beta2" linters: - kubeapilinter diff --git a/apis/kueue/v1beta2/admissioncheck_types.go b/apis/kueue/v1beta2/admissioncheck_types.go index d82d4e891ff..2b3ea2b692d 100644 --- a/apis/kueue/v1beta2/admissioncheck_types.go +++ b/apis/kueue/v1beta2/admissioncheck_types.go @@ -50,6 +50,8 @@ type AdmissionCheckSpec struct { // not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. // +kubebuilder:validation:Required // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable" + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:MinLength=1 ControllerName string `json:"controllerName"` // retryDelayMinutes specifies how long to keep the workload suspended after @@ -85,11 +87,13 @@ type AdmissionCheckParametersReference struct { type AdmissionCheckStatus struct { // conditions hold the latest available observations of the AdmissionCheck // current state. + // This is limited to at most 16 separate conditions. // +optional // +listType=map // +listMapKey=type // +patchStrategy=merge // +patchMergeKey=type + // +kubebuilder:validation:MaxItems=16 Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index 01c067c75e6..20649b41b57 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -111,6 +111,7 @@ type ClusterQueueSpec struct { // admissionChecks lists the AdmissionChecks required by this ClusterQueue. // Cannot be used along with AdmissionCheckStrategy. + // Admission checks are limited to at most 64 items. // +optional AdmissionChecks []AdmissionCheckReference `json:"admissionChecks,omitempty"` //nolint:kubeapilinter // field is being removed @@ -149,6 +150,7 @@ type AdmissionChecksStrategy struct { // admissionChecks is a list of strategies for AdmissionChecks // +listType=map // +listMapKey=name + // +kubebuilder:validation:MaxItems=64 AdmissionChecks []AdmissionCheckStrategyRule `json:"admissionChecks,omitempty"` } @@ -161,6 +163,7 @@ type AdmissionCheckStrategyRule struct { // If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. // +optional // +listType=set + // +kubebuilder:validation:MaxItems=64 OnFlavors []ResourceFlavorReference `json:"onFlavors,omitempty"` } @@ -272,14 +275,17 @@ type ResourceFlavorReference string type ClusterQueueStatus struct { // conditions hold the latest available observations of the ClusterQueue // current state. + // conditions are limited to 16 elements. // +optional // +listType=map // +listMapKey=type // +patchStrategy=merge // +patchMergeKey=type + // +kubebuilder:validation:MaxItems=16 Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` // flavorsReservation are the reserved quotas, by flavor, currently in use by the // workloads assigned to this ClusterQueue. + // flavorsReservation are limited to 64 elements. // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=64 @@ -329,7 +335,7 @@ type ClusterQueuePendingWorkloadsStatus struct { // clusterQueuePendingWorkload contains the list of top pending workloads. // +listType=atomic // +optional - Head []ClusterQueuePendingWorkload `json:"clusterQueuePendingWorkload"` + Head []ClusterQueuePendingWorkload `json:"clusterQueuePendingWorkload"` //nolint:kubeapilinter // field is being removed // lastChangeTime indicates the time of the last change of the structure. LastChangeTime metav1.Time `json:"lastChangeTime"` @@ -339,9 +345,11 @@ type ClusterQueuePendingWorkloadsStatus struct { // in the cluster queue. type ClusterQueuePendingWorkload struct { // name indicates the name of the pending workload. + // +kubebuilder:validation:MaxLength=256 Name string `json:"name"` // namespace indicates the name of the pending workload. + // +kubebuilder:validation:MaxLength=63 Namespace string `json:"namespace"` } diff --git a/apis/kueue/v1beta2/fairsharing_types.go b/apis/kueue/v1beta2/fairsharing_types.go index cc89f5e7fda..1cf1e3250e5 100644 --- a/apis/kueue/v1beta2/fairsharing_types.go +++ b/apis/kueue/v1beta2/fairsharing_types.go @@ -78,6 +78,7 @@ type AdmissionScope struct { // - UsageBasedAdmissionFairSharing // - NoAdmissionFairSharing // + // +kubebuilder:validation:Enum=UsageBasedAdmissionFairSharing;NoAdmissionFairSharing // +required AdmissionMode AdmissionMode `json:"admissionMode"` } diff --git a/apis/kueue/v1beta2/localqueue_types.go b/apis/kueue/v1beta2/localqueue_types.go index d0fe9bd0cc7..04c396208cd 100644 --- a/apis/kueue/v1beta2/localqueue_types.go +++ b/apis/kueue/v1beta2/localqueue_types.go @@ -106,6 +106,7 @@ type TopologyInfo struct { // +kubebuilder:validation:Required // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:items:MaxLength=317 Levels []string `json:"levels"` } @@ -113,11 +114,13 @@ type TopologyInfo struct { type LocalQueueStatus struct { // conditions hold the latest available observations of the LocalQueue // current state. + // conditions are limited to 16 items. // +optional // +listType=map // +listMapKey=type // +patchStrategy=merge // +patchMergeKey=type + // +kubebuilder:validation:MaxItems=16 Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` // pendingWorkloads is the number of Workloads in the LocalQueue not yet admitted to a ClusterQueue diff --git a/apis/kueue/v1beta2/multikueue_types.go b/apis/kueue/v1beta2/multikueue_types.go index 97e99a5fc41..38ea931f7a6 100644 --- a/apis/kueue/v1beta2/multikueue_types.go +++ b/apis/kueue/v1beta2/multikueue_types.go @@ -49,6 +49,7 @@ type KubeConfig struct { // // If LocationType is Secret then Location is the name of the secret inside the namespace in // which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. + // +kubebuilder:validation:MaxLength=256 Location string `json:"location"` // locationType of the KubeConfig. @@ -66,11 +67,13 @@ type MultiKueueClusterSpec struct { type MultiKueueClusterStatus struct { // conditions hold the latest available observations of the MultiKueueCluster // current state. + // conditions are limited to 16 elements. // +optional // +listType=map // +listMapKey=type // +patchStrategy=merge // +patchMergeKey=type + // +kubebuilder:validation:MaxItems=16 Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } @@ -111,6 +114,7 @@ type MultiKueueConfigSpec struct { // +listType=set // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=10 + // +kubebuilder:validation:items:MaxLength=256 Clusters []string `json:"clusters"` } diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index a04782f7412..48d0ecc68cf 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -92,15 +92,19 @@ type PodSetTopologyRequest struct { // required indicates the topology level required by the PodSet, as // indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet // annotation. + // This is limited to 63 characters. // // +optional + // +kubebuilder:validation:MaxLength=63 Required *string `json:"required,omitempty"` // preferred indicates the topology level preferred by the PodSet, as // indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet // annotation. + // This is limited to 63 characters. // // +optional + // +kubebuilder:validation:MaxLength=63 Preferred *string `json:"preferred,omitempty"` // unconstrained indicates that Kueue has the freedom to schedule the PodSet within @@ -116,10 +120,14 @@ type PodSetTopologyRequest struct { // - kubernetes job this is: kubernetes.io/job-completion-index // - JobSet: kubernetes.io/job-completion-index (inherited from Job) // - Kubeflow: training.kubeflow.org/replica-index + // This is limited to 317 characters. + // +kubebuilder:validation:MaxLength=317 PodIndexLabel *string `json:"podIndexLabel,omitempty"` // subGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) // within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. + // This is limited to 317 characters. + // +kubebuilder:validation:MaxLength=317 SubGroupIndexLabel *string `json:"subGroupIndexLabel,omitempty"` // subGroupCount indicates the count of replicated Jobs (groups) within a PodSet. @@ -130,12 +138,15 @@ type PodSetTopologyRequest struct { // PodSets with the same `PodSetGroupName` should be assigned the same ResourceFlavor // // +optional + // +kubebuilder:validation:MaxLength=256 PodSetGroupName *string `json:"podSetGroupName,omitempty"` // podSetSliceRequiredTopology indicates the topology level required by the PodSet slice, as // indicated by the `kueue.x-k8s.io/podset-slice-required-topology` annotation. // + // This is limited to 63 // +optional + // +kubebuilder:validation:MaxLength=63 PodSetSliceRequiredTopology *string `json:"podSetSliceRequiredTopology,omitempty"` // podSetSliceSize indicates the size of a subgroup of pods in a PodSet for which @@ -251,6 +262,8 @@ type PodSetAssignment struct { // DelayedTopologyRequestState indicates the state of the delayed TopologyRequest. // +enum +// +kubebuilder:validation:enum=Pending;Ready +// +kubebuilder:validation:MaxLength=7 type DelayedTopologyRequestState string const ( @@ -270,6 +283,7 @@ type TopologyAssignment struct { // +listType=atomic // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:items:MaxLength=317 Levels []string `json:"levels"` // domains is a list of topology assignments split by topology domains at @@ -277,6 +291,7 @@ type TopologyAssignment struct { // // +required // +listType=atomic + // +kubebuilder:validation:MaxItems=100000 Domains []TopologyDomainAssignment `json:"domains"` } @@ -289,6 +304,7 @@ type TopologyDomainAssignment struct { // +listType=atomic // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 + // +kubebuilder:validation:items:MaxLength=63 Values []string `json:"values"` // count indicates the number of Pods to be scheduled in the topology @@ -358,12 +374,14 @@ type WorkloadStatus struct { // - Finished: the associated workload finished running (failed or succeeded). // - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have // succeeded. + // conditions are limited to 16 items. // // +optional // +listType=map // +listMapKey=type // +patchStrategy=merge // +patchMergeKey=type + // +kubebuilder:validation:MaxItems=16 Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` // admission holds the parameters of the admission of the workload by a @@ -424,6 +442,7 @@ type WorkloadStatus struct { // // +listType=atomic // +kubebuilder:validation:MaxItems=10 + // +kubebuilder:validation:items:MaxLength=256 // +optional NominatedClusterNames []string `json:"nominatedClusterNames,omitempty"` @@ -435,6 +454,7 @@ type WorkloadStatus struct { // // This field is reset after the Workload is evicted. // +optional + // +kubebuilder:validation:MaxLength=256 ClusterName *string `json:"clusterName,omitempty"` // unhealthyNodes holds the failed nodes running at least one pod of this workload @@ -445,6 +465,7 @@ type WorkloadStatus struct { // +optional // +listType=map // +listMapKey=name + // +kubebuilder:validation:MaxItems=8 UnhealthyNodes []UnhealthyNode `json:"unhealthyNodes,omitempty"` } @@ -458,6 +479,7 @@ type SchedulingStats struct { // +patchStrategy=merge // +patchMergeKey=reason // +patchMergeKey=underlyingCause + // +kubebuilder:validation:MaxItems=256 Evictions []WorkloadSchedulingStatsEviction `json:"evictions,omitempty"` } @@ -493,6 +515,7 @@ type UnhealthyNode struct { // // +required // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=63 Name string `json:"name"` } diff --git a/apis/kueue/v1beta2/workloadpriorityclass_types.go b/apis/kueue/v1beta2/workloadpriorityclass_types.go index 33140eb5f2b..fc48d7d1a5e 100644 --- a/apis/kueue/v1beta2/workloadpriorityclass_types.go +++ b/apis/kueue/v1beta2/workloadpriorityclass_types.go @@ -39,7 +39,9 @@ type WorkloadPriorityClass struct { // description is an arbitrary string that usually provides guidelines on // when this workloadPriorityClass should be used. + // The description is limited to a maximum of 2048 characters. // +optional + // +kubebuilder:validation:MaxLength=2048 Description string `json:"description,omitempty"` } diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml index 6bf25e5743d..3327436148b 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml @@ -205,6 +205,8 @@ spec: description: |- controllerName identifies the controller that processes the AdmissionCheck, not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. + maxLength: 253 + minLength: 1 type: string x-kubernetes-validations: - message: field is immutable @@ -253,6 +255,7 @@ spec: description: |- conditions hold the latest available observations of the AdmissionCheck current state. + This is limited to at most 16 separate conditions. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: @@ -306,6 +309,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml index 6066d734808..5aad8e9f42b 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml @@ -842,6 +842,7 @@ spec: description: |- admissionChecks lists the AdmissionChecks required by this ClusterQueue. Cannot be used along with AdmissionCheckStrategy. + Admission checks are limited to at most 64 items. items: description: AdmissionCheckReference is the name of an AdmissionCheck. maxLength: 316 @@ -870,11 +871,13 @@ spec: maxLength: 253 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string + maxItems: 64 type: array x-kubernetes-list-type: set required: - name type: object + maxItems: 64 type: array x-kubernetes-list-map-keys: - name @@ -889,6 +892,9 @@ spec: in the AdmissionScope. Possible values are: - UsageBasedAdmissionFairSharing - NoAdmissionFairSharing + enum: + - UsageBasedAdmissionFairSharing + - NoAdmissionFairSharing type: string required: - admissionMode @@ -1286,6 +1292,7 @@ spec: description: |- conditions hold the latest available observations of the ClusterQueue current state. + conditions are limited to 16 elements. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: @@ -1339,6 +1346,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type @@ -1390,6 +1398,7 @@ spec: description: |- flavorsReservation are the reserved quotas, by flavor, currently in use by the workloads assigned to this ClusterQueue. + flavorsReservation are limited to 64 elements. items: properties: name: @@ -1516,9 +1525,11 @@ spec: properties: name: description: name indicates the name of the pending workload. + maxLength: 256 type: string namespace: description: namespace indicates the name of the pending workload. + maxLength: 63 type: string required: - name diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml index 4299eb41c4d..60727b73e3f 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml @@ -544,6 +544,7 @@ spec: description: |- conditions hold the latest available observations of the LocalQueue current state. + conditions are limited to 16 items. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: @@ -597,6 +598,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type @@ -712,6 +714,7 @@ spec: levels: description: levels define the levels of topology. items: + maxLength: 317 type: string maxItems: 16 minItems: 1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml index 5ae9b67d06f..efdd0b52730 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml @@ -207,6 +207,7 @@ spec: If LocationType is Secret then Location is the name of the secret inside the namespace in which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. + maxLength: 256 type: string locationType: default: Secret @@ -229,6 +230,7 @@ spec: description: |- conditions hold the latest available observations of the MultiKueueCluster current state. + conditions are limited to 16 elements. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: @@ -282,6 +284,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml index ffbb7296148..586fa771bf5 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueconfigs.yaml @@ -103,6 +103,7 @@ spec: clusters: description: clusters is a list of MultiKueueClusters names where the workloads from the ClusterQueue should be distributed. items: + maxLength: 256 type: string maxItems: 10 minItems: 1 diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml index 801fe8502fb..09435bdd59e 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloadpriorityclasses.yaml @@ -102,6 +102,8 @@ spec: description: |- description is an arbitrary string that usually provides guidelines on when this workloadPriorityClass should be used. + The description is limited to a maximum of 2048 characters. + maxLength: 2048 type: string kind: description: |- diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index c607621733e..43336aa3d68 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -17046,22 +17046,22 @@ spec: description: topologyRequest defines the topology request for the PodSet. properties: podIndexLabel: - description: |- - podIndexLabel indicates the name of the label indexing the pods. - For example, in the context of - - kubernetes job this is: kubernetes.io/job-completion-index - - JobSet: kubernetes.io/job-completion-index (inherited from Job) - - Kubeflow: training.kubeflow.org/replica-index + description: "podIndexLabel indicates the name of the label indexing the pods.\nFor example, in the context of\n- kubernetes job this is: kubernetes.io/job-completion-index\n- JobSet: kubernetes.io/job-completion-index (inherited from Job)\n- Kubeflow: training.kubeflow.org/replica-index\n\tThis is limited to 317 characters." + maxLength: 317 type: string podSetGroupName: description: |- podSetGroupName indicates the name of the group of PodSets to which this PodSet belongs to. PodSets with the same `PodSetGroupName` should be assigned the same ResourceFlavor + maxLength: 256 type: string podSetSliceRequiredTopology: description: |- podSetSliceRequiredTopology indicates the topology level required by the PodSet slice, as indicated by the `kueue.x-k8s.io/podset-slice-required-topology` annotation. + + This is limited to 63 + maxLength: 63 type: string podSetSliceSize: description: |- @@ -17075,12 +17075,16 @@ spec: preferred indicates the topology level preferred by the PodSet, as indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet annotation. + This is limited to 63 characters. + maxLength: 63 type: string required: description: |- required indicates the topology level required by the PodSet, as indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet annotation. + This is limited to 63 characters. + maxLength: 63 type: string subGroupCount: description: |- @@ -17089,9 +17093,8 @@ spec: format: int32 type: integer subGroupIndexLabel: - description: |- - subGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) - within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. + description: "subGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups)\nwithin a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index.\n\tThis is limited to 317 characters." + maxLength: 317 type: string unconstrained: description: |- @@ -17199,6 +17202,7 @@ spec: Kueue schedules the second pass of scheduling for each workload with at least one PodSet which has delayedTopologyRequest=true and without topologyAssignment. + maxLength: 7 type: string flavors: additionalProperties: @@ -17294,6 +17298,7 @@ spec: domain. The values correspond to the consecutive topology levels, from the highest to the lowest. items: + maxLength: 63 type: string maxItems: 16 minItems: 1 @@ -17303,6 +17308,7 @@ spec: - count - values type: object + maxItems: 100000 type: array x-kubernetes-list-type: atomic levels: @@ -17311,6 +17317,7 @@ spec: topology (i.e. node label keys), from the highest to the lowest level of the topology. items: + maxLength: 317 type: string maxItems: 16 minItems: 1 @@ -17467,6 +17474,7 @@ spec: affinity or propagation policies across workload slices. This field is reset after the Workload is evicted. + maxLength: 256 type: string conditions: description: |- @@ -17479,6 +17487,7 @@ spec: - Finished: the associated workload finished running (failed or succeeded). - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have succeeded. + conditions are limited to 16 items. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: @@ -17532,6 +17541,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type @@ -17543,6 +17553,7 @@ spec: `status.clusterName` is set. This field is optional. items: + maxLength: 256 type: string maxItems: 10 type: array @@ -17655,6 +17666,7 @@ spec: - reason - underlyingCause type: object + maxItems: 256 type: array x-kubernetes-list-map-keys: - reason @@ -17671,10 +17683,12 @@ spec: properties: name: description: name is the name of the unhealthy node. + maxLength: 63 type: string required: - name type: object + maxItems: 8 type: array x-kubernetes-list-map-keys: - name diff --git a/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml b/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml index cad6d1aa984..aec0bad2bc6 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml @@ -184,6 +184,8 @@ spec: description: |- controllerName identifies the controller that processes the AdmissionCheck, not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. + maxLength: 253 + minLength: 1 type: string x-kubernetes-validations: - message: field is immutable @@ -232,6 +234,7 @@ spec: description: |- conditions hold the latest available observations of the AdmissionCheck current state. + This is limited to at most 16 separate conditions. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -286,6 +289,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml index 390941107c7..f81e10975ac 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml @@ -827,6 +827,7 @@ spec: description: |- admissionChecks lists the AdmissionChecks required by this ClusterQueue. Cannot be used along with AdmissionCheckStrategy. + Admission checks are limited to at most 64 items. items: description: AdmissionCheckReference is the name of an AdmissionCheck. maxLength: 316 @@ -857,11 +858,13 @@ spec: maxLength: 253 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string + maxItems: 64 type: array x-kubernetes-list-type: set required: - name type: object + maxItems: 64 type: array x-kubernetes-list-map-keys: - name @@ -877,6 +880,9 @@ spec: in the AdmissionScope. Possible values are: - UsageBasedAdmissionFairSharing - NoAdmissionFairSharing + enum: + - UsageBasedAdmissionFairSharing + - NoAdmissionFairSharing type: string required: - admissionMode @@ -1280,6 +1286,7 @@ spec: description: |- conditions hold the latest available observations of the ClusterQueue current state. + conditions are limited to 16 elements. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -1334,6 +1341,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type @@ -1387,6 +1395,7 @@ spec: description: |- flavorsReservation are the reserved quotas, by flavor, currently in use by the workloads assigned to this ClusterQueue. + flavorsReservation are limited to 64 elements. items: properties: name: @@ -1516,10 +1525,12 @@ spec: properties: name: description: name indicates the name of the pending workload. + maxLength: 256 type: string namespace: description: namespace indicates the name of the pending workload. + maxLength: 63 type: string required: - name diff --git a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml index 10233d7cf17..ef44784b709 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml @@ -526,6 +526,7 @@ spec: description: |- conditions hold the latest available observations of the LocalQueue current state. + conditions are limited to 16 items. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -580,6 +581,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type @@ -703,6 +705,7 @@ spec: levels: description: levels define the levels of topology. items: + maxLength: 317 type: string maxItems: 16 minItems: 1 diff --git a/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml b/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml index 8519cad4759..22943efa928 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml @@ -186,6 +186,7 @@ spec: If LocationType is Secret then Location is the name of the secret inside the namespace in which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. + maxLength: 256 type: string locationType: default: Secret @@ -208,6 +209,7 @@ spec: description: |- conditions hold the latest available observations of the MultiKueueCluster current state. + conditions are limited to 16 elements. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -262,6 +264,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type diff --git a/config/components/crd/bases/kueue.x-k8s.io_multikueueconfigs.yaml b/config/components/crd/bases/kueue.x-k8s.io_multikueueconfigs.yaml index 8883c32d46c..e629a7235ab 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_multikueueconfigs.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_multikueueconfigs.yaml @@ -83,6 +83,7 @@ spec: description: clusters is a list of MultiKueueClusters names where the workloads from the ClusterQueue should be distributed. items: + maxLength: 256 type: string maxItems: 10 minItems: 1 diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloadpriorityclasses.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloadpriorityclasses.yaml index 60636a003d2..659b0e9cd77 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloadpriorityclasses.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloadpriorityclasses.yaml @@ -82,6 +82,8 @@ spec: description: |- description is an arbitrary string that usually provides guidelines on when this workloadPriorityClass should be used. + The description is limited to a maximum of 2048 characters. + maxLength: 2048 type: string kind: description: |- diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index f500d797619..f57543ccd4a 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -18026,22 +18026,27 @@ spec: the PodSet. properties: podIndexLabel: - description: |- - podIndexLabel indicates the name of the label indexing the pods. - For example, in the context of - - kubernetes job this is: kubernetes.io/job-completion-index - - JobSet: kubernetes.io/job-completion-index (inherited from Job) - - Kubeflow: training.kubeflow.org/replica-index + description: "podIndexLabel indicates the name of the label + indexing the pods.\nFor example, in the context of\n- + kubernetes job this is: kubernetes.io/job-completion-index\n- + JobSet: kubernetes.io/job-completion-index (inherited + from Job)\n- Kubeflow: training.kubeflow.org/replica-index\n\tThis + is limited to 317 characters." + maxLength: 317 type: string podSetGroupName: description: |- podSetGroupName indicates the name of the group of PodSets to which this PodSet belongs to. PodSets with the same `PodSetGroupName` should be assigned the same ResourceFlavor + maxLength: 256 type: string podSetSliceRequiredTopology: description: |- podSetSliceRequiredTopology indicates the topology level required by the PodSet slice, as indicated by the `kueue.x-k8s.io/podset-slice-required-topology` annotation. + + This is limited to 63 + maxLength: 63 type: string podSetSliceSize: description: |- @@ -18055,12 +18060,16 @@ spec: preferred indicates the topology level preferred by the PodSet, as indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet annotation. + This is limited to 63 characters. + maxLength: 63 type: string required: description: |- required indicates the topology level required by the PodSet, as indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet annotation. + This is limited to 63 characters. + maxLength: 63 type: string subGroupCount: description: |- @@ -18069,9 +18078,12 @@ spec: format: int32 type: integer subGroupIndexLabel: - description: |- - subGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) - within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. + description: "subGroupIndexLabel indicates the name of the + label indexing the instances of replicated Jobs (groups)\nwithin + a PodSet. For example, in the context of JobSet this is + jobset.sigs.k8s.io/job-index.\n\tThis is limited to 317 + characters." + maxLength: 317 type: string unconstrained: description: |- @@ -18181,6 +18193,7 @@ spec: Kueue schedules the second pass of scheduling for each workload with at least one PodSet which has delayedTopologyRequest=true and without topologyAssignment. + maxLength: 7 type: string flavors: additionalProperties: @@ -18279,6 +18292,7 @@ spec: domain. The values correspond to the consecutive topology levels, from the highest to the lowest. items: + maxLength: 63 type: string maxItems: 16 minItems: 1 @@ -18288,6 +18302,7 @@ spec: - count - values type: object + maxItems: 100000 type: array x-kubernetes-list-type: atomic levels: @@ -18296,6 +18311,7 @@ spec: topology (i.e. node label keys), from the highest to the lowest level of the topology. items: + maxLength: 317 type: string maxItems: 16 minItems: 1 @@ -18465,6 +18481,7 @@ spec: affinity or propagation policies across workload slices. This field is reset after the Workload is evicted. + maxLength: 256 type: string conditions: description: |- @@ -18477,6 +18494,7 @@ spec: - Finished: the associated workload finished running (failed or succeeded). - PodsReady: at least `.spec.podSets[*].count` Pods are ready or have succeeded. + conditions are limited to 16 items. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -18531,6 +18549,7 @@ spec: - status - type type: object + maxItems: 16 type: array x-kubernetes-list-map-keys: - type @@ -18542,6 +18561,7 @@ spec: `status.clusterName` is set. This field is optional. items: + maxLength: 256 type: string maxItems: 10 type: array @@ -18659,6 +18679,7 @@ spec: - reason - underlyingCause type: object + maxItems: 256 type: array x-kubernetes-list-map-keys: - reason @@ -18675,10 +18696,12 @@ spec: properties: name: description: name is the name of the unhealthy node. + maxLength: 63 type: string required: - name type: object + maxItems: 8 type: array x-kubernetes-list-map-keys: - name diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md index 563503ba4fd..8c6c32457d0 100644 --- a/site/content/en/docs/reference/kueue.v1beta2.md +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -380,7 +380,8 @@ Changing the value of workloadPriorityClass doesn't affect the priority of workl

description is an arbitrary string that usually provides guidelines on -when this workloadPriorityClass should be used.

+when this workloadPriorityClass should be used. +The description is limited to a maximum of 2048 characters.

@@ -593,7 +594,8 @@ This may be an empty string.

conditions hold the latest available observations of the AdmissionCheck -current state.

+current state. +This is limited to at most 16 separate conditions.

@@ -1062,7 +1064,8 @@ before borrowing or preempting in the flavor being evaluated.

admissionChecks lists the AdmissionChecks required by this ClusterQueue. -Cannot be used along with AdmissionCheckStrategy.

+Cannot be used along with AdmissionCheckStrategy. +Admission checks are limited to at most 64 items.

admissionChecksStrategy
@@ -1127,7 +1130,8 @@ if FairSharing is enabled in the Kueue configuration.

conditions hold the latest available observations of the ClusterQueue -current state.

+current state. +conditions are limited to 16 elements.

flavorsReservation
@@ -1135,7 +1139,8 @@ current state.

flavorsReservation are the reserved quotas, by flavor, currently in use by the -workloads assigned to this ClusterQueue.

+workloads assigned to this ClusterQueue. +flavorsReservation are limited to 64 elements.

flavorsUsage
@@ -1789,7 +1794,8 @@ if AdmissionFairSharing is enabled in the Kueue configuration.

conditions hold the latest available observations of the LocalQueue -current state.

+current state. +conditions are limited to 16 items.

pendingWorkloads
@@ -1904,7 +1910,8 @@ Deprecated: Flavors is deprecated and marked for removal in v1beta2.

conditions hold the latest available observations of the MultiKueueCluster -current state.

+current state. +conditions are limited to 16 elements.

@@ -2208,7 +2215,8 @@ and the application of resource.excludeResourcePrefixes and resource.transformat

required indicates the topology level required by the PodSet, as indicated by the kueue.x-k8s.io/podset-required-topology PodSet -annotation.

+annotation. +This is limited to 63 characters.

preferred
@@ -2217,7 +2225,8 @@ annotation.

preferred indicates the topology level preferred by the PodSet, as indicated by the kueue.x-k8s.io/podset-preferred-topology PodSet -annotation.

+annotation. +This is limited to 63 characters.

unconstrained
@@ -2238,7 +2247,8 @@ For example, in the context of

  • kubernetes job this is: kubernetes.io/job-completion-index
  • JobSet: kubernetes.io/job-completion-index (inherited from Job)
  • -
  • Kubeflow: training.kubeflow.org/replica-index
  • +
  • Kubeflow: training.kubeflow.org/replica-index +This is limited to 317 characters.
@@ -2247,7 +2257,8 @@ For example, in the context of

subGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) -within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index.

+within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. +This is limited to 317 characters.

subGroupCount [Required]
@@ -2272,6 +2283,7 @@ PodSets with the same PodSetGroupName should be assigned the same R

podSetSliceRequiredTopology indicates the topology level required by the PodSet slice, as indicated by the kueue.x-k8s.io/podset-slice-required-topology annotation.

+

This is limited to 63

podSetSliceSize
@@ -3273,7 +3285,8 @@ current state.

  • Admitted: the Workload was admitted through a ClusterQueue.
  • Finished: the associated workload finished running (failed or succeeded).
  • PodsReady: at least .spec.podSets[*].count Pods are ready or have -succeeded.
  • +succeeded. +conditions are limited to 16 items. From fc3c1c47350160b2340f2a4d94b2e3129b7e93c3 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 31 Oct 2025 14:12:02 +0530 Subject: [PATCH 063/119] Fix feature gates tables. (#7467) --- site/content/en/docs/installation/_index.md | 8 +-- .../content/zh-CN/docs/installation/_index.md | 65 ++++++++++--------- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 1673f063406..8c268c65e86 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -264,7 +264,7 @@ spec: ### Feature gates for alpha and beta features | Feature | Default | Stage | Since | Until | -| --------------------------------------------- |---------|-------|-------|-------| +|-----------------------------------------------|---------|-------|-------|-------| | `FlavorFungibility` | `true` | Beta | 0.5 | | | `MultiKueue` | `false` | Alpha | 0.6 | 0.8 | | `MultiKueue` | `true` | Beta | 0.9 | | @@ -297,16 +297,16 @@ spec: | `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | | `SanitizePodSets` | `true` | Beta | 0.13 | | +| `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.13 | | {{% alert title="Note" color="primary" %}} -The SanitizePodSets feature is available starting from versions 0.13.8 and 0.14.3. +The SanitizePodSets and MultiKueueAllowInsecureKubeconfigs features are available starting from versions 0.13.8 and 0.14.3. {{% /alert %}} -| `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.15 | | ### Feature gates for graduated or deprecated features | Feature | Default | Stage | Since | Until | -|---------------------------------------| ------- | ---------- | ----- | ----- | +|---------------------------------------|---------|------------|-------|-------| | `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | | `ProvisioningACC` | `true` | Beta | 0.7 | | | `ProvisioningACC` | `true` | GA | 0.14 | | diff --git a/site/content/zh-CN/docs/installation/_index.md b/site/content/zh-CN/docs/installation/_index.md index cc341b66700..ea521665cb0 100644 --- a/site/content/zh-CN/docs/installation/_index.md +++ b/site/content/zh-CN/docs/installation/_index.md @@ -263,36 +263,41 @@ spec: ### Alpha 和 Beta 级别特性的特性门控 {#feature-gates-for-alpha-and-beta-features} -| 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | -| --------------------------------------------- | ------- | ----- |----------| -------- | -| `FlavorFungibility` | `true` | Beta | 0.5 | | -| `MultiKueue` | `false` | Alpha | 0.6 | 0.8 | -| `MultiKueue` | `true` | Beta | 0.9 | | -| `MultiKueueBatchJobWithManagedBy` | `false` | Alpha | 0.8 | | -| `PartialAdmission` | `false` | Alpha | 0.4 | 0.4 | -| `PartialAdmission` | `true` | Beta | 0.5 | | -| `VisibilityOnDemand` | `false` | Alpha | 0.6 | 0.8 | -| `VisibilityOnDemand` | `true` | Beta | 0.9 | | -| `PrioritySortingWithinCohort` | `true` | Beta | 0.6 | | -| `LendingLimit` | `false` | Alpha | 0.6 | 0.8 | -| `LendingLimit` | `true` | Beta | 0.9 | | -| `TopologyAwareScheduling` | `false` | Alpha | 0.9 | | -| `LocalQueueDefaulting` | `false` | Alpha | 0.10 | 0.11 | -| `LocalQueueDefaulting` | `true` | Beta | 0.12 | | -| `LocalQueueMetrics` | `false` | Alpha | 0.10 | | -| `HierarchicalCohort` | `true` | Beta | 0.11 | | -| `ObjectRetentionPolicies` | `false` | Alpha | 0.12 | 0.12 | -| `ObjectRetentionPolicies` | `true` | Beta | 0.13 | | -| `TASFailedNodeReplacement` | `false` | Alpha | 0.12 | | -| `AdmissionFairSharing` | `false` | Alpha | 0.12 | | -| `TASFailedNodeReplacementFailFast` | `false` | Alpha | 0.12 | | -| `TASReplaceNodeOnPodTermination` | `false` | Alpha | 0.13 | | -| `ElasticJobsViaWorkloadSlices` | `false` | Alpha | 0.13 | | -| `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | -| `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | -| `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | -| `SanitizePodSets` | `true` | Beta | 0.13 | | -| `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.15 | | +| 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | +| --------------------------------------------- | ------- | ----- |---------|------| +| `FlavorFungibility` | `true` | Beta | 0.5 | | +| `MultiKueue` | `false` | Alpha | 0.6 | 0.8 | +| `MultiKueue` | `true` | Beta | 0.9 | | +| `MultiKueueBatchJobWithManagedBy` | `false` | Alpha | 0.8 | | +| `MultiKueueBatchJobWithManagedBy` | `true` | Beta | 0.15 | | +| `PartialAdmission` | `false` | Alpha | 0.4 | 0.4 | +| `PartialAdmission` | `true` | Beta | 0.5 | | +| `VisibilityOnDemand` | `false` | Alpha | 0.6 | 0.8 | +| `VisibilityOnDemand` | `true` | Beta | 0.9 | | +| `PrioritySortingWithinCohort` | `true` | Beta | 0.6 | | +| `LendingLimit` | `false` | Alpha | 0.6 | 0.8 | +| `LendingLimit` | `true` | Beta | 0.9 | | +| `TopologyAwareScheduling` | `false` | Alpha | 0.9 | | +| `TopologyAwareScheduling` | `true` | Beta | 0.14 | | +| `LocalQueueDefaulting` | `false` | Alpha | 0.10 | 0.11 | +| `LocalQueueDefaulting` | `true` | Beta | 0.12 | | +| `LocalQueueMetrics` | `false` | Alpha | 0.10 | | +| `HierarchicalCohort` | `true` | Beta | 0.11 | | +| `ObjectRetentionPolicies` | `false` | Alpha | 0.12 | 0.12 | +| `ObjectRetentionPolicies` | `true` | Beta | 0.13 | | +| `TASFailedNodeReplacement` | `false` | Alpha | 0.12 | | +| `TASFailedNodeReplacement` | `true` | Beta | 0.14 | | +| `AdmissionFairSharing` | `false` | Alpha | 0.12 | | +| `TASFailedNodeReplacementFailFast` | `false` | Alpha | 0.12 | | +| `TASFailedNodeReplacementFailFast` | `true` | Beta | 0.14 | | +| `TASReplaceNodeOnPodTermination` | `false` | Alpha | 0.13 | | +| `TASReplaceNodeOnPodTermination` | `true` | Beta | 0.14 | | +| `ElasticJobsViaWorkloadSlices` | `false` | Alpha | 0.13 | | +| `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | +| `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | +| `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | +| `SanitizePodSets` | `true` | Beta | 0.13 | | +| `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.13 | | ### 已毕业或已弃用特性的特性门控 {#feature-gates-for-graduated-or-deprecated-features} From 4ce6906db809f5648e018ff167f9ab07b41ad442 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 31 Oct 2025 15:18:05 +0530 Subject: [PATCH 064/119] Sync feature gate tables. (#7475) --- .../content/zh-CN/docs/installation/_index.md | 94 +++++++++---------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/site/content/zh-CN/docs/installation/_index.md b/site/content/zh-CN/docs/installation/_index.md index ea521665cb0..5527497dbc0 100644 --- a/site/content/zh-CN/docs/installation/_index.md +++ b/site/content/zh-CN/docs/installation/_index.md @@ -263,56 +263,56 @@ spec: ### Alpha 和 Beta 级别特性的特性门控 {#feature-gates-for-alpha-and-beta-features} -| 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | -| --------------------------------------------- | ------- | ----- |---------|------| -| `FlavorFungibility` | `true` | Beta | 0.5 | | -| `MultiKueue` | `false` | Alpha | 0.6 | 0.8 | -| `MultiKueue` | `true` | Beta | 0.9 | | -| `MultiKueueBatchJobWithManagedBy` | `false` | Alpha | 0.8 | | -| `MultiKueueBatchJobWithManagedBy` | `true` | Beta | 0.15 | | -| `PartialAdmission` | `false` | Alpha | 0.4 | 0.4 | -| `PartialAdmission` | `true` | Beta | 0.5 | | -| `VisibilityOnDemand` | `false` | Alpha | 0.6 | 0.8 | -| `VisibilityOnDemand` | `true` | Beta | 0.9 | | -| `PrioritySortingWithinCohort` | `true` | Beta | 0.6 | | -| `LendingLimit` | `false` | Alpha | 0.6 | 0.8 | -| `LendingLimit` | `true` | Beta | 0.9 | | -| `TopologyAwareScheduling` | `false` | Alpha | 0.9 | | -| `TopologyAwareScheduling` | `true` | Beta | 0.14 | | -| `LocalQueueDefaulting` | `false` | Alpha | 0.10 | 0.11 | -| `LocalQueueDefaulting` | `true` | Beta | 0.12 | | -| `LocalQueueMetrics` | `false` | Alpha | 0.10 | | -| `HierarchicalCohort` | `true` | Beta | 0.11 | | -| `ObjectRetentionPolicies` | `false` | Alpha | 0.12 | 0.12 | -| `ObjectRetentionPolicies` | `true` | Beta | 0.13 | | -| `TASFailedNodeReplacement` | `false` | Alpha | 0.12 | | -| `TASFailedNodeReplacement` | `true` | Beta | 0.14 | | -| `AdmissionFairSharing` | `false` | Alpha | 0.12 | | -| `TASFailedNodeReplacementFailFast` | `false` | Alpha | 0.12 | | -| `TASFailedNodeReplacementFailFast` | `true` | Beta | 0.14 | | -| `TASReplaceNodeOnPodTermination` | `false` | Alpha | 0.13 | | -| `TASReplaceNodeOnPodTermination` | `true` | Beta | 0.14 | | -| `ElasticJobsViaWorkloadSlices` | `false` | Alpha | 0.13 | | -| `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | -| `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | -| `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | -| `SanitizePodSets` | `true` | Beta | 0.13 | | -| `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.13 | | +| 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | +|-----------------------------------------------|---------|-------|------|------| +| `FlavorFungibility` | `true` | Beta | 0.5 | | +| `MultiKueue` | `false` | Alpha | 0.6 | 0.8 | +| `MultiKueue` | `true` | Beta | 0.9 | | +| `MultiKueueBatchJobWithManagedBy` | `false` | Alpha | 0.8 | 0.15 | +| `MultiKueueBatchJobWithManagedBy` | `true` | Beta | 0.15 | | +| `PartialAdmission` | `false` | Alpha | 0.4 | 0.4 | +| `PartialAdmission` | `true` | Beta | 0.5 | | +| `VisibilityOnDemand` | `false` | Alpha | 0.6 | 0.8 | +| `VisibilityOnDemand` | `true` | Beta | 0.9 | | +| `PrioritySortingWithinCohort` | `true` | Beta | 0.6 | | +| `LendingLimit` | `false` | Alpha | 0.6 | 0.8 | +| `LendingLimit` | `true` | Beta | 0.9 | | +| `TopologyAwareScheduling` | `false` | Alpha | 0.9 | 0.13 | +| `TopologyAwareScheduling` | `true` | Beta | 0.14 | | +| `LocalQueueDefaulting` | `false` | Alpha | 0.10 | 0.11 | +| `LocalQueueDefaulting` | `true` | Beta | 0.12 | | +| `LocalQueueMetrics` | `false` | Alpha | 0.10 | | +| `HierarchicalCohort` | `true` | Beta | 0.11 | | +| `ObjectRetentionPolicies` | `false` | Alpha | 0.12 | 0.12 | +| `ObjectRetentionPolicies` | `true` | Beta | 0.13 | | +| `TASFailedNodeReplacement` | `false` | Alpha | 0.12 | 0.13 | +| `TASFailedNodeReplacement` | `true` | Beta | 0.14 | | +| `AdmissionFairSharing` | `false` | Alpha | 0.12 | | +| `TASFailedNodeReplacementFailFast` | `false` | Alpha | 0.12 | 0.13 | +| `TASFailedNodeReplacementFailFast` | `true` | Beta | 0.14 | | +| `TASReplaceNodeOnPodTermination` | `false` | Alpha | 0.13 | 0.13 | +| `TASReplaceNodeOnPodTermination` | `true` | Beta | 0.14 | | +| `ElasticJobsViaWorkloadSlices` | `false` | Alpha | 0.13 | | +| `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | +| `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | +| `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | +| `SanitizePodSets` | `true` | Beta | 0.13 | | +| `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.13 | | ### 已毕业或已弃用特性的特性门控 {#feature-gates-for-graduated-or-deprecated-features} -| 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | -|---------------------------------------| ------- | ---------- |---------|---------| -| `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | -| `ProvisioningACC` | `true` | Beta | 0.7 | | -| `ProvisioningACC` | `true` | GA | 0.14 | | -| `ConfigurableResourceTransformations` | `false` | Alpha | 0.9 | 0.9 | -| `ConfigurableResourceTransformations` | `true` | Beta | 0.10 | 0.13 | -| `ConfigurableResourceTransformations` | `true` | GA | 0.14 | | -| `TASProfileMostFreeCapacity` | `false` | Deprecated | 0.11 | 0.13 | -| `TASProfileLeastFreeCapacity` | `false` | Deprecated | 0.11 | | -| `TASProfileMixed` | `false` | Deprecated | 0.11 | | +| 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | +|---------------------------------------|---------|------------|------|------| +| `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | +| `ProvisioningACC` | `true` | Beta | 0.7 | | +| `ProvisioningACC` | `true` | GA | 0.14 | | +| `ConfigurableResourceTransformations` | `false` | Alpha | 0.9 | 0.9 | +| `ConfigurableResourceTransformations` | `true` | Beta | 0.10 | 0.13 | +| `ConfigurableResourceTransformations` | `true` | GA | 0.14 | | +| `TASProfileMostFreeCapacity` | `false` | Deprecated | 0.11 | 0.13 | +| `TASProfileLeastFreeCapacity` | `false` | Deprecated | 0.11 | | +| `TASProfileMixed` | `false` | Deprecated | 0.11 | | ## 接下来是什么 {#whats-next} -- 阅读 [`Configuration` 的 API 参考](/docs/reference/kueue-config.v1beta1/#Configuration) +- 阅读 [`Configuration` 的 API 参考](/docs/reference/kueue-config.v1beta1/#Configuration) \ No newline at end of file From 6121826ec41a5cf9d33c464bb34f97687ab4b881 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 31 Oct 2025 16:58:02 +0530 Subject: [PATCH 065/119] Drop graduated ProvisioningACC feature gate. (#7465) --- pkg/features/kube_features.go | 10 ---------- site/content/en/docs/installation/_index.md | 3 --- .../reference/components-tools/feature-gate-removed.md | 5 ++++- site/content/zh-CN/docs/installation/_index.md | 5 ++--- 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 5e629a3441a..5bb4d2dc9f0 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -41,12 +41,6 @@ const ( // Enables flavor fungibility. FlavorFungibility featuregate.Feature = "FlavorFungibility" - // owner: @trasc - // kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/1136-provisioning-request-support - // - // Enables Provisioning Admission Check Controller. - ProvisioningACC featuregate.Feature = "ProvisioningACC" - // owner: @pbundyra // kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/168-2-pending-workloads-visibility // @@ -229,10 +223,6 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned FlavorFungibility: { {Version: version.MustParse("0.5"), Default: true, PreRelease: featuregate.Beta}, }, - ProvisioningACC: { - {Version: version.MustParse("0.5"), Default: true, PreRelease: featuregate.Beta}, - {Version: version.MustParse("0.14"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 0.15 - }, VisibilityOnDemand: { {Version: version.MustParse("0.6"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("0.9"), Default: true, PreRelease: featuregate.Beta}, diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 8c268c65e86..87230a7fcd8 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -307,9 +307,6 @@ The SanitizePodSets and MultiKueueAllowInsecureKubeconfigs features are availabl | Feature | Default | Stage | Since | Until | |---------------------------------------|---------|------------|-------|-------| -| `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | -| `ProvisioningACC` | `true` | Beta | 0.7 | | -| `ProvisioningACC` | `true` | GA | 0.14 | | | `ConfigurableResourceTransformations` | `false` | Alpha | 0.9 | 0.9 | | `ConfigurableResourceTransformations` | `true` | Beta | 0.10 | 0.13 | | `ConfigurableResourceTransformations` | `true` | GA | 0.14 | | diff --git a/site/content/en/docs/reference/components-tools/feature-gate-removed.md b/site/content/en/docs/reference/components-tools/feature-gate-removed.md index d8b87755f78..012f5a56150 100644 --- a/site/content/en/docs/reference/components-tools/feature-gate-removed.md +++ b/site/content/en/docs/reference/components-tools/feature-gate-removed.md @@ -35,4 +35,7 @@ In the following table: | `QueueVisibility` | `false` | Alpha | 0.5 | 0.9 | | `QueueVisibility` | `false` | Deprecated | 0.9 | 0.14 | | `ManagedJobsNamespaceSelector` | `true` | Beta | 0.10 | 0.13 | -| `ManagedJobsNamespaceSelector` | `true` | GA | 0.13 | 0.15 | \ No newline at end of file +| `ManagedJobsNamespaceSelector` | `true` | GA | 0.13 | 0.15 | +| `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | +| `ProvisioningACC` | `true` | Beta | 0.7 | 0.14 | +| `ProvisioningACC` | `true` | GA | 0.14 | 0.15 | diff --git a/site/content/zh-CN/docs/installation/_index.md b/site/content/zh-CN/docs/installation/_index.md index 5527497dbc0..35635f8e5fc 100644 --- a/site/content/zh-CN/docs/installation/_index.md +++ b/site/content/zh-CN/docs/installation/_index.md @@ -303,9 +303,8 @@ spec: | 功能 | 默认值 | 阶段 | 起始版本 | 截止版本 | |---------------------------------------|---------|------------|------|------| -| `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | -| `ProvisioningACC` | `true` | Beta | 0.7 | | -| `ProvisioningACC` | `true` | GA | 0.14 | | +| `ManagedJobsNamespaceSelector` | `true` | Beta | 0.10 | 0.13 | +| `ManagedJobsNamespaceSelector` | `true` | GA | 0.13 | | | `ConfigurableResourceTransformations` | `false` | Alpha | 0.9 | 0.9 | | `ConfigurableResourceTransformations` | `true` | Beta | 0.10 | 0.13 | | `ConfigurableResourceTransformations` | `true` | GA | 0.14 | | From 04238980dc0eb29cde7b19d86596e72706b5dce2 Mon Sep 17 00:00:00 2001 From: Daniel Henkel <9447057+dhenkel92@users.noreply.github.com> Date: Fri, 31 Oct 2025 13:22:05 +0100 Subject: [PATCH 066/119] docs(kep): Create delayed admission check retries KEP (#6210) --- .../README.md | 566 ++++++++++++++++++ .../kep.yaml | 33 + .../reconciler-flow.png | Bin 0 -> 13495 bytes 3 files changed, 599 insertions(+) create mode 100644 keps/3258-delayed-admission-check-retries/README.md create mode 100644 keps/3258-delayed-admission-check-retries/kep.yaml create mode 100644 keps/3258-delayed-admission-check-retries/reconciler-flow.png diff --git a/keps/3258-delayed-admission-check-retries/README.md b/keps/3258-delayed-admission-check-retries/README.md new file mode 100644 index 00000000000..a9761e19fa7 --- /dev/null +++ b/keps/3258-delayed-admission-check-retries/README.md @@ -0,0 +1,566 @@ +# KEP-3258: Delayed Admission Check Retries + + +- [Summary](#summary) +- [Motivation](#motivation) + - [Goals](#goals) + - [Non-Goals](#non-goals) +- [Proposal](#proposal) + - [User Stories](#user-stories) + - [Story 1: External Admission Check with Backoff](#story-1-external-admission-check-with-backoff) + - [Story 2: Control Plane Load Management](#story-2-control-plane-load-management) + - [Story 3: Multi-tenancy with Different Retry Strategies](#story-3-multi-tenancy-with-different-retry-strategies) + - [Notes/Constraints/Caveats](#notesconstraintscaveats) + - [Race Condition Between Admission Check Updates and Workload Eviction](#race-condition-between-admission-check-updates-and-workload-eviction) + - [Risks and Mitigations](#risks-and-mitigations) +- [Design Details](#design-details) + - [API Changes](#api-changes) + - [Delayed Processing and State Reconciliation](#delayed-processing-and-state-reconciliation) + - [Caveats](#caveats) + - [Documentation Requirements](#documentation-requirements) + - [Test Plan](#test-plan) + - [Unit Tests](#unit-tests) + - [Integration tests](#integration-tests) + - [Graduation Criteria](#graduation-criteria) +- [Implementation History](#implementation-history) +- [Drawbacks](#drawbacks) +- [Future outlook](#future-outlook) +- [Alternatives](#alternatives) + - [Alternative to sync AC results: Wait for All Admission Checks](#alternative-to-sync-ac-results-wait-for-all-admission-checks) + + +## Summary + +This KEP proposes a mechanism to allow external admission check controllers to specify retry delays when marking a workload's admission check state as `Retry`. Currently, when an admission check is set to `Retry`, Kueue immediately evicts the workload and requeues it with minimal delay, which can cause excessive load on external controllers and the control plane. + +## Motivation + +Kueue supports custom admission checks that allow external controllers to influence workload admission decisions. When an external controller sets an admission check state to `Retry`, Kueue currently: + +1. Evicts the workload +2. Releases the quota reservation +3. Immediately requeues the workload with minimal delay + +This immediate retry behavior can cause several issues: + +- Excessive load on external admission check controllers +- Unnecessary churn in the control plane +- Inefficient resource utilization when the retry condition is expected to persist for a known duration +- Artificial reduction of available quotas + +Currently, there's a shared `.status.requeueState` field used by both the [provisioning request admission check](https://kueue.sigs.k8s.io/docs/admission-check-controllers/provisioning/#provisioningrequestconfig) and the [WaitForPodsReady](https://kueue.sigs.k8s.io/docs/tasks/manage/setup_wait_for_pods_ready/) feature. It already serves these purposes well but will need some adjustments to support the new use case. + +### Goals + +- Provide a mechanism for external admission check controllers to specify retry delays +- Allow each admission check to independently manage its retry timing +- Support proper coordination of multiple admission checks with different retry delays + +### Non-Goals + +- Change the behavior of other admission check states (`Ready`, `Rejected`, `Pending`) +- Reactivate the deprecated `AdmissionCheck.Spec.RetryDelayMinutes` field +- Modify the FIFO ordering guarantees for workloads without retry delays +- Change the retry behavior for admission checks that don't specify a delay + +## Proposal + +Add new fields to the `AdmissionCheckState` status to support per-admission-check retry delays: + +- `requeueAfterSeconds`: A duration, based on eviction time, indicating when this admission check should be retried +- `retryCount`: A counter tracking how many times this admission check has been retried so far + +When multiple admission checks are in the `Retry` state with different delays, Kueue will use the maximum delay across all checks and save it into `requeueState.requeueAt`. +Besides that, the old behavior remains unchanged. When all admission checks are in the `Ready` state, the workload will be admitted; if any of them are in the `Rejected` state, the workload will be rejected. + +### User Stories + +#### Story 1: External Admission Check with Backoff + +As a developer of an external admission check controller that validates cloud resource availability, I want to implement exponential backoff when resources are temporarily unavailable, so that I don't overwhelm the cloud API with requests. + +When my controller detects that required cloud resources are not available, it can: + +1. Set the admission check state to `Retry` +2. Calculate an appropriate backoff delay (e.g., 30s, 1m, 2m, 4m) +3. Set `requeueAfterSeconds` to the calculated future timestamp +4. Track retry attempts using `retryCount` + +#### Story 2: Control Plane Load Management + +As a cluster administrator, I have an admission check that validates whether the Kubernetes control plane is healthy before admitting large workloads. When the control plane is under stress, I want workloads to be delayed with longer intervals to allow recovery. + +The admission check controller can: + +1. Detect control plane stress (high API server latency, etcd performance issues) +2. Set workloads to `Retry` with delays proportional to the stress level +3. Prevent thundering herd effects by spreading retry times + +#### Story 3: Multi-tenancy with Different Retry Strategies + +As a platform engineer supporting multiple teams, I want different admission checks to have independent retry strategies based on their specific requirements without interfering with each other. + +For example: + +- Security scanning admission check: retry after 5 minutes if scanner is busy +- Resource provisioning check: retry after 1 minute if capacity is being added +- Compliance validation check: retry after 10 minutes if external audit service is down + +### Notes/Constraints/Caveats + +- The retry delay is advisory; the actual delay might be slightly longer due to reconciliation timing +- Workloads with very long retry delays might appear to be "stuck" to users +- When multiple admission checks specify different retry delays, the workload will not be retried until all delays have expired +- The retry mechanism does not guarantee strict FIFO ordering when delays are involved +- There is a potential race condition where Kueue might reset admission check states before all controllers have set their retry delays +- The `requeueState.requeueAt` field is used not only by admission checks but also by features like [WaitForPodsReady](https://kueue.sigs.k8s.io/docs/tasks/manage/setup_wait_for_pods_ready/#requeuing-strategy) + +#### Race Condition Between Admission Check Updates and Workload Eviction + +The current implementation has critical issues with how admission checks are processed. The system uses a small state machine that reacts immediately to changes in the admission check state. When one admission check sets the status to `Retry`, Kueue evicts the workload (and resets the AC state) right away. This immediate action prevents other admission checks from updating their status afterward. If another check sets the status to `Retry` after eviction, it can break Kueue’s internal state. Therefore, the order of events must be strictly followed. The challenge here is that to wait for the maximum time, we need to know all potential delays in advance. + +```mermaid +sequenceDiagram + autonumber + actor user as User + participant kueue as Kueue + participant kube as Kube API Server + participant ac1 as Admission Check 1 + participant ac2 as Admission Check 2 + + user ->> kube: Create Workload + kube -->> kueue: New WL event + activate kueue + kueue -> kueue: Reserve Quota + kueue -> kueue: Attach admission check information + kueue ->> kube: Push WL update + deactivate kueue + kube -->> ac1: WL update event + activate ac1 + kube -->> ac2: WL update event + activate ac2 + ac1 ->> ac1: Perform checks + ac2 ->> ac2: Perform checks + ac2 ->> kube: Set AC to retry + deactivate ac2 + kube -->> kueue: WL update event + kueue ->> kube: Reset admission check states + kueue ->> kube: Evict Workload + activate kueue + ac1 -x kube: Try set AC to retry + deactivate ac1 + kueue -> kueue: Requeue Workload + kueue -> kueue: Reserve Quota + kueue ->> kube: Push WL update + deactivate kueue +``` + +The diagram above shows the current flow of a workload with two admission checks: + +- Both admission checks will set the state to `Retry` +- Kueue will take actions after the first one applies its result (steps 10-13) +- The second admission check should not update its status after the workload got evicted (step 14) + - Otherwise, it will cause issues in Kueue's internal state machine + +This is a fundamental problem for delayed retries as we can never know the maximum delay if Kueue reacts to the first response. + +```mermaid +sequenceDiagram + autonumber + actor user as User + participant kueue as Kueue + participant kube as Kube API Server + participant ac1 as Admission Check 1 + participant ac2 as Admission Check 2 + + user ->> kube: Create Workload + kube -->> kueue: New WL event + activate kueue + kueue -> kueue: Reserve Quota + kueue -> kueue: Attach admission check information + kueue ->> kube: Push WL update + deactivate kueue + kube -->> ac1: WL update event + activate ac1 + kube -->> ac2: WL update event + activate ac2 + ac1 ->> ac1: Perform checks + ac2 ->> ac2: Perform checks + ac2 ->> kube: Set AC to retry with 1m delay + deactivate ac2 + kube -->> kueue: WL update event + kueue ->> kube: Reset admission check states + kueue ->> kube: Evict Workload + activate kueue + ac1 -x kube: Try set AC to retry with 2m delay + deactivate ac1 + kueue -> kueue: Wait for 1m + kueue -> kueue: Requeue Workload + kueue -> kueue: Reserve Quota + kueue ->> kube: Push WL update + deactivate kueue +``` + +This updated version of the diagram includes potential delays: + +- Admission check 1 sets the status to retry with a delay of one minute (step 10) +- Kueue will immediately take action and evict the workload (step 12 + 13) +- Shortly after, admission check 2 wants to set its state to retry too, but can't as the workload was evicted already (step 14) + - However, it would have set a higher retry delay than admission check one + - This means that we'll always only use the delay of the fastest responding check + +**What happens if the second AC updates it's state at the wrong time?** + +When the admission check controller is too fast and doesn't implement proper safeguards, it can lead to invalid states where a workload won't be reconsidered by Kueue anymore. +The following sequence of events illustrates how this issue can eventually break a workload: + +1. User creates a workload. +2. Kueue reserves quota and applies changes in two steps: + 2.1 Applies a patch to set resource flavour information and sets `QuotaReserved=true`. + 2.2 Applies an update to `add admission checks to the status in a pending state`. +3. The admission check controller runs because `QuotaReserved=true` and `AdmissionCheckState=Pending`: + 3.1 It applies a patch to `set the admission check state to retry`. +4. Kueue `resets the admission check state to Pending` and sets `Evicted=true` in a single patch. +5. The admission check controller runs again under the same conditions: + 5.1 It `sets the admission check state back to retry`. +6. Kueue does not notice the change and sets `QuotaReserved=false` and `Requeue=false` in a single patch + 6.1 Evicted remains true, and QuotaReserved is false. + 6.2 The admission check state stays in Retry. +7. Kueue sets `Requeued=true`. +8. Kueue can't reserve quota again due to the invalid state. It tries to schedule the workload but always sees the admission check in a Retry state. + 8.1 The message on the QuotaReserved status: The workload has failed admission checks. + +At this point, the workload should theoretically be picked up by the scheduler again. However, it's ignored because the admission check remains in a Retry state that is never cleared. +We mitigated this issue by being more cautious with updates in our custom admission checks. Specifically, by avoiding state changes at the wrong time. However, this behavior was not immediately obvious. + +### Risks and Mitigations + +**Risk**: Breaking backward compatibility with existing admission check controllers\ +**Mitigation**: All fields are optional; controllers that don't set them will maintain current behavior + +**Risk**: Workloads appearing stuck due to long retry delays \ +**Mitigation**: Add clear status messages that indicate the retry time, expose metrics to monitor retry delays, and document the recommended maximum delay values. + +**Risk**: Resource leakage if workloads are forgotten during long delays \ +**Mitigation**: Ensure proper cleanup mechanisms and consider maximum delay limits + +**Risk**: Race conditions when multiple admission checks set delays \ +**Mitigation**: Allow and support admission checks to update the state after a workload has been evicted. + +## Design Details + +### API Changes + +Add new fields to `AdmissionCheckState` in `workload_types.go`: + +```go +type AdmissionCheckState struct { + // lastTransitionTime is the last time the condition transitioned from one status to another. + // This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + // +required + LastTransitionTime metav1.Time `json:"lastTransitionTime"` + + // Existing fields remain unchanged... + + // RequeueAfterSeconds indicates how long to wait at least before + // retrying to admit the workload. + // When State=Retry, admission check controllers should set this + // to implement delays between retry attempts. + // + // If nil when State=Retry, Kueue will retry immediately. + // If set, Kueue will add the workload back to the queue after + // lastTransitionTime + RequeueAfterSeconds + // + // +optional + RequeueAfterSeconds *metav1.Duration `json:"requeueAfterSeconds,omitempty"` + + // RetryCount tracks retry attempts for this admission check. + // Kueue automatically increments this when transitioning from + // Retry→Pending. Reset to 0 when check succeeds or workload + // is admitted. + // + // +optional + RetryCount *int32 `json:"retryCount,omitempty"` +} +``` + +To get the final timestamp, Kueue adds the relative RequeueAfterSeconds duration to the LastTransitionTime field from the spec. + +**Resetting admission check requeue state** + +When the delay expires, all fields will be automatically reset before the workload is requeued. +This includes both the `requeueState` and the new `AdmissionCheckState.RequeueAfterSeconds` field. + +Additionally, the system will automatically reset requeueAfterSeconds and retryCount when a workload is deactivated. +This allows the cluster administrator to manually re-enable the workload, and it will start with a fresh configuration automatically. + +**Example: Workload Status with Multiple Admission Checks** + +```yaml +apiVersion: kueue.x-k8s.io/v1beta1 +kind: Workload +metadata: + name: ml-training-job + namespace: research +status: + requeueState: + requeueAt: "2024-02-07T00:10:00Z" + + admissionChecks: + # Check 1: Budget check with daily reset + - name: budget-check + state: Retry + lastTransitionTime: "2024-02-06T10:10:00Z" + message: "Daily budget exhausted. Will retry at midnight." + requeueAfterSeconds: 50400 // in 14h, which is approx midnight + retryCount: 1 + + # Check 2: GPU availability with exponential backoff + - name: gpu-availability + state: Retry + lastTransitionTime: "2024-02-06T10:11:00Z" + message: "No A100 GPUs available. Retrying in 8m (attempt 4)" + requeueAfterSeconds: 480 # 8 minutes from now + retryCount: 4 + + # Check 3: License server check - immediate retry + - name: license-check + state: Retry + lastTransitionTime: "2024-02-06T10:20:00Z" + message: "License server temporarily unavailable" + # No requeueAfterSeconds set - will retry immediately +# In this example, Kueue will wait until 00:10:00Z (the longest delay) +# before re-evaluating all admission checks +``` + +#### Delayed Processing and State Reconciliation + +To address the issues mentioned in [Race Condition Between Admission Check Updates and Workload Eviction](#race-condition-between-admission-check-updates-and-workload-eviction), the implementation will be changed to allow admission check updates until all available delays have expired. The workload should still be evicted immediately to release all reserved quota. However, Kueue will no longer reset the admission check states right away. Instead, it will wait until `max(AdmissionCheckState.lastTransitionTime + AdmissionCheckState.RequeueAfterSeconds)` is reached. Only then will the admission checks be reset to pending, and the workload marked as `Requeued=true`. + +Furthermore, instead of scheduling a timer in `pkg/controller/core/workload_controller.go:807` that sets the requeuing in the future, the controller should rely on the Reconcile function for managing delays. Whenever the workload gets re-submitted to be reconciled, it should search for the latest delay across all admission checks. If the calculated retry time is in the future, the reconciler should reprocess the workload with a delay. If it's in the past, the reconciler should add the workload back into the scheduling rotation, by setting the `Requeued=true` condition. + +![This is how the reconciler would work high level](reconciler-flow.png "Reconciler flow") + +This diagram shows the updated approach where admission check one sets the state to Retry with a delay of one minute (step 10), and Kueue evicts the workload (step 12). Crucially, admission check two is able to push its update even after the workload has been evicted, and Kueue will take this new delay into account. Only after all available retry delays have expired Kueue will set the `Requeued=true` condition, which then triggers the actual requeuing of the workload. This approach ensures that all admission checks have the opportunity to set their retry delays, and Kueue will respect the maximum delay across all checks. + +```mermaid +sequenceDiagram + autonumber + actor user as User + participant kueue as Kueue + participant kube as Kube API Server + participant ac1 as Admission Check 1 + participant ac2 as Admission Check 2 + + user ->> kube: Create Workload + kube -->> kueue: New WL event + activate kueue + kueue -> kueue: Reserve Quota + kueue -> kueue: Attach admission check information + kueue ->> kube: Push WL update + deactivate kueue + kube -->> ac1: WL update event + activate ac1 + kube -->> ac2: WL update event + activate ac2 + ac1 ->> ac1: Perform checks + ac2 ->> ac2: Perform checks + ac2 ->> kube: Set AC to retry with 1m delay + deactivate ac2 + kube -->> kueue: WL update event + kueue ->> kube: Evict Workload + activate kueue + kueue --> kueue: Don't retry as retry after is not done (in 1m) + ac1 ->> kube: Set AC to retry with 2m delay + kueue --> kueue: Don't retry as retry after is not done (in 2m) + deactivate ac1 + kueue -> kueue: Set Requeued=true condition on Workload + kueue -> kueue: Reset admission check status + kueue -> kueue: Reserve Quota + kueue ->> kube: Push WL update + deactivate kueue +``` + +##### Caveats + +In the new case, there's still a race condition if one admission check is fast and doesn't set any delay, which means that the workload will be retries immediately. +However, this shouldn't be a problem, since the update can be posted at any time. This means the second admission check will eventually set the retry with a delay, ensuring it's not reworked immediately again. +That said, it still means we go through a couple of unnecessary cycles, which should be okay. + +### Documentation Requirements + +To ensure proper implementation of admission check controllers, the following documentation should be added: + +- **Admission Check Flow Guide**: Detailed documentation on the expected flow and behavior of admission checks, including: + - Proper handling of state transitions + - Best practices for setting retry delays + - Guidelines on when and how to update admission check states + - Examples of correct implementation patterns + +- **Migration Guide**: Step-by-step instructions for migrating from the old `requeueState` mechanism to the new per-admission-check retry system + +### Test Plan + +#### Unit Tests + +- Test retry delay calculation with single and multiple admission checks +- Test backward compatibility with existing retry behavior + +#### Integration tests + +- End-to-end test with external admission check setting retry delays +- Test interaction between multiple admission checks with different delays +- Test workload flow through retry cycles with delays +- Test that the workload state remains consistent when admission checks post updates after workload eviction +- Verify that late admission check updates are properly handled without corrupting the state machine + +### Graduation Criteria + +**Alpha:** + +- `ACRequeueAfter` feature gate (disabled by default in alpha) +- rework delaying mechanism to support multiple updates to `requeueState.requeueAt` +- add new fields (`RequeueAfterSeconds` and `RetryCount`) to admission check state +- update kueue logic to take new admission check state filed into account +- first iteration of admission check documentation + +**Beta:** + +- `ACRequeueAfter` feature gate in beta (enabled by default) +- fix all reported bugs +- enhance admission check documentation based on community feedback +- re-evaluate the migration of the `ProvisioningRequest` to the new API +- add metrics to track retry behavior + +**GA:** + +- `ACRequeueAfter` feature gate in stable (locked and enabled by default) +- fix all reported bugs + +## Implementation History + +- 2025-07-21: Initial KEP proposal + +## Drawbacks + +- Adds complexity to the admission check state machine +- Potential for workloads to be delayed for extended periods +- Will break strict FIFO ordering guarantees in some scenarios + +## Future outlook + +With this KEP, each admission check controller must implement its own delay logic, leading to redundant effort and inconsistent behavior. + +To reduce this friction, a future extension could add automatic retry delay handling directly in Kueue. This would involve enhancing the `AdmissionCheck` CRD with a configurable retry delay, such as: + +``` +type AdmissionCheckSpec struct { + Retry *AdmissionCheckRetry +} + +type AdmissionCheckRetryStrategy string + +const ( + AdmissionCheckRetryStrategyStatic AdmissionCheckRetryStrategy = "static" + AdmissionCheckRetryStrategyBackoff AdmissionCheckRetryStrategy = "backoff" +) + +type AdmissionCheckRetry struct { + Strategy AdmissionCheckRetryStrategy + BaseDelay time.Duration + Faktor *float64 + Jitter *float64 +} +``` + +When an admission check includes this configuration and sets its state to Retry, Kueue would compute the delay and populate the `requeueAfterSeconds`` field defined in this KEP. + +## Alternatives + +1. **Global retry configuration per AdmissionCheck**: Reactivate and enhance the deprecated `retryDelayMinutes` field in the AdmissionCheck spec. This was rejected because: + - It doesn't allow for dynamic retry strategies + - All workloads would use the same fixed delay + +2. **Shared retry configuration**: Continue using the existing `requeueState` mechanism. This was rejected because: + - It creates race conditions between multiple admission checks + - Only one admission check can effectively control the retry delay + +3. **Separate AdmissionCheckRetryState status field**: Add a new top-level status field to the Workload that contains all retry-related information for admission checks: + + ```go + type WorkloadStatus struct { + // Existing fields... + + // New field for retry state + AdmissionCheckRetryState *AdmissionCheckRetryState `json:"admissionCheckRetryState,omitempty"` + } + + type AdmissionCheckRetryState struct { + // Map of admission check name to retry information + RetryInfo map[string]RetryInfo `json:"retryInfo"` + } + + type RetryInfo struct { + RequeueAfter *metav1.Time `json:"requeueAfter,omitempty"` + RetryCount *int32 `json:"retryCount,omitempty"` + } + ``` + + This approach would provide centralized retry information that cannot be accidentally erased when updating individual admission check states. It would create a clear separation between admission check state and retry metadata, and could potentially make it easier to query and monitor retry states across workloads. + + **This was rejected because:** + - Creates confusion about where retry information should be set and read + - Adds another top-level status field that admission check controllers need to update + - Increases API complexity with multiple locations for related information + - Requires admission check controllers to update two different parts of the workload status + - Makes it harder to understand the complete state of an admission check at a glance + +### Alternative to sync AC results: Wait for All Admission Checks + +One alternative approach would be to have Kueue wait for all admission checks to complete before taking any eviction action. This would ensure that all admission checks have the opportunity to set their retry delays, and Kueue could then use the maximum delay across all checks. + +```mermaid +sequenceDiagram + autonumber + actor user as User + participant kueue as Kueue + participant kube as Kube API Server + participant ac1 as Admission Check 1 + participant ac2 as Admission Check 2 + + user ->> kube: Create Workload + kube -->> kueue: New WL event + activate kueue + kueue -> kueue: Reserve Quota + kueue -> kueue: Attach admission check information + kueue ->> kube: Push WL update + deactivate kueue + kube -->> ac1: WL update event + activate ac1 + kube -->> ac2: WL update event + activate ac2 + ac1 ->> ac1: Perform checks + ac2 ->> ac2: Perform checks + ac2 ->> kube: Set AC to retry with 1m delay + deactivate ac2 + kube -->> kueue: WL update event + kueue -> kueue: Ignore as not all ACs are done + ac1 ->> kube: Set AC to retry with 2m delay + deactivate ac1 + kube -->> kueue: WL update event + kueue ->> kube: Reset admission check states + kueue ->> kube: Evict Workload + activate kueue + kueue -> kueue: Wait for 2m + kueue -> kueue: Reserve Quota + kueue ->> kube: Push WL update + deactivate kueue +``` + +This diagram shows the alternative flow where admission check one sets the state to retry with delay, and Kueue ignores it initially. Only after admission check two returns a result does Kueue take action and start to evict the workload. However, this approach has significant disadvantages: + +**Breaking Backwards Compatibility**: The behavior fundamentally changes from the current implementation. Where Kueue would evict or reject a workload immediately upon the first admission check failure, it would now wait for all admission checks to respond. This could affect existing workload behaviors and expectations. + +**Risk of Stuck Workloads**: If one of the admission check controllers becomes unresponsive or is unable to set the status, a workload would be stuck indefinitely in the admission phase. This also means that the quota reservation would be held forever, potentially starving other workloads in the queue. This creates a new failure mode that doesn't exist in the current implementation. + +Due to these significant drawbacks, this alternative was rejected in favor of the solution described before. diff --git a/keps/3258-delayed-admission-check-retries/kep.yaml b/keps/3258-delayed-admission-check-retries/kep.yaml new file mode 100644 index 00000000000..bfed6fdf190 --- /dev/null +++ b/keps/3258-delayed-admission-check-retries/kep.yaml @@ -0,0 +1,33 @@ +title: Delayed Admission Check Retries +kep-number: 3258 +authors: + - "@dhenkel92" +status: provisional +creation-date: 2025-09-04 +reviewers: + - "@mimowo" + - "@PBundyra" +approvers: + - "@mimowo" + +# The target maturity stage. Valid values are: +# - "alpha" +# - "beta" +# - "stable" +# To determine the maturity level for the feature please see: +# https://github.com/kubernetes-sigs/kueue/blob/main/docs/concepts/features.md#feature-gates-and-maturity +stage: beta + +latest-milestone: "v0.15" + +milestone: + alpha: "v0.15" + beta: "v0.15" + stable: TBD + +# List the feature gate name and the components for which it must be enabled +feature-gates: {} +# No associated feature gates. The feature can be disabled through the Configuration API. +disable-supported: true + +metrics: {} diff --git a/keps/3258-delayed-admission-check-retries/reconciler-flow.png b/keps/3258-delayed-admission-check-retries/reconciler-flow.png new file mode 100644 index 0000000000000000000000000000000000000000..0c81e96ab4cd09a9d6d92529f7f9d496f00bb3c9 GIT binary patch literal 13495 zcmdUWcT`kCvnPlGl9Mt+mMn1q5dq0CARsws7;?@zisT^-ISY(Plq5kwiNXK^Lrwz% zl5+;hJAQA!x4Y-;J8#e1eS3EQnA1I7^{ZR=-oDjU)zvX-D)NMQRCpK|7=&*WWHm4_ zFyR;&Sh~1a=#lE_jwN)%08!9*i-F+|M<`otzYZ!`1DLQ-jH@CJML+pDH1M>@u!QnC2H#ghc+xq(Y z*VorSe*6dv3u|a-K%>XT#)^xJWo2a-7Z+1fQch1#d3bmL0D!Zz^Zvo%`T2#ev1LV7 zotW%fZ9_;|Wlhe9kMeJIvT_SY$0w0UWK&aHhvc3Wc(>v-9`&2ZOf2xgFj#Bif(xkP6dW*22W=c>#?Z)+cZ>uIl)i)$WuMsn?Oiv93O9yH;hOidYqTDel`S@ zTgW%fS)ZIadR8DXe&Z?B7^4%!|5n9^f803H$vl3)N*D#~W!Ynx&!mJH0!%;oi8e-f zC6{G7v(hR4Aitw^3ki?*Cn&vM0gPNA_zDgZA(D&qh|o5P1k9ii8mz0kSJbze4?*UEH!0#7-c=yd%XUP19uNdiHzhC2iFHz>86IV2??Wbq7_8M4jA2 z7_n>u7V+*5f5-1tjx>Kp#E?zUZC-Y^Md%P)XrJHVO5&l2>K6aTFrG z1Rz|V)%UqPeu?^h>9yFOR~6agh{dupKJ1Y|vW}Gj`X+cw-X}L>o@M4PpS7ZBt%1j@ z7*Z2}9)B+uvFt?ibY?qvO}hi|B|khJAbAqQF!V;;4tN{M(Djv<%#Py?qekYt?r=G9 zgWxH(B}J^_GXVp5FQ9LIrBs&r&DN&SBhrkY1q;P99%XzomCY}jm;y)l*b^Hxtn0z? ze4*hIJ!Gy?5+75SO#2{X~01OYmp=1S#fI%tF9%Iu(h~ zquQh|Sb#5bMWGkvg8h^|?3(k304G1mezlwiH=~W^ocR^+dX*u*(7d0=9iig<%=gFA zT6+p*)kItUjYU*(KNx)y83nDARvHxc=-13?*00PSRYgI~_WBanex*8r;DYGOaSi0R zkzCGy|Rl0L(6=JGA6%cLb~n>+UoI*M6T~Jz2Tz=@urcDu%R2i*@A*i1tLhKU zBgq#2p`8=McTQXTyX$i=kcdVp=?zwMw@;cPtt~Xva$k6cg-3_Hd>VQCi7ZVSGeabz z2WtsyM9|MD4me6FP{eC3YdRR+mPs1K$8==@KK;)o3|s|1;$QxNganui?)<#r;WzC) z@3J4IXDO2C_3pACpLW3;ZK{Tdnm25D%qDylEvk@sQY3*m@3fybjU6?GT$?rQ2$YCi zU8QK!`rQ&j@90pE9P*sQM)ziT^c>C>UbH9fg*BMmXRb>?0?a;7bN}TXE=%}!x2z>^ zA`uMTTaww{y|LIWdHQ1KIQD~8fv4chSg+rR#?axnCs)=k$mp38XJqwdCpUvH_{l<) z%b`HIr0)9J%+smO{p>G!=0;;gUGaW==rG$)<6Fcc{-byB_BBNXn$s2g1YtH+S`uwd$(~MS|XOR%rgw+6AY#O7hqV(~V%n zlwi*zx|%|<UJ_n7wxD z+O*@pw0gxKsmi~U2)KD185}lB4dwY1UDAtdamSo>*Sj}lHl!=*ea}M~^2hYg!C;c% z^`mGT+@mzqDE3`J^Ahi!5cBI<9ZO{!ml={@=|Hhgt;|Cmn^WAC_@=sPH|uuN^&cZ* zu&e3LNLCzw)ta&`HLPReZUq~rKV_;Vnzp0@g2RTxUWscDzECk`vC^+iO!rs(`i(5v z+QDP$zqbJDG6Z%laFafIrVQ zXE@Mlw|Y>q+Jywcw&8bme23i`;v3NM6l-twLhR0unB9hV{6Z-7@alxJ>IUESHaM39 zyWN#U_oo*!TJtzK&?BF2O)m`Sw5_^UYB~e3Z1vNQqRC&^-*dWSD+qrS3>4`$S8Sa= z3Hg1#w6OBA7uQwMQPtipChjkUZwyO#{yuDheq)&wJOLQO5k8W8bbd)o+TXVGmbJG( zWxHZOWJ`>m!qb^pRS=LUpdqV&H+MKxVrM@4lem>zIq zH;%+!2@&ba;-EgDMB!_Ne=;NR40}7Cn$RHLu*!~wFm*0_H)pAn!%0kKV2MtjM;D#_ zgL4GN5blcL?P%mp!C0ww;toWO-~B!AlHegDiu-mzNZoczD45ib?`~}_W_iZDKu0U& z;TwwPvxBUes4|Ey-S2 zh}aTU(p99WQB3sqMIRRL~X_jGh0SZ?Aj++xlnQii=%IsbnjrR z$26D}8^EtEOTHa6>JqrFmpx4qoHJ16MF3WMu&wj>N(@ zyD%8M=!0}po^t%FZw4jS6@so#!c5(&n7`Z!@juxd>btk@+&b~f=UMt=g4I?)Ik(x< zF95^4m>cbq=_zFiYCaPk>?y7CNik$l^Kyut)SntocP@fDC%_sfe{zWQX;5@b~e(`tGoIm!{QB(Ruk{5eL~>(;Ehj1OGc zz-sS~tG1g==+(+;J0H()VWnsGO`XTh%|h32nG9D)W@OZ#PcYXXU*;ympw+}NoW=&# zVd}3xxG)(Z(rocbKek}fy(7(43wfC6CgyBq(N0wQ%Sw0S zHc;33dm09jI>I>xzXb>%TsAME_`72&puTwymTdU5<%ZUTx86GeDu{Te6?k^^j=+cV+gv9WXLfrTnA|UI z<T&g>4z7Qxzw;4NRTG)clgeFd*V|q^h z7T{*>E&{Ph4m8fZ2d?%`odQXRy_zU9WF$qf!Pv8W-5WnuyL&<_qZ7Y}WQO;Ke!~70 zwV>rA^S(MxU58FOZa@)>i%0*}ve3tNEPDJoy$sW+#Lhahps69!wIhv6!x>_xpj~Im z5XjCB2CarJ!2{#uC(*|RFB&UvLH4hnd@XCQ^^yx!*FR)MrP!-K$4&>@shd4KD?TK9 zY}&w1d0h+!SLNCGwiJrUlUi8StjHDNo-)44wiMPOWqX;5P_iIae zlfxDH8q1FdiY`4np^X}ee`B9z*JmX9zGc^4ZzdH)Lqlnb^cSO|8Ye*r2DO->H;pq6t%1N6ZdV@@)gn=dJdJYwIS&33;kC2&(K6ykOnShADweePWFF4F8Qyq&rTm8sO>@4 zA#$VkJHaO=$f;E@Wfm_a0H`eA+xG`9dy1G*i5#!xySjS&?CRXhk*#}c1`x!98r|}g zmq+yd(R_C0;Yg3Qy$QH22HXlZqP-wbsQ3q?a#3+G$84TPxu`Ta1Pk7d@aFrNkT}W> zYN18-B#DTFdws@l#2cfeyxv3a_=ZAy&q1XLNuzr)#Vrs^%Rq_GT0W`JJK=vc!8ZU@ zXARZ1KK2GBM53WUKSTG13+whd1g^)no#+IN9BsBUS=`+Nl3q{jT?*dfG!dLZIa-Jp zmu$3_Z8E1JAwz;Kt)KGuBF#MCh~f&Z_xydWnKp^Z;ykXsNII7LK)v9|YN>ouzmWgj zsrEmg$MHjb4N5Ev)QXNiwjmTRUc%K<>oPXpbK;{IUl1`^PxQ0v)TOWE*4BXpL^4GH z4TI+3Vp~4<8^+@rsoBQC;PPwOIG#{X3}bzApLgCRReIhl^Hi1u7_0iasK0=Tb=23q zE^uY76T9T;2{dBQ+$DYeD?rlyltXex*TI@| z!B{nfLN*o#{Y)9v+(*llPzYCa%clMMQaJ@GoBzXvXC2a^x9SJDZtk0ii_7Vk;bf-( z8Ok0$_iM?Mz+>!S*GgV-ea4WSU{SEtTu%E6h{hE>x@4@F%3u$={9UHA67zsRk5Eb2 z{_SkrjTU!({9lxU(m5@OITj;f%o{$A7l~R8j{{RUv`Y2d;<{c3r} z`QTN6_n;S&QV9cx)yk{je)jUU2799t16)N-{Xjmh_8M&;>r#Sd)_-tLTr)khk5?4X z5LT{Wg9D<7vz+CVc)owWRVQHq!)W+yfvxpSS!CqG^CL>4l7swH(ZDmC_cAq#O1J9v zr&biDJOd+p3f|ztJ{q|3V5tO??)|UtxW_pI?k8?}(-|^adYRzpAcjXKG5@EYHvo6**7Jzrr$i}^1S zimnmt?~0w$*6}RAMM&UL<-ic09S=v&q65KGh~phIJa_dByY z=W{NntrgFN@9Puf0!8Q#@!vC;;uuV9Gj3Renw`S%qN`}%LMFKC(xVG(u>%InpiA8)U?II5dj;;TL}C`<8RbYHT}<6t0?gLOBz z$xaKV_TXt-(N)rQx8kZ|`&V(uN3!Hfb8GBwx;T!SG4o5PU_$pknIuR}HmfA}CC~74 z1?^8(Ocrk%l>lSs zU7h@B1-41CXm`GHarSn7jfK(u5}r|u_3z(egkSysL)KM_Ro}>78r5Da7p)HBWpOrR z)=bwGY=}6U15P$Ih5sFgIm(Ceu&7W29`VmMH&hs0)uWfj@3c4Z+|tn8h>G_*fc|F@ z^JkBBryU>^yf0e*g;MZ~TEvetv2uH?l?pCPF*yt~`K7}<1f>y-&WJ~PPX&t$Y$Fwj zI^4fak@Z?X-#z^31>QQ|XE&<<-6-(xQlE_34^hlLh+g~4SDeFI_Fqj?!n)hkZflI0 zpC-DlXmhHF!rdoGKD&Vb^32!l3XP3?NjZ*B^Q|R~^5A;5&&XY=&s=6?%s=^)yRC2U ziMHMG`CaeRN8q3*F?1IcRHKW9p0d!c9#Smf5S;N*5Qm759OBGJl9rGO09QZ%M`s1( z=NxiJRI+03+4Aqr$iHFq|EAgfZ%g>E_&xK_XghEI3wkR!y*2-h2G!+H*tt3X?bD6S z_m|bZjWs4RAQ6EfOmQovg&@ZK*eG*LqpbC_OYY7_Pakg&r|b;~YIB4EYT3xd^*hU% z9o#!Rxi-U|?d7KOdK4JW$Ino`t5kI%h!z=ot~D^w!wqmQ!7W_@U1=oFeEkl4uI-+6?q9ZZGSGi*p{$o`HGerj^kRl5a^i@u z_x=CEZ!2IE|D{I*psCEEn>A=KIrYWgOapuf)iY1&y9@4WL};r z^foop+Sr8fWH*vtq8B_;Tv4lDIlfG<=@E$xy(G3%9{YVZ^eaE6Ru#RaMSmkD*fhCf z%Cnh3y_99-N&nJ)$;N#_iu#@DW}@ndby@H7z&AQn7uA9>JwGZaJ%01J@my#Wt0JTR zgR(cogvJpiv8q<)eSTIk=<_0g<@dFV4Nkxm$7*wr_~!XL+cu^kwN_1w2{tq=^*|+B9f{^vX%hv4Z%>S#^Iz_3 z0i2RI6xI8E*t%@H(KK+I=woosMzaR<@xpV^94i`^bf6MCn(kLM@#k{CTVPHcnMNhH z7~~0VMPCN8(MmjQi+HVTKOcBe2;9rxP>jz$5a)*n{Uxrq5)ay{`l;ZfR$rm9{Bx%H zoM({AzzcOCZP80#bVmwEyD220(srE$C7SkuCLf41!evab_86U*Lu;51nLsQh%Ze@fH1aFjQKIru0ssHdYr zJ!bKG+6&s$Y{(4`2$R1K*q)^=hkWO21ayw~7qziDCBGm6^o%(KxgTm z-ko)>p30YZ;?3}oHci4(M-`RH^xRp)cHXIBg|M3UH3rsupMnIosRf4E*%{=0PqrFy ztzt$}?$EDCP4DsJ{i0HigmdvBbniocc5?#J@+Esl#qUKo%_qS^EUDRjjICg2UH|z&ebw7^Vzku=;ZSpHS&rqHA7LOD}MKa8IUXvgahd zpZ(8ADyZa9@O5e|j8}~NJZf#)HMOE>WBn?qkD)YGSf@O+Y;d>{pTZ7UO>= zOrmzy+GQGq57%m!d1dERsy>YGm9FYaI;0T)M&pLXvETL!M4&~Y%W`Lu-{4XZdgZfI z80-Y71t9r5#R^t)uAh6pIS}O=!_#J%WPm@Xv9cea5xOo^?Va#Imf zJbA~yjVv9Zj>)@xko08y+)WDyK9nfM3Ff0)Nxm$kdb{6IDsMOi{bf=klni&Bq%T{T zTL5&jclJbI6Mk{0UCD=%HN4YlkbD-{9u#894R;!rVPKMv7Y)CyXM$fh@#k%?NV>C~ zPrJMhytvM|tI#xt91sNHiIKV1G4}30l#rG`I|+6HhtUC;)h#A!S{XU!qFQzUPF@L; z4NO}socv--xzvlb)`N8IszOB&f9hp_K`aH4*F8%aDsh`=B&YpcGTk^Uhg#Xh+O? z1k2=hh46i8@9%~W9;P9NA6z~U9~H+Vw7l4WF78Fr4yJ+`5^X#Ui1QP9|Bp{lFD*-7 z)PL!koE1HE&0cw$Z%A9M`QRvG%XWn?PP+*xBWx=oOaO%)6U%Z zCky**qt(hMi#Z~!4>ruZr}E(5AwJU&tHO-DSWxHt$j6~Me)ke5?ZH~6E`i{5TjAW6 z;SU78f2PTVHm!+$TC1GD-XZtq00gY2dz4M(=%ict;-3J+Tbm2&i7DsU``ZI#?;q&u zSCD}ZUBT=ka28=O-_K)*c*`aJbqU$t1|8~)nWq2XVkKK8P)=}AE&_L1Zx0Eg%Td43 zbw2iS$i4@nHvy`=Lk=%tYnQ`&m#P;;I<4@3Y^}DYz zbGK)0bQ0thI5~%Xs`SOa93cDvu9n3emw=uEb(33Em2f0^_5SH8SLx0{^~TwsDw>7r zRT0C}n3gaca(r#p@j}vsvPK2IF`N~!^~A~5gj$}WzLqICp`4^{n&wQKYGqzF2Qrr+YKAR5oK+yM~y<-njaE{wU^&kuCulb$eL#F!ztfy`{3KIwC-uz8OBs zSed0NkHxLOui-@5l%WtHY!0)@kNe_DnBx*)D6l=~V{ZGCP6A2^=&6gU?RJe&=W+jI zlB~5p85e0(`*NC=AL-T8q1&FXASV>2>3&!|AYyho^xErE$tjOPv)xd;PD86lmIYoa zGGvoR$mzn3x7}GPy14ZvEv`gQFqZ|)=;<2sI_Z}fY$xnp+O!f}*K_3c^po)VO4T1O z@6IH)m9(2xD8kkzcPOovPDohInJrbG7szB*Gq->J@afrIo;I81_}-13*J>*vpzM38 z(WT$1yBCL((VF1HZT{D*22aP{)+qKk{9!5~f>T^_Ya6J6=ZIe$erSD)5=A$-By>8{DXBUXhIGPUNGFJXFsDdKLw5K z^H|_JlzkA$%%;RrzwiCAjUFuR{mSad)gT_oA7YZg)ht>u?e4r(UDX(0ZwHl{%!DFOZ83z>Tq@dr^V6EmhnzN*=0d`aysT!`wA z)sC8rGKmCK0KaV$%wXXv&WVghFVeQ+-r3sy1Wj1LPVBpvkcs32`zKAWh&q~T5;07b z_lCF|nM!Lr;w;E-)3SRBWPa^@WWtW<0Cit@tV0XxCY&rE-p23QNG}Nfh;UzEP*?Dx zairwk;;C-l6e;>PTU-7K{c+59H59=r7O(7)uL+B%yYwwBCA-OB!5gx_ws8pkkp&5W z%^lXY6KYhBy7@?7P}nP0g8|aznk* zm&5U6o#%VgdgrH2MbGw2{0uHUk6RNiU6v1)E|erb2f+1$bchP>~H(hpSj^P7ScJsMAaG6RLiz8@zxv{flZ_4z&`SI?yT3cnIrx*sfOFJ@3>4DIM+u^<&Gxt#x*hVdk+ji zeVHJckAe{NU87hR)%!fIp=dhwhA5ezb2}fyflGIwJQH$H7>##A;{(sno}ugCOb-Hu zwgpy+sj-gl{Cn?~8ss3oXbr@Lp&I$3vglg49UA} z!X$3srl@akJjE6l$c8)$@Ut&2xV_p=XosdBALm_eA!gC}h zT7EG&uHPrXKU~Fw0FPG274nk=fw5Pgyrbf|r+Wk^|FJY#)4QmfydWR3pu z&s1hrJ&X!;%0@zaDM8tT1lF>8eVaX{0O5=A>#ICRG*2=V?E&Kq_4$RQRaYN)C=S{N zz5%ONr%Vg=Y?|DOrrj^<{uPC03swTf!d6eLzjdNC&|H{ID{@5CmdRKmBZAMKb6dY? zOBz*fqqU}5&Qs*=N-cBeOyBvaQ~pS{-jVyfU%%ymH^>8?D(noF_IX0cKIJH z5V8ttE0;a(wXNhmX)O-~qVOx^?$e|1Xi<+V-U=;J6uEqKY4Dx9>MBcUyIVt7=Wxur zmBTA}bGJ8lOjW#Ay;xEfn#}hf56o$ND5O1-!k)Y8esK>#VL5zn3%*7t--#>Ft_J^t zm(XC-dny!Wh0+J%Nm^-}4NBHG#wM`fmcGyyD(W5)!Xp6rm59*LPEIX-!$>LWWT%kM zQrhrvko6rg))>`N)?4`k&J%w}d7Ld>kt3i7UZx1=*jrd51AO> zE{RnEr$q3?-_4o-BNd_Tkl3)5-L`m{D^csuF=1Ay=uZ;k>ESbZe5UChh z*3Z#f)ntBTlBcW9C?)J9%bE)%Hhcu$i_RR)27HWFiEIyetV~G9XvPeu>d5(;0^2rs z=R2?i=^IEVQcmirDah)hct8U7iBh;_M9H*mxC^{2Bo<9HuU3trB*ip=OImCSyUMnG zc0siYO;c}g@{;E)l43MSHr<&+eY=w1FZ3wD)P7Rhrxv_S;^x9$iG6#UmyE7X)Q$!P z{@hcr`ZR2$>mbjU0Gz;mRzLL@(0H(85XPf&%Ie^BMi_cx;SN!YQM*ZJv|7iTyUl}2(LXvC%=c0MdS3^{ly zh+xRPuX3`b{u)9teUW%GdCN3^FhqHg*yVn&a4uD~CU1|`d3;Cuf;#{!q@;KqK;geQ z*w8uR687g$vC|U;IuM7o1y2PL9aVY2I;Y8NUv*n;wtz!l z4e|dN4p&n-2A-kLL@)ke92Pd!pKl6)VUz0D|E@)zh4~fNl=OP zg!tg+BF}{fI4xz>-wqFUdR|&jryK0CsN41lLw$_XBRRUu;jXT9%C&qBjW>=$9bewR)DO+3|3n?aFe+QW?9lS1~Z0`OM9AiTk=Iw&db{n?Y~P1Y=<* z)W1}>s(a2<3ER)aLz|zh(Sb!cop38c6SJhJn+91o@Td;d7z$&5Fe6*Q5w$SEQIi`1 z^eU%h%GzL%?`a^bh$S6Vvs(5y56U;zCZ4r|<*ge^eok6>I?|gvX;6foD3Mnn_krUU z+h8$`{=RUbbRM}*+YAN)1uO67$SMz4hYW0kmi6~QuTo-o`R^B58AYj3tt?2qI$jTZ z0l}e`)$^7yZQV>sW49_1X7r8aLnjPQ<>}bY}lG`u2YEd>dzL@O8 z#p}hIUgwih_4?7FL&@-{nd%{zQL+UW9|va)Bxo;LWO!tPzO4J{cD4;#gDQaDV>&;j z?k<(>Y24yx4;LqO&!fznL0l_;^79_{jT4?(trr?-e764Bw%1HFZ1h!VVEs-->OXF7 z;Z)%rPhkg}-mrkH zMBcE^6G}jVPUKjxnCL{RT6WeM;S?NkghE~KA|HW?UNCkyX#4$;x($+aqQNqylm%Bn zUu!ER{|25s0_RutQ2z9pT5NA8E{%t%ZFb#Qrm6cOys4c%8aw)?gv3u?w#mi-ok+v^ z=Q}yz5qOi2z$e19)*pJ&ib#Ee&{u=%u0Y#oxk<)1X!)ZF6KN#Dc`!Ii-D&2jj*nU5 zGJOa-FQZ0{U>JQ{*r6N55)lU-P%?E@UaqKriaSX!cOo2hc7fU8MzBQs$o10L~FP*KZ;+HewS3$uHl0 z+6QvuY&7PkEYn=Fo^(Sm5*tU?`P8nfrA<>QQeA4YwHLnYEv$<6;-?sU<#s+_muDc_ zJy@};0DO(u0j!Y>;Vsm2J$d=N2Dc&(>6T!aL+)~?K4_Z(aC{A=n^{DV{_75Z>3^VB z8!Gzy4x97{FrU4?Fhd8Pj_?yeAK?%Y;zcxq+J=W51|%jNPV61=S9S(pNW||IYWsYM zF8Lki`A-M%pOp#W?}y)Z{Z=mV^!5^d8Eb8v>rs@mvkxu6T{r#5SrZkNdwNOM!mVK) zS$y=d78XxAJx@z>Pb(3KyA`^@;0N&wa)X4p`Cn=A@relVi@f0D1c5|AAnm4n*Z%?F x;y@IEmla+_3xs&U^jq$k0AxC2{-pHxQR!N(^`!9Qd%@qIu literal 0 HcmV?d00001 From 8c973f587a93532aebc818bc55824594e309931b Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 31 Oct 2025 17:52:12 +0530 Subject: [PATCH 067/119] v1beta2: drop types related to QueueVisibility (#7447) --- apis/kueue/v1beta1/clusterqueue_conversion.go | 4 + apis/kueue/v1beta1/zz_generated.conversion.go | 82 ++----------------- apis/kueue/v1beta2/clusterqueue_types.go | 31 ------- apis/kueue/v1beta2/zz_generated.deepcopy.go | 41 ---------- .../crd/kueue.x-k8s.io_clusterqueues.yaml | 37 --------- .../v1beta2/clusterqueuependingworkload.go | 47 ----------- .../clusterqueuependingworkloadsstatus.go | 56 ------------- .../kueue/v1beta2/clusterqueuestatus.go | 23 ++---- client-go/applyconfiguration/utils.go | 4 - .../bases/kueue.x-k8s.io_clusterqueues.yaml | 40 --------- pkg/cache/queue/manager.go | 60 +------------- .../core/clusterqueue_controller.go | 1 - .../core/clusterqueue_controller_test.go | 1 - .../en/docs/reference/kueue.v1beta2.md | 77 ----------------- .../core/clusterqueue_controller_test.go | 15 ++-- .../scheduler/podsready/scheduler_test.go | 5 +- .../scheduler/workload_controller_test.go | 2 +- 17 files changed, 30 insertions(+), 496 deletions(-) delete mode 100644 client-go/applyconfiguration/kueue/v1beta2/clusterqueuependingworkload.go delete mode 100644 client-go/applyconfiguration/kueue/v1beta2/clusterqueuependingworkloadsstatus.go diff --git a/apis/kueue/v1beta1/clusterqueue_conversion.go b/apis/kueue/v1beta1/clusterqueue_conversion.go index 0b30c586416..e316084b955 100644 --- a/apis/kueue/v1beta1/clusterqueue_conversion.go +++ b/apis/kueue/v1beta1/clusterqueue_conversion.go @@ -45,3 +45,7 @@ func Convert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec(in *v1beta2.Cl out.Cohort = CohortReference(in.CohortName) return autoConvert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec(in, out, s) } + +func Convert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(in *ClusterQueueStatus, out *v1beta2.ClusterQueueStatus, s conversionapi.Scope) error { + return autoConvert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(in, out, s) +} diff --git a/apis/kueue/v1beta1/zz_generated.conversion.go b/apis/kueue/v1beta1/zz_generated.conversion.go index 7fb80707c5c..061c8f635de 100644 --- a/apis/kueue/v1beta1/zz_generated.conversion.go +++ b/apis/kueue/v1beta1/zz_generated.conversion.go @@ -178,26 +178,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*ClusterQueuePendingWorkload)(nil), (*v1beta2.ClusterQueuePendingWorkload)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload(a.(*ClusterQueuePendingWorkload), b.(*v1beta2.ClusterQueuePendingWorkload), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueuePendingWorkload)(nil), (*ClusterQueuePendingWorkload)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload(a.(*v1beta2.ClusterQueuePendingWorkload), b.(*ClusterQueuePendingWorkload), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*ClusterQueuePendingWorkloadsStatus)(nil), (*v1beta2.ClusterQueuePendingWorkloadsStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus(a.(*ClusterQueuePendingWorkloadsStatus), b.(*v1beta2.ClusterQueuePendingWorkloadsStatus), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueuePendingWorkloadsStatus)(nil), (*ClusterQueuePendingWorkloadsStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus(a.(*v1beta2.ClusterQueuePendingWorkloadsStatus), b.(*ClusterQueuePendingWorkloadsStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*ClusterQueuePreemption)(nil), (*v1beta2.ClusterQueuePreemption)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_ClusterQueuePreemption_To_v1beta2_ClusterQueuePreemption(a.(*ClusterQueuePreemption), b.(*v1beta2.ClusterQueuePreemption), scope) }); err != nil { @@ -208,11 +188,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*ClusterQueueStatus)(nil), (*v1beta2.ClusterQueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(a.(*ClusterQueueStatus), b.(*v1beta2.ClusterQueueStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*v1beta2.ClusterQueueStatus)(nil), (*ClusterQueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_ClusterQueueStatus_To_v1beta1_ClusterQueueStatus(a.(*v1beta2.ClusterQueueStatus), b.(*ClusterQueueStatus), scope) }); err != nil { @@ -793,6 +768,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*ClusterQueueStatus)(nil), (*v1beta2.ClusterQueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(a.(*ClusterQueueStatus), b.(*v1beta2.ClusterQueueStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*LocalQueueStatus)(nil), (*v1beta2.LocalQueueStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_LocalQueueStatus_To_v1beta2_LocalQueueStatus(a.(*LocalQueueStatus), b.(*v1beta2.LocalQueueStatus), scope) }); err != nil { @@ -1173,50 +1153,6 @@ func Convert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(in *v1beta2.Cl return autoConvert_v1beta2_ClusterQueueList_To_v1beta1_ClusterQueueList(in, out, s) } -func autoConvert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload(in *ClusterQueuePendingWorkload, out *v1beta2.ClusterQueuePendingWorkload, s conversion.Scope) error { - out.Name = in.Name - out.Namespace = in.Namespace - return nil -} - -// Convert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload is an autogenerated conversion function. -func Convert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload(in *ClusterQueuePendingWorkload, out *v1beta2.ClusterQueuePendingWorkload, s conversion.Scope) error { - return autoConvert_v1beta1_ClusterQueuePendingWorkload_To_v1beta2_ClusterQueuePendingWorkload(in, out, s) -} - -func autoConvert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload(in *v1beta2.ClusterQueuePendingWorkload, out *ClusterQueuePendingWorkload, s conversion.Scope) error { - out.Name = in.Name - out.Namespace = in.Namespace - return nil -} - -// Convert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload is an autogenerated conversion function. -func Convert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload(in *v1beta2.ClusterQueuePendingWorkload, out *ClusterQueuePendingWorkload, s conversion.Scope) error { - return autoConvert_v1beta2_ClusterQueuePendingWorkload_To_v1beta1_ClusterQueuePendingWorkload(in, out, s) -} - -func autoConvert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus(in *ClusterQueuePendingWorkloadsStatus, out *v1beta2.ClusterQueuePendingWorkloadsStatus, s conversion.Scope) error { - out.Head = *(*[]v1beta2.ClusterQueuePendingWorkload)(unsafe.Pointer(&in.Head)) - out.LastChangeTime = in.LastChangeTime - return nil -} - -// Convert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus is an autogenerated conversion function. -func Convert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus(in *ClusterQueuePendingWorkloadsStatus, out *v1beta2.ClusterQueuePendingWorkloadsStatus, s conversion.Scope) error { - return autoConvert_v1beta1_ClusterQueuePendingWorkloadsStatus_To_v1beta2_ClusterQueuePendingWorkloadsStatus(in, out, s) -} - -func autoConvert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus(in *v1beta2.ClusterQueuePendingWorkloadsStatus, out *ClusterQueuePendingWorkloadsStatus, s conversion.Scope) error { - out.Head = *(*[]ClusterQueuePendingWorkload)(unsafe.Pointer(&in.Head)) - out.LastChangeTime = in.LastChangeTime - return nil -} - -// Convert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus is an autogenerated conversion function. -func Convert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus(in *v1beta2.ClusterQueuePendingWorkloadsStatus, out *ClusterQueuePendingWorkloadsStatus, s conversion.Scope) error { - return autoConvert_v1beta2_ClusterQueuePendingWorkloadsStatus_To_v1beta1_ClusterQueuePendingWorkloadsStatus(in, out, s) -} - func autoConvert_v1beta1_ClusterQueuePreemption_To_v1beta2_ClusterQueuePreemption(in *ClusterQueuePreemption, out *v1beta2.ClusterQueuePreemption, s conversion.Scope) error { out.ReclaimWithinCohort = v1beta2.PreemptionPolicy(in.ReclaimWithinCohort) out.BorrowWithinCohort = (*v1beta2.BorrowWithinCohort)(unsafe.Pointer(in.BorrowWithinCohort)) @@ -1278,16 +1214,11 @@ func autoConvert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(in *Cl out.PendingWorkloads = in.PendingWorkloads out.ReservingWorkloads = in.ReservingWorkloads out.AdmittedWorkloads = in.AdmittedWorkloads - out.PendingWorkloadsStatus = (*v1beta2.ClusterQueuePendingWorkloadsStatus)(unsafe.Pointer(in.PendingWorkloadsStatus)) + // WARNING: in.PendingWorkloadsStatus requires manual conversion: does not exist in peer-type out.FairSharing = (*v1beta2.FairSharingStatus)(unsafe.Pointer(in.FairSharing)) return nil } -// Convert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus is an autogenerated conversion function. -func Convert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(in *ClusterQueueStatus, out *v1beta2.ClusterQueueStatus, s conversion.Scope) error { - return autoConvert_v1beta1_ClusterQueueStatus_To_v1beta2_ClusterQueueStatus(in, out, s) -} - func autoConvert_v1beta2_ClusterQueueStatus_To_v1beta1_ClusterQueueStatus(in *v1beta2.ClusterQueueStatus, out *ClusterQueueStatus, s conversion.Scope) error { out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) out.FlavorsReservation = *(*[]FlavorUsage)(unsafe.Pointer(&in.FlavorsReservation)) @@ -1295,7 +1226,6 @@ func autoConvert_v1beta2_ClusterQueueStatus_To_v1beta1_ClusterQueueStatus(in *v1 out.PendingWorkloads = in.PendingWorkloads out.ReservingWorkloads = in.ReservingWorkloads out.AdmittedWorkloads = in.AdmittedWorkloads - out.PendingWorkloadsStatus = (*ClusterQueuePendingWorkloadsStatus)(unsafe.Pointer(in.PendingWorkloadsStatus)) out.FairSharing = (*FairSharingStatus)(unsafe.Pointer(in.FairSharing)) return nil } diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index 20649b41b57..40ba8d10b46 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -315,15 +315,6 @@ type ClusterQueueStatus struct { // +optional AdmittedWorkloads int32 `json:"admittedWorkloads"` - // pendingWorkloadsStatus contains the information exposed about the current - // status of the pending workloads in the cluster queue. - // Deprecated: This field is no longer effective since v0.14.0, which means Kueue no longer stores and updates information. - // You can migrate to VisibilityOnDemand - // (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) - // instead. - // +optional - PendingWorkloadsStatus *ClusterQueuePendingWorkloadsStatus `json:"pendingWorkloadsStatus"` - // fairSharing contains the current state for this ClusterQueue // when participating in Fair Sharing. // This is recorded only when Fair Sharing is enabled in the Kueue configuration. @@ -331,28 +322,6 @@ type ClusterQueueStatus struct { FairSharing *FairSharingStatus `json:"fairSharing,omitempty"` } -type ClusterQueuePendingWorkloadsStatus struct { - // clusterQueuePendingWorkload contains the list of top pending workloads. - // +listType=atomic - // +optional - Head []ClusterQueuePendingWorkload `json:"clusterQueuePendingWorkload"` //nolint:kubeapilinter // field is being removed - - // lastChangeTime indicates the time of the last change of the structure. - LastChangeTime metav1.Time `json:"lastChangeTime"` -} - -// ClusterQueuePendingWorkload contains the information identifying a pending workload -// in the cluster queue. -type ClusterQueuePendingWorkload struct { - // name indicates the name of the pending workload. - // +kubebuilder:validation:MaxLength=256 - Name string `json:"name"` - - // namespace indicates the name of the pending workload. - // +kubebuilder:validation:MaxLength=63 - Namespace string `json:"namespace"` -} - type FlavorUsage struct { // name of the flavor. Name ResourceFlavorReference `json:"name"` diff --git a/apis/kueue/v1beta2/zz_generated.deepcopy.go b/apis/kueue/v1beta2/zz_generated.deepcopy.go index 281deaf91f6..94e774cc731 100644 --- a/apis/kueue/v1beta2/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta2/zz_generated.deepcopy.go @@ -351,42 +351,6 @@ func (in *ClusterQueueList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterQueuePendingWorkload) DeepCopyInto(out *ClusterQueuePendingWorkload) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterQueuePendingWorkload. -func (in *ClusterQueuePendingWorkload) DeepCopy() *ClusterQueuePendingWorkload { - if in == nil { - return nil - } - out := new(ClusterQueuePendingWorkload) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterQueuePendingWorkloadsStatus) DeepCopyInto(out *ClusterQueuePendingWorkloadsStatus) { - *out = *in - if in.Head != nil { - in, out := &in.Head, &out.Head - *out = make([]ClusterQueuePendingWorkload, len(*in)) - copy(*out, *in) - } - in.LastChangeTime.DeepCopyInto(&out.LastChangeTime) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterQueuePendingWorkloadsStatus. -func (in *ClusterQueuePendingWorkloadsStatus) DeepCopy() *ClusterQueuePendingWorkloadsStatus { - if in == nil { - return nil - } - out := new(ClusterQueuePendingWorkloadsStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterQueuePreemption) DeepCopyInto(out *ClusterQueuePreemption) { *out = *in @@ -493,11 +457,6 @@ func (in *ClusterQueueStatus) DeepCopyInto(out *ClusterQueueStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.PendingWorkloadsStatus != nil { - in, out := &in.PendingWorkloadsStatus, &out.PendingWorkloadsStatus - *out = new(ClusterQueuePendingWorkloadsStatus) - (*in).DeepCopyInto(*out) - } if in.FairSharing != nil { in, out := &in.FairSharing, &out.FairSharing *out = new(FairSharingStatus) diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml index 5aad8e9f42b..dd77c7d8d1c 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml @@ -1507,43 +1507,6 @@ spec: admitted to this clusterQueue. format: int32 type: integer - pendingWorkloadsStatus: - description: |- - pendingWorkloadsStatus contains the information exposed about the current - status of the pending workloads in the cluster queue. - Deprecated: This field is no longer effective since v0.14.0, which means Kueue no longer stores and updates information. - You can migrate to VisibilityOnDemand - (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) - instead. - properties: - clusterQueuePendingWorkload: - description: clusterQueuePendingWorkload contains the list of top pending workloads. - items: - description: |- - ClusterQueuePendingWorkload contains the information identifying a pending workload - in the cluster queue. - properties: - name: - description: name indicates the name of the pending workload. - maxLength: 256 - type: string - namespace: - description: namespace indicates the name of the pending workload. - maxLength: 63 - type: string - required: - - name - - namespace - type: object - type: array - x-kubernetes-list-type: atomic - lastChangeTime: - description: lastChangeTime indicates the time of the last change of the structure. - format: date-time - type: string - required: - - lastChangeTime - type: object reservingWorkloads: description: |- reservingWorkloads is the number of workloads currently reserving quota in this diff --git a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuependingworkload.go b/client-go/applyconfiguration/kueue/v1beta2/clusterqueuependingworkload.go deleted file mode 100644 index 4de34db0d28..00000000000 --- a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuependingworkload.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1beta2 - -// ClusterQueuePendingWorkloadApplyConfiguration represents a declarative configuration of the ClusterQueuePendingWorkload type for use -// with apply. -type ClusterQueuePendingWorkloadApplyConfiguration struct { - Name *string `json:"name,omitempty"` - Namespace *string `json:"namespace,omitempty"` -} - -// ClusterQueuePendingWorkloadApplyConfiguration constructs a declarative configuration of the ClusterQueuePendingWorkload type for use with -// apply. -func ClusterQueuePendingWorkload() *ClusterQueuePendingWorkloadApplyConfiguration { - return &ClusterQueuePendingWorkloadApplyConfiguration{} -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *ClusterQueuePendingWorkloadApplyConfiguration) WithName(value string) *ClusterQueuePendingWorkloadApplyConfiguration { - b.Name = &value - return b -} - -// WithNamespace sets the Namespace field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Namespace field is set to the value of the last call. -func (b *ClusterQueuePendingWorkloadApplyConfiguration) WithNamespace(value string) *ClusterQueuePendingWorkloadApplyConfiguration { - b.Namespace = &value - return b -} diff --git a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuependingworkloadsstatus.go b/client-go/applyconfiguration/kueue/v1beta2/clusterqueuependingworkloadsstatus.go deleted file mode 100644 index 24bfa8c3077..00000000000 --- a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuependingworkloadsstatus.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1beta2 - -import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// ClusterQueuePendingWorkloadsStatusApplyConfiguration represents a declarative configuration of the ClusterQueuePendingWorkloadsStatus type for use -// with apply. -type ClusterQueuePendingWorkloadsStatusApplyConfiguration struct { - Head []ClusterQueuePendingWorkloadApplyConfiguration `json:"clusterQueuePendingWorkload,omitempty"` - LastChangeTime *v1.Time `json:"lastChangeTime,omitempty"` -} - -// ClusterQueuePendingWorkloadsStatusApplyConfiguration constructs a declarative configuration of the ClusterQueuePendingWorkloadsStatus type for use with -// apply. -func ClusterQueuePendingWorkloadsStatus() *ClusterQueuePendingWorkloadsStatusApplyConfiguration { - return &ClusterQueuePendingWorkloadsStatusApplyConfiguration{} -} - -// WithHead adds the given value to the Head field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Head field. -func (b *ClusterQueuePendingWorkloadsStatusApplyConfiguration) WithHead(values ...*ClusterQueuePendingWorkloadApplyConfiguration) *ClusterQueuePendingWorkloadsStatusApplyConfiguration { - for i := range values { - if values[i] == nil { - panic("nil value passed to WithHead") - } - b.Head = append(b.Head, *values[i]) - } - return b -} - -// WithLastChangeTime sets the LastChangeTime field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the LastChangeTime field is set to the value of the last call. -func (b *ClusterQueuePendingWorkloadsStatusApplyConfiguration) WithLastChangeTime(value v1.Time) *ClusterQueuePendingWorkloadsStatusApplyConfiguration { - b.LastChangeTime = &value - return b -} diff --git a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuestatus.go b/client-go/applyconfiguration/kueue/v1beta2/clusterqueuestatus.go index aa8bc5ff027..5e3d8efdf52 100644 --- a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuestatus.go +++ b/client-go/applyconfiguration/kueue/v1beta2/clusterqueuestatus.go @@ -24,14 +24,13 @@ import ( // ClusterQueueStatusApplyConfiguration represents a declarative configuration of the ClusterQueueStatus type for use // with apply. type ClusterQueueStatusApplyConfiguration struct { - Conditions []v1.ConditionApplyConfiguration `json:"conditions,omitempty"` - FlavorsReservation []FlavorUsageApplyConfiguration `json:"flavorsReservation,omitempty"` - FlavorsUsage []FlavorUsageApplyConfiguration `json:"flavorsUsage,omitempty"` - PendingWorkloads *int32 `json:"pendingWorkloads,omitempty"` - ReservingWorkloads *int32 `json:"reservingWorkloads,omitempty"` - AdmittedWorkloads *int32 `json:"admittedWorkloads,omitempty"` - PendingWorkloadsStatus *ClusterQueuePendingWorkloadsStatusApplyConfiguration `json:"pendingWorkloadsStatus,omitempty"` - FairSharing *FairSharingStatusApplyConfiguration `json:"fairSharing,omitempty"` + Conditions []v1.ConditionApplyConfiguration `json:"conditions,omitempty"` + FlavorsReservation []FlavorUsageApplyConfiguration `json:"flavorsReservation,omitempty"` + FlavorsUsage []FlavorUsageApplyConfiguration `json:"flavorsUsage,omitempty"` + PendingWorkloads *int32 `json:"pendingWorkloads,omitempty"` + ReservingWorkloads *int32 `json:"reservingWorkloads,omitempty"` + AdmittedWorkloads *int32 `json:"admittedWorkloads,omitempty"` + FairSharing *FairSharingStatusApplyConfiguration `json:"fairSharing,omitempty"` } // ClusterQueueStatusApplyConfiguration constructs a declarative configuration of the ClusterQueueStatus type for use with @@ -103,14 +102,6 @@ func (b *ClusterQueueStatusApplyConfiguration) WithAdmittedWorkloads(value int32 return b } -// WithPendingWorkloadsStatus sets the PendingWorkloadsStatus field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the PendingWorkloadsStatus field is set to the value of the last call. -func (b *ClusterQueueStatusApplyConfiguration) WithPendingWorkloadsStatus(value *ClusterQueuePendingWorkloadsStatusApplyConfiguration) *ClusterQueueStatusApplyConfiguration { - b.PendingWorkloadsStatus = value - return b -} - // WithFairSharing sets the FairSharing field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the FairSharing field is set to the value of the last call. diff --git a/client-go/applyconfiguration/utils.go b/client-go/applyconfiguration/utils.go index 673a24f15a7..f6ba18558fd 100644 --- a/client-go/applyconfiguration/utils.go +++ b/client-go/applyconfiguration/utils.go @@ -197,10 +197,6 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &kueuev1beta2.BorrowWithinCohortApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("ClusterQueue"): return &kueuev1beta2.ClusterQueueApplyConfiguration{} - case v1beta2.SchemeGroupVersion.WithKind("ClusterQueuePendingWorkload"): - return &kueuev1beta2.ClusterQueuePendingWorkloadApplyConfiguration{} - case v1beta2.SchemeGroupVersion.WithKind("ClusterQueuePendingWorkloadsStatus"): - return &kueuev1beta2.ClusterQueuePendingWorkloadsStatusApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("ClusterQueuePreemption"): return &kueuev1beta2.ClusterQueuePreemptionApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("ClusterQueueSpec"): diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml index f81e10975ac..fed13a7bec1 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml @@ -1506,46 +1506,6 @@ spec: admitted to this clusterQueue. format: int32 type: integer - pendingWorkloadsStatus: - description: |- - pendingWorkloadsStatus contains the information exposed about the current - status of the pending workloads in the cluster queue. - Deprecated: This field is no longer effective since v0.14.0, which means Kueue no longer stores and updates information. - You can migrate to VisibilityOnDemand - (https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) - instead. - properties: - clusterQueuePendingWorkload: - description: clusterQueuePendingWorkload contains the list of - top pending workloads. - items: - description: |- - ClusterQueuePendingWorkload contains the information identifying a pending workload - in the cluster queue. - properties: - name: - description: name indicates the name of the pending workload. - maxLength: 256 - type: string - namespace: - description: namespace indicates the name of the pending - workload. - maxLength: 63 - type: string - required: - - name - - namespace - type: object - type: array - x-kubernetes-list-type: atomic - lastChangeTime: - description: lastChangeTime indicates the time of the last change - of the structure. - format: date-time - type: string - required: - - lastChangeTime - type: object reservingWorkloads: description: |- reservingWorkloads is the number of workloads currently reserving quota in this diff --git a/pkg/cache/queue/manager.go b/pkg/cache/queue/manager.go index d853d7455a0..42adebef794 100644 --- a/pkg/cache/queue/manager.go +++ b/pkg/cache/queue/manager.go @@ -23,7 +23,6 @@ import ( "sync" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" @@ -109,9 +108,6 @@ type Manager struct { statusChecker StatusChecker localQueues map[queue.LocalQueueReference]*LocalQueue - snapshotsMutex sync.RWMutex - snapshots map[kueue.ClusterQueueReference][]kueue.ClusterQueuePendingWorkload - workloadOrdering workload.Ordering workloadInfoOptions []workload.InfoOption @@ -131,12 +127,10 @@ type Manager struct { func NewManager(client client.Client, checker StatusChecker, options ...Option) *Manager { m := &Manager{ - clock: realClock, - client: client, - statusChecker: checker, - localQueues: make(map[queue.LocalQueueReference]*LocalQueue), - snapshotsMutex: sync.RWMutex{}, - snapshots: make(map[kueue.ClusterQueueReference][]kueue.ClusterQueuePendingWorkload, 0), + clock: realClock, + client: client, + statusChecker: checker, + localQueues: make(map[queue.LocalQueueReference]*LocalQueue), workloadOrdering: workload.Ordering{ PodsReadyRequeuingTimestamp: config.EvictionTimestamp, }, @@ -759,52 +753,6 @@ func (m *Manager) ClusterQueueFromLocalQueue(localQueueKey queue.LocalQueueRefer return "", false } -// UpdateSnapshot computes the new snapshot and replaces if it differs from the -// previous version. It returns true if the snapshot was actually updated. -func (m *Manager) UpdateSnapshot(cqName kueue.ClusterQueueReference, maxCount int32) bool { - cq := m.getClusterQueue(cqName) - if cq == nil { - return false - } - newSnapshot := make([]kueue.ClusterQueuePendingWorkload, 0) - for index, info := range cq.Snapshot() { - if int32(index) >= maxCount { - break - } - if info == nil { - continue - } - newSnapshot = append(newSnapshot, kueue.ClusterQueuePendingWorkload{ - Name: info.Obj.Name, - Namespace: info.Obj.Namespace, - }) - } - prevSnapshot := m.GetSnapshot(cqName) - if !equality.Semantic.DeepEqual(prevSnapshot, newSnapshot) { - m.setSnapshot(cqName, newSnapshot) - return true - } - return false -} - -func (m *Manager) setSnapshot(cqName kueue.ClusterQueueReference, workloads []kueue.ClusterQueuePendingWorkload) { - m.snapshotsMutex.Lock() - defer m.snapshotsMutex.Unlock() - m.snapshots[cqName] = workloads -} - -func (m *Manager) GetSnapshot(cqName kueue.ClusterQueueReference) []kueue.ClusterQueuePendingWorkload { - m.snapshotsMutex.RLock() - defer m.snapshotsMutex.RUnlock() - return m.snapshots[cqName] -} - -func (m *Manager) DeleteSnapshot(cq *kueue.ClusterQueue) { - m.snapshotsMutex.Lock() - defer m.snapshotsMutex.Unlock() - delete(m.snapshots, kueue.ClusterQueueReference(cq.Name)) -} - // DeleteSecondPassWithoutLock deletes the pending workload from the second // pass queue. func (m *Manager) DeleteSecondPassWithoutLock(w *kueue.Workload) { diff --git a/pkg/controller/core/clusterqueue_controller.go b/pkg/controller/core/clusterqueue_controller.go index 8774c220733..7170e0c93ea 100644 --- a/pkg/controller/core/clusterqueue_controller.go +++ b/pkg/controller/core/clusterqueue_controller.go @@ -312,7 +312,6 @@ func (r *ClusterQueueReconciler) Delete(e event.TypedDeleteEvent[*kueue.ClusterQ r.log.V(2).Info("ClusterQueue delete event", "clusterQueue", klog.KObj(e.Object)) r.cache.DeleteClusterQueue(e.Object) r.qManager.DeleteClusterQueue(e.Object) - r.qManager.DeleteSnapshot(e.Object) metrics.ClearClusterQueueResourceMetrics(e.Object.Name) r.log.V(2).Info("Cleared resource metrics for deleted ClusterQueue.", "clusterQueue", klog.KObj(e.Object)) diff --git a/pkg/controller/core/clusterqueue_controller_test.go b/pkg/controller/core/clusterqueue_controller_test.go index 4ceb06213d6..60f56203623 100644 --- a/pkg/controller/core/clusterqueue_controller_test.go +++ b/pkg/controller/core/clusterqueue_controller_test.go @@ -236,7 +236,6 @@ func TestUpdateCqStatusIfChanged(t *testing.T) { } configCmpOpts := cmp.Options{ cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime"), - cmpopts.IgnoreFields(kueue.ClusterQueuePendingWorkloadsStatus{}, "LastChangeTime"), cmpopts.EquateEmpty(), } if diff := cmp.Diff(tc.wantCqStatus, cq.Status, configCmpOpts...); len(diff) != 0 { diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md index 8c6c32457d0..e2cd40bf8bf 100644 --- a/site/content/en/docs/reference/kueue.v1beta2.md +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -806,71 +806,6 @@ policy can be preempted by the borrowing workload.

    -## `ClusterQueuePendingWorkload` {#kueue-x-k8s-io-v1beta2-ClusterQueuePendingWorkload} - - -**Appears in:** - -- [ClusterQueuePendingWorkloadsStatus](#kueue-x-k8s-io-v1beta2-ClusterQueuePendingWorkloadsStatus) - - -

    ClusterQueuePendingWorkload contains the information identifying a pending workload -in the cluster queue.

    - - - - - - - - - - - - - - -
    FieldDescription
    name [Required]
    -string -
    -

    name indicates the name of the pending workload.

    -
    namespace [Required]
    -string -
    -

    namespace indicates the name of the pending workload.

    -
    - -## `ClusterQueuePendingWorkloadsStatus` {#kueue-x-k8s-io-v1beta2-ClusterQueuePendingWorkloadsStatus} - - -**Appears in:** - -- [ClusterQueueStatus](#kueue-x-k8s-io-v1beta2-ClusterQueueStatus) - - - - - - - - - - - - - - - -
    FieldDescription
    clusterQueuePendingWorkload
    -[]ClusterQueuePendingWorkload -
    -

    clusterQueuePendingWorkload contains the list of top pending workloads.

    -
    lastChangeTime [Required]
    -k8s.io/apimachinery/pkg/apis/meta/v1.Time -
    -

    lastChangeTime indicates the time of the last change of the structure.

    -
    - ## `ClusterQueuePreemption` {#kueue-x-k8s-io-v1beta2-ClusterQueuePreemption} @@ -1175,18 +1110,6 @@ clusterQueue.

    clusterQueue and haven't finished yet.

    -pendingWorkloadsStatus
    -ClusterQueuePendingWorkloadsStatus - - -

    pendingWorkloadsStatus contains the information exposed about the current -status of the pending workloads in the cluster queue. -Deprecated: This field is no longer effective since v0.14.0, which means Kueue no longer stores and updates information. -You can migrate to VisibilityOnDemand -(https://kueue.sigs.k8s.io/docs/tasks/manage/monitor_pending_workloads/pending_workloads_on_demand/) -instead.

    - - fairSharing
    FairSharingStatus diff --git a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go index 565d4771da1..cdc05a481fc 100644 --- a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go +++ b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go @@ -17,7 +17,6 @@ limitations under the License. package core import ( - "github.com/google/go-cmp/cmp/cmpopts" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -47,8 +46,6 @@ const ( flavorCPUArchB = "arch-b" ) -var ignorePendingWorkloadsStatus = cmpopts.IgnoreFields(kueue.ClusterQueueStatus{}, "PendingWorkloadsStatus") - var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { var ( ns *corev1.Namespace @@ -195,7 +192,7 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin Message: "Can't admit new workloads: references missing ResourceFlavor(s): on-demand,spot,model-a,model-b.", }, }, - }, util.IgnoreConditionTimestampsAndObservedGeneration, ignorePendingWorkloadsStatus)) + }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) // Workloads are inadmissible because ResourceFlavors don't exist here yet. util.ExpectPendingWorkloadsMetric(clusterQueue, 0, 5) @@ -289,7 +286,7 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin Message: "Can admit new workloads", }, }, - }, util.IgnoreConditionTimestampsAndObservedGeneration, ignorePendingWorkloadsStatus)) + }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPendingWorkloadsMetric(clusterQueue, 1, 0) util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 4) @@ -325,7 +322,7 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin Message: "Can admit new workloads", }, }, - }, util.IgnoreConditionTimestampsAndObservedGeneration, ignorePendingWorkloadsStatus)) + }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPendingWorkloadsMetric(clusterQueue, 1, 0) util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 4) @@ -355,7 +352,7 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin Message: "Can admit new workloads", }, }, - }, util.IgnoreConditionTimestampsAndObservedGeneration, ignorePendingWorkloadsStatus)) + }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPendingWorkloadsMetric(clusterQueue, 0, 0) util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 0) @@ -388,7 +385,7 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin Message: "Can't admit new workloads: references missing ResourceFlavor(s): on-demand,spot,model-a,model-b.", }, }, - }, util.IgnoreConditionTimestampsAndObservedGeneration, ignorePendingWorkloadsStatus)) + }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPendingWorkloadsMetric(clusterQueue, 0, 1) @@ -416,7 +413,7 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin Message: "Can't admit new workloads: references missing ResourceFlavor(s): on-demand,spot,model-a,model-b.", }, }, - }, util.IgnoreConditionTimestampsAndObservedGeneration, ignorePendingWorkloadsStatus)) + }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPendingWorkloadsMetric(clusterQueue, 0, 0) util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 0) diff --git a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go index 26e070038aa..225ae08530e 100644 --- a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go +++ b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go @@ -40,7 +40,6 @@ import ( var ( ignoreCQConditions = cmpopts.IgnoreFields(kueue.ClusterQueueStatus{}, "Conditions") - ignorePendingWorkloadsStatus = cmpopts.IgnoreFields(kueue.ClusterQueueStatus{}, "PendingWorkloadsStatus") defaultRequeuingBackoffLimitCount *int32 = nil ) @@ -402,7 +401,7 @@ var _ = ginkgo.Describe("SchedulerWithWaitForPodsReady", func() { Total: resource.MustParse("2"), }}, }}, - }, ignoreCQConditions, ignorePendingWorkloadsStatus)) + }, ignoreCQConditions)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("wait for the timeout to be exceeded") @@ -439,7 +438,7 @@ var _ = ginkgo.Describe("SchedulerWithWaitForPodsReady", func() { Total: resource.MustParse("0"), }}, }}, - }, ignoreCQConditions, ignorePendingWorkloadsStatus)) + }, ignoreCQConditions)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("verify the active workload metric is decreased for the cluster queue") diff --git a/test/integration/singlecluster/scheduler/workload_controller_test.go b/test/integration/singlecluster/scheduler/workload_controller_test.go index cf3df01a4f9..dd3d2e888f0 100644 --- a/test/integration/singlecluster/scheduler/workload_controller_test.go +++ b/test/integration/singlecluster/scheduler/workload_controller_test.go @@ -37,7 +37,7 @@ import ( ) var ignoreCqCondition = cmpopts.IgnoreFields(kueue.ClusterQueueStatus{}, "Conditions") -var ignoreInClusterQueueStatus = cmpopts.IgnoreFields(kueue.ClusterQueueStatus{}, "PendingWorkloadsStatus", "FlavorsUsage", "AdmittedWorkloads") +var ignoreInClusterQueueStatus = cmpopts.IgnoreFields(kueue.ClusterQueueStatus{}, "FlavorsUsage", "AdmittedWorkloads") const pseudoCPU = "kueue.x-k8s.io/cpu" From 83effda4fa35582c11d47d007dec3ecbb8f13b66 Mon Sep 17 00:00:00 2001 From: Obinna Odirionye Date: Fri, 31 Oct 2025 16:12:03 +0300 Subject: [PATCH 068/119] v1beta2: Remove deprecated retryDelayMinutes field and fix conversion (#7407) - Remove retryDelayMinutes field from AdmissionCheckSpec in v1beta2 - Add conversion function for AdmissionCheckSpec from v1beta1 to v1beta2 - Update CRD, generated code, and documentation - Update integration tests to reflect changes --- .../v1beta1/admissioncheck_conversion.go | 30 +++++++++++++ apis/kueue/v1beta1/zz_generated.conversion.go | 42 ++++++++++++------- apis/kueue/v1beta2/admissioncheck_types.go | 8 ---- apis/kueue/v1beta2/zz_generated.deepcopy.go | 5 --- .../crd/kueue.x-k8s.io_admissionchecks.yaml | 9 ---- .../kueue/v1beta2/admissioncheckspec.go | 13 +----- .../bases/kueue.x-k8s.io_admissionchecks.yaml | 9 ---- .../en/docs/reference/kueue.v1beta2.md | 10 ----- .../webhook/core/admissioncheck_test.go | 4 +- 9 files changed, 61 insertions(+), 69 deletions(-) create mode 100644 apis/kueue/v1beta1/admissioncheck_conversion.go diff --git a/apis/kueue/v1beta1/admissioncheck_conversion.go b/apis/kueue/v1beta1/admissioncheck_conversion.go new file mode 100644 index 00000000000..5897784e18e --- /dev/null +++ b/apis/kueue/v1beta1/admissioncheck_conversion.go @@ -0,0 +1,30 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + conversionapi "k8s.io/apimachinery/pkg/conversion" + + "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +//lint:file-ignore ST1003 "generated Convert_* calls below use underscores" +//revive:disable:var-naming + +func Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(in *AdmissionCheckSpec, out *v1beta2.AdmissionCheckSpec, s conversionapi.Scope) error { + return autoConvert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(in, out, s) +} diff --git a/apis/kueue/v1beta1/zz_generated.conversion.go b/apis/kueue/v1beta1/zz_generated.conversion.go index 061c8f635de..cf9b51bfa8b 100644 --- a/apis/kueue/v1beta1/zz_generated.conversion.go +++ b/apis/kueue/v1beta1/zz_generated.conversion.go @@ -78,11 +78,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*AdmissionCheckSpec)(nil), (*v1beta2.AdmissionCheckSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(a.(*AdmissionCheckSpec), b.(*v1beta2.AdmissionCheckSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*v1beta2.AdmissionCheckSpec)(nil), (*AdmissionCheckSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_AdmissionCheckSpec_To_v1beta1_AdmissionCheckSpec(a.(*v1beta2.AdmissionCheckSpec), b.(*AdmissionCheckSpec), scope) }); err != nil { @@ -763,6 +758,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*AdmissionCheckSpec)(nil), (*v1beta2.AdmissionCheckSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(a.(*AdmissionCheckSpec), b.(*v1beta2.AdmissionCheckSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*ClusterQueueSpec)(nil), (*v1beta2.ClusterQueueSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(a.(*ClusterQueueSpec), b.(*v1beta2.ClusterQueueSpec), scope) }); err != nil { @@ -857,7 +857,17 @@ func Convert_v1beta2_AdmissionCheck_To_v1beta1_AdmissionCheck(in *v1beta2.Admiss func autoConvert_v1beta1_AdmissionCheckList_To_v1beta2_AdmissionCheckList(in *AdmissionCheckList, out *v1beta2.AdmissionCheckList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]v1beta2.AdmissionCheck)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]v1beta2.AdmissionCheck, len(*in)) + for i := range *in { + if err := Convert_v1beta1_AdmissionCheck_To_v1beta2_AdmissionCheck(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -868,7 +878,17 @@ func Convert_v1beta1_AdmissionCheckList_To_v1beta2_AdmissionCheckList(in *Admiss func autoConvert_v1beta2_AdmissionCheckList_To_v1beta1_AdmissionCheckList(in *v1beta2.AdmissionCheckList, out *AdmissionCheckList, s conversion.Scope) error { out.ListMeta = in.ListMeta - out.Items = *(*[]AdmissionCheck)(unsafe.Pointer(&in.Items)) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AdmissionCheck, len(*in)) + for i := range *in { + if err := Convert_v1beta2_AdmissionCheck_To_v1beta1_AdmissionCheck(&(*in)[i], &(*out)[i], s); err != nil { + return err + } + } + } else { + out.Items = nil + } return nil } @@ -903,19 +923,13 @@ func Convert_v1beta2_AdmissionCheckParametersReference_To_v1beta1_AdmissionCheck func autoConvert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(in *AdmissionCheckSpec, out *v1beta2.AdmissionCheckSpec, s conversion.Scope) error { out.ControllerName = in.ControllerName - out.RetryDelayMinutes = (*int64)(unsafe.Pointer(in.RetryDelayMinutes)) + // WARNING: in.RetryDelayMinutes requires manual conversion: does not exist in peer-type out.Parameters = (*v1beta2.AdmissionCheckParametersReference)(unsafe.Pointer(in.Parameters)) return nil } -// Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec is an autogenerated conversion function. -func Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(in *AdmissionCheckSpec, out *v1beta2.AdmissionCheckSpec, s conversion.Scope) error { - return autoConvert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(in, out, s) -} - func autoConvert_v1beta2_AdmissionCheckSpec_To_v1beta1_AdmissionCheckSpec(in *v1beta2.AdmissionCheckSpec, out *AdmissionCheckSpec, s conversion.Scope) error { out.ControllerName = in.ControllerName - out.RetryDelayMinutes = (*int64)(unsafe.Pointer(in.RetryDelayMinutes)) out.Parameters = (*AdmissionCheckParametersReference)(unsafe.Pointer(in.Parameters)) return nil } diff --git a/apis/kueue/v1beta2/admissioncheck_types.go b/apis/kueue/v1beta2/admissioncheck_types.go index 2b3ea2b692d..00f2b1cca57 100644 --- a/apis/kueue/v1beta2/admissioncheck_types.go +++ b/apis/kueue/v1beta2/admissioncheck_types.go @@ -54,14 +54,6 @@ type AdmissionCheckSpec struct { // +kubebuilder:validation:MinLength=1 ControllerName string `json:"controllerName"` - // retryDelayMinutes specifies how long to keep the workload suspended after - // a failed check (after it transitioned to False). When the delay period has passed, the check - // state goes to "Unknown". The default is 15 min. - // +optional - // +kubebuilder:default=15 - // Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. - RetryDelayMinutes *int64 `json:"retryDelayMinutes,omitempty"` - // parameters identifies a configuration with additional parameters for the // check. // +optional diff --git a/apis/kueue/v1beta2/zz_generated.deepcopy.go b/apis/kueue/v1beta2/zz_generated.deepcopy.go index 94e774cc731..5308dd3dd21 100644 --- a/apis/kueue/v1beta2/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta2/zz_generated.deepcopy.go @@ -125,11 +125,6 @@ func (in *AdmissionCheckParametersReference) DeepCopy() *AdmissionCheckParameter // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AdmissionCheckSpec) DeepCopyInto(out *AdmissionCheckSpec) { *out = *in - if in.RetryDelayMinutes != nil { - in, out := &in.RetryDelayMinutes, &out.RetryDelayMinutes - *out = new(int64) - **out = **in - } if in.Parameters != nil { in, out := &in.Parameters, &out.Parameters *out = new(AdmissionCheckParametersReference) diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml index 3327436148b..d486a0f8774 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml @@ -236,15 +236,6 @@ spec: - kind - name type: object - retryDelayMinutes: - default: 15 - description: |- - retryDelayMinutes specifies how long to keep the workload suspended after - a failed check (after it transitioned to False). When the delay period has passed, the check - state goes to "Unknown". The default is 15 min. - Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. - format: int64 - type: integer required: - controllerName type: object diff --git a/client-go/applyconfiguration/kueue/v1beta2/admissioncheckspec.go b/client-go/applyconfiguration/kueue/v1beta2/admissioncheckspec.go index ddc8be3eec5..68da40cbdba 100644 --- a/client-go/applyconfiguration/kueue/v1beta2/admissioncheckspec.go +++ b/client-go/applyconfiguration/kueue/v1beta2/admissioncheckspec.go @@ -20,9 +20,8 @@ package v1beta2 // AdmissionCheckSpecApplyConfiguration represents a declarative configuration of the AdmissionCheckSpec type for use // with apply. type AdmissionCheckSpecApplyConfiguration struct { - ControllerName *string `json:"controllerName,omitempty"` - RetryDelayMinutes *int64 `json:"retryDelayMinutes,omitempty"` - Parameters *AdmissionCheckParametersReferenceApplyConfiguration `json:"parameters,omitempty"` + ControllerName *string `json:"controllerName,omitempty"` + Parameters *AdmissionCheckParametersReferenceApplyConfiguration `json:"parameters,omitempty"` } // AdmissionCheckSpecApplyConfiguration constructs a declarative configuration of the AdmissionCheckSpec type for use with @@ -39,14 +38,6 @@ func (b *AdmissionCheckSpecApplyConfiguration) WithControllerName(value string) return b } -// WithRetryDelayMinutes sets the RetryDelayMinutes field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the RetryDelayMinutes field is set to the value of the last call. -func (b *AdmissionCheckSpecApplyConfiguration) WithRetryDelayMinutes(value int64) *AdmissionCheckSpecApplyConfiguration { - b.RetryDelayMinutes = &value - return b -} - // WithParameters sets the Parameters field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the Parameters field is set to the value of the last call. diff --git a/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml b/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml index aec0bad2bc6..dd5b207380c 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml @@ -215,15 +215,6 @@ spec: - kind - name type: object - retryDelayMinutes: - default: 15 - description: |- - retryDelayMinutes specifies how long to keep the workload suspended after - a failed check (after it transitioned to False). When the delay period has passed, the check - state goes to "Unknown". The default is 15 min. - Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2. - format: int64 - type: integer required: - controllerName type: object diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md index e2cd40bf8bf..5ed42dce44c 100644 --- a/site/content/en/docs/reference/kueue.v1beta2.md +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -498,16 +498,6 @@ The description is limited to a maximum of 2048 characters.

    not necessarily a Kubernetes Pod or Deployment name. Cannot be empty.

    -retryDelayMinutes
    -int64 - - -

    retryDelayMinutes specifies how long to keep the workload suspended after -a failed check (after it transitioned to False). When the delay period has passed, the check -state goes to "Unknown". The default is 15 min. -Deprecated: retryDelayMinutes has already been deprecated since v0.8 and will be removed in v1beta2.

    - - parameters
    AdmissionCheckParametersReference diff --git a/test/integration/singlecluster/webhook/core/admissioncheck_test.go b/test/integration/singlecluster/webhook/core/admissioncheck_test.go index 4b1b3993638..93a44473400 100644 --- a/test/integration/singlecluster/webhook/core/admissioncheck_test.go +++ b/test/integration/singlecluster/webhook/core/admissioncheck_test.go @@ -24,7 +24,6 @@ import ( "github.com/onsi/gomega" "github.com/onsi/gomega/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -67,8 +66,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { Name: "foo", }, Spec: kueue.AdmissionCheckSpec{ - ControllerName: "ac-controller", - RetryDelayMinutes: ptr.To[int64](15), + ControllerName: "ac-controller", }, }, ), From 8970bdbdbb10b9c7cf64e2fb1fe133739986d049 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 31 Oct 2025 18:42:10 +0530 Subject: [PATCH 069/119] v1beta2: drop deprecated Flavors field from LocalQueueStatus (#7449) --- .golangci.yaml | 6 - apis/kueue/v1beta1/zz_generated.conversion.go | 41 +------ apis/kueue/v1beta2/localqueue_types.go | 44 ------- apis/kueue/v1beta2/zz_generated.deepcopy.go | 46 -------- .../crd/kueue.x-k8s.io_localqueues.yaml | 94 --------------- .../kueue/v1beta2/localqueueflavorstatus.go | 93 --------------- .../kueue/v1beta2/localqueuestatus.go | 28 ++--- .../kueue/v1beta2/topologyinfo.go | 53 --------- client-go/applyconfiguration/utils.go | 4 - .../crd/bases/kueue.x-k8s.io_localqueues.yaml | 99 ---------------- .../kep.yaml | 5 +- pkg/cache/scheduler/cache.go | 40 ------- pkg/controller/core/localqueue_controller.go | 1 - pkg/features/kube_features.go | 9 -- .../components-tools/feature-gate-removed.md | 2 + .../en/docs/reference/kueue.v1beta2.md | 100 ---------------- .../core/localqueue_controller_test.go | 108 ------------------ .../integration/singlecluster/tas/tas_test.go | 22 ---- 18 files changed, 13 insertions(+), 782 deletions(-) delete mode 100644 client-go/applyconfiguration/kueue/v1beta2/localqueueflavorstatus.go delete mode 100644 client-go/applyconfiguration/kueue/v1beta2/topologyinfo.go diff --git a/.golangci.yaml b/.golangci.yaml index 16c9f0296a7..e43fff90754 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -90,12 +90,6 @@ linters: - linters: - staticcheck text: 'SA1019: j.Status.RayClusterStatus.State is deprecated: the State field is replaced by the Conditions field.' - - linters: - - staticcheck - text: 'SA1019: kueue.LocalQueueFlavorStatus is deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.' - - linters: - - staticcheck - text: 'SA1019: stats.Flavors is deprecated: Flavors is deprecated and marked for removal in v1beta2.' - linters: - fatcontext path: ^test/* diff --git a/apis/kueue/v1beta1/zz_generated.conversion.go b/apis/kueue/v1beta1/zz_generated.conversion.go index cf9b51bfa8b..12d5833795e 100644 --- a/apis/kueue/v1beta1/zz_generated.conversion.go +++ b/apis/kueue/v1beta1/zz_generated.conversion.go @@ -298,16 +298,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*LocalQueueFlavorStatus)(nil), (*v1beta2.LocalQueueFlavorStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus(a.(*LocalQueueFlavorStatus), b.(*v1beta2.LocalQueueFlavorStatus), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta2.LocalQueueFlavorStatus)(nil), (*LocalQueueFlavorStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus(a.(*v1beta2.LocalQueueFlavorStatus), b.(*LocalQueueFlavorStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*LocalQueueFlavorUsage)(nil), (*v1beta2.LocalQueueFlavorUsage)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage(a.(*LocalQueueFlavorUsage), b.(*v1beta2.LocalQueueFlavorUsage), scope) }); err != nil { @@ -1509,34 +1499,6 @@ func Convert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(in *v1beta2.LocalQueue, ou return autoConvert_v1beta2_LocalQueue_To_v1beta1_LocalQueue(in, out, s) } -func autoConvert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus(in *LocalQueueFlavorStatus, out *v1beta2.LocalQueueFlavorStatus, s conversion.Scope) error { - out.Name = v1beta2.ResourceFlavorReference(in.Name) - out.Resources = *(*[]corev1.ResourceName)(unsafe.Pointer(&in.Resources)) - out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels)) - out.NodeTaints = *(*[]corev1.Taint)(unsafe.Pointer(&in.NodeTaints)) - out.Topology = (*v1beta2.TopologyInfo)(unsafe.Pointer(in.Topology)) - return nil -} - -// Convert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus is an autogenerated conversion function. -func Convert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus(in *LocalQueueFlavorStatus, out *v1beta2.LocalQueueFlavorStatus, s conversion.Scope) error { - return autoConvert_v1beta1_LocalQueueFlavorStatus_To_v1beta2_LocalQueueFlavorStatus(in, out, s) -} - -func autoConvert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus(in *v1beta2.LocalQueueFlavorStatus, out *LocalQueueFlavorStatus, s conversion.Scope) error { - out.Name = ResourceFlavorReference(in.Name) - out.Resources = *(*[]corev1.ResourceName)(unsafe.Pointer(&in.Resources)) - out.NodeLabels = *(*map[string]string)(unsafe.Pointer(&in.NodeLabels)) - out.NodeTaints = *(*[]corev1.Taint)(unsafe.Pointer(&in.NodeTaints)) - out.Topology = (*TopologyInfo)(unsafe.Pointer(in.Topology)) - return nil -} - -// Convert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus is an autogenerated conversion function. -func Convert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus(in *v1beta2.LocalQueueFlavorStatus, out *LocalQueueFlavorStatus, s conversion.Scope) error { - return autoConvert_v1beta2_LocalQueueFlavorStatus_To_v1beta1_LocalQueueFlavorStatus(in, out, s) -} - func autoConvert_v1beta1_LocalQueueFlavorUsage_To_v1beta2_LocalQueueFlavorUsage(in *LocalQueueFlavorUsage, out *v1beta2.LocalQueueFlavorUsage, s conversion.Scope) error { out.Name = v1beta2.ResourceFlavorReference(in.Name) out.Resources = *(*[]v1beta2.LocalQueueResourceUsage)(unsafe.Pointer(&in.Resources)) @@ -1654,7 +1616,7 @@ func autoConvert_v1beta1_LocalQueueStatus_To_v1beta2_LocalQueueStatus(in *LocalQ out.AdmittedWorkloads = in.AdmittedWorkloads out.FlavorsReservation = *(*[]v1beta2.LocalQueueFlavorUsage)(unsafe.Pointer(&in.FlavorsReservation)) // WARNING: in.FlavorUsage requires manual conversion: does not exist in peer-type - out.Flavors = *(*[]v1beta2.LocalQueueFlavorStatus)(unsafe.Pointer(&in.Flavors)) + // WARNING: in.Flavors requires manual conversion: does not exist in peer-type out.FairSharing = (*v1beta2.FairSharingStatus)(unsafe.Pointer(in.FairSharing)) return nil } @@ -1666,7 +1628,6 @@ func autoConvert_v1beta2_LocalQueueStatus_To_v1beta1_LocalQueueStatus(in *v1beta out.AdmittedWorkloads = in.AdmittedWorkloads out.FlavorsReservation = *(*[]LocalQueueFlavorUsage)(unsafe.Pointer(&in.FlavorsReservation)) // WARNING: in.FlavorsUsage requires manual conversion: does not exist in peer-type - out.Flavors = *(*[]LocalQueueFlavorStatus)(unsafe.Pointer(&in.Flavors)) out.FairSharing = (*FairSharingStatus)(unsafe.Pointer(in.FairSharing)) return nil } diff --git a/apis/kueue/v1beta2/localqueue_types.go b/apis/kueue/v1beta2/localqueue_types.go index 04c396208cd..fc964c1a4a1 100644 --- a/apis/kueue/v1beta2/localqueue_types.go +++ b/apis/kueue/v1beta2/localqueue_types.go @@ -56,42 +56,6 @@ type LocalQueueSpec struct { FairSharing *FairSharing `json:"fairSharing,omitempty"` } -// Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2. -type LocalQueueFlavorStatus struct { - // name of the flavor. - // +required - // +kubebuilder:validation:Required - Name ResourceFlavorReference `json:"name"` - - // resources used in the flavor. - // +listType=set - // +kubebuilder:validation:MaxItems=16 - // +optional - Resources []corev1.ResourceName `json:"resources,omitempty"` - - // nodeLabels are labels that associate the ResourceFlavor with Nodes that - // have the same labels. - // +mapType=atomic - // +kubebuilder:validation:MaxProperties=8 - // +optional - NodeLabels map[string]string `json:"nodeLabels,omitempty"` - - // nodeTaints are taints that the nodes associated with this ResourceFlavor - // have. - // +listType=atomic - // +kubebuilder:validation:MaxItems=8 - // +optional - NodeTaints []corev1.Taint `json:"nodeTaints,omitempty"` - - // topology is the topology that associated with this ResourceFlavor. - // - // This is a beta field and requires enabling the TopologyAwareScheduling - // feature gate. - // - // +optional - Topology *TopologyInfo `json:"topology,omitempty"` -} - type TopologyInfo struct { // name is the name of the topology. // @@ -153,14 +117,6 @@ type LocalQueueStatus struct { // +optional FlavorsUsage []LocalQueueFlavorUsage `json:"flavorsUsage"` - // flavors lists all currently available ResourceFlavors in specified ClusterQueue. - // +listType=map - // +listMapKey=name - // +kubebuilder:validation:MaxItems=16 - // +optional - // Deprecated: Flavors is deprecated and marked for removal in v1beta2. - Flavors []LocalQueueFlavorStatus `json:"flavors,omitempty"` - // fairSharing contains the information about the current status of fair sharing. // +optional FairSharing *FairSharingStatus `json:"fairSharing,omitempty"` diff --git a/apis/kueue/v1beta2/zz_generated.deepcopy.go b/apis/kueue/v1beta2/zz_generated.deepcopy.go index 5308dd3dd21..40980e392f7 100644 --- a/apis/kueue/v1beta2/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta2/zz_generated.deepcopy.go @@ -716,45 +716,6 @@ func (in *LocalQueue) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LocalQueueFlavorStatus) DeepCopyInto(out *LocalQueueFlavorStatus) { - *out = *in - if in.Resources != nil { - in, out := &in.Resources, &out.Resources - *out = make([]corev1.ResourceName, len(*in)) - copy(*out, *in) - } - if in.NodeLabels != nil { - in, out := &in.NodeLabels, &out.NodeLabels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.NodeTaints != nil { - in, out := &in.NodeTaints, &out.NodeTaints - *out = make([]corev1.Taint, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Topology != nil { - in, out := &in.Topology, &out.Topology - *out = new(TopologyInfo) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalQueueFlavorStatus. -func (in *LocalQueueFlavorStatus) DeepCopy() *LocalQueueFlavorStatus { - if in == nil { - return nil - } - out := new(LocalQueueFlavorStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LocalQueueFlavorUsage) DeepCopyInto(out *LocalQueueFlavorUsage) { *out = *in @@ -874,13 +835,6 @@ func (in *LocalQueueStatus) DeepCopyInto(out *LocalQueueStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Flavors != nil { - in, out := &in.Flavors, &out.Flavors - *out = make([]LocalQueueFlavorStatus, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } if in.FairSharing != nil { in, out := &in.FairSharing, &out.FairSharing *out = new(FairSharingStatus) diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml index 60727b73e3f..62e490de2d7 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml @@ -643,100 +643,6 @@ spec: required: - weightedShare type: object - flavors: - description: |- - flavors lists all currently available ResourceFlavors in specified ClusterQueue. - Deprecated: Flavors is deprecated and marked for removal in v1beta2. - items: - description: 'Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.' - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint was added. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - resources: - description: resources used in the flavor. - items: - description: ResourceName is the name identifying various resources in a ResourceList. - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - topology: - description: |- - topology is the topology that associated with this ResourceFlavor. - - This is a beta field and requires enabling the TopologyAwareScheduling - feature gate. - properties: - levels: - description: levels define the levels of topology. - items: - maxLength: 317 - type: string - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - name: - description: name is the name of the topology. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - levels - - name - type: object - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map flavorsReservation: description: |- flavorsReservation are the reserved quotas, by flavor currently in use by the diff --git a/client-go/applyconfiguration/kueue/v1beta2/localqueueflavorstatus.go b/client-go/applyconfiguration/kueue/v1beta2/localqueueflavorstatus.go deleted file mode 100644 index 396673d5b04..00000000000 --- a/client-go/applyconfiguration/kueue/v1beta2/localqueueflavorstatus.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1beta2 - -import ( - v1 "k8s.io/api/core/v1" - corev1 "k8s.io/client-go/applyconfigurations/core/v1" - kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" -) - -// LocalQueueFlavorStatusApplyConfiguration represents a declarative configuration of the LocalQueueFlavorStatus type for use -// with apply. -type LocalQueueFlavorStatusApplyConfiguration struct { - Name *kueuev1beta2.ResourceFlavorReference `json:"name,omitempty"` - Resources []v1.ResourceName `json:"resources,omitempty"` - NodeLabels map[string]string `json:"nodeLabels,omitempty"` - NodeTaints []corev1.TaintApplyConfiguration `json:"nodeTaints,omitempty"` - Topology *TopologyInfoApplyConfiguration `json:"topology,omitempty"` -} - -// LocalQueueFlavorStatusApplyConfiguration constructs a declarative configuration of the LocalQueueFlavorStatus type for use with -// apply. -func LocalQueueFlavorStatus() *LocalQueueFlavorStatusApplyConfiguration { - return &LocalQueueFlavorStatusApplyConfiguration{} -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *LocalQueueFlavorStatusApplyConfiguration) WithName(value kueuev1beta2.ResourceFlavorReference) *LocalQueueFlavorStatusApplyConfiguration { - b.Name = &value - return b -} - -// WithResources adds the given value to the Resources field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Resources field. -func (b *LocalQueueFlavorStatusApplyConfiguration) WithResources(values ...v1.ResourceName) *LocalQueueFlavorStatusApplyConfiguration { - for i := range values { - b.Resources = append(b.Resources, values[i]) - } - return b -} - -// WithNodeLabels puts the entries into the NodeLabels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the NodeLabels field, -// overwriting an existing map entries in NodeLabels field with the same key. -func (b *LocalQueueFlavorStatusApplyConfiguration) WithNodeLabels(entries map[string]string) *LocalQueueFlavorStatusApplyConfiguration { - if b.NodeLabels == nil && len(entries) > 0 { - b.NodeLabels = make(map[string]string, len(entries)) - } - for k, v := range entries { - b.NodeLabels[k] = v - } - return b -} - -// WithNodeTaints adds the given value to the NodeTaints field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the NodeTaints field. -func (b *LocalQueueFlavorStatusApplyConfiguration) WithNodeTaints(values ...*corev1.TaintApplyConfiguration) *LocalQueueFlavorStatusApplyConfiguration { - for i := range values { - if values[i] == nil { - panic("nil value passed to WithNodeTaints") - } - b.NodeTaints = append(b.NodeTaints, *values[i]) - } - return b -} - -// WithTopology sets the Topology field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Topology field is set to the value of the last call. -func (b *LocalQueueFlavorStatusApplyConfiguration) WithTopology(value *TopologyInfoApplyConfiguration) *LocalQueueFlavorStatusApplyConfiguration { - b.Topology = value - return b -} diff --git a/client-go/applyconfiguration/kueue/v1beta2/localqueuestatus.go b/client-go/applyconfiguration/kueue/v1beta2/localqueuestatus.go index c16e4ee961a..988cf865564 100644 --- a/client-go/applyconfiguration/kueue/v1beta2/localqueuestatus.go +++ b/client-go/applyconfiguration/kueue/v1beta2/localqueuestatus.go @@ -24,14 +24,13 @@ import ( // LocalQueueStatusApplyConfiguration represents a declarative configuration of the LocalQueueStatus type for use // with apply. type LocalQueueStatusApplyConfiguration struct { - Conditions []v1.ConditionApplyConfiguration `json:"conditions,omitempty"` - PendingWorkloads *int32 `json:"pendingWorkloads,omitempty"` - ReservingWorkloads *int32 `json:"reservingWorkloads,omitempty"` - AdmittedWorkloads *int32 `json:"admittedWorkloads,omitempty"` - FlavorsReservation []LocalQueueFlavorUsageApplyConfiguration `json:"flavorsReservation,omitempty"` - FlavorsUsage []LocalQueueFlavorUsageApplyConfiguration `json:"flavorsUsage,omitempty"` - Flavors []LocalQueueFlavorStatusApplyConfiguration `json:"flavors,omitempty"` - FairSharing *FairSharingStatusApplyConfiguration `json:"fairSharing,omitempty"` + Conditions []v1.ConditionApplyConfiguration `json:"conditions,omitempty"` + PendingWorkloads *int32 `json:"pendingWorkloads,omitempty"` + ReservingWorkloads *int32 `json:"reservingWorkloads,omitempty"` + AdmittedWorkloads *int32 `json:"admittedWorkloads,omitempty"` + FlavorsReservation []LocalQueueFlavorUsageApplyConfiguration `json:"flavorsReservation,omitempty"` + FlavorsUsage []LocalQueueFlavorUsageApplyConfiguration `json:"flavorsUsage,omitempty"` + FairSharing *FairSharingStatusApplyConfiguration `json:"fairSharing,omitempty"` } // LocalQueueStatusApplyConfiguration constructs a declarative configuration of the LocalQueueStatus type for use with @@ -103,19 +102,6 @@ func (b *LocalQueueStatusApplyConfiguration) WithFlavorsUsage(values ...*LocalQu return b } -// WithFlavors adds the given value to the Flavors field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Flavors field. -func (b *LocalQueueStatusApplyConfiguration) WithFlavors(values ...*LocalQueueFlavorStatusApplyConfiguration) *LocalQueueStatusApplyConfiguration { - for i := range values { - if values[i] == nil { - panic("nil value passed to WithFlavors") - } - b.Flavors = append(b.Flavors, *values[i]) - } - return b -} - // WithFairSharing sets the FairSharing field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the FairSharing field is set to the value of the last call. diff --git a/client-go/applyconfiguration/kueue/v1beta2/topologyinfo.go b/client-go/applyconfiguration/kueue/v1beta2/topologyinfo.go deleted file mode 100644 index 15e80353bd2..00000000000 --- a/client-go/applyconfiguration/kueue/v1beta2/topologyinfo.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1beta2 - -import ( - kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" -) - -// TopologyInfoApplyConfiguration represents a declarative configuration of the TopologyInfo type for use -// with apply. -type TopologyInfoApplyConfiguration struct { - Name *kueuev1beta2.TopologyReference `json:"name,omitempty"` - Levels []string `json:"levels,omitempty"` -} - -// TopologyInfoApplyConfiguration constructs a declarative configuration of the TopologyInfo type for use with -// apply. -func TopologyInfo() *TopologyInfoApplyConfiguration { - return &TopologyInfoApplyConfiguration{} -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *TopologyInfoApplyConfiguration) WithName(value kueuev1beta2.TopologyReference) *TopologyInfoApplyConfiguration { - b.Name = &value - return b -} - -// WithLevels adds the given value to the Levels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Levels field. -func (b *TopologyInfoApplyConfiguration) WithLevels(values ...string) *TopologyInfoApplyConfiguration { - for i := range values { - b.Levels = append(b.Levels, values[i]) - } - return b -} diff --git a/client-go/applyconfiguration/utils.go b/client-go/applyconfiguration/utils.go index f6ba18558fd..ad820f3afd7 100644 --- a/client-go/applyconfiguration/utils.go +++ b/client-go/applyconfiguration/utils.go @@ -223,8 +223,6 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &kueuev1beta2.KubeConfigApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("LocalQueue"): return &kueuev1beta2.LocalQueueApplyConfiguration{} - case v1beta2.SchemeGroupVersion.WithKind("LocalQueueFlavorStatus"): - return &kueuev1beta2.LocalQueueFlavorStatusApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("LocalQueueFlavorUsage"): return &kueuev1beta2.LocalQueueFlavorUsageApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("LocalQueueResourceUsage"): @@ -285,8 +283,6 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &kueuev1beta2.TopologyAssignmentApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("TopologyDomainAssignment"): return &kueuev1beta2.TopologyDomainAssignmentApplyConfiguration{} - case v1beta2.SchemeGroupVersion.WithKind("TopologyInfo"): - return &kueuev1beta2.TopologyInfoApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("TopologyLevel"): return &kueuev1beta2.TopologyLevelApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("TopologySpec"): diff --git a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml index ef44784b709..25d21369b3a 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml @@ -629,105 +629,6 @@ spec: required: - weightedShare type: object - flavors: - description: |- - flavors lists all currently available ResourceFlavors in specified ClusterQueue. - Deprecated: Flavors is deprecated and marked for removal in v1beta2. - items: - description: 'Deprecated: LocalQueueFlavorStatus is deprecated and - marked for removal in v1beta2.' - properties: - name: - description: name of the flavor. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - nodeLabels: - additionalProperties: - type: string - description: |- - nodeLabels are labels that associate the ResourceFlavor with Nodes that - have the same labels. - maxProperties: 8 - type: object - x-kubernetes-map-type: atomic - nodeTaints: - description: |- - nodeTaints are taints that the nodes associated with this ResourceFlavor - have. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to - a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the - taint was added. - format: date-time - type: string - value: - description: The taint value corresponding to the taint - key. - type: string - required: - - effect - - key - type: object - maxItems: 8 - type: array - x-kubernetes-list-type: atomic - resources: - description: resources used in the flavor. - items: - description: ResourceName is the name identifying various - resources in a ResourceList. - type: string - maxItems: 16 - type: array - x-kubernetes-list-type: set - topology: - description: |- - topology is the topology that associated with this ResourceFlavor. - - This is a beta field and requires enabling the TopologyAwareScheduling - feature gate. - properties: - levels: - description: levels define the levels of topology. - items: - maxLength: 317 - type: string - maxItems: 16 - minItems: 1 - type: array - x-kubernetes-list-type: atomic - name: - description: name is the name of the topology. - maxLength: 253 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - required: - - levels - - name - type: object - required: - - name - type: object - maxItems: 16 - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map flavorsReservation: description: |- flavorsReservation are the reserved quotas, by flavor currently in use by the diff --git a/keps/3122-expose-flavors-in-localqueue-status/kep.yaml b/keps/3122-expose-flavors-in-localqueue-status/kep.yaml index ecb385990b2..a03435d10b7 100644 --- a/keps/3122-expose-flavors-in-localqueue-status/kep.yaml +++ b/keps/3122-expose-flavors-in-localqueue-status/kep.yaml @@ -14,16 +14,17 @@ approvers: - "@tenzen-y" # The target maturity stage in the current dev cycle for this KEP. -stage: beta +stage: deprecated # The most recent milestone for which work toward delivery of this KEP has been # done. This can be the current (upcoming) milestone, if it is being actively # worked on. -latest-milestone: "v0.9" +latest-milestone: "v0.15" # The milestone at which this feature was, or is targeted to be, at each stage. milestone: beta: "v0.9" + deprecated: "v0.15" # List the feature gate name and the components for which it must be enabled feature-gates: diff --git a/pkg/cache/scheduler/cache.go b/pkg/cache/scheduler/cache.go index ac9eabcd495..31906ab4f28 100644 --- a/pkg/cache/scheduler/cache.go +++ b/pkg/cache/scheduler/cache.go @@ -25,7 +25,6 @@ import ( "sync" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -803,8 +802,6 @@ type LocalQueueUsageStats struct { ReservingWorkloads int AdmittedResources []kueue.LocalQueueFlavorUsage AdmittedWorkloads int - // Deprecated: Flavors is deprecated and marked for removal in v1beta2. - Flavors []kueue.LocalQueueFlavorStatus } func (c *Cache) LocalQueueUsage(qObj *kueue.LocalQueue) (*LocalQueueUsageStats, error) { @@ -820,48 +817,11 @@ func (c *Cache) LocalQueueUsage(qObj *kueue.LocalQueue) (*LocalQueueUsageStats, return nil, errQNotFound } - var flavors []kueue.LocalQueueFlavorStatus - - if features.Enabled(features.ExposeFlavorsInLocalQueue) { - resourcesInFlavor := make(map[kueue.ResourceFlavorReference][]corev1.ResourceName) - for _, rg := range cqImpl.ResourceGroups { - for _, rgFlavor := range rg.Flavors { - if _, ok := resourcesInFlavor[rgFlavor]; !ok { - resourcesInFlavor[rgFlavor] = make([]corev1.ResourceName, 0, len(rg.CoveredResources)) - } - resourcesInFlavor[rgFlavor] = append(resourcesInFlavor[rgFlavor], sets.List(rg.CoveredResources)...) - } - } - - for _, rg := range cqImpl.ResourceGroups { - for _, rgFlavor := range rg.Flavors { - flavor := kueue.LocalQueueFlavorStatus{Name: rgFlavor} - if rif, ok := resourcesInFlavor[rgFlavor]; ok { - flavor.Resources = append(flavor.Resources, rif...) - } - if rf, ok := c.resourceFlavors[rgFlavor]; ok { - flavor.NodeLabels = rf.Spec.NodeLabels - flavor.NodeTaints = rf.Spec.NodeTaints - if handleTASFlavor(rf) { - if cache := c.tasCache.Get(rgFlavor); cache != nil { - flavor.Topology = &kueue.TopologyInfo{ - Name: cache.flavor.TopologyName, - Levels: cache.topology.Levels, - } - } - } - } - flavors = append(flavors, flavor) - } - } - } - return &LocalQueueUsageStats{ ReservedResources: filterLocalQueueUsage(qImpl.totalReserved, cqImpl.ResourceGroups), ReservingWorkloads: qImpl.reservingWorkloads, AdmittedResources: filterLocalQueueUsage(qImpl.admittedUsage, cqImpl.ResourceGroups), AdmittedWorkloads: qImpl.admittedWorkloads, - Flavors: flavors, }, nil } diff --git a/pkg/controller/core/localqueue_controller.go b/pkg/controller/core/localqueue_controller.go index 0af72d5a774..c65f9381acb 100644 --- a/pkg/controller/core/localqueue_controller.go +++ b/pkg/controller/core/localqueue_controller.go @@ -477,7 +477,6 @@ func (r *LocalQueueReconciler) UpdateStatusIfChanged( queue.Status.AdmittedWorkloads = int32(stats.AdmittedWorkloads) queue.Status.FlavorsReservation = stats.ReservedResources queue.Status.FlavorsUsage = stats.AdmittedResources - queue.Status.Flavors = stats.Flavors if len(conditionStatus) != 0 && len(reason) != 0 && len(msg) != 0 { meta.SetStatusCondition(&queue.Status.Conditions, metav1.Condition{ Type: kueue.LocalQueueActive, diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 5bb4d2dc9f0..624332842de 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -84,12 +84,6 @@ const ( // the resource requests of a Workload ConfigurableResourceTransformations featuregate.Feature = "ConfigurableResourceTransformations" - // owner: @mbobrovskyi - // - // Enable the Flavors status field in the LocalQueue, allowing users to view - // all currently available ResourceFlavors for the LocalQueue. - ExposeFlavorsInLocalQueue featuregate.Feature = "ExposeFlavorsInLocalQueue" - // owner: @kpostoffice // kep: https://github.com/kubernetes-sigs/kueue/tree/main/keps/1833-metrics-for-local-queue // @@ -251,9 +245,6 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned {Version: version.MustParse("0.10"), Default: true, PreRelease: featuregate.Beta}, {Version: version.MustParse("0.14"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 0.16 }, - ExposeFlavorsInLocalQueue: { - {Version: version.MustParse("0.9"), Default: true, PreRelease: featuregate.Beta}, - }, LocalQueueMetrics: { {Version: version.MustParse("0.10"), Default: false, PreRelease: featuregate.Alpha}, }, diff --git a/site/content/en/docs/reference/components-tools/feature-gate-removed.md b/site/content/en/docs/reference/components-tools/feature-gate-removed.md index 012f5a56150..9a5c0122901 100644 --- a/site/content/en/docs/reference/components-tools/feature-gate-removed.md +++ b/site/content/en/docs/reference/components-tools/feature-gate-removed.md @@ -39,3 +39,5 @@ In the following table: | `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | | `ProvisioningACC` | `true` | Beta | 0.7 | 0.14 | | `ProvisioningACC` | `true` | GA | 0.14 | 0.15 | +| `ExposeFlavorsInLocalQueue` | `false` | Beta | 0.9 | 0.15 | +| `ExposeFlavorsInLocalQueue` | `false` | Deprecated | 0.15 | 0.15 | diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md index 5ed42dce44c..ad144496515 100644 --- a/site/content/en/docs/reference/kueue.v1beta2.md +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -1502,64 +1502,6 @@ which the kueue controller manager is running. The config should be stored in th -## `LocalQueueFlavorStatus` {#kueue-x-k8s-io-v1beta2-LocalQueueFlavorStatus} - - -**Appears in:** - -- [LocalQueueStatus](#kueue-x-k8s-io-v1beta2-LocalQueueStatus) - - -

    Deprecated: LocalQueueFlavorStatus is deprecated and marked for removal in v1beta2.

    - - - - - - - - - - - - - - - - - - - - - - - -
    FieldDescription
    name [Required]
    -ResourceFlavorReference -
    -

    name of the flavor.

    -
    resources
    -[]k8s.io/api/core/v1.ResourceName -
    -

    resources used in the flavor.

    -
    nodeLabels
    -map[string]string -
    -

    nodeLabels are labels that associate the ResourceFlavor with Nodes that -have the same labels.

    -
    nodeTaints
    -[]k8s.io/api/core/v1.Taint -
    -

    nodeTaints are taints that the nodes associated with this ResourceFlavor -have.

    -
    topology
    -TopologyInfo -
    -

    topology is the topology that associated with this ResourceFlavor.

    -

    This is a beta field and requires enabling the TopologyAwareScheduling -feature gate.

    -
    - ## `LocalQueueFlavorUsage` {#kueue-x-k8s-io-v1beta2-LocalQueueFlavorUsage} @@ -1750,14 +1692,6 @@ workloads assigned to this LocalQueue.

    workloads assigned to this LocalQueue.

    -flavors
    -[]LocalQueueFlavorStatus - - -

    flavors lists all currently available ResourceFlavors in specified ClusterQueue. -Deprecated: Flavors is deprecated and marked for removal in v1beta2.

    - - fairSharing
    FairSharingStatus @@ -2566,8 +2500,6 @@ this time would be reset to null.

    - [FlavorUsage](#kueue-x-k8s-io-v1beta2-FlavorUsage) -- [LocalQueueFlavorStatus](#kueue-x-k8s-io-v1beta2-LocalQueueFlavorStatus) - - [LocalQueueFlavorUsage](#kueue-x-k8s-io-v1beta2-LocalQueueFlavorUsage) - [PodSetAssignment](#kueue-x-k8s-io-v1beta2-PodSetAssignment) @@ -2910,37 +2842,6 @@ domain indicated by the values field.

    -## `TopologyInfo` {#kueue-x-k8s-io-v1beta2-TopologyInfo} - - -**Appears in:** - -- [LocalQueueFlavorStatus](#kueue-x-k8s-io-v1beta2-LocalQueueFlavorStatus) - - - - - - - - - - - - - - - -
    FieldDescription
    name [Required]
    -TopologyReference -
    -

    name is the name of the topology.

    -
    levels [Required]
    -[]string -
    -

    levels define the levels of topology.

    -
    - ## `TopologyLevel` {#kueue-x-k8s-io-v1beta2-TopologyLevel} @@ -2981,7 +2882,6 @@ level.

    - [ResourceFlavorSpec](#kueue-x-k8s-io-v1beta2-ResourceFlavorSpec) -- [TopologyInfo](#kueue-x-k8s-io-v1beta2-TopologyInfo)

    TopologyReference is the name of the Topology.

    diff --git a/test/integration/singlecluster/controller/core/localqueue_controller_test.go b/test/integration/singlecluster/controller/core/localqueue_controller_test.go index 982b5909a53..1d33cd7995b 100644 --- a/test/integration/singlecluster/controller/core/localqueue_controller_test.go +++ b/test/integration/singlecluster/controller/core/localqueue_controller_test.go @@ -144,10 +144,6 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: emptyUsage, FlavorsUsage: emptyUsage, - Flavors: []kueue.LocalQueueFlavorStatus{ - {Name: flavorModelD, Resources: []corev1.ResourceName{"example.com/gpu"}}, - {Name: flavorModelC, Resources: []corev1.ResourceName{"example.com/gpu"}}, - }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -171,19 +167,6 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: emptyUsage, FlavorsUsage: emptyUsage, - Flavors: []kueue.LocalQueueFlavorStatus{ - { - Name: flavorModelD, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-d"}, - }, - { - Name: flavorModelC, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-c"}, - NodeTaints: []corev1.Taint{{Key: "spot", Value: "true", Effect: "NoSchedule"}}, - }, - }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -275,19 +258,6 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: emptyUsage, FlavorsUsage: emptyUsage, - Flavors: []kueue.LocalQueueFlavorStatus{ - { - Name: flavorModelD, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-d"}, - }, - { - Name: flavorModelC, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-c"}, - NodeTaints: []corev1.Taint{{Key: "spot", Value: "true", Effect: "NoSchedule"}}, - }, - }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -341,19 +311,6 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: fullUsage, FlavorsUsage: emptyUsage, - Flavors: []kueue.LocalQueueFlavorStatus{ - { - Name: flavorModelD, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-d"}, - }, - { - Name: flavorModelC, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-c"}, - NodeTaints: []corev1.Taint{{Key: "spot", Value: "true", Effect: "NoSchedule"}}, - }, - }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -386,19 +343,6 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: fullUsage, FlavorsUsage: fullUsage, - Flavors: []kueue.LocalQueueFlavorStatus{ - { - Name: flavorModelD, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-d"}, - }, - { - Name: flavorModelC, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-c"}, - NodeTaints: []corev1.Taint{{Key: "spot", Value: "true", Effect: "NoSchedule"}}, - }, - }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -428,19 +372,6 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: emptyUsage, FlavorsUsage: emptyUsage, - Flavors: []kueue.LocalQueueFlavorStatus{ - { - Name: flavorModelD, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-d"}, - }, - { - Name: flavorModelC, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-c"}, - NodeTaints: []corev1.Taint{{Key: "spot", Value: "true", Effect: "NoSchedule"}}, - }, - }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -509,19 +440,6 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: emptyUsage, FlavorsUsage: emptyUsage, - Flavors: []kueue.LocalQueueFlavorStatus{ - { - Name: flavorModelD, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-d"}, - }, - { - Name: flavorModelC, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-c"}, - NodeTaints: []corev1.Taint{{Key: "spot", Value: "true", Effect: "NoSchedule"}}, - }, - }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -569,19 +487,6 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: fullUsage, FlavorsUsage: emptyUsage, - Flavors: []kueue.LocalQueueFlavorStatus{ - { - Name: flavorModelD, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-d"}, - }, - { - Name: flavorModelC, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-c"}, - NodeTaints: []corev1.Taint{{Key: "spot", Value: "true", Effect: "NoSchedule"}}, - }, - }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -608,19 +513,6 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: fullUsage, FlavorsUsage: fullUsage, - Flavors: []kueue.LocalQueueFlavorStatus{ - { - Name: flavorModelD, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-d"}, - }, - { - Name: flavorModelC, - Resources: []corev1.ResourceName{"example.com/gpu"}, - NodeLabels: map[string]string{"example.com/gpu": "model-c"}, - NodeTaints: []corev1.Taint{{Key: "spot", Value: "true", Effect: "NoSchedule"}}, - }, - }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/tas/tas_test.go b/test/integration/singlecluster/tas/tas_test.go index f14338fcfa3..78dee73f868 100644 --- a/test/integration/singlecluster/tas/tas_test.go +++ b/test/integration/singlecluster/tas/tas_test.go @@ -355,16 +355,6 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { } }) - ginkgo.It("should expose the TopologyName in LocalQueue status", func() { - gomega.Eventually(func(g gomega.Gomega) { - createdLocalQueue := &kueue.LocalQueue{} - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(localQueue), createdLocalQueue)).Should(gomega.Succeed()) - g.Expect(createdLocalQueue.Status.Flavors).Should(gomega.HaveLen(1)) - g.Expect(createdLocalQueue.Status.Flavors[0].Topology).ShouldNot(gomega.BeNil()) - g.Expect(createdLocalQueue.Status.Flavors[0].Topology.Levels).Should(gomega.Equal(utiltas.Levels(topology))) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) - }) - ginkgo.It("should not admit workload which does not fit to the required topology domain", func() { ginkgo.By("creating a workload which requires rack, but does not fit in any", func() { wl1 := utiltestingapi.MakeWorkload("wl1-inadmissible", ns.Name). @@ -2801,18 +2791,6 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { } }) - ginkgo.It("should expose the TopologyName in LocalQueue status", func() { - gomega.Eventually(func(g gomega.Gomega) { - createdLocalQueue := &kueue.LocalQueue{} - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(localQueue), createdLocalQueue)).Should(gomega.Succeed()) - g.Expect(createdLocalQueue.Status.Flavors).Should(gomega.HaveLen(2)) - for _, flavor := range createdLocalQueue.Status.Flavors { - g.Expect(flavor.Topology).ShouldNot(gomega.BeNil()) - g.Expect(flavor.Topology.Levels).Should(gomega.Equal(utiltas.Levels(topology))) - } - }, util.Timeout, util.Interval).Should(gomega.Succeed()) - }) - ginkgo.It("should admit workload which fits in a required topology domain", func() { var wl1 *kueue.Workload ginkgo.By("creating a workload which requires block and can fit", func() { From 825c4e3c7444ffeba2d0e47692b9af581639d72a Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 31 Oct 2025 18:42:18 +0530 Subject: [PATCH 070/119] v1beta2: remove all unnecessary wrappers for v1beta1 (#7481) --- pkg/util/testing/v1beta1/wrappers.go | 1156 +------------------------- 1 file changed, 26 insertions(+), 1130 deletions(-) diff --git a/pkg/util/testing/v1beta1/wrappers.go b/pkg/util/testing/v1beta1/wrappers.go index 777ea0e724c..1e52033729c 100644 --- a/pkg/util/testing/v1beta1/wrappers.go +++ b/pkg/util/testing/v1beta1/wrappers.go @@ -18,7 +18,6 @@ package v1beta1 import ( "fmt" - "maps" "time" corev1 "k8s.io/api/core/v1" @@ -31,30 +30,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" - utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) -// MakeDefaultOneLevelTopology creates a default topology with hostname level. -func MakeDefaultOneLevelTopology(name string) *kueue.Topology { - return MakeTopology(name). - Levels(corev1.LabelHostname). - Obj() -} - -// MakeDefaultTwoLevelTopology creates a default topology with block and rack levels. -func MakeDefaultTwoLevelTopology(name string) *kueue.Topology { - return MakeTopology(name). - Levels(utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel). - Obj() -} - -// MakeDefaultThreeLevelTopology creates a default topology with block, rack and hostname levels. -func MakeDefaultThreeLevelTopology(name string) *kueue.Topology { - return MakeTopology(name). - Levels(utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel, corev1.LabelHostname). - Obj() -} - type WorkloadWrapper struct{ kueue.Workload } // MakeWorkload creates a wrapper for a Workload with a single pod @@ -70,14 +47,6 @@ func MakeWorkload(name, ns string) *WorkloadWrapper { }} } -// MakeWorkloadWithGeneratedName creates a wrapper for a Workload with a single pod -// with a single container. -func MakeWorkloadWithGeneratedName(namePrefix, ns string) *WorkloadWrapper { - wl := MakeWorkload("", ns) - wl.GenerateName = namePrefix - return wl -} - func (w *WorkloadWrapper) Obj() *kueue.Workload { return &w.Workload } @@ -86,252 +55,44 @@ func (w *WorkloadWrapper) Clone() *WorkloadWrapper { return &WorkloadWrapper{Workload: *w.DeepCopy()} } -func (w *WorkloadWrapper) UID(uid types.UID) *WorkloadWrapper { - w.Workload.UID = uid - return w -} - -// Generation sets the generation of the Workload. -func (w *WorkloadWrapper) Generation(num int64) *WorkloadWrapper { - w.ObjectMeta.Generation = num - return w -} - -func (w *WorkloadWrapper) Name(name string) *WorkloadWrapper { - w.Workload.Name = name - return w -} - func (w *WorkloadWrapper) Finalizers(fin ...string) *WorkloadWrapper { w.ObjectMeta.Finalizers = fin return w } -func (w *WorkloadWrapper) Request(r corev1.ResourceName, q string) *WorkloadWrapper { - w.Spec.PodSets[0].Template.Spec.Containers[0].Resources.Requests[r] = resource.MustParse(q) - return w -} - -func (w *WorkloadWrapper) Limit(r corev1.ResourceName, q string) *WorkloadWrapper { - res := &w.Spec.PodSets[0].Template.Spec.Containers[0].Resources - if res.Limits == nil { - res.Limits = corev1.ResourceList{ - r: resource.MustParse(q), - } - } else { - res.Limits[r] = resource.MustParse(q) - } - return w -} - -func (w *WorkloadWrapper) RequestAndLimit(r corev1.ResourceName, q string) *WorkloadWrapper { - return w.Request(r, q).Limit(r, q) -} - func (w *WorkloadWrapper) Queue(q kueue.LocalQueueName) *WorkloadWrapper { w.Spec.QueueName = q return w } -func (w *WorkloadWrapper) Active(a bool) *WorkloadWrapper { - w.Spec.Active = ptr.To(a) - return w -} - -// SimpleReserveQuota reserves the quota for all the requested resources in one flavor. -// It assumes one podset with one container. -func (w *WorkloadWrapper) SimpleReserveQuota(cq, flavor string, now time.Time) *WorkloadWrapper { - admission := MakeAdmission(cq, w.Spec.PodSets[0].Name) - resReq := make(corev1.ResourceList) - flavors := make(map[corev1.ResourceName]kueue.ResourceFlavorReference) - for res, val := range w.Spec.PodSets[0].Template.Spec.Containers[0].Resources.Requests { - val.Mul(int64(w.Spec.PodSets[0].Count)) - resReq[res] = val - flavors[res] = kueue.ResourceFlavorReference(flavor) - } - admission.PodSetAssignments[0].Count = ptr.To(w.Spec.PodSets[0].Count) - admission.PodSetAssignments[0].Flavors = flavors - admission.PodSetAssignments[0].ResourceUsage = resReq - - return w.ReserveQuotaAt(admission.Obj(), now) -} - // ReserveQuota sets workload admission and adds a "QuotaReserved" status condition func (w *WorkloadWrapper) ReserveQuota(a *kueue.Admission) *WorkloadWrapper { - return w.ReserveQuotaAt(a, time.Now()) -} - -// ReserveQuotaAt sets workload admission and adds a "QuotaReserved" status condition -func (w *WorkloadWrapper) ReserveQuotaAt(a *kueue.Admission, now time.Time) *WorkloadWrapper { w.Status.Admission = a w.Status.Conditions = []metav1.Condition{{ Type: kueue.WorkloadQuotaReserved, Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(now), + LastTransitionTime: metav1.NewTime(time.Now()), Reason: "AdmittedByTest", Message: fmt.Sprintf("Admitted by ClusterQueue %s", w.Status.Admission.ClusterQueue), }} return w } -// QuotaReservedTime - sets the LastTransitionTime of the QuotaReserved condition if found. -func (w *WorkloadWrapper) QuotaReservedTime(t time.Time) *WorkloadWrapper { - cond := apimeta.FindStatusCondition(w.Status.Conditions, kueue.WorkloadQuotaReserved) - cond.LastTransitionTime = metav1.NewTime(t) - return w -} - -func (w *WorkloadWrapper) Admitted(a bool) *WorkloadWrapper { - return w.AdmittedAt(a, time.Now()) -} - -func (w *WorkloadWrapper) AdmittedAt(a bool, t time.Time) *WorkloadWrapper { - cond := metav1.Condition{ - Type: kueue.WorkloadAdmitted, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(t), - Reason: "ByTest", - Message: fmt.Sprintf("Admitted by ClusterQueue %s", w.Status.Admission.ClusterQueue), - } - if !a { - cond.Status = metav1.ConditionFalse - } - apimeta.SetStatusCondition(&w.Status.Conditions, cond) - return w -} - -func (w *WorkloadWrapper) Finished() *WorkloadWrapper { - return w.FinishedAt(time.Now()) -} - -func (w *WorkloadWrapper) FinishedAt(t time.Time) *WorkloadWrapper { - cond := metav1.Condition{ - Type: kueue.WorkloadFinished, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(t), - Reason: "ByTest", - Message: "Finished by test", - } - apimeta.SetStatusCondition(&w.Status.Conditions, cond) - return w -} - -func (w *WorkloadWrapper) Evicted() *WorkloadWrapper { - cond := metav1.Condition{ - Type: kueue.WorkloadEvicted, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.Now(), - Reason: "ByTest", - Message: "Evicted by test", - } - apimeta.SetStatusCondition(&w.Status.Conditions, cond) - return w -} - -func (w *WorkloadWrapper) Creation(t time.Time) *WorkloadWrapper { - w.CreationTimestamp = metav1.NewTime(t) - return w -} - -func (w *WorkloadWrapper) PriorityClass(priorityClassName string) *WorkloadWrapper { - w.Spec.PriorityClassName = priorityClassName - return w -} - -func (w *WorkloadWrapper) RuntimeClass(name string) *WorkloadWrapper { - for i := range w.Spec.PodSets { - w.Spec.PodSets[i].Template.Spec.RuntimeClassName = &name - } - return w -} - -func (w *WorkloadWrapper) Priority(priority int32) *WorkloadWrapper { - w.Spec.Priority = &priority - return w -} - -func (w *WorkloadWrapper) PriorityClassSource(source string) *WorkloadWrapper { - w.Spec.PriorityClassSource = source - return w -} - func (w *WorkloadWrapper) PodSets(podSets ...kueue.PodSet) *WorkloadWrapper { w.Spec.PodSets = podSets return w } -func (w *WorkloadWrapper) Toleration(t corev1.Toleration) *WorkloadWrapper { - w.Spec.PodSets[0].Template.Spec.Tolerations = append(w.Spec.PodSets[0].Template.Spec.Tolerations, t) - return w -} - -func (w *WorkloadWrapper) NodeSelector(kv map[string]string) *WorkloadWrapper { - w.Spec.PodSets[0].Template.Spec.NodeSelector = kv - return w -} - func (w *WorkloadWrapper) Condition(condition metav1.Condition) *WorkloadWrapper { apimeta.SetStatusCondition(&w.Status.Conditions, condition) return w } -func (w *WorkloadWrapper) AdmissionCheck(ac kueue.AdmissionCheckState) *WorkloadWrapper { - w.Status.AdmissionChecks = append(w.Status.AdmissionChecks, ac) - return w -} - -func (w *WorkloadWrapper) ResourceRequests(rr ...kueue.PodSetRequest) *WorkloadWrapper { - w.Status.ResourceRequests = rr - return w -} - -func (w *WorkloadWrapper) SetOrReplaceCondition(condition metav1.Condition) *WorkloadWrapper { - existingCondition := apimeta.FindStatusCondition(w.Status.Conditions, condition.Type) - if existingCondition != nil { - apimeta.RemoveStatusCondition(&w.Status.Conditions, condition.Type) - } - apimeta.SetStatusCondition(&w.Status.Conditions, condition) - return w -} - -func (w *WorkloadWrapper) ReclaimablePods(rps ...kueue.ReclaimablePod) *WorkloadWrapper { - w.Status.ReclaimablePods = rps - return w -} - -func (w *WorkloadWrapper) Labels(l map[string]string) *WorkloadWrapper { - w.ObjectMeta.Labels = l - return w -} - func (w *WorkloadWrapper) Label(k, v string) *WorkloadWrapper { - if w.ObjectMeta.Labels == nil { - w.ObjectMeta.Labels = make(map[string]string) + if w.Labels == nil { + w.Labels = make(map[string]string) } - w.ObjectMeta.Labels[k] = v - return w -} - -func (w *WorkloadWrapper) Annotation(k, v string) *WorkloadWrapper { - if w.ObjectMeta.Annotations == nil { - w.ObjectMeta.Annotations = make(map[string]string) - } - w.ObjectMeta.Annotations[k] = v - return w -} - -func (w *WorkloadWrapper) AdmissionChecks(checks ...kueue.AdmissionCheckState) *WorkloadWrapper { - w.Status.AdmissionChecks = checks - return w -} - -func (w *WorkloadWrapper) Admission(admission *kueue.Admission) *WorkloadWrapper { - w.Status.Admission = admission - return w -} - -func (w *WorkloadWrapper) Conditions(conditions ...metav1.Condition) *WorkloadWrapper { - w.Status.Conditions = conditions + w.Labels[k] = v return w } @@ -340,79 +101,6 @@ func (w *WorkloadWrapper) ControllerReference(gvk schema.GroupVersionKind, name, return w } -func (w *WorkloadWrapper) OwnerReference(gvk schema.GroupVersionKind, name, uid string) *WorkloadWrapper { - AppendOwnerReference(&w.Workload, gvk, name, uid, nil, nil) - return w -} - -func (w *WorkloadWrapper) Annotations(annotations map[string]string) *WorkloadWrapper { - w.ObjectMeta.Annotations = annotations - return w -} - -func (w *WorkloadWrapper) UnhealthyNodes(nodeNames ...string) *WorkloadWrapper { - for _, nodeName := range nodeNames { - w.Status.UnhealthyNodes = append(w.Status.UnhealthyNodes, kueue.UnhealthyNode{Name: nodeName}) - } - return w -} - -// DeletionTimestamp sets a deletion timestamp for the workload. -func (w *WorkloadWrapper) DeletionTimestamp(t time.Time) *WorkloadWrapper { - w.Workload.DeletionTimestamp = ptr.To(metav1.NewTime(t).Rfc3339Copy()) - return w -} - -func (w *WorkloadWrapper) RequeueState(count *int32, requeueAt *metav1.Time) *WorkloadWrapper { - if count == nil && requeueAt == nil { - w.Status.RequeueState = nil - return w - } - if w.Status.RequeueState == nil { - w.Status.RequeueState = &kueue.RequeueState{} - } - if count != nil { - w.Status.RequeueState.Count = count - } - if requeueAt != nil { - w.Status.RequeueState.RequeueAt = requeueAt - } - return w -} - -func (w *WorkloadWrapper) ResourceVersion(v string) *WorkloadWrapper { - w.SetResourceVersion(v) - return w -} - -func (w *WorkloadWrapper) MaximumExecutionTimeSeconds(v int32) *WorkloadWrapper { - w.Spec.MaximumExecutionTimeSeconds = &v - return w -} - -func (w *WorkloadWrapper) PastAdmittedTime(v int32) *WorkloadWrapper { - w.Status.AccumulatedPastExexcutionTimeSeconds = &v - return w -} - -func (w *WorkloadWrapper) SchedulingStatsEviction(evictionState kueue.WorkloadSchedulingStatsEviction) *WorkloadWrapper { - if w.Status.SchedulingStats == nil { - w.Status.SchedulingStats = &kueue.SchedulingStats{} - } - w.Status.SchedulingStats.Evictions = append(w.Status.SchedulingStats.Evictions, evictionState) - return w -} - -func (w *WorkloadWrapper) ClusterName(clusterName string) *WorkloadWrapper { - w.Status.ClusterName = &clusterName - return w -} - -func (w *WorkloadWrapper) NominatedClusterNames(nominatedClusterNames ...string) *WorkloadWrapper { - w.Status.NominatedClusterNames = nominatedClusterNames - return w -} - func AppendOwnerReference(obj client.Object, gvk schema.GroupVersionKind, name, uid string, controller, blockDeletion *bool) { obj.SetOwnerReferences(append(obj.GetOwnerReferences(), metav1.OwnerReference{ APIVersion: gvk.GroupVersion().String(), @@ -448,66 +136,6 @@ func MakePodSet(name kueue.PodSetReference, count int) *PodSetWrapper { } } -func (p *PodSetWrapper) PodSpec(ps corev1.PodSpec) *PodSetWrapper { - p.Template.Spec = ps - return p -} - -func (p *PodSetWrapper) PriorityClass(pc string) *PodSetWrapper { - p.Template.Spec.PriorityClassName = pc - return p -} - -func (p *PodSetWrapper) RuntimeClass(name string) *PodSetWrapper { - p.Template.Spec.RuntimeClassName = &name - return p -} - -func (p *PodSetWrapper) RestartPolicy(policy corev1.RestartPolicy) *PodSetWrapper { - p.Template.Spec.RestartPolicy = policy - return p -} - -func (p *PodSetWrapper) RequiredTopologyRequest(level string) *PodSetWrapper { - if p.TopologyRequest == nil { - p.TopologyRequest = &kueue.PodSetTopologyRequest{} - } - p.TopologyRequest.Required = &level - return p -} - -func (p *PodSetWrapper) PodSetGroup(name string) *PodSetWrapper { - if p.TopologyRequest == nil { - p.TopologyRequest = &kueue.PodSetTopologyRequest{} - } - p.TopologyRequest.PodSetGroupName = &name - return p -} - -func (p *PodSetWrapper) SliceRequiredTopologyRequest(level string) *PodSetWrapper { - if p.TopologyRequest == nil { - p.TopologyRequest = &kueue.PodSetTopologyRequest{} - } - p.TopologyRequest.PodSetSliceRequiredTopology = &level - return p -} - -func (p *PodSetWrapper) SliceSizeTopologyRequest(size int32) *PodSetWrapper { - if p.TopologyRequest == nil { - p.TopologyRequest = &kueue.PodSetTopologyRequest{} - } - p.TopologyRequest.PodSetSliceSize = &size - return p -} - -func (p *PodSetWrapper) PreferredTopologyRequest(level string) *PodSetWrapper { - if p.TopologyRequest == nil { - p.TopologyRequest = &kueue.PodSetTopologyRequest{} - } - p.TopologyRequest.Preferred = &level - return p -} - func (p *PodSetWrapper) PodIndexLabel(label *string) *PodSetWrapper { if p.TopologyRequest == nil { p.TopologyRequest = &kueue.PodSetTopologyRequest{} @@ -516,143 +144,20 @@ func (p *PodSetWrapper) PodIndexLabel(label *string) *PodSetWrapper { return p } -func (p *PodSetWrapper) SubGroupIndexLabel(label *string) *PodSetWrapper { - if p.TopologyRequest == nil { - p.TopologyRequest = &kueue.PodSetTopologyRequest{} - } - p.TopologyRequest.SubGroupIndexLabel = label - return p -} - -func (p *PodSetWrapper) SubGroupCount(count *int32) *PodSetWrapper { - if p.TopologyRequest == nil { - p.TopologyRequest = &kueue.PodSetTopologyRequest{} - } - p.TopologyRequest.SubGroupCount = count - return p -} - func (p *PodSetWrapper) Obj() *kueue.PodSet { return &p.PodSet } -func (p *PodSetWrapper) Clone() *PodSetWrapper { - return &PodSetWrapper{PodSet: *p.DeepCopy()} -} - func (p *PodSetWrapper) Request(r corev1.ResourceName, q string) *PodSetWrapper { p.Template.Spec.Containers[0].Resources.Requests[r] = resource.MustParse(q) return p } -func (p *PodSetWrapper) Limit(r corev1.ResourceName, q string) *PodSetWrapper { - if p.Template.Spec.Containers[0].Resources.Limits == nil { - p.Template.Spec.Containers[0].Resources.Limits = corev1.ResourceList{} - } - p.Template.Spec.Containers[0].Resources.Limits[r] = resource.MustParse(q) - return p -} - func (p *PodSetWrapper) Image(image string) *PodSetWrapper { p.Template.Spec.Containers[0].Image = image return p } -func (p *PodSetWrapper) SetMinimumCount(mc int32) *PodSetWrapper { - p.MinCount = &mc - return p -} - -func (p *PodSetWrapper) Toleration(t corev1.Toleration) *PodSetWrapper { - p.Template.Spec.Tolerations = append(p.Template.Spec.Tolerations, t) - return p -} - -func (p *PodSetWrapper) Containers(containers ...corev1.Container) *PodSetWrapper { - p.Template.Spec.Containers = containers - return p -} - -func (p *PodSetWrapper) InitContainers(containers ...corev1.Container) *PodSetWrapper { - p.Template.Spec.InitContainers = containers - return p -} - -func (p *PodSetWrapper) NodeSelector(kv map[string]string) *PodSetWrapper { - p.Template.Spec.NodeSelector = kv - return p -} - -func (p *PodSetWrapper) RequiredDuringSchedulingIgnoredDuringExecution(nodeSelectorTerms []corev1.NodeSelectorTerm) *PodSetWrapper { - if p.Template.Spec.Affinity == nil { - p.Template.Spec.Affinity = &corev1.Affinity{} - } - if p.Template.Spec.Affinity.NodeAffinity == nil { - p.Template.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{} - } - if p.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { - p.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{} - } - p.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append( - p.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, - nodeSelectorTerms..., - ) - return p -} - -func (p *PodSetWrapper) NodeName(name string) *PodSetWrapper { - p.Template.Spec.NodeName = name - return p -} - -func (p *PodSetWrapper) Labels(kv map[string]string) *PodSetWrapper { - p.Template.Labels = kv - return p -} - -func (p *PodSetWrapper) Annotations(kv map[string]string) *PodSetWrapper { - p.Template.Annotations = kv - return p -} - -func (p *PodSetWrapper) SchedulingGates(sg ...corev1.PodSchedulingGate) *PodSetWrapper { - p.Template.Spec.SchedulingGates = sg - return p -} - -func (p *PodSetWrapper) PodOverHead(resources corev1.ResourceList) *PodSetWrapper { - p.Template.Spec.Overhead = resources - return p -} - -func (p *PodSetWrapper) ResourceClaimTemplate(claimName, templateName string) *PodSetWrapper { - p.Template.Spec.ResourceClaims = append(p.Template.Spec.ResourceClaims, corev1.PodResourceClaim{ - Name: claimName, - ResourceClaimTemplateName: ptr.To(templateName), - }) - if len(p.Template.Spec.Containers) > 0 { - p.Template.Spec.Containers[0].Resources.Claims = append( - p.Template.Spec.Containers[0].Resources.Claims, - corev1.ResourceClaim{Name: claimName}, - ) - } - return p -} - -func (p *PodSetWrapper) ResourceClaim(claimName, resourceClaimName string) *PodSetWrapper { - p.Template.Spec.ResourceClaims = append(p.Template.Spec.ResourceClaims, corev1.PodResourceClaim{ - Name: claimName, - ResourceClaimName: ptr.To(resourceClaimName), - }) - if len(p.Template.Spec.Containers) > 0 { - p.Template.Spec.Containers[0].Resources.Claims = append( - p.Template.Spec.Containers[0].Resources.Claims, - corev1.ResourceClaim{Name: claimName}, - ) - } - return p -} - // AdmissionWrapper wraps an Admission type AdmissionWrapper struct{ kueue.Admission } @@ -671,163 +176,42 @@ func MakeAdmission(cq string, podSetNames ...kueue.PodSetReference) *AdmissionWr var psFlavors []kueue.PodSetAssignment for _, name := range podSetNames { psFlavors = append(psFlavors, MakePodSetAssignment(name).Obj()) - } - wrap.PodSetAssignments = psFlavors - return wrap -} - -func (w *AdmissionWrapper) Obj() *kueue.Admission { - return &w.Admission -} - -func (w *AdmissionWrapper) PodSets(podSets ...kueue.PodSetAssignment) *AdmissionWrapper { - w.PodSetAssignments = podSets - return w -} - -// LocalQueueWrapper wraps a Queue. -type LocalQueueWrapper struct{ kueue.LocalQueue } - -// MakeLocalQueue creates a wrapper for a LocalQueue. -func MakeLocalQueue(name, ns string) *LocalQueueWrapper { - return &LocalQueueWrapper{kueue.LocalQueue{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - }, - }} -} - -// Creation sets the creation timestamp of the LocalQueue. -func (q *LocalQueueWrapper) Creation(t time.Time) *LocalQueueWrapper { - q.CreationTimestamp = metav1.NewTime(t) - return q -} - -// Label sets the label on the LocalQueue. -func (q *LocalQueueWrapper) Label(k, v string) *LocalQueueWrapper { - if q.Labels == nil { - q.Labels = make(map[string]string) - } - q.Labels[k] = v - return q -} - -// Obj returns the inner LocalQueue. -func (q *LocalQueueWrapper) Obj() *kueue.LocalQueue { - return &q.LocalQueue -} - -// ClusterQueue updates the clusterQueue the queue points to. -func (q *LocalQueueWrapper) ClusterQueue(c string) *LocalQueueWrapper { - q.Spec.ClusterQueue = kueue.ClusterQueueReference(c) - return q -} - -// StopPolicy sets the stop policy. -func (q *LocalQueueWrapper) StopPolicy(p kueue.StopPolicy) *LocalQueueWrapper { - q.Spec.StopPolicy = &p - return q -} - -// FairSharing sets the fair sharing config. -func (q *LocalQueueWrapper) FairSharing(fs *kueue.FairSharing) *LocalQueueWrapper { - q.Spec.FairSharing = fs - return q -} - -// PendingWorkloads updates the pendingWorkloads in status. -func (q *LocalQueueWrapper) PendingWorkloads(n int32) *LocalQueueWrapper { - q.Status.PendingWorkloads = n - return q -} - -// ReservingWorkloads updates the reservingWorkloads in status. -func (q *LocalQueueWrapper) ReservingWorkloads(n int32) *LocalQueueWrapper { - q.Status.ReservingWorkloads = n - return q -} - -// AdmittedWorkloads updates the admittedWorkloads in status. -func (q *LocalQueueWrapper) AdmittedWorkloads(n int32) *LocalQueueWrapper { - q.Status.AdmittedWorkloads = n - return q -} - -// Condition sets a condition on the LocalQueue. -func (q *LocalQueueWrapper) Condition(conditionType string, status metav1.ConditionStatus, reason, message string, generation int64) *LocalQueueWrapper { - apimeta.SetStatusCondition(&q.Status.Conditions, metav1.Condition{ - Type: conditionType, - Status: status, - Reason: reason, - Message: message, - ObservedGeneration: generation, - }) - return q -} - -func (q *LocalQueueWrapper) Active(status metav1.ConditionStatus) *LocalQueueWrapper { - apimeta.SetStatusCondition(&q.Status.Conditions, metav1.Condition{ - Type: kueue.LocalQueueActive, - Status: status, - Reason: "Ready", - Message: "Can submit new workloads to localQueue", - }) - return q -} - -// FairSharingStatus updates the fairSharing in status. -func (q *LocalQueueWrapper) FairSharingStatus(status *kueue.FairSharingStatus) *LocalQueueWrapper { - q.Status.FairSharing = status - return q + } + wrap.PodSetAssignments = psFlavors + return wrap } -// Generation sets the generation of the LocalQueue. -func (q *LocalQueueWrapper) Generation(num int64) *LocalQueueWrapper { - q.ObjectMeta.Generation = num - return q +func (w *AdmissionWrapper) Obj() *kueue.Admission { + return &w.Admission } -// GeneratedName sets the prefix for the server to generate unique name. -// No name should be given in the MakeClusterQueue for the GeneratedName to work. -func (q *LocalQueueWrapper) GeneratedName(name string) *LocalQueueWrapper { - q.GenerateName = name - return q +func (w *AdmissionWrapper) PodSets(podSets ...kueue.PodSetAssignment) *AdmissionWrapper { + w.PodSetAssignments = podSets + return w } -type CohortWrapper struct { - kueue.Cohort -} +// LocalQueueWrapper wraps a Queue. +type LocalQueueWrapper struct{ kueue.LocalQueue } -func MakeCohort(name kueue.CohortReference) *CohortWrapper { - return &CohortWrapper{kueue.Cohort{ +// MakeLocalQueue creates a wrapper for a LocalQueue. +func MakeLocalQueue(name, ns string) *LocalQueueWrapper { + return &LocalQueueWrapper{kueue.LocalQueue{ ObjectMeta: metav1.ObjectMeta{ - Name: string(name), + Name: name, + Namespace: ns, }, }} } -func (c *CohortWrapper) Obj() *kueue.Cohort { - return &c.Cohort -} - -func (c *CohortWrapper) Parent(parentName kueue.CohortReference) *CohortWrapper { - c.Spec.ParentName = parentName - return c -} - -// ResourceGroup adds a ResourceGroup with flavors. -func (c *CohortWrapper) ResourceGroup(flavors ...kueue.FlavorQuotas) *CohortWrapper { - c.Spec.ResourceGroups = append(c.Spec.ResourceGroups, ResourceGroup(flavors...)) - return c +// Obj returns the inner LocalQueue. +func (q *LocalQueueWrapper) Obj() *kueue.LocalQueue { + return &q.LocalQueue } -func (c *CohortWrapper) FairWeight(w resource.Quantity) *CohortWrapper { - if c.Spec.FairSharing == nil { - c.Spec.FairSharing = &kueue.FairSharing{} - } - c.Spec.FairSharing.Weight = &w - return c +// ClusterQueue updates the clusterQueue the queue points to. +func (q *LocalQueueWrapper) ClusterQueue(c string) *LocalQueueWrapper { + q.Spec.ClusterQueue = kueue.ClusterQueueReference(c) + return q } // ClusterQueueWrapper wraps a ClusterQueue. @@ -851,10 +235,6 @@ func MakeClusterQueue(name string) *ClusterQueueWrapper { }} } -func (c *ClusterQueueWrapper) Clone() *ClusterQueueWrapper { - return &ClusterQueueWrapper{ClusterQueue: *c.DeepCopy()} -} - // Obj returns the inner ClusterQueue. func (c *ClusterQueueWrapper) Obj() *kueue.ClusterQueue { return &c.ClusterQueue @@ -866,39 +246,6 @@ func (c *ClusterQueueWrapper) Cohort(cohort kueue.CohortReference) *ClusterQueue return c } -func (c *ClusterQueueWrapper) AdmissionCheckStrategy(acs ...kueue.AdmissionCheckStrategyRule) *ClusterQueueWrapper { - if c.Spec.AdmissionChecksStrategy == nil { - c.Spec.AdmissionChecksStrategy = &kueue.AdmissionChecksStrategy{} - } - c.Spec.AdmissionChecksStrategy.AdmissionChecks = acs - return c -} - -func (c *ClusterQueueWrapper) AdmissionMode(am kueue.AdmissionMode) *ClusterQueueWrapper { - if c.Spec.AdmissionScope == nil { - c.Spec.AdmissionScope = &kueue.AdmissionScope{} - } - c.Spec.AdmissionScope.AdmissionMode = am - return c -} - -func (c *ClusterQueueWrapper) Active(status metav1.ConditionStatus) *ClusterQueueWrapper { - apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{ - Type: kueue.ClusterQueueActive, - Status: status, - Reason: "By test", - Message: "by test", - }) - return c -} - -// GeneratedName sets the prefix for the server to generate unique name. -// No name should be given in the MakeClusterQueue for the GeneratedName to work. -func (c *ClusterQueueWrapper) GeneratedName(name string) *ClusterQueueWrapper { - c.GenerateName = name - return c -} - // ResourceGroup creates a ResourceGroup with the given FlavorQuotas. func ResourceGroup(flavors ...kueue.FlavorQuotas) kueue.ResourceGroup { rg := kueue.ResourceGroup{ @@ -930,99 +277,12 @@ func (c *ClusterQueueWrapper) ResourceGroup(flavors ...kueue.FlavorQuotas) *Clus return c } -// AdmissionChecks replaces the queue additional checks -func (c *ClusterQueueWrapper) AdmissionChecks(checks ...kueue.AdmissionCheckReference) *ClusterQueueWrapper { - c.Spec.AdmissionChecks = checks - return c -} - -// QueueingStrategy sets the queueing strategy in this ClusterQueue. -func (c *ClusterQueueWrapper) QueueingStrategy(strategy kueue.QueueingStrategy) *ClusterQueueWrapper { - c.Spec.QueueingStrategy = strategy - return c -} - -// NamespaceSelector sets the namespace selector. -func (c *ClusterQueueWrapper) NamespaceSelector(s *metav1.LabelSelector) *ClusterQueueWrapper { - c.Spec.NamespaceSelector = s - return c -} - // Preemption sets the preemption policies. func (c *ClusterQueueWrapper) Preemption(p kueue.ClusterQueuePreemption) *ClusterQueueWrapper { c.Spec.Preemption = &p return c } -// FlavorFungibility sets the flavorFungibility policies. -func (c *ClusterQueueWrapper) FlavorFungibility(p kueue.FlavorFungibility) *ClusterQueueWrapper { - c.Spec.FlavorFungibility = &p - return c -} - -// StopPolicy sets the stop policy. -func (c *ClusterQueueWrapper) StopPolicy(p kueue.StopPolicy) *ClusterQueueWrapper { - c.Spec.StopPolicy = &p - return c -} - -// DeletionTimestamp sets a deletion timestamp for the cluster queue. -func (c *ClusterQueueWrapper) DeletionTimestamp(t time.Time) *ClusterQueueWrapper { - c.ClusterQueue.DeletionTimestamp = ptr.To(metav1.NewTime(t).Rfc3339Copy()) - return c -} - -func (c *ClusterQueueWrapper) Label(k, v string) *ClusterQueueWrapper { - if c.Labels == nil { - c.Labels = make(map[string]string) - } - c.Labels[k] = v - return c -} - -func (c *ClusterQueueWrapper) FairWeight(w resource.Quantity) *ClusterQueueWrapper { - if c.Spec.FairSharing == nil { - c.Spec.FairSharing = &kueue.FairSharing{} - } - c.Spec.FairSharing.Weight = &w - return c -} - -// Condition sets a condition on the ClusterQueue. -func (c *ClusterQueueWrapper) Condition(conditionType string, status metav1.ConditionStatus, reason, message string) *ClusterQueueWrapper { - apimeta.SetStatusCondition(&c.Status.Conditions, metav1.Condition{ - Type: conditionType, - Status: status, - Reason: reason, - Message: message, - }) - return c -} - -// Generation sets the generation of the ClusterQueue. -func (c *ClusterQueueWrapper) Generation(num int64) *ClusterQueueWrapper { - c.ObjectMeta.Generation = num - return c -} - -// Creation sets the creation timestamp of the ClusterQueue. -func (c *ClusterQueueWrapper) Creation(t time.Time) *ClusterQueueWrapper { - c.CreationTimestamp = metav1.NewTime(t) - return c -} - -// PendingWorkloads sets the pendingWorkloads in status. -func (c *ClusterQueueWrapper) PendingWorkloads(n int32) *ClusterQueueWrapper { - c.Status.PendingWorkloads = n - return c -} - -// AdmittedWorkloads sets the admittedWorkloads in status. -func (c *ClusterQueueWrapper) AdmittedWorkloads(n int32) *ClusterQueueWrapper { - c.Status.AdmittedWorkloads = n - return c -} - // FlavorQuotasWrapper wraps a FlavorQuotas object. type FlavorQuotasWrapper struct{ kueue.FlavorQuotas } @@ -1111,120 +371,12 @@ func (rf *ResourceFlavorWrapper) Obj() *kueue.ResourceFlavor { return &rf.ResourceFlavor } -// TopologyName sets the topology name -func (rf *ResourceFlavorWrapper) TopologyName(name string) *ResourceFlavorWrapper { - rf.Spec.TopologyName = ptr.To(kueue.TopologyReference(name)) - return rf -} - -// Label sets the label on the ResourceFlavor. -func (rf *ResourceFlavorWrapper) Label(k, v string) *ResourceFlavorWrapper { - if rf.Labels == nil { - rf.Labels = map[string]string{} - } - rf.Labels[k] = v - return rf -} - // NodeLabel add a label kueue and value pair to the ResourceFlavor. func (rf *ResourceFlavorWrapper) NodeLabel(k, v string) *ResourceFlavorWrapper { rf.Spec.NodeLabels[k] = v return rf } -// Taint adds a taint to the ResourceFlavor. -func (rf *ResourceFlavorWrapper) Taint(t corev1.Taint) *ResourceFlavorWrapper { - rf.Spec.NodeTaints = append(rf.Spec.NodeTaints, t) - return rf -} - -// Toleration adds a taint to the ResourceFlavor. -func (rf *ResourceFlavorWrapper) Toleration(t corev1.Toleration) *ResourceFlavorWrapper { - rf.Spec.Tolerations = append(rf.Spec.Tolerations, t) - return rf -} - -// Creation sets the creation timestamp of the LocalQueue. -func (rf *ResourceFlavorWrapper) Creation(t time.Time) *ResourceFlavorWrapper { - rf.CreationTimestamp = metav1.NewTime(t) - return rf -} - -// TopologyWrapper wraps a Topology. -type TopologyWrapper struct{ kueue.Topology } - -// MakeTopology creates a wrapper for a Topology. -func MakeTopology(name string) *TopologyWrapper { - return &TopologyWrapper{kueue.Topology{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - }} -} - -// Levels sets the levels for a Topology. -func (t *TopologyWrapper) Levels(levels ...string) *TopologyWrapper { - t.Spec.Levels = make([]kueue.TopologyLevel, len(levels)) - for i, level := range levels { - t.Spec.Levels[i] = kueue.TopologyLevel{ - NodeLabel: level, - } - } - return t -} - -func (t *TopologyWrapper) Obj() *kueue.Topology { - return &t.Topology -} - -type TopologyDomainAssignmentWrapper struct { - kueue.TopologyDomainAssignment -} - -func MakeTopologyDomainAssignment(values []string, count int32) *TopologyDomainAssignmentWrapper { - return &TopologyDomainAssignmentWrapper{ - TopologyDomainAssignment: kueue.TopologyDomainAssignment{ - Values: values, - Count: count, - }, - } -} - -func (t *TopologyDomainAssignmentWrapper) Obj() kueue.TopologyDomainAssignment { - return t.TopologyDomainAssignment -} - -type TopologyAssignmentWrapper struct { - kueue.TopologyAssignment -} - -func MakeTopologyAssignment(levels []string) *TopologyAssignmentWrapper { - return &TopologyAssignmentWrapper{ - TopologyAssignment: kueue.TopologyAssignment{ - Levels: levels, - }, - } -} - -func (t *TopologyAssignmentWrapper) Levels(levels ...string) *TopologyAssignmentWrapper { - t.TopologyAssignment.Levels = levels - return t -} - -func (t *TopologyAssignmentWrapper) Domains(domains ...kueue.TopologyDomainAssignment) *TopologyAssignmentWrapper { - t.TopologyAssignment.Domains = domains - return t -} - -func (t *TopologyAssignmentWrapper) Domain(domain kueue.TopologyDomainAssignment) *TopologyAssignmentWrapper { - t.TopologyAssignment.Domains = append(t.TopologyAssignment.Domains, domain) - return t -} - -func (t *TopologyAssignmentWrapper) Obj() *kueue.TopologyAssignment { - return &t.TopologyAssignment -} - type PodSetAssignmentWrapper struct { kueue.PodSetAssignment } @@ -1260,101 +412,10 @@ func (p *PodSetAssignmentWrapper) ResourceUsage(resourceName corev1.ResourceName return p } -func (p *PodSetAssignmentWrapper) Count(count int32) *PodSetAssignmentWrapper { - p.PodSetAssignment.Count = ptr.To(count) - return p -} - -func (p *PodSetAssignmentWrapper) TopologyAssignment(ta *kueue.TopologyAssignment) *PodSetAssignmentWrapper { - p.PodSetAssignment.TopologyAssignment = ta - return p -} - -func (p *PodSetAssignmentWrapper) DelayedTopologyRequest(state kueue.DelayedTopologyRequestState) *PodSetAssignmentWrapper { - p.PodSetAssignment.DelayedTopologyRequest = ptr.To(state) - return p -} - func (p *PodSetAssignmentWrapper) Assignment(r corev1.ResourceName, f kueue.ResourceFlavorReference, value string) *PodSetAssignmentWrapper { return p.Flavor(r, f).ResourceUsage(r, value) } -type AdmissionCheckWrapper struct{ kueue.AdmissionCheck } - -func MakeAdmissionCheck(name string) *AdmissionCheckWrapper { - return &AdmissionCheckWrapper{ - AdmissionCheck: kueue.AdmissionCheck{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - }, - } -} - -type AdmissionCheckStrategyRuleWrapper struct { - kueue.AdmissionCheckStrategyRule -} - -func MakeAdmissionCheckStrategyRule(name kueue.AdmissionCheckReference, flavors ...kueue.ResourceFlavorReference) *AdmissionCheckStrategyRuleWrapper { - if len(flavors) == 0 { - flavors = make([]kueue.ResourceFlavorReference, 0) - } - return &AdmissionCheckStrategyRuleWrapper{ - AdmissionCheckStrategyRule: kueue.AdmissionCheckStrategyRule{ - Name: name, - OnFlavors: flavors, - }, - } -} - -func (acs *AdmissionCheckStrategyRuleWrapper) OnFlavors(flavors []kueue.ResourceFlavorReference) *AdmissionCheckStrategyRuleWrapper { - acs.AdmissionCheckStrategyRule.OnFlavors = flavors - return acs -} - -func (acs *AdmissionCheckStrategyRuleWrapper) Obj() *kueue.AdmissionCheckStrategyRule { - return &acs.AdmissionCheckStrategyRule -} - -func (ac *AdmissionCheckWrapper) Active(status metav1.ConditionStatus) *AdmissionCheckWrapper { - apimeta.SetStatusCondition(&ac.Status.Conditions, metav1.Condition{ - Type: kueue.AdmissionCheckActive, - Status: status, - Reason: "ByTest", - Message: "by test", - }) - return ac -} - -func (ac *AdmissionCheckWrapper) Condition(cond metav1.Condition) *AdmissionCheckWrapper { - apimeta.SetStatusCondition(&ac.Status.Conditions, cond) - return ac -} - -// Generation sets the generation of the AdmissionCheck. -func (ac *AdmissionCheckWrapper) Generation(num int64) *AdmissionCheckWrapper { - ac.ObjectMeta.Generation = num - return ac -} - -func (ac *AdmissionCheckWrapper) ControllerName(c string) *AdmissionCheckWrapper { - ac.Spec.ControllerName = c - return ac -} - -func (ac *AdmissionCheckWrapper) Parameters(apigroup, kind, name string) *AdmissionCheckWrapper { - ac.Spec.Parameters = &kueue.AdmissionCheckParametersReference{ - APIGroup: apigroup, - Kind: kind, - Name: name, - } - return ac -} - -func (ac *AdmissionCheckWrapper) Obj() *kueue.AdmissionCheck { - return &ac.AdmissionCheck -} - // WorkloadPriorityClassWrapper wraps a WorkloadPriorityClass. type WorkloadPriorityClassWrapper struct { kueue.WorkloadPriorityClass @@ -1379,168 +440,3 @@ func (p *WorkloadPriorityClassWrapper) PriorityValue(v int32) *WorkloadPriorityC func (p *WorkloadPriorityClassWrapper) Obj() *kueue.WorkloadPriorityClass { return &p.WorkloadPriorityClass } - -type MultiKueueConfigWrapper struct { - kueue.MultiKueueConfig -} - -func MakeMultiKueueConfig(name string) *MultiKueueConfigWrapper { - return &MultiKueueConfigWrapper{ - MultiKueueConfig: kueue.MultiKueueConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - }, - } -} - -func (mkc *MultiKueueConfigWrapper) Obj() *kueue.MultiKueueConfig { - return &mkc.MultiKueueConfig -} - -func (mkc *MultiKueueConfigWrapper) Clusters(clusters ...string) *MultiKueueConfigWrapper { - mkc.Spec.Clusters = append(mkc.Spec.Clusters, clusters...) - return mkc -} - -type MultiKueueClusterWrapper struct { - kueue.MultiKueueCluster -} - -func MakeMultiKueueCluster(name string) *MultiKueueClusterWrapper { - return &MultiKueueClusterWrapper{ - MultiKueueCluster: kueue.MultiKueueCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - }, - } -} - -func (mkc *MultiKueueClusterWrapper) Obj() *kueue.MultiKueueCluster { - return &mkc.MultiKueueCluster -} - -func (mkc *MultiKueueClusterWrapper) KubeConfig(locationType kueue.LocationType, location string) *MultiKueueClusterWrapper { - mkc.Spec.KubeConfig = kueue.KubeConfig{ - Location: location, - LocationType: locationType, - } - return mkc -} - -func (mkc *MultiKueueClusterWrapper) Active(state metav1.ConditionStatus, reason, message string, generation int64) *MultiKueueClusterWrapper { - cond := metav1.Condition{ - Type: kueue.MultiKueueClusterActive, - Status: state, - Reason: reason, - Message: message, - ObservedGeneration: generation, - } - apimeta.SetStatusCondition(&mkc.Status.Conditions, cond) - return mkc -} - -// Generation sets the generation of the MultiKueueCluster. -func (mkc *MultiKueueClusterWrapper) Generation(num int64) *MultiKueueClusterWrapper { - mkc.ObjectMeta.Generation = num - return mkc -} - -// ProvisioningRequestConfigWrapper wraps a ProvisioningRequestConfig -type ProvisioningRequestConfigWrapper struct { - kueue.ProvisioningRequestConfig -} - -// MakeProvisioningRequestConfig creates a wrapper for a ProvisioningRequestConfig. -func MakeProvisioningRequestConfig(name string) *ProvisioningRequestConfigWrapper { - return &ProvisioningRequestConfigWrapper{kueue.ProvisioningRequestConfig{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }}, - } -} - -func (prc *ProvisioningRequestConfigWrapper) ProvisioningClass(pc string) *ProvisioningRequestConfigWrapper { - prc.Spec.ProvisioningClassName = pc - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) Parameters(parameters map[string]kueue.Parameter) *ProvisioningRequestConfigWrapper { - if prc.Spec.Parameters == nil { - prc.Spec.Parameters = make(map[string]kueue.Parameter, len(parameters)) - } - - maps.Copy(prc.Spec.Parameters, parameters) - - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) WithParameter(key string, value kueue.Parameter) *ProvisioningRequestConfigWrapper { - if prc.Spec.Parameters == nil { - prc.Spec.Parameters = make(map[string]kueue.Parameter, 1) - } - - prc.Spec.Parameters[key] = value - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) ManagedResources(r []corev1.ResourceName) *ProvisioningRequestConfigWrapper { - prc.Spec.ManagedResources = r - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) WithManagedResource(managedResource corev1.ResourceName) *ProvisioningRequestConfigWrapper { - prc.Spec.ManagedResources = append(prc.Spec.ManagedResources, managedResource) - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) RetryStrategy(retryStrategy *kueue.ProvisioningRequestRetryStrategy) *ProvisioningRequestConfigWrapper { - prc.Spec.RetryStrategy = retryStrategy - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) PodSetUpdate(update kueue.ProvisioningRequestPodSetUpdates) *ProvisioningRequestConfigWrapper { - prc.Spec.PodSetUpdates = &update - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) BaseBackoff(backoffBaseSeconds int32) *ProvisioningRequestConfigWrapper { - if prc.Spec.RetryStrategy == nil { - prc.Spec.RetryStrategy = &kueue.ProvisioningRequestRetryStrategy{} - } - - prc.Spec.RetryStrategy.BackoffBaseSeconds = &backoffBaseSeconds - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) MaxBackoff(backoffMaxSeconds int32) *ProvisioningRequestConfigWrapper { - if prc.Spec.RetryStrategy == nil { - prc.Spec.RetryStrategy = &kueue.ProvisioningRequestRetryStrategy{} - } - - prc.Spec.RetryStrategy.BackoffMaxSeconds = &backoffMaxSeconds - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) RetryLimit(backoffLimitCount int32) *ProvisioningRequestConfigWrapper { - if prc.Spec.RetryStrategy == nil { - prc.Spec.RetryStrategy = &kueue.ProvisioningRequestRetryStrategy{} - } - - prc.Spec.RetryStrategy.BackoffLimitCount = &backoffLimitCount - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) PodSetMergePolicy(mode kueue.ProvisioningRequestConfigPodSetMergePolicy) *ProvisioningRequestConfigWrapper { - prc.Spec.PodSetMergePolicy = &mode - return prc -} - -func (prc *ProvisioningRequestConfigWrapper) Clone() *ProvisioningRequestConfigWrapper { - return &ProvisioningRequestConfigWrapper{ProvisioningRequestConfig: *prc.DeepCopy()} -} - -func (prc *ProvisioningRequestConfigWrapper) Obj() *kueue.ProvisioningRequestConfig { - return &prc.ProvisioningRequestConfig -} From 5225f29db1acccb6110fcf5667bcdcde73748c1d Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 31 Oct 2025 19:12:03 +0530 Subject: [PATCH 071/119] Replace preemption stub with interceptor function in TestSchedule. (#7473) --- pkg/scheduler/scheduler_test.go | 500 +++++++++++++++++++++++++-- pkg/util/testing/v1beta2/wrappers.go | 5 + 2 files changed, 469 insertions(+), 36 deletions(-) diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 8560a4cb55d..2275dc1000d 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -197,8 +197,6 @@ func TestSchedule(t *testing.T) { wantLeft map[kueue.ClusterQueueReference][]workload.Reference // wantInadmissibleLeft is the workload keys that are left in the inadmissible state after this cycle. wantInadmissibleLeft map[kueue.ClusterQueueReference][]workload.Reference - // wantPreempted is the keys of the workloads that get preempted in the scheduling cycle. - wantPreempted sets.Set[workload.Reference] // wantEvents ignored if empty, the Message is ignored (it contains the duration) wantEvents []utiltesting.EventRecord // eventCmpOpts are the cmp options to compare recorded events. @@ -277,7 +275,6 @@ func TestSchedule(t *testing.T) { Obj()). Obj(), }, - wantPreempted: sets.Set[workload.Reference]{}, wantAssignments: map[workload.Reference]kueue.Admission{ "eng-alpha/admitted": { ClusterQueue: "other-alpha", @@ -1237,6 +1234,8 @@ func TestSchedule(t *testing.T) { "preempt workloads in ClusterQueue and cohort": { workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Queue("main"). Request(corev1.ResourceCPU, "20"). Obj(), @@ -1283,6 +1282,21 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "on-demand", "60000m"). Obj()). Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("use-all-spot", "eng-alpha"). Request(corev1.ResourceCPU, "100"). @@ -1309,8 +1323,25 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "on-demand", "10000m"). Obj()). Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Queue("main"). Request(corev1.ResourceCPU, "20"). Condition(metav1.Condition{ @@ -1332,7 +1363,6 @@ func TestSchedule(t *testing.T) { // Preemptor is not admitted in this cycle. "eng-beta": {"eng-beta/preemptor"}, }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/borrower", "eng-beta/low-2"), wantAssignments: map[workload.Reference]kueue.Admission{ "eng-alpha/use-all-spot": *utiltestingapi.MakeAdmission("eng-alpha"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). @@ -1384,6 +1414,8 @@ func TestSchedule(t *testing.T) { }, workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(-1). Queue("other"). Request(corev1.ResourceCPU, "1"). @@ -1428,8 +1460,25 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "on-demand", "100"). Obj()). Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(-1). Queue("other"). Request(corev1.ResourceCPU, "1"). @@ -1455,7 +1504,6 @@ func TestSchedule(t *testing.T) { wantInadmissibleLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/pending"}, }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/use-all"), wantAssignments: map[workload.Reference]kueue.Admission{ // Removal from cache for the preempted workloads is deferred until we receive Workload updates "eng-alpha/use-all": *utiltestingapi.MakeAdmission("other-alpha"). @@ -1848,6 +1896,8 @@ func TestSchedule(t *testing.T) { "partial admission single variable pod set, preempt first": { workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("new", "eng-beta"). + UID("wl-new"). + JobUID("job-new"). Queue("main"). Priority(4). PodSets(*utiltestingapi.MakePodSet("one", 20). @@ -1865,6 +1915,8 @@ func TestSchedule(t *testing.T) { }, wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("new", "eng-beta"). + UID("wl-new"). + JobUID("job-new"). Queue("main"). Priority(4). PodSets(*utiltestingapi.MakePodSet("one", 20). @@ -1891,6 +1943,21 @@ func TestSchedule(t *testing.T) { Request("example.com/gpu", "1"). Obj()). ReserveQuotaAt(utiltestingapi.MakeAdmission("eng-beta").PodSets(utiltestingapi.MakePodSetAssignment("one").Assignment("example.com/gpu", "model-a", "10").Count(10).Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, wantAssignments: map[workload.Reference]kueue.Admission{ @@ -1904,7 +1971,6 @@ func TestSchedule(t *testing.T) { }, }, }, - wantPreempted: sets.New[workload.Reference]("eng-beta/old"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "eng-beta": {"eng-beta/new"}, }, @@ -1912,6 +1978,8 @@ func TestSchedule(t *testing.T) { "partial admission single variable pod set, preempt with partial admission": { workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("new", "eng-beta"). + UID("wl-new"). + JobUID("job-new"). Queue("main"). Priority(4). PodSets(*utiltestingapi.MakePodSet("one", 30). @@ -1929,6 +1997,8 @@ func TestSchedule(t *testing.T) { }, wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("new", "eng-beta"). + UID("wl-new"). + JobUID("job-new"). Queue("main"). Priority(4). PodSets(*utiltestingapi.MakePodSet("one", 30). @@ -1955,6 +2025,21 @@ func TestSchedule(t *testing.T) { Request("example.com/gpu", "1"). Obj()). ReserveQuotaAt(utiltestingapi.MakeAdmission("eng-beta").PodSets(utiltestingapi.MakePodSetAssignment("one").Assignment("example.com/gpu", "model-a", "10").Count(10).Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, wantAssignments: map[workload.Reference]kueue.Admission{ @@ -1968,7 +2053,6 @@ func TestSchedule(t *testing.T) { }, }, }, - wantPreempted: sets.New[workload.Reference]("eng-beta/old"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "eng-beta": {"eng-beta/new"}, }, @@ -3897,6 +3981,8 @@ func TestSchedule(t *testing.T) { Obj(), now). Obj(), *utiltestingapi.MakeWorkload("incoming", "eng-alpha"). + UID("wl-incoming"). + JobUID("job-incoming"). Priority(0). Queue("other"). Request(corev1.ResourceCPU, "2"). @@ -3912,6 +3998,21 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "on-demand", "1"). Obj()). Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a2", "eng-alpha"). Priority(-2). @@ -3922,6 +4023,21 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "on-demand", "1"). Obj()). Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a3", "eng-alpha"). Priority(-1). @@ -3934,6 +4050,8 @@ func TestSchedule(t *testing.T) { Obj(), now). Obj(), *utiltestingapi.MakeWorkload("incoming", "eng-alpha"). + UID("wl-incoming"). + JobUID("job-incoming"). Priority(0). Queue("other"). Request(corev1.ResourceCPU, "2"). @@ -3982,7 +4100,6 @@ func TestSchedule(t *testing.T) { Obj(), now). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/a1", "eng-alpha/a2"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/incoming"}, }, @@ -4167,6 +4284,8 @@ func TestSchedule(t *testing.T) { Request(corev1.ResourceCPU, "20"). SimpleReserveQuota("eng-gamma", "on-demand", now).Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Queue("main"). Request(corev1.ResourceCPU, "30").Obj(), }, @@ -4176,7 +4295,23 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("eng-alpha", "spot", now).Obj(), *utiltestingapi.MakeWorkload("alpha1", "eng-alpha").UID("alpha1"). Request(corev1.ResourceCPU, "20"). - SimpleReserveQuota("eng-alpha", "on-demand", now).Obj(), + SimpleReserveQuota("eng-alpha", "on-demand", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortFairSharing", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), *utiltestingapi.MakeWorkload("alpha2", "eng-alpha").UID("alpha2"). Request(corev1.ResourceCPU, "20"). SimpleReserveQuota("eng-alpha", "on-demand", now).Obj(), @@ -4187,6 +4322,8 @@ func TestSchedule(t *testing.T) { Request(corev1.ResourceCPU, "20"). SimpleReserveQuota("eng-alpha", "on-demand", now).Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Queue("main"). Request(corev1.ResourceCPU, "30"). Condition(metav1.Condition{ @@ -4205,7 +4342,23 @@ func TestSchedule(t *testing.T) { Obj(), *utiltestingapi.MakeWorkload("gamma1", "eng-gamma").UID("gamma1"). Request(corev1.ResourceCPU, "10"). - SimpleReserveQuota("eng-gamma", "on-demand", now).Obj(), + SimpleReserveQuota("eng-gamma", "on-demand", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortFairSharing", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), *utiltestingapi.MakeWorkload("gamma2", "eng-gamma").UID("gamma2"). Request(corev1.ResourceCPU, "20"). SimpleReserveQuota("eng-gamma", "on-demand", now).Obj(), @@ -4216,7 +4369,6 @@ func TestSchedule(t *testing.T) { Request(corev1.ResourceCPU, "20"). SimpleReserveQuota("eng-gamma", "on-demand", now).Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/alpha1", "eng-gamma/gamma1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ // Preemptor is not admitted in this cycle. "eng-beta": {"eng-beta/preemptor"}, @@ -4321,11 +4473,15 @@ func TestSchedule(t *testing.T) { Obj(), now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "2"). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "2"). @@ -4341,8 +4497,25 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "default", "2"). Obj()). Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "2"). @@ -4369,8 +4542,25 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "default", "2"). Obj()). Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "2"). @@ -4389,7 +4579,6 @@ func TestSchedule(t *testing.T) { }). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/a1", "eng-beta/b1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/preemptor"}, "other-beta": {"eng-beta/preemptor"}, @@ -4458,6 +4647,8 @@ func TestSchedule(t *testing.T) { Request(corev1.ResourceCPU, "1"). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(99). Queue("other"). Request(corev1.ResourceCPU, "2"). @@ -4502,8 +4693,25 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "default", "2"). Obj()). Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(99). Queue("other"). Request(corev1.ResourceCPU, "2"). @@ -4522,7 +4730,6 @@ func TestSchedule(t *testing.T) { }). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-beta/b1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-beta": {"eng-beta/preemptor"}, }, @@ -4613,6 +4820,8 @@ func TestSchedule(t *testing.T) { Obj(), now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). @@ -4633,8 +4842,25 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "default", "2"). Obj()). Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). @@ -4681,7 +4907,6 @@ func TestSchedule(t *testing.T) { }). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/a1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/preemptor"}, "other-beta": {"eng-beta/pretending-preemptor"}, @@ -4781,16 +5006,22 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-gamma"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). @@ -4805,8 +5036,25 @@ func TestSchedule(t *testing.T) { PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "3"). Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). @@ -4831,8 +5079,25 @@ func TestSchedule(t *testing.T) { ReserveQuotaAt(utiltestingapi.MakeAdmission("other-beta"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-beta"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). @@ -4857,8 +5122,25 @@ func TestSchedule(t *testing.T) { ReserveQuotaAt(utiltestingapi.MakeAdmission("other-gamma"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "3").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-gamma"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). @@ -4877,7 +5159,6 @@ func TestSchedule(t *testing.T) { }). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/a1", "eng-beta/b1", "eng-gamma/c1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/preemptor"}, "other-beta": {"eng-beta/preemptor"}, @@ -4988,6 +5269,8 @@ func TestSchedule(t *testing.T) { Assignment(corev1.ResourceCPU, "default", "9").Obj()).Obj(), now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). @@ -5009,8 +5292,25 @@ func TestSchedule(t *testing.T) { PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment("alpha-resource", "default", "1"). Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request(corev1.ResourceCPU, "3"). @@ -5066,9 +5366,23 @@ func TestSchedule(t *testing.T) { ReserveQuotaAt(utiltestingapi.MakeAdmission("other-gamma"). PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). Assignment(corev1.ResourceCPU, "default", "9").Obj()).Obj(), now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortFairSharing", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/a1", "eng-gamma/c1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/preemptor"}, "other-beta": {"eng-beta/pretending-preemptor"}, @@ -5305,6 +5619,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-beta", "spot", now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request("gpu", "6"). @@ -5318,6 +5634,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-alpha", "on-demand", now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request("gpu", "6"). @@ -5340,9 +5658,23 @@ func TestSchedule(t *testing.T) { Queue("other"). Request("gpu", "5"). SimpleReserveQuota("other-beta", "spot", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-beta/b1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/preemptor"}, }, @@ -5404,6 +5736,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-beta", "spot", now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request("gpu", "8"). @@ -5417,6 +5751,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-alpha", "on-demand", now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request("gpu", "8"). @@ -5439,9 +5775,23 @@ func TestSchedule(t *testing.T) { Queue("other"). Request("gpu", "5"). SimpleReserveQuota("other-beta", "spot", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortFairSharing", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-beta/b1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/preemptor"}, }, @@ -5506,6 +5856,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-beta", "spot", now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request("gpu", "6"). @@ -5517,6 +5869,21 @@ func TestSchedule(t *testing.T) { Queue("other"). Request("gpu", "5"). SimpleReserveQuota("other-alpha", "on-demand", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a2", "eng-alpha"). Priority(50). @@ -5525,6 +5892,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-alpha", "spot", now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request("gpu", "6"). @@ -5549,7 +5918,6 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-beta", "spot", now). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/a1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/preemptor"}, }, @@ -5621,6 +5989,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-beta", "spot", now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request("gpu", "5"). @@ -5632,6 +6002,21 @@ func TestSchedule(t *testing.T) { Queue("other"). Request("gpu", "6"). SimpleReserveQuota("other-alpha", "on-demand", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a2", "eng-alpha"). Priority(50). @@ -5640,6 +6025,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-alpha", "spot", now). Obj(), *utiltestingapi.MakeWorkload("preemptor", "eng-alpha"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(100). Queue("other"). Request("gpu", "5"). @@ -5665,7 +6052,6 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("other-beta", "spot", now). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-alpha/a1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "other-alpha": {"eng-alpha/preemptor"}, }, @@ -5759,6 +6145,8 @@ func TestSchedule(t *testing.T) { Request("gpu", "10"). Obj(), *utiltestingapi.MakeWorkload("WL2", "eng-beta"). + UID("wl-WL2"). + JobUID("job-WL2"). Creation(now.Add(time.Second)). Queue("lq"). Request("gpu", "10"). @@ -5801,6 +6189,8 @@ func TestSchedule(t *testing.T) { }). Obj(), *utiltestingapi.MakeWorkload("WL2", "eng-beta"). + UID("wl-WL2"). + JobUID("job-WL2"). Creation(now.Add(time.Second)). Queue("lq"). Request("gpu", "10"). @@ -5823,6 +6213,21 @@ func TestSchedule(t *testing.T) { Priority(0). Request("gpu", "5"). SimpleReserveQuota("CQ3", "on-demand", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-WL2, JobUID: job-WL2) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-WL2, JobUID: job-WL2) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("Admitted-Workload-3", "eng-gamma"). Queue("lq"). @@ -5831,7 +6236,6 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("CQ3", "on-demand", now). Obj(), }, - wantPreempted: sets.New[workload.Reference]("eng-gamma/Admitted-Workload-2"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "CQ2": {"eng-beta/WL2"}, }, @@ -6790,6 +7194,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("queue1", "on-demand", now). Obj(), *utiltestingapi.MakeWorkload("a2", "default"). + UID("wl-a2"). + JobUID("job-a2"). Priority(99). Queue("queue1"). Request("gpu", "1"). @@ -6802,8 +7208,25 @@ func TestSchedule(t *testing.T) { Queue("queue1"). Request("gpu", "2"). SimpleReserveQuota("queue1", "on-demand", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a2", "default"). + UID("wl-a2"). + JobUID("job-a2"). Priority(99). Queue("queue1"). Request("gpu", "1"). @@ -6822,7 +7245,6 @@ func TestSchedule(t *testing.T) { }). Obj(), }, - wantPreempted: sets.New[workload.Reference]("default/a1"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "queue1": {"default/a2"}, }, @@ -6897,6 +7319,8 @@ func TestSchedule(t *testing.T) { SimpleReserveQuota("queue2", "on-demand", now). Obj(), *utiltestingapi.MakeWorkload("a1", "default"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(99). Queue("queue1"). Request("gpu", "1"). @@ -6904,6 +7328,8 @@ func TestSchedule(t *testing.T) { }, wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("a1", "default"). + UID("wl-preemptor"). + JobUID("job-preemptor"). Priority(99). Queue("queue1"). Request("gpu", "1"). @@ -6927,9 +7353,23 @@ func TestSchedule(t *testing.T) { Queue("queue2"). Request("gpu", "2"). SimpleReserveQuota("queue2", "on-demand", now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortFairSharing", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, - wantPreempted: sets.New[workload.Reference]("default/a2"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "queue1": {"default/a1"}, }, @@ -7021,15 +7461,6 @@ func TestSchedule(t *testing.T) { func() { wg.Done() }, )) - var mu sync.Mutex - gotPreempted := sets.New[workload.Reference]() - scheduler.preemptor.OverrideApply(func(_ context.Context, w *kueue.Workload, _, _ string) error { - mu.Lock() - gotPreempted.Insert(workload.Key(w)) - mu.Unlock() - return nil - }) - ctx, cancel := context.WithTimeout(ctx, queueingTimeout) go qManager.CleanUpOnContext(ctx) defer cancel() @@ -7037,10 +7468,6 @@ func TestSchedule(t *testing.T) { scheduler.schedule(ctx) wg.Wait() - if diff := cmp.Diff(tc.wantPreempted, gotPreempted); diff != "" { - t.Errorf("Unexpected preemptions (-want,+got):\n%s", diff) - } - // Verify assignments in cache. gotAssignments := make(map[workload.Reference]kueue.Admission) snapshot, err := cqCache.Snapshot(ctx) @@ -7070,6 +7497,7 @@ func TestSchedule(t *testing.T) { cmpopts.EquateEmpty(), cmpopts.IgnoreFields(kueue.AdmissionCheckState{}, "LastTransitionTime"), cmpopts.IgnoreFields(kueue.Workload{}, "ObjectMeta.ResourceVersion", "ObjectMeta.CreationTimestamp"), + cmpopts.SortSlices(func(a, b metav1.Condition) bool { return a.Type < b.Type }), } if diff := cmp.Diff(tc.wantWorkloads, gotWorkloads.Items, defaultWorkloadCmpOpts); diff != "" { diff --git a/pkg/util/testing/v1beta2/wrappers.go b/pkg/util/testing/v1beta2/wrappers.go index 47a3472a400..7a97ff80bdc 100644 --- a/pkg/util/testing/v1beta2/wrappers.go +++ b/pkg/util/testing/v1beta2/wrappers.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/pkg/controller/constants" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) @@ -91,6 +92,10 @@ func (w *WorkloadWrapper) UID(uid types.UID) *WorkloadWrapper { return w } +func (w *WorkloadWrapper) JobUID(uid string) *WorkloadWrapper { + return w.Label(constants.JobUIDLabel, uid) +} + // Generation sets the generation of the Workload. func (w *WorkloadWrapper) Generation(num int64) *WorkloadWrapper { w.ObjectMeta.Generation = num From f483b4f6a119117164d5bb073be1ceb00c0f9693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Fri, 31 Oct 2025 16:10:06 +0100 Subject: [PATCH 072/119] Extend kubeconfig validation tests (#7483) to include a status of admission check --- test/integration/multikueue/setup_test.go | 111 ++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/test/integration/multikueue/setup_test.go b/test/integration/multikueue/setup_test.go index 85a44cc3e02..d91929db2d5 100644 --- a/test/integration/multikueue/setup_test.go +++ b/test/integration/multikueue/setup_test.go @@ -441,6 +441,21 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, secret) }) }) + ac := utiltestingapi.MakeAdmissionCheck("testing-ac"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "testing-config"). + Obj() + ginkgo.By("creating the admission check", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, ac)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, ac) }) + }) + + config := utiltestingapi.MakeMultiKueueConfig("testing-config").Clusters("testing-cluster").Obj() + ginkgo.By("creating the config", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, config)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, config) }) + }) + cluster := utiltestingapi.MakeMultiKueueCluster("testing-cluster").KubeConfig(kueue.SecretLocationType, "testing-secret").Obj() ginkgo.By("creating the cluster, its Active state is updated", func() { gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, cluster)).Should(gomega.Succeed()) @@ -459,6 +474,20 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, }, util.IgnoreConditionTimestampsAndObservedGeneration))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) + + ginkgo.By("wait for the check's active state update", func() { + updatedAc := kueue.AdmissionCheck{} + acKey := client.ObjectKeyFromObject(ac) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatedAc)).To(gomega.Succeed()) + g.Expect(updatedAc.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.AdmissionCheckActive, + Status: metav1.ConditionFalse, + Reason: "NoUsableClusters", + Message: "Inactive clusters: [testing-cluster]", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) }) w1Kubeconfig, err := worker1TestCluster.kubeConfigBytes() @@ -486,6 +515,19 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, }, util.IgnoreConditionTimestampsAndObservedGeneration))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) + ginkgo.By("wait for the check's active state update", func() { + updatedAc := kueue.AdmissionCheck{} + acKey := client.ObjectKeyFromObject(ac) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatedAc)).To(gomega.Succeed()) + g.Expect(updatedAc.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.AdmissionCheckActive, + Status: metav1.ConditionTrue, + Reason: "Active", + Message: "The admission check is active", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) }) }) @@ -511,6 +553,21 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, gomega.Expect(os.WriteFile(fsKubeConfig, w1KubeconfigInvalidBytes, 0666)).Should(gomega.Succeed()) }) + ac := utiltestingapi.MakeAdmissionCheck("testing-ac"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "testing-config"). + Obj() + ginkgo.By("creating the admission check", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, ac)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, ac) }) + }) + + config := utiltestingapi.MakeMultiKueueConfig("testing-config").Clusters("testing-cluster").Obj() + ginkgo.By("creating the config", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, config)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, config) }) + }) + cluster := utiltestingapi.MakeMultiKueueCluster("testing-cluster").KubeConfig(kueue.PathLocationType, fsKubeConfig).Obj() ginkgo.By("creating the cluster, its Active state is updated", func() { gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, cluster)).Should(gomega.Succeed()) @@ -529,6 +586,19 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, }, util.IgnoreConditionTimestampsAndObservedGeneration))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) + ginkgo.By("wait for the check's active state update", func() { + updatedAc := kueue.AdmissionCheck{} + acKey := client.ObjectKeyFromObject(ac) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatedAc)).To(gomega.Succeed()) + g.Expect(updatedAc.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.AdmissionCheckActive, + Status: metav1.ConditionFalse, + Reason: "NoUsableClusters", + Message: "Inactive clusters: [testing-cluster]", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) }) w1Kubeconfig, err := worker1TestCluster.kubeConfigBytes() @@ -549,6 +619,19 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, }, util.IgnoreConditionTimestampsAndObservedGeneration))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) + ginkgo.By("wait for the check's active state update", func() { + updatedAc := kueue.AdmissionCheck{} + acKey := client.ObjectKeyFromObject(ac) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatedAc)).To(gomega.Succeed()) + g.Expect(updatedAc.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.AdmissionCheckActive, + Status: metav1.ConditionTrue, + Reason: "Active", + Message: "The admission check is active", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) }) }) @@ -591,6 +674,21 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, secret) }) }) + ac := utiltestingapi.MakeAdmissionCheck("testing-ac"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", "testing-config"). + Obj() + ginkgo.By("creating the admission check", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, ac)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, ac) }) + }) + + config := utiltestingapi.MakeMultiKueueConfig("testing-config").Clusters("testing-cluster").Obj() + ginkgo.By("creating the config", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, config)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, config) }) + }) + cluster := utiltestingapi.MakeMultiKueueCluster("testing-cluster").KubeConfig(kueue.SecretLocationType, "testing-secret").Obj() ginkgo.By("creating the cluster, its Active state is updated", func() { gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, cluster)).Should(gomega.Succeed()) @@ -609,6 +707,19 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, }, util.IgnoreConditionTimestampsAndObservedGeneration))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) + ginkgo.By("wait for the check's active state update", func() { + updatedAc := kueue.AdmissionCheck{} + acKey := client.ObjectKeyFromObject(ac) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatedAc)).To(gomega.Succeed()) + g.Expect(updatedAc.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.AdmissionCheckActive, + Status: metav1.ConditionTrue, + Reason: "Active", + Message: "The admission check is active", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) }) }) }) From 0827c4597435475dc635ab670a41707098a99acb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Irving=20Mondrag=C3=B3n?= Date: Mon, 3 Nov 2025 09:00:04 +0100 Subject: [PATCH 073/119] Prevent StatefulSet scale-up while workload is being deleted (#7479) --- .../jobs/statefulset/statefulset_webhook.go | 18 +++++++++-- .../statefulset/statefulset_webhook_test.go | 31 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/pkg/controller/jobs/statefulset/statefulset_webhook.go b/pkg/controller/jobs/statefulset/statefulset_webhook.go index 0b9e36c8fc1..8e03c1528ad 100644 --- a/pkg/controller/jobs/statefulset/statefulset_webhook.go +++ b/pkg/controller/jobs/statefulset/statefulset_webhook.go @@ -174,8 +174,22 @@ func (wh *Webhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Ob )...) } - if oldReplicas == 0 && newReplicas > 0 && newStatefulSet.Status.Replicas > 0 { - allErrs = append(allErrs, field.Forbidden(replicasPath, "scaling down is still in progress")) + if oldReplicas == 0 && newReplicas > 0 { + if newStatefulSet.Status.Replicas > 0 { + // Block if pods are still terminating + allErrs = append(allErrs, field.Forbidden(replicasPath, "scaling down is still in progress")) + } else { + // Block if workload is still being deleted + workloadName := GetWorkloadName(oldStatefulSet.GetName()) + wlKey := client.ObjectKey{Namespace: oldStatefulSet.GetNamespace(), Name: workloadName} + var wl kueue.Workload + err := wh.client.Get(ctx, wlKey, &wl) + if client.IgnoreNotFound(err) != nil { + return nil, err + } else if err == nil { + allErrs = append(allErrs, field.Forbidden(replicasPath, "workload from previous scale-down is still being deleted")) + } + } } } diff --git a/pkg/controller/jobs/statefulset/statefulset_webhook_test.go b/pkg/controller/jobs/statefulset/statefulset_webhook_test.go index a1136351898..a2285c4c168 100644 --- a/pkg/controller/jobs/statefulset/statefulset_webhook_test.go +++ b/pkg/controller/jobs/statefulset/statefulset_webhook_test.go @@ -780,6 +780,37 @@ func TestValidateUpdate(t *testing.T) { }, }.ToAggregate(), }, + "scale up from zero blocked by existing workload": { + objs: []runtime.Object{ + utiltestingapi.MakeWorkload(GetWorkloadName("test-statefulset"), "test-ns").Obj(), + }, + oldObj: testingstatefulset.MakeStatefulSet("test-statefulset", "test-ns"). + Queue("test-queue"). + Replicas(0). + Obj(), + newObj: testingstatefulset.MakeStatefulSet("test-statefulset", "test-ns"). + Queue("test-queue"). + Replicas(3). + Obj(), + wantErr: field.ErrorList{ + &field.Error{ + Type: field.ErrorTypeForbidden, + Field: replicasPath.String(), + }, + }.ToAggregate(), + }, + "scale up from zero allowed when workload does not exist": { + oldObj: testingstatefulset.MakeStatefulSet("test-sts", "test-ns"). + UID("test-sts-uid"). + Queue("test-queue"). + Replicas(0). + Obj(), + newObj: testingstatefulset.MakeStatefulSet("test-sts", "test-ns"). + UID("test-sts-uid"). + Queue("test-queue"). + Replicas(3). + Obj(), + }, } for name, tc := range testCases { From d96094c7c84aa33c104139c047e91981046f87db Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 3 Nov 2025 03:54:05 -0500 Subject: [PATCH 074/119] Promote AdmissionFairSharing to beta (#7463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the AdmissionFairSharing feature to beta status for v0.15: - Update KEP status to implementable and stage to beta - Enable feature gate by default in beta (v0.15) - Update documentation to reflect beta status - Simplify test setup for feature gate configuration - Add beta milestone to feature gate table 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- keps/4136-admission-fair-sharing/kep.yaml | 7 ++++--- pkg/features/kube_features.go | 1 + pkg/scheduler/scheduler_afs_test.go | 4 +--- site/content/en/docs/concepts/admission_fair_sharing.md | 6 +++--- site/content/en/docs/installation/_index.md | 1 + 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/keps/4136-admission-fair-sharing/kep.yaml b/keps/4136-admission-fair-sharing/kep.yaml index 3c3288d8c50..bcbcd8bea5c 100644 --- a/keps/4136-admission-fair-sharing/kep.yaml +++ b/keps/4136-admission-fair-sharing/kep.yaml @@ -3,7 +3,7 @@ kep-number: 4136 authors: - "@mwielgus" - "@pbundyra" -status: draft +status: implementable creation-date: 2025-02-03 reviewers: - "@mimowo" @@ -16,16 +16,17 @@ sea-also: - "KEP-1714" # The target maturity stage in the current dev cycle for this KEP. -stage: alpha +stage: beta # The most recent milestone for which work toward delivery of this KEP has been # done. This can be the current (upcoming) milestone, if it is being actively # worked on. -latest-milestone: "v0.12" +latest-milestone: "v0.15" # The milestone at which this feature was, or is targeted to be, at each stage. milestone: alpha: "v0.12" + beta: "v0.15" # The following PRR answers are required at alpha release # List the feature gate name and the components for which it must be enabled diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 624332842de..5cc93a14f54 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -265,6 +265,7 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned }, AdmissionFairSharing: { {Version: version.MustParse("0.12"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("0.15"), Default: true, PreRelease: featuregate.Beta}, }, ObjectRetentionPolicies: { {Version: version.MustParse("0.12"), Default: false, PreRelease: featuregate.Alpha}, diff --git a/pkg/scheduler/scheduler_afs_test.go b/pkg/scheduler/scheduler_afs_test.go index facd16482bf..8ccbf77ed17 100644 --- a/pkg/scheduler/scheduler_afs_test.go +++ b/pkg/scheduler/scheduler_afs_test.go @@ -518,9 +518,7 @@ func TestScheduleForAFS(t *testing.T) { for _, enabled := range []bool{false, true} { t.Run(fmt.Sprintf("%s WorkloadRequestUseMergePatch enabled: %t", name, enabled), func(t *testing.T) { features.SetFeatureGateDuringTest(t, features.WorkloadRequestUseMergePatch, enabled) - if tc.enableFairSharing { - features.SetFeatureGateDuringTest(t, features.AdmissionFairSharing, true) - } + features.SetFeatureGateDuringTest(t, features.AdmissionFairSharing, tc.enableFairSharing) for i, q := range queues { if resList, found := tc.initialUsage[q.Name]; found { diff --git a/site/content/en/docs/concepts/admission_fair_sharing.md b/site/content/en/docs/concepts/admission_fair_sharing.md index 3bd5853f121..9df69c23f0d 100644 --- a/site/content/en/docs/concepts/admission_fair_sharing.md +++ b/site/content/en/docs/concepts/admission_fair_sharing.md @@ -6,12 +6,12 @@ description: > A mechanism for ordering workloads based on the historical resource usage of their source LocalQueues, giving preference to those that have consumed fewer resources over time. --- -{{< feature-state state="alpha" for_version="v0.12" >}} +{{< feature-state state="beta" for_version="v0.15" >}} {{% alert title="Note" color="primary" %}} -`AdmissionFairSharing` is currently an alpha feature and is not enabled by default. +`AdmissionFairSharing` is currently an beta feature and is enabled by default. -You can enable it by editing the `AdmissionFairSharing` feature gate. Check the [Installation](/docs/installation/#change-the-feature-gates-configuration) guide for details on feature gate configuration. +You can disable it by editing the `AdmissionFairSharing` feature gate. Check the [Installation](/docs/installation/#change-the-feature-gates-configuration) guide for details on feature gate configuration. {{% /alert %}} diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 87230a7fcd8..7118340d929 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -288,6 +288,7 @@ spec: | `TASFailedNodeReplacement` | `false` | Alpha | 0.12 | 0.13 | | `TASFailedNodeReplacement` | `true` | Beta | 0.14 | | | `AdmissionFairSharing` | `false` | Alpha | 0.12 | | +| `AdmissionFairSharing` | `true` | Beta | 0.15 | | | `TASFailedNodeReplacementFailFast` | `false` | Alpha | 0.12 | 0.13 | | `TASFailedNodeReplacementFailFast` | `true` | Beta | 0.14 | | | `TASReplaceNodeOnPodTermination` | `false` | Alpha | 0.13 | 0.13 | From cedc24164232d5fbfdba683f726bc317837a0b19 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Mon, 3 Nov 2025 16:59:36 +0530 Subject: [PATCH 075/119] Replace preemption stub with interceptor function in TestLastSchedulingContext. (#7502) --- pkg/scheduler/scheduler_test.go | 63 ++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 2275dc1000d..dc709757451 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -31,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/client-go/tools/record" "k8s.io/component-base/metrics/testutil" @@ -7892,7 +7891,6 @@ func TestLastSchedulingContext(t *testing.T) { cqs []kueue.ClusterQueue workloads []kueue.Workload deleteWorkloads []client.ObjectKey - wantPreempted sets.Set[workload.Reference] wantWorkloads []kueue.Workload wantAdmissionsOnSecondSchedule map[workload.Reference]kueue.Admission }{ @@ -7937,7 +7935,6 @@ func TestLastSchedulingContext(t *testing.T) { Namespace: metav1.NamespaceDefault, Name: "low-1", }}, - wantPreempted: sets.Set[workload.Reference]{}, wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("low-1", "default"). Queue("main"). @@ -7997,7 +7994,6 @@ func TestLastSchedulingContext(t *testing.T) { Request(corev1.ResourceCPU, "20"). Obj(), }, - wantPreempted: sets.Set[workload.Reference]{}, wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("borrower", "default"). Queue("main-alpha"). @@ -8110,7 +8106,6 @@ func TestLastSchedulingContext(t *testing.T) { Request(corev1.ResourceCPU, "20"). Obj(), }, - wantPreempted: sets.Set[workload.Reference]{}, wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("placeholder", "default"). Request(corev1.ResourceCPU, "50"). @@ -8213,7 +8208,6 @@ func TestLastSchedulingContext(t *testing.T) { Request(corev1.ResourceCPU, "20"). Obj(), }, - wantPreempted: sets.Set[workload.Reference]{}, wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("placeholder", "default"). Request(corev1.ResourceCPU, "40"). @@ -8318,6 +8312,8 @@ func TestLastSchedulingContext(t *testing.T) { AdmittedAt(true, now). Obj(), *utiltestingapi.MakeWorkload("new", "default"). + UID("wl-new"). + JobUID("job-new"). Queue("main-theta"). Request(corev1.ResourceCPU, "20"). Obj(), @@ -8326,9 +8322,10 @@ func TestLastSchedulingContext(t *testing.T) { Namespace: metav1.NamespaceDefault, Name: "placeholder-alpha", }}, - wantPreempted: sets.New[workload.Reference]("default/placeholder-alpha"), wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("new", "default"). + UID("wl-new"). + JobUID("job-new"). Queue("main-theta"). Request(corev1.ResourceCPU, "20"). Condition(metav1.Condition{ @@ -8354,6 +8351,21 @@ func TestLastSchedulingContext(t *testing.T) { Obj()). Obj(), now). AdmittedAt(true, now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("placeholder-theta-spot", "default"). Request(corev1.ResourceCPU, "100"). @@ -8447,6 +8459,8 @@ func TestLastSchedulingContext(t *testing.T) { AdmittedAt(true, now). Obj(), *utiltestingapi.MakeWorkload("new", "default"). + UID("wl-new"). + JobUID("job-new"). Queue("main-beta"). Request(corev1.ResourceCPU, "22"). Obj(), @@ -8455,7 +8469,6 @@ func TestLastSchedulingContext(t *testing.T) { Namespace: metav1.NamespaceDefault, Name: "alpha2", }}, - wantPreempted: sets.New[workload.Reference]("default/alpha2"), wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("alpha1", "default"). Request(corev1.ResourceCPU, "22"). @@ -8466,6 +8479,21 @@ func TestLastSchedulingContext(t *testing.T) { Request(corev1.ResourceCPU, "22"). SimpleReserveQuota("eng-cohort-alpha", "on-demand", now). AdmittedAt(true, now). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("alpha3", "default"). Request(corev1.ResourceCPU, "22"). @@ -8478,6 +8506,8 @@ func TestLastSchedulingContext(t *testing.T) { AdmittedAt(true, now). Obj(), *utiltestingapi.MakeWorkload("new", "default"). + UID("wl-new"). + JobUID("job-new"). Queue("main-beta"). Request(corev1.ResourceCPU, "22"). Condition(metav1.Condition{ @@ -8574,15 +8604,6 @@ func TestLastSchedulingContext(t *testing.T) { func() { wg.Done() }, )) - var mu sync.Mutex - gotPreempted := sets.New[workload.Reference]() - scheduler.preemptor.OverrideApply(func(_ context.Context, w *kueue.Workload, _, _ string) error { - mu.Lock() - gotPreempted.Insert(workload.Key(w)) - mu.Unlock() - return nil - }) - ctx, cancel := context.WithTimeout(ctx, queueingTimeout) go qManager.CleanUpOnContext(ctx) defer cancel() @@ -8590,10 +8611,6 @@ func TestLastSchedulingContext(t *testing.T) { scheduler.schedule(ctx) wg.Wait() - if diff := cmp.Diff(tc.wantPreempted, gotPreempted); diff != "" { - t.Errorf("Unexpected preemptions (-want,+got):\n%s", diff) - } - gotWorkloads := &kueue.WorkloadList{} err := cl.List(ctx, gotWorkloads) if err != nil { @@ -8603,6 +8620,7 @@ func TestLastSchedulingContext(t *testing.T) { defaultWorkloadCmpOpts := cmp.Options{ cmpopts.EquateEmpty(), cmpopts.IgnoreFields(kueue.Workload{}, "ObjectMeta.ResourceVersion"), + cmpopts.SortSlices(func(a, b metav1.Condition) bool { return a.Type < b.Type }), } if diff := cmp.Diff(tc.wantWorkloads, gotWorkloads.Items, defaultWorkloadCmpOpts); diff != "" { @@ -8635,9 +8653,6 @@ func TestLastSchedulingContext(t *testing.T) { wg.Wait() } - if diff := cmp.Diff(tc.wantPreempted, gotPreempted); diff != "" { - t.Errorf("Unexpected preemptions (-want,+got):\n%s", diff) - } // Verify assignments in cache. gotAssignments := make(map[workload.Reference]kueue.Admission) snapshot, err := cqCache.Snapshot(ctx) From ef12e7260f5e079ca0aa162856f2838dae1f0daa Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 3 Nov 2025 09:18:11 -0500 Subject: [PATCH 076/119] Enable nomaps and nobools kube api linter (#7489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enable no maps and no bools but disable the failures. * Update apis/kueue/v1beta2/workload_types.go Co-authored-by: Michał Woźniak --------- Co-authored-by: Michał Woźniak --- .golangci-kal.yml | 4 ++-- apis/kueue/v1beta2/fairsharing_types.go | 2 +- apis/kueue/v1beta2/workload_types.go | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.golangci-kal.yml b/.golangci-kal.yml index a7cf9268690..f1c8e0e3537 100644 --- a/.golangci-kal.yml +++ b/.golangci-kal.yml @@ -20,10 +20,10 @@ linters: - "integers" # Ensure only int32 and int64 are used for integers. - "jsontags" # Ensure every field has a json tag. - "maxlength" # Ensure all strings and arrays have maximum lengths/maximum items. - # - "nobools" # Bools do not evolve over time, should use enums instead. + - "nobools" # Bools do not evolve over time, should use enums instead. - "nodurations" # Prevents usage of `Duration` types. - "nofloats" # Ensure floats are not used. - # - "nomaps" # Ensure maps are not used. + - "nomaps" # Ensure maps are not used. - "nonullable" # Ensure that types and fields do not have the nullable marker. - "notimestamp" # Prevents usage of 'Timestamp' fields # - "optionalfields" # Ensure that all fields marked as optional adhere to being pointers and diff --git a/apis/kueue/v1beta2/fairsharing_types.go b/apis/kueue/v1beta2/fairsharing_types.go index 1cf1e3250e5..b0fb0445ba2 100644 --- a/apis/kueue/v1beta2/fairsharing_types.go +++ b/apis/kueue/v1beta2/fairsharing_types.go @@ -65,7 +65,7 @@ type AdmissionFairSharingStatus struct { // with decaying function applied. // The value is populated if usage consumption functionality is enabled in Kueue config. // +required - ConsumedResources corev1.ResourceList `json:"consumedResources"` + ConsumedResources corev1.ResourceList `json:"consumedResources"` //nolint:kubeapilinter // map type is required for standard Kubernetes ResourceList // lastUpdate is the time when share and consumed resources were updated. // +required diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index 48d0ecc68cf..431dd7ea69d 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -75,7 +75,7 @@ type WorkloadSpec struct { // // Defaults to true // +kubebuilder:default=true - Active *bool `json:"active,omitempty"` + Active *bool `json:"active,omitempty"` //nolint:kubeapilinter // bool for the ease of use, inspired by k8s Job's Suspend field. // maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, // the workload can be admitted before it's automatically deactivated. @@ -113,7 +113,7 @@ type PodSetTopologyRequest struct { // // +optional // +kubebuilder:validation:Type=boolean - Unconstrained *bool `json:"unconstrained,omitempty"` + Unconstrained *bool `json:"unconstrained,omitempty"` //nolint:kubeapilinter // disabling to avoid changing existing API // podIndexLabel indicates the name of the label indexing the pods. // For example, in the context of @@ -190,7 +190,7 @@ type PodSetAssignment struct { // Beside what is provided in podSet's specs, this calculation takes into account // the LimitRange defaults and RuntimeClass overheads at the moment of admission. // This field will not change in case of quota reclaim. - ResourceUsage corev1.ResourceList `json:"resourceUsage,omitempty"` + ResourceUsage corev1.ResourceList `json:"resourceUsage,omitempty"` //nolint:kubeapilinter // map type is required for standard Kubernetes ResourceList // count is the number of pods taken into account at admission time. // This field will not change in case of quota reclaim. @@ -627,7 +627,7 @@ type PodSetRequest struct { // the LimitRange defaults and RuntimeClass overheads at the moment of consideration // and the application of resource.excludeResourcePrefixes and resource.transformations. // +optional - Resources corev1.ResourceList `json:"resources,omitempty"` + Resources corev1.ResourceList `json:"resources,omitempty"` //nolint:kubeapilinter // map type is required for standard Kubernetes ResourceList } const ( From 4aa6d05fa182bedb6c68997daa4affd559d01bf0 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Mon, 3 Nov 2025 21:30:07 +0530 Subject: [PATCH 077/119] Replace preemption stub with interceptor function in scheduler TAS unit tests. (#7501) --- pkg/scheduler/scheduler_tas_test.go | 1586 +++++++++++++++++++++++++-- 1 file changed, 1496 insertions(+), 90 deletions(-) diff --git a/pkg/scheduler/scheduler_tas_test.go b/pkg/scheduler/scheduler_tas_test.go index ef3369d10bb..5f469cdcc86 100644 --- a/pkg/scheduler/scheduler_tas_test.go +++ b/pkg/scheduler/scheduler_tas_test.go @@ -2502,6 +2502,7 @@ func TestScheduleForTAS(t *testing.T) { } func TestScheduleForTASPreemption(t *testing.T) { + now := time.Now().Truncate(time.Second) singleNode := testingnode.MakeNode("x1"). Label("tas-node", "true"). Label(corev1.LabelHostname, "x1"). @@ -2557,6 +2558,7 @@ func TestScheduleForTASPreemption(t *testing.T) { resourceFlavors []kueue.ResourceFlavor clusterQueues []kueue.ClusterQueue workloads []kueue.Workload + wantWorkloads []kueue.Workload // wantNewAssignments is a summary of all new admissions in the cache after this cycle. wantNewAssignments map[workload.Reference]kueue.Admission @@ -2564,8 +2566,6 @@ func TestScheduleForTASPreemption(t *testing.T) { wantLeft map[kueue.ClusterQueueReference][]workload.Reference // wantInadmissibleLeft is the workload keys that are left in the inadmissible state after this cycle. wantInadmissibleLeft map[kueue.ClusterQueueReference][]workload.Reference - // wantPreempted is the keys of the workloads that get preempted in the scheduling cycle. - wantPreempted sets.Set[workload.Reference] // wantEvents asserts on the events, the comparison options are passed by eventCmpOpts wantEvents []utiltesting.EventRecord // eventCmpOpts are the comparison options for the events @@ -2587,6 +2587,8 @@ func TestScheduleForTASPreemption(t *testing.T) { }, workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("foo", "default"). + UID("wl-foo"). + JobUID("job-foo"). Queue("tas-main"). Priority(3). PodSets(*utiltestingapi.MakePodSet("one", 10). @@ -2597,7 +2599,7 @@ func TestScheduleForTASPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). Queue("tas-main"). Priority(1). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-main"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "5"). @@ -2606,21 +2608,84 @@ func TestScheduleForTASPreemption(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "5"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("foo", "default"). + UID("wl-foo"). + JobUID("job-foo"). + Queue("tas-main"). + Priority(3). + PodSets(*utiltestingapi.MakePodSet("one", 10). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: insufficient unused quota for cpu in flavor tas-default, 5 more needed. Pending the preemption of 1 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10"), + }, + }). + Obj(), + *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). + Queue("tas-main"). + Priority(1). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-main"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "5"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"z1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "5"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, - wantPreempted: sets.New[workload.Reference]("default/low-priority-admitted"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-main": {"default/foo"}, }, wantEvents: []utiltesting.EventRecord{ + utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: UNKNOWN, JobUID: UNKNOWN) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). Obj(), utiltesting.MakeEventRecord("default", "foo", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: insufficient unused quota for cpu in flavor tas-default, 5 more needed. Pending the preemption of 1 workload(s)`). @@ -2636,6 +2701,8 @@ func TestScheduleForTASPreemption(t *testing.T) { clusterQueues: []kueue.ClusterQueue{defaultClusterQueueWithPreemption}, workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("foo", "default"). + UID("wl-foo"). + JobUID("job-foo"). Queue("tas-main"). Priority(3). PodSets(*utiltestingapi.MakePodSet("one", 1). @@ -2646,7 +2713,7 @@ func TestScheduleForTASPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). Queue("tas-main"). Priority(1). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-main"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "5"). @@ -2655,21 +2722,84 @@ func TestScheduleForTASPreemption(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "5"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("foo", "default"). + UID("wl-foo"). + JobUID("job-foo"). + Queue("tas-main"). + Priority(3). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "2"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: topology \"tas-single-level\" doesn't allow to fit any of 1 pod(s). Pending the preemption of 1 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }). + Obj(), + *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). + Queue("tas-main"). + Priority(1). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-main"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "5"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "5"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, - wantPreempted: sets.New[workload.Reference]("default/low-priority-admitted"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-main": {"default/foo"}, }, wantEvents: []utiltesting.EventRecord{ + utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: UNKNOWN, JobUID: UNKNOWN) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). Obj(), utiltesting.MakeEventRecord("default", "foo", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: topology "tas-single-level" doesn't allow to fit any of 1 pod(s). Pending the preemption of 1 workload(s)`). @@ -2689,6 +2819,8 @@ func TestScheduleForTASPreemption(t *testing.T) { clusterQueues: []kueue.ClusterQueue{defaultClusterQueueWithPreemption}, workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("high-priority-waiting", "default"). + UID("wl-high-priority-waiting"). + JobUID("job-high-priority-waiting"). Queue("tas-main"). Priority(3). PodSets(*utiltestingapi.MakePodSet("one", 1). @@ -2699,7 +2831,7 @@ func TestScheduleForTASPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). Queue("tas-main"). Priority(1). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-main"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "5"). @@ -2708,21 +2840,84 @@ func TestScheduleForTASPreemption(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "5"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("high-priority-waiting", "default"). + UID("wl-high-priority-waiting"). + JobUID("job-high-priority-waiting"). + Queue("tas-main"). + Priority(3). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "2"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: topology \"tas-single-level\" doesn't allow to fit any of 1 pod(s). Pending the preemption of 1 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }). + Obj(), + *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). + Queue("tas-main"). + Priority(1). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-main"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "5"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "5"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), }, - wantPreempted: sets.New[workload.Reference]("default/low-priority-admitted"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-main": {"default/high-priority-waiting"}, }, wantEvents: []utiltesting.EventRecord{ + utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). + Message("Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue"). + Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: UNKNOWN, JobUID: UNKNOWN) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue"). Obj(), utiltesting.MakeEventRecord("default", "high-priority-waiting", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: topology "tas-single-level" doesn't allow to fit any of 1 pod(s). Pending the preemption of 1 workload(s)`). @@ -2738,6 +2933,8 @@ func TestScheduleForTASPreemption(t *testing.T) { clusterQueues: []kueue.ClusterQueue{defaultClusterQueueWithPreemption}, workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("foo", "default"). + UID("wl-foo"). + JobUID("job-foo"). Queue("tas-main"). Priority(3). PodSets(*utiltestingapi.MakePodSet("one", 1). @@ -2748,7 +2945,7 @@ func TestScheduleForTASPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("mid-priority-admitted", "default"). Queue("tas-main"). Priority(2). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-main"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "2"). @@ -2757,8 +2954,9 @@ func TestScheduleForTASPreemption(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "2"). @@ -2767,7 +2965,7 @@ func TestScheduleForTASPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). Queue("tas-main"). Priority(1). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-main"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "2"). @@ -2776,21 +2974,104 @@ func TestScheduleForTASPreemption(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "2"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("foo", "default"). + UID("wl-foo"). + JobUID("job-foo"). + Queue("tas-main"). + Priority(3). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "2"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: topology \"tas-single-level\" doesn't allow to fit any of 1 pod(s). Pending the preemption of 1 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }). + Obj(), + *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). + Queue("tas-main"). + Priority(1). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-main"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "2"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "2"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("mid-priority-admitted", "default"). + Queue("tas-main"). + Priority(2). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-main"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "2"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "2"). Obj()). Obj(), }, - wantPreempted: sets.New[workload.Reference]("default/low-priority-admitted"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-main": {"default/foo"}, }, wantEvents: []utiltesting.EventRecord{ + utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: UNKNOWN, JobUID: UNKNOWN) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). Obj(), utiltesting.MakeEventRecord("default", "foo", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: topology "tas-single-level" doesn't allow to fit any of 1 pod(s). Pending the preemption of 1 workload(s)`). @@ -2808,6 +3089,8 @@ func TestScheduleForTASPreemption(t *testing.T) { clusterQueues: []kueue.ClusterQueue{defaultClusterQueueWithPreemption}, workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("foo", "default"). + UID("wl-foo"). + JobUID("job-foo"). Queue("tas-main"). Priority(3). PodSets(*utiltestingapi.MakePodSet("one", 2). @@ -2818,7 +3101,7 @@ func TestScheduleForTASPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("mid-priority-admitted", "default"). Queue("tas-main"). Priority(2). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-main"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "4"). @@ -2827,8 +3110,9 @@ func TestScheduleForTASPreemption(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "4"). @@ -2837,7 +3121,7 @@ func TestScheduleForTASPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). Queue("tas-main"). Priority(1). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-main"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "4"). @@ -2846,21 +3130,104 @@ func TestScheduleForTASPreemption(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "4"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("foo", "default"). + UID("wl-foo"). + JobUID("job-foo"). + Queue("tas-main"). + Priority(3). + PodSets(*utiltestingapi.MakePodSet("one", 2). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: topology \"tas-single-level\" allows to fit only 1 out of 2 pod(s). Pending the preemption of 1 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }). + Obj(), + *utiltestingapi.MakeWorkload("low-priority-admitted", "default"). + Queue("tas-main"). + Priority(1). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-main"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "4"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "4"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("mid-priority-admitted", "default"). + Queue("tas-main"). + Priority(2). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-main"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "4"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "4"). Obj()). Obj(), }, - wantPreempted: sets.New[workload.Reference]("default/low-priority-admitted"), wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-main": {"default/foo"}, }, wantEvents: []utiltesting.EventRecord{ + utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: UNKNOWN, JobUID: UNKNOWN) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). Obj(), utiltesting.MakeEventRecord("default", "foo", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: topology "tas-single-level" allows to fit only 1 out of 2 pod(s). Pending the preemption of 1 workload(s)`). @@ -2897,7 +3264,7 @@ func TestScheduleForTASPreemption(t *testing.T) { *utiltestingapi.MakeWorkload("mid-priority-admitted", "default"). Queue("tas-main"). Priority(2). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-main"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "4"). @@ -2906,13 +3273,65 @@ func TestScheduleForTASPreemption(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "4"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("low-priority-which-would-fit", "default"). + Queue("tas-main"). + Priority(1). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("mid-priority-admitted", "default"). + Queue("tas-main"). + Priority(2). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-main"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "4"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "4"). Obj()). Obj(), + *utiltestingapi.MakeWorkload("mid-priority-waiting", "default"). + Queue("tas-main"). + Priority(2). + PodSets(*utiltestingapi.MakePodSet("one", 2). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: topology \"tas-single-level\" allows to fit only 1 out of 2 pod(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }). + Obj(), }, wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-main": {"default/low-priority-which-would-fit"}, @@ -2945,7 +3364,9 @@ func TestScheduleForTASPreemption(t *testing.T) { &kueue.LocalQueueList{Items: queues}). WithObjects( utiltesting.MakeNamespace("default"), - ) + ). + WithInterceptorFuncs(interceptor.Funcs{SubResourcePatch: utiltesting.TreatSSAAsStrategicMerge}). + WithStatusSubresource(&kueue.Workload{}) _ = tasindexer.SetupIndexes(ctx, utiltesting.AsIndexer(clientBuilder)) cl := clientBuilder.Build() recorder := &utiltesting.EventRecorder{} @@ -2983,22 +3404,13 @@ func TestScheduleForTASPreemption(t *testing.T) { initiallyAdmittedWorkloads.Insert(workload.Key(&w)) } } - scheduler := New(qManager, cqCache, cl, recorder) + scheduler := New(qManager, cqCache, cl, recorder, WithClock(t, testingclock.NewFakeClock(now))) wg := sync.WaitGroup{} scheduler.setAdmissionRoutineWrapper(routine.NewWrapper( func() { wg.Add(1) }, func() { wg.Done() }, )) - var mu sync.Mutex - gotPreempted := sets.New[workload.Reference]() - scheduler.preemptor.OverrideApply(func(_ context.Context, w *kueue.Workload, _, _ string) error { - mu.Lock() - gotPreempted.Insert(workload.Key(w)) - mu.Unlock() - return nil - }) - ctx, cancel := context.WithTimeout(ctx, queueingTimeout) go qManager.CleanUpOnContext(ctx) defer cancel() @@ -3009,10 +3421,26 @@ func TestScheduleForTASPreemption(t *testing.T) { if err != nil { t.Fatalf("unexpected error while building snapshot: %v", err) } - if diff := cmp.Diff(tc.wantPreempted, gotPreempted); diff != "" { - t.Errorf("Unexpected preemptions (-want,+got):\n%s", diff) + + gotWorkloads := &kueue.WorkloadList{} + err = cl.List(ctx, gotWorkloads) + if err != nil { + t.Fatalf("Unexpected list workloads error: %v", err) } - gotAssignments := make(map[workload.Reference]kueue.Admission) + + defaultWorkloadCmpOpts := cmp.Options{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(kueue.Workload{}, "ObjectMeta.ResourceVersion"), + cmpopts.SortSlices(func(a, b metav1.Condition) bool { + return a.Type < b.Type + }), + } + + if diff := cmp.Diff(tc.wantWorkloads, gotWorkloads.Items, defaultWorkloadCmpOpts); diff != "" { + t.Errorf("Unexpected scheduled workloads (-want,+got):\n%s", diff) + } + + gotAssignments := make(map[workload.Reference]kueue.Admission) for cqName, c := range snapshot.ClusterQueues() { for name, w := range c.Workloads { if initiallyAdmittedWorkloads.Has(workload.Key(w.Obj)) { @@ -3048,6 +3476,7 @@ func TestScheduleForTASPreemption(t *testing.T) { } func TestScheduleForTASCohorts(t *testing.T) { + now := time.Now().Truncate(time.Second) defaultNodeX1 := *testingnode.MakeNode("x1"). Label("tas-node", "true"). Label(corev1.LabelHostname, "x1"). @@ -3115,6 +3544,7 @@ func TestScheduleForTASCohorts(t *testing.T) { resourceFlavors []kueue.ResourceFlavor clusterQueues []kueue.ClusterQueue workloads []kueue.Workload + wantWorkloads []kueue.Workload // wantNewAssignments is a summary of all new admissions in the cache after this cycle. wantNewAssignments map[workload.Reference]kueue.Admission @@ -3122,8 +3552,6 @@ func TestScheduleForTASCohorts(t *testing.T) { wantLeft map[kueue.ClusterQueueReference][]workload.Reference // wantInadmissibleLeft is the workload keys that are left in the inadmissible state after this cycle. wantInadmissibleLeft map[kueue.ClusterQueueReference][]workload.Reference - // wantPreempted is the keys of the workloads that get preempted in the scheduling cycle. - wantPreempted sets.Set[workload.Reference] // wantEvents asserts on the events, the comparison options are passed by eventCmpOpts wantEvents []utiltesting.EventRecord // eventCmpOpts are the comparison options for the events @@ -3143,6 +3571,41 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj(), }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "default"). + Queue("tas-lq-a"). + PodSets(*utiltestingapi.MakePodSet("one", 6). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionTrue, + Reason: "QuotaReserved", + Message: "Quota reserved in ClusterQueue tas-cq-a", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(now), + }). + Admission( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Count(6). + Assignment(corev1.ResourceCPU, "tas-default", "6"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 5).Obj()). + Obj()). + Obj()). + Obj(), + ). + Obj(), + }, wantNewAssignments: map[workload.Reference]kueue.Admission{ "default/a1": *utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). @@ -3171,7 +3634,7 @@ func TestScheduleForTASCohorts(t *testing.T) { workloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("a1-admitted", "default"). Queue("tas-lq-a"). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "5"). @@ -3181,14 +3644,17 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 5). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). Obj(), *utiltestingapi.MakeWorkload("b1", "default"). + UID("wl-b1"). + JobUID("job-b1"). Queue("tas-lq-b"). PodSets(*utiltestingapi.MakePodSet("one", 4). PreferredTopologyRequest(corev1.LabelHostname). @@ -3196,14 +3662,73 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj(), }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1-admitted", "default"). + Queue("tas-lq-a"). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "5"). + Count(5). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 5).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 5). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + UID("wl-b1"). + JobUID("job-b1"). + Queue("tas-lq-b"). + PodSets(*utiltestingapi.MakePodSet("one", 4). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: insufficient unused quota for cpu in flavor tas-default, 1 more needed. Pending the preemption of 1 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + }). + Obj(), + }, wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-cq-b": {"default/b1"}, }, - wantPreempted: sets.New[workload.Reference]("default/a1-admitted"), - eventCmpOpts: cmp.Options{eventIgnoreMessage}, + eventCmpOpts: cmp.Options{eventIgnoreMessage}, wantEvents: []utiltesting.EventRecord{ utiltesting.MakeEventRecord("default", "b1", "Pending", corev1.EventTypeWarning).Obj(), utiltesting.MakeEventRecord("default", "a1-admitted", "Preempted", corev1.EventTypeNormal).Obj(), + utiltesting.MakeEventRecord("default", "a1-admitted", "EvictedDueToPreempted", corev1.EventTypeNormal).Obj(), }, }, "reclaim within cohort; single workload is preempted out three candidates": { @@ -3218,7 +3743,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a1-admitted", "default"). Queue("tas-lq-a"). Priority(1). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "5"). @@ -3228,8 +3753,9 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 5). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). @@ -3238,7 +3764,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a2-admitted", "default"). Queue("tas-lq-a"). Priority(2). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "2"). @@ -3248,8 +3774,9 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 2). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). @@ -3258,7 +3785,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a3-admitted", "default"). Queue("tas-lq-a"). Priority(2). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "1"). @@ -3267,28 +3794,133 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + UID("wl-b1"). + JobUID("job-b1"). + Queue("tas-lq-b"). + Priority(3). + PodSets(*utiltestingapi.MakePodSet("one", 3). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1-admitted", "default"). + Queue("tas-lq-a"). + Priority(1). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "5"). + Count(5). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 5).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 5). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("a2-admitted", "default"). + Queue("tas-lq-a"). + Priority(2). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "2"). + Count(2). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 2).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 2). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("a3-admitted", "default"). + Queue("tas-lq-a"). + Priority(2). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "1"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). Obj(), *utiltestingapi.MakeWorkload("b1", "default"). + UID("wl-b1"). + JobUID("job-b1"). Queue("tas-lq-b"). Priority(3). PodSets(*utiltestingapi.MakePodSet("one", 3). PreferredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: insufficient unused quota for cpu in flavor tas-default, 3 more needed. Pending the preemption of 1 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + }, + }). Obj(), }, wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-cq-b": {"default/b1"}, }, - wantPreempted: sets.New[workload.Reference]("default/a1-admitted"), - eventCmpOpts: cmp.Options{eventIgnoreMessage}, + eventCmpOpts: cmp.Options{eventIgnoreMessage}, wantEvents: []utiltesting.EventRecord{ + utiltesting.MakeEventRecord("default", "a1-admitted", "EvictedDueToPreempted", corev1.EventTypeNormal).Obj(), utiltesting.MakeEventRecord("default", "a1-admitted", "Preempted", corev1.EventTypeNormal).Obj(), utiltesting.MakeEventRecord("default", "b1", "Pending", corev1.EventTypeWarning).Obj(), }, @@ -3306,7 +3938,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a1-admitted", "default"). Queue("tas-lq-a"). Priority(3). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "5"). @@ -3316,8 +3948,9 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 4). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). @@ -3326,7 +3959,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a2-admitted", "default"). Queue("tas-lq-a"). Priority(2). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "3"). @@ -3336,14 +3969,88 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 3). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + UID("wl-b1"). + JobUID("job-b1"). + Queue("tas-lq-b"). + Priority(3). + PodSets(*utiltestingapi.MakePodSet("one", 4). + SetMinimumCount(3). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Request(corev1.ResourceMemory, "1"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1-admitted", "default"). + Queue("tas-lq-a"). + Priority(3). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "5"). + Count(5). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 5).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 4). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("a2-admitted", "default"). + Queue("tas-lq-a"). + Priority(2). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "3"). + Count(3). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 3).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 3). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("b1", "default"). + UID("wl-b1"). + JobUID("job-b1"). Queue("tas-lq-b"). Priority(3). PodSets(*utiltestingapi.MakePodSet("one", 4). @@ -3352,14 +4059,28 @@ func TestScheduleForTASCohorts(t *testing.T) { Request(corev1.ResourceCPU, "1"). Request(corev1.ResourceMemory, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: insufficient unused quota for cpu in flavor tas-default, 2 more needed. Pending the preemption of 1 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + corev1.ResourceMemory: resource.MustParse("4"), + }, + }). Obj(), }, wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-cq-b": {"default/b1"}, }, - wantPreempted: sets.New[workload.Reference]("default/a2-admitted"), - eventCmpOpts: cmp.Options{eventIgnoreMessage}, + eventCmpOpts: cmp.Options{eventIgnoreMessage}, wantEvents: []utiltesting.EventRecord{ + utiltesting.MakeEventRecord("default", "a2-admitted", "EvictedDueToPreempted", corev1.EventTypeNormal).Obj(), utiltesting.MakeEventRecord("default", "a2-admitted", "Preempted", corev1.EventTypeNormal).Obj(), utiltesting.MakeEventRecord("default", "b1", "Pending", corev1.EventTypeWarning).Obj(), }, @@ -3376,7 +4097,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a1-admitted", "default"). Queue("tas-lq-a"). Priority(2). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "5"). @@ -3387,8 +4108,9 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 5). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). @@ -3397,7 +4119,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a2-admitted", "default"). Queue("tas-lq-a"). Priority(1). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "1"). @@ -3406,8 +4128,9 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). @@ -3416,7 +4139,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a3-admitted", "default"). Queue("tas-lq-a"). Priority(1). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "1"). @@ -3425,20 +4148,148 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + UID("wl-b1"). + JobUID("job-b1"). + Queue("tas-lq-b"). + Priority(3). + PodSets(*utiltestingapi.MakePodSet("one", 3). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("c1", "default"). + Queue("tas-lq-c"). + Priority(1). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1-admitted", "default"). + Queue("tas-lq-a"). + Priority(2). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "5"). + Count(5). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 3).Obj()). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 2).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 5). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("a2-admitted", "default"). + Queue("tas-lq-a"). + Priority(1). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "1"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 1). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). + Obj(), + *utiltestingapi.MakeWorkload("a3-admitted", "default"). + Queue("tas-lq-a"). + Priority(1). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "1"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 1). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InCohortReclamation", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("b1", "default"). + UID("wl-b1"). + JobUID("job-b1"). Queue("tas-lq-b"). Priority(3). PodSets(*utiltestingapi.MakePodSet("one", 3). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: topology \"tas-single-level\" allows to fit only 1 out of 3 pod(s). Pending the preemption of 2 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + }, + }). Obj(), *utiltestingapi.MakeWorkload("c1", "default"). Queue("tas-lq-c"). @@ -3447,15 +4298,29 @@ func TestScheduleForTASCohorts(t *testing.T) { PreferredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "Workload no longer fits after processing another workload", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + }). Obj(), }, wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-cq-b": {"default/b1"}, "tas-cq-c": {"default/c1"}, }, - wantPreempted: sets.New[workload.Reference]("default/a2-admitted", "default/a3-admitted"), - eventCmpOpts: cmp.Options{eventIgnoreMessage}, + eventCmpOpts: cmp.Options{eventIgnoreMessage}, wantEvents: []utiltesting.EventRecord{ + utiltesting.MakeEventRecord("default", "a2-admitted", "EvictedDueToPreempted", corev1.EventTypeNormal).Obj(), + utiltesting.MakeEventRecord("default", "a3-admitted", "EvictedDueToPreempted", corev1.EventTypeNormal).Obj(), utiltesting.MakeEventRecord("default", "b1", "Pending", corev1.EventTypeWarning).Obj(), utiltesting.MakeEventRecord("default", "c1", "Pending", corev1.EventTypeWarning).Obj(), utiltesting.MakeEventRecord("default", "a2-admitted", "Preempted", corev1.EventTypeNormal).Obj(), @@ -3485,6 +4350,74 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj(), }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "default"). + Priority(2). + Queue("tas-lq-a"). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "5"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionTrue, + Reason: "QuotaReserved", + Message: "Quota reserved in ClusterQueue tas-cq-a", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(now), + }). + Admission( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Count(1). + Assignment(corev1.ResourceCPU, "tas-default", "5"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + ). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + Priority(1). + Queue("tas-lq-b"). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceMemory, "5"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionTrue, + Reason: "QuotaReserved", + Message: "Quota reserved in ClusterQueue tas-cq-b", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(now), + }). + Admission( + utiltestingapi.MakeAdmission("tas-cq-b"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Count(1). + Assignment(corev1.ResourceMemory, "tas-default", "5"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + ). + Obj(), + }, wantNewAssignments: map[workload.Reference]kueue.Admission{ "default/a1": *utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). @@ -3534,6 +4467,74 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj(), }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "default"). + Priority(2). + Queue("tas-lq-a"). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionTrue, + Reason: "QuotaReserved", + Message: "Quota reserved in ClusterQueue tas-cq-a", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(now), + }). + Admission( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Count(1). + Assignment(corev1.ResourceCPU, "tas-default", "1"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + ). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + Priority(1). + Queue("tas-lq-b"). + PodSets(*utiltestingapi.MakePodSet("one", 2). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionTrue, + Reason: "QuotaReserved", + Message: "Quota reserved in ClusterQueue tas-cq-b", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(now), + }). + Admission( + utiltestingapi.MakeAdmission("tas-cq-b"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Count(2). + Assignment(corev1.ResourceCPU, "tas-default", "2"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 2).Obj()). + Obj()). + Obj()). + Obj(), + ). + Obj(), + }, wantNewAssignments: map[workload.Reference]kueue.Admission{ "default/a1": *utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). @@ -3584,6 +4585,62 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj(), }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "default"). + Priority(2). + Queue("tas-lq-a"). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionTrue, + Reason: "QuotaReserved", + Message: "Quota reserved in ClusterQueue tas-cq-a", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(now), + }). + Admission( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Count(1). + Assignment(corev1.ResourceCPU, "tas-default", "1"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + ). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + Priority(1). + Queue("tas-lq-b"). + PodSets(*utiltestingapi.MakePodSet("one", 3). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "Workload no longer fits after processing another workload", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + }, + }). + Obj(), + }, wantNewAssignments: map[workload.Reference]kueue.Admission{ "default/a1": *utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). @@ -3627,6 +4684,62 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj(), }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "default"). + Priority(2). + Queue("tas-lq-a"). + PodSets(*utiltestingapi.MakePodSet("one", 5). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionTrue, + Reason: "QuotaReserved", + Message: "Quota reserved in ClusterQueue tas-cq-a", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(now), + }). + Admission( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Count(5). + Assignment(corev1.ResourceCPU, "tas-default", "5"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 5).Obj()). + Obj()). + Obj()). + Obj(), + ). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + Priority(1). + Queue("tas-lq-b"). + PodSets(*utiltestingapi.MakePodSet("one", 5). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "Workload no longer fits after processing another workload", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5"), + }, + }). + Obj(), + }, wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-cq-b": {"default/b1"}, }, @@ -3673,6 +4786,62 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj(), }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1", "default"). + Priority(2). + Queue("tas-lq-a"). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "2"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionTrue, + Reason: "QuotaReserved", + Message: "Quota reserved in ClusterQueue tas-cq-a", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(now), + }). + Admission( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Count(1). + Assignment(corev1.ResourceCPU, "tas-default", "2"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"x1"}, 1).Obj()). + Obj()). + Obj()). + Obj(), + ). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + Priority(1). + Queue("tas-lq-b"). + PodSets(*utiltestingapi.MakePodSet("one", 1). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "2"). + Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "Workload no longer fits after processing another workload", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + }, + }). + Obj(), + }, wantNewAssignments: map[workload.Reference]kueue.Admission{ "default/a1": *utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). @@ -3711,7 +4880,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a1-admitted", "default"). Queue("tas-lq-a"). Priority(1). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "2"). @@ -3721,20 +4890,92 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 2). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("a2", "default"). + UID("wl-a2"). + JobUID("job-a2"). + Queue("tas-lq-a"). + Priority(2). + PodSets(*utiltestingapi.MakePodSet("one", 4). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + Queue("tas-lq-b"). + Priority(1). + PodSets(*utiltestingapi.MakePodSet("one", 3). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1-admitted", "default"). + Queue("tas-lq-a"). + Priority(1). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "2"). + Count(2). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 2).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 2). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + Reason: "Preempted", + Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadPreempted, + Status: metav1.ConditionTrue, + Reason: "InClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue", + LastTransitionTime: metav1.NewTime(now), + }). + SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). Obj(), *utiltestingapi.MakeWorkload("a2", "default"). + UID("wl-a2"). + JobUID("job-a2"). Queue("tas-lq-a"). Priority(2). PodSets(*utiltestingapi.MakePodSet("one", 4). PreferredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: topology \"tas-single-level\" allows to fit only 3 out of 4 pod(s). Pending the preemption of 1 workload(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + }). Obj(), *utiltestingapi.MakeWorkload("b1", "default"). Queue("tas-lq-b"). @@ -3743,15 +4984,28 @@ func TestScheduleForTASCohorts(t *testing.T) { PreferredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "Workload no longer fits after processing another workload", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + }, + }). Obj(), }, wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ "tas-cq-a": {"default/a2"}, "tas-cq-b": {"default/b1"}, }, - wantPreempted: sets.New[workload.Reference]("default/a1-admitted"), - eventCmpOpts: cmp.Options{eventIgnoreMessage}, + eventCmpOpts: cmp.Options{eventIgnoreMessage}, wantEvents: []utiltesting.EventRecord{ + utiltesting.MakeEventRecord("default", "a1-admitted", "EvictedDueToPreempted", corev1.EventTypeNormal).Obj(), utiltesting.MakeEventRecord("default", "a2", "Pending", corev1.EventTypeWarning).Obj(), utiltesting.MakeEventRecord("default", "a1-admitted", "Preempted", corev1.EventTypeNormal).Obj(), utiltesting.MakeEventRecord("default", "b1", "Pending", corev1.EventTypeWarning).Obj(), @@ -3775,7 +5029,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a1-admitted", "default"). Queue("tas-lq-a"). Priority(2). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "2"). @@ -3785,8 +5039,48 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 2). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("a2", "default"). + Queue("tas-lq-a"). + Priority(2). + PodSets(*utiltestingapi.MakePodSet("one", 4). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + Queue("tas-lq-b"). + Priority(1). + PodSets(*utiltestingapi.MakePodSet("one", 3). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1-admitted", "default"). + Queue("tas-lq-a"). + Priority(2). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "2"). + Count(2). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 2).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 2). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). @@ -3799,6 +5093,19 @@ func TestScheduleForTASCohorts(t *testing.T) { PreferredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: topology \"tas-single-level\" allows to fit only 3 out of 4 pod(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + }). Obj(), *utiltestingapi.MakeWorkload("b1", "default"). Queue("tas-lq-b"). @@ -3807,6 +5114,19 @@ func TestScheduleForTASCohorts(t *testing.T) { PreferredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "Workload no longer fits after processing another workload", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("3"), + }, + }). Obj(), }, wantLeft: map[kueue.ClusterQueueReference][]workload.Reference{ @@ -3839,7 +5159,7 @@ func TestScheduleForTASCohorts(t *testing.T) { *utiltestingapi.MakeWorkload("a1-admitted", "default"). Queue("tas-lq-a"). Priority(2). - ReserveQuota( + ReserveQuotaAt( utiltestingapi.MakeAdmission("tas-cq-a"). PodSets(utiltestingapi.MakePodSetAssignment("one"). Assignment(corev1.ResourceCPU, "tas-default", "2"). @@ -3849,8 +5169,48 @@ func TestScheduleForTASCohorts(t *testing.T) { Obj()). Obj()). Obj(), + now, ). - Admitted(true). + AdmittedAt(true, now). + PodSets(*utiltestingapi.MakePodSet("one", 2). + RequiredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("a2", "default"). + Queue("tas-lq-a"). + Priority(2). + PodSets(*utiltestingapi.MakePodSet("one", 4). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + *utiltestingapi.MakeWorkload("b1", "default"). + Queue("tas-lq-b"). + Priority(1). + PodSets(*utiltestingapi.MakePodSet("one", 3). + PreferredTopologyRequest(corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj()). + Obj(), + }, + wantWorkloads: []kueue.Workload{ + *utiltestingapi.MakeWorkload("a1-admitted", "default"). + Queue("tas-lq-a"). + Priority(2). + ReserveQuotaAt( + utiltestingapi.MakeAdmission("tas-cq-a"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Assignment(corev1.ResourceCPU, "tas-default", "2"). + Count(2). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 2).Obj()). + Obj()). + Obj()). + Obj(), + now, + ). + AdmittedAt(true, now). PodSets(*utiltestingapi.MakePodSet("one", 2). RequiredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). @@ -3863,6 +5223,19 @@ func TestScheduleForTASCohorts(t *testing.T) { PreferredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionFalse, + Reason: "Pending", + Message: "couldn't assign flavors to pod set one: topology \"tas-single-level\" allows to fit only 3 out of 4 pod(s)", + LastTransitionTime: metav1.NewTime(now), + }). + ResourceRequests(kueue.PodSetRequest{ + Name: "one", + Resources: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + }, + }). Obj(), *utiltestingapi.MakeWorkload("b1", "default"). Queue("tas-lq-b"). @@ -3871,6 +5244,31 @@ func TestScheduleForTASCohorts(t *testing.T) { PreferredTopologyRequest(corev1.LabelHostname). Request(corev1.ResourceCPU, "1"). Obj()). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadQuotaReserved, + Status: metav1.ConditionTrue, + Reason: "QuotaReserved", + Message: "Quota reserved in ClusterQueue tas-cq-b", + LastTransitionTime: metav1.NewTime(now), + }). + SetOrReplaceCondition(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(now), + }). + Admission( + utiltestingapi.MakeAdmission("tas-cq-b"). + PodSets(utiltestingapi.MakePodSetAssignment("one"). + Count(3). + Assignment(corev1.ResourceCPU, "tas-default", "3"). + TopologyAssignment(utiltestingapi.MakeTopologyAssignment(utiltas.Levels(&defaultSingleLevelTopology)). + Domain(utiltestingapi.MakeTopologyDomainAssignment([]string{"y1"}, 3).Obj()). + Obj()). + Obj()). + Obj(), + ). Obj(), }, wantInadmissibleLeft: map[kueue.ClusterQueueReference][]workload.Reference{ @@ -3918,6 +5316,7 @@ func TestScheduleForTASCohorts(t *testing.T) { WithObjects( utiltesting.MakeNamespace("default"), ). + WithInterceptorFuncs(interceptor.Funcs{SubResourcePatch: utiltesting.TreatSSAAsStrategicMerge}). WithStatusSubresource(&kueue.Workload{}) _ = tasindexer.SetupIndexes(ctx, utiltesting.AsIndexer(clientBuilder)) cl := clientBuilder.Build() @@ -3956,22 +5355,13 @@ func TestScheduleForTASCohorts(t *testing.T) { initiallyAdmittedWorkloads.Insert(workload.Key(&w)) } } - scheduler := New(qManager, cqCache, cl, recorder) + scheduler := New(qManager, cqCache, cl, recorder, WithClock(t, testingclock.NewFakeClock(now))) wg := sync.WaitGroup{} scheduler.setAdmissionRoutineWrapper(routine.NewWrapper( func() { wg.Add(1) }, func() { wg.Done() }, )) - var mu sync.Mutex - gotPreempted := sets.New[workload.Reference]() - scheduler.preemptor.OverrideApply(func(_ context.Context, w *kueue.Workload, _, _ string) error { - mu.Lock() - gotPreempted.Insert(workload.Key(w)) - mu.Unlock() - return nil - }) - ctx, cancel := context.WithTimeout(ctx, queueingTimeout) go qManager.CleanUpOnContext(ctx) defer cancel() @@ -3982,9 +5372,25 @@ func TestScheduleForTASCohorts(t *testing.T) { if err != nil { t.Fatalf("unexpected error while building snapshot: %v", err) } - if diff := cmp.Diff(tc.wantPreempted, gotPreempted); diff != "" { - t.Errorf("Unexpected preemptions (-want,+got):\n%s", diff) + + gotWorkloads := &kueue.WorkloadList{} + err = cl.List(ctx, gotWorkloads) + if err != nil { + t.Fatalf("Unexpected list workloads error: %v", err) + } + + defaultWorkloadCmpOpts := cmp.Options{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(kueue.Workload{}, "ObjectMeta.ResourceVersion"), + cmpopts.SortSlices(func(a, b metav1.Condition) bool { + return a.Type < b.Type + }), + } + + if diff := cmp.Diff(tc.wantWorkloads, gotWorkloads.Items, defaultWorkloadCmpOpts); diff != "" { + t.Errorf("Unexpected scheduled workloads (-want,+got):\n%s", diff) } + gotAssignments := make(map[workload.Reference]kueue.Admission) for cqName, c := range snapshot.ClusterQueues() { for name, w := range c.Workloads { From c7d7ef36a8a925237b6a43f37eaf3f4a9946204e Mon Sep 17 00:00:00 2001 From: Singularity23x0 Date: Tue, 4 Nov 2025 15:47:28 +0000 Subject: [PATCH 078/119] Completed the initial draft of Delete event refactor. --- apis/kueue/v1beta2/constants.go | 1 + pkg/controller/core/workload_controller.go | 64 +++++----------------- pkg/util/testing/v1beta2/wrappers.go | 6 +- 3 files changed, 20 insertions(+), 51 deletions(-) diff --git a/apis/kueue/v1beta2/constants.go b/apis/kueue/v1beta2/constants.go index c40820066e7..b5fc6f60f3d 100644 --- a/apis/kueue/v1beta2/constants.go +++ b/apis/kueue/v1beta2/constants.go @@ -18,6 +18,7 @@ package v1beta2 const ( ResourceInUseFinalizerName = "kueue.x-k8s.io/resource-in-use" + SafeDeleteFinalizerName = "kueue.x-k8s.io/delete-safeguard" DefaultPodSetName PodSetReference = "main" // ElasticJobSchedulingGate is the name of the scheduling gate applied to Pods diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index 5d32a16b886..173c407013c 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -166,49 +166,42 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c log.V(2).Info("Reconcile Workload") if !wl.DeletionTimestamp.IsZero() { - // Deletion requested (some finalizers must be present). Try to finalize and remove. + log = log.WithValues("deletionTimestamp", wl.DeletionTimestamp) + log.Info("Workload marked for deletion.") + hasInUseFinalizer := controllerutil.ContainsFinalizer(&wl, kueue.ResourceInUseFinalizerName) hasSafeDeleteFinlaizer := controllerutil.ContainsFinalizer(&wl, kueue.SafeDeleteFinalizerName) switch { case hasInUseFinalizer && hasSafeDeleteFinlaizer: { - // Manual delete request by a user. + log.Info("Manual deletion by a user detected.") if len(wl.OwnerReferences) == 0 { - // Delete legal - proceed. - err := r.cleanUp(ctx, &wl, log) - return ctrl.Result{}, err + return ctrl.Result{}, r.cleanUp(ctx, &wl, log) } else { - // Delete illegal at this time, ignore and continue the reconciliation loop. - log.Info("Manual delete by user when workload still has owners. Delete request ignored.") + log.Info("Uable to delete. Workload still has owners.", "owners", wl.OwnerReferences) } } case !hasInUseFinalizer && hasSafeDeleteFinlaizer: { - // Natural deleteion. - err := r.cleanUp(ctx, &wl, log) - return ctrl.Result{}, err + return ctrl.Result{}, r.cleanUp(ctx, &wl, log) } case hasInUseFinalizer && !hasSafeDeleteFinlaizer: { - err := errors.New("Illegal finalizer configuration for workload") - log.Error(err, "Illegal finalizer configuration for workload") + err := errors.New("Illegal finalizer configuration for workload.") + log.Error(err, "Illegal finalizer configuration for workload.") return ctrl.Result{}, err } default: { - // Unknown finalizer preventing deletion. Continuing the reconciliation loop. - log.Info("Unknown finalizers set for workload.") + log.Info("Unknown finalizer(s) preventing workload deletion.") } } } - // TODO: integrate cache and queue updates based on newObject status (make "oldObject status"-agnostic). - finishedCond := apimeta.FindStatusCondition(wl.Status.Conditions, kueue.WorkloadFinished) if finishedCond != nil && finishedCond.Status == metav1.ConditionTrue { - // (new) workload is in state FINISHED - if !features.Enabled(features.ObjectRetentionPolicies) || r.workloadRetention == nil || r.workloadRetention.afterFinished == nil { + log.Info("Unable to determine workload retention scheme.") return ctrl.Result{}, nil } @@ -510,25 +503,21 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } func (r *WorkloadReconciler) cleanUp(ctx context.Context, wl *kueue.Workload, log logr.Logger) error { - log.Info("Cleaning up for workload deletion") + log.Info("Finalizing workload before deletion.") defer r.notifyWatchers(wl, nil) var resultError error = nil if workload.HasQuotaReservation(wl) { - var cleanupErrors []error r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, wl, func() { if err := r.cache.DeleteWorkload(log, wl); err != nil { - append(cleanupErrors, err) + resultError = err log.Error(err, "Failed to delete workload from cache") } }) - if len(cleanupErrors) > 0 { - resultError = errors.Join(cleanupErrors...) - } } else { r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, wl, func() { if err := r.cache.DeleteWorkload(log, wl); err != nil { - log.Info("Failed to delete workload from cache. This may be indended behavior.", "error", err) + log.Info("Failed to delete workload from cache.", "Error", err, "Note", "this may be intended behavior") } }) } @@ -825,37 +814,12 @@ func (r *WorkloadReconciler) Create(e event.TypedCreateEvent[*kueue.Workload]) b } func (r *WorkloadReconciler) Delete(e event.TypedDeleteEvent[*kueue.Workload]) bool { - // TODO: delete commented - - // defer r.notifyWatchers(e.Object, nil) status := "unknown" if !e.DeleteStateUnknown { status = workload.Status(e.Object) } log := r.log.WithValues("workload", klog.KObj(e.Object), "queue", e.Object.Spec.QueueName, "status", status) log.V(2).Info("Workload delete event") - // ctx := ctrl.LoggerInto(context.Background(), log) - - // // When assigning a clusterQueue to a workload, we assume it in the cache. If - // // the state is unknown, the workload could have been assumed, and we need - // // to clear it from the cache. - // if workload.HasQuotaReservation(e.Object) || e.DeleteStateUnknown { - // // trigger the move of associated inadmissibleWorkloads if required. - // r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, e.Object, func() { - // // Delete the workload from cache while holding the queues lock - // // to guarantee that requeued workloads are taken into account before - // // the next scheduling cycle. - // if err := r.cache.DeleteWorkload(log, e.Object); err != nil { - // if !e.DeleteStateUnknown { - // log.Error(err, "Failed to delete workload from cache") - // } - // } - // }) - // } - - // // Even if the state is unknown, the last cached state tells us whether the - // // workload was in the queues and should be cleared from them. - // r.queues.DeleteWorkload(e.Object) return true } diff --git a/pkg/util/testing/v1beta2/wrappers.go b/pkg/util/testing/v1beta2/wrappers.go index 7a97ff80bdc..6b96e7f381d 100644 --- a/pkg/util/testing/v1beta2/wrappers.go +++ b/pkg/util/testing/v1beta2/wrappers.go @@ -62,7 +62,11 @@ type WorkloadWrapper struct{ kueue.Workload } // with a single container. func MakeWorkload(name, ns string) *WorkloadWrapper { return &WorkloadWrapper{kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Finalizers: []string{kueue.SafeDeleteFinalizerName}, + }, Spec: kueue.WorkloadSpec{ PodSets: []kueue.PodSet{ *MakePodSet(kueue.DefaultPodSetName, 1).Obj(), From 1cef4e85e068f340dfd0857c0dace660d48c7bb6 Mon Sep 17 00:00:00 2001 From: Singularity23x0 Date: Wed, 5 Nov 2025 14:06:58 +0000 Subject: [PATCH 079/119] Finished deletion refactor. Added unit tests. --- pkg/cache/queue/manager.go | 4 +- pkg/cache/queue/manager_test.go | 2 +- pkg/cache/scheduler/cache.go | 4 + .../core/workload_controller_test.go | 181 +++++++++++++++--- pkg/util/testing/v1beta2/wrappers.go | 4 + 5 files changed, 162 insertions(+), 33 deletions(-) diff --git a/pkg/cache/queue/manager.go b/pkg/cache/queue/manager.go index 42adebef794..75e1ca17f08 100644 --- a/pkg/cache/queue/manager.go +++ b/pkg/cache/queue/manager.go @@ -723,7 +723,7 @@ func (m *Manager) GetClusterQueueNames() []kueue.ClusterQueueReference { return m.hm.ClusterQueuesNames() } -func (m *Manager) getClusterQueue(cqName kueue.ClusterQueueReference) *ClusterQueue { +func (m *Manager) GetClusterQueue(cqName kueue.ClusterQueueReference) *ClusterQueue { m.RLock() defer m.RUnlock() return m.hm.ClusterQueue(cqName) @@ -735,7 +735,7 @@ func (m *Manager) getClusterQueueLockless(cqName kueue.ClusterQueueReference) (v } func (m *Manager) PendingWorkloadsInfo(cqName kueue.ClusterQueueReference) []*workload.Info { - cq := m.getClusterQueue(cqName) + cq := m.GetClusterQueue(cqName) if cq == nil { return nil } diff --git a/pkg/cache/queue/manager_test.go b/pkg/cache/queue/manager_test.go index 7674d5d69f5..5988c7e4c29 100644 --- a/pkg/cache/queue/manager_test.go +++ b/pkg/cache/queue/manager_test.go @@ -148,7 +148,7 @@ func TestUpdateClusterQueue(t *testing.T) { t.Fatalf("Failed adding clusterQueue %s: %v", cq.Name, err) } // Increase the popCycle to ensure that the workload will be added as inadmissible. - manager.getClusterQueue(kueue.ClusterQueueReference(cq.Name)).popCycle++ + manager.GetClusterQueue(kueue.ClusterQueueReference(cq.Name)).popCycle++ } for _, q := range queues { if err := manager.AddLocalQueue(ctx, q); err != nil { diff --git a/pkg/cache/scheduler/cache.go b/pkg/cache/scheduler/cache.go index 31906ab4f28..40584bc91d2 100644 --- a/pkg/cache/scheduler/cache.go +++ b/pkg/cache/scheduler/cache.go @@ -530,6 +530,10 @@ func (c *Cache) GetCacheLocalQueue(cqName kueue.ClusterQueueReference, lq *kueue return nil, errQNotFound } +func (c *Cache) GetClusterQueue(cqName kueue.ClusterQueueReference) *clusterQueue { + return c.hm.ClusterQueues()[cqName] +} + func (c *Cache) UpdateLocalQueue(oldQ, newQ *kueue.LocalQueue) error { if oldQ.Spec.ClusterQueue == newQ.Spec.ClusterQueue { return nil diff --git a/pkg/controller/core/workload_controller_test.go b/pkg/controller/core/workload_controller_test.go index f7c6c3be9ec..601649a3cac 100644 --- a/pkg/controller/core/workload_controller_test.go +++ b/pkg/controller/core/workload_controller_test.go @@ -48,6 +48,7 @@ import ( utilqueue "sigs.k8s.io/kueue/pkg/util/queue" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" ) @@ -419,6 +420,7 @@ func TestReconcile(t *testing.T) { wantErrorMsg string wantEvents []utiltesting.EventRecord wantResult reconcile.Result + wantWorkloadCached bool reconcilerOpts []Option }{ "reconcile DRA ResourceClaim should be rejected as inadmissible": { @@ -729,34 +731,6 @@ func TestReconcile(t *testing.T) { }). Obj(), }, - "remove finalizer for finished workload": { - workload: utiltestingapi.MakeWorkload("unit-test", "ns").Finalizers(kueue.ResourceInUseFinalizerName). - Condition(metav1.Condition{ - Type: "Finished", - Status: "True", - }). - DeletionTimestamp(testStartTime). - Obj(), - wantWorkload: nil, - }, - "don't remove finalizer for owned finished workload": { - workload: utiltestingapi.MakeWorkload("unit-test", "ns").Finalizers(kueue.ResourceInUseFinalizerName). - Condition(metav1.Condition{ - Type: "Finished", - Status: "True", - }). - ControllerReference(batchv1.SchemeGroupVersion.WithKind("Job"), "job", "test-uid"). - DeletionTimestamp(testStartTime). - Obj(), - wantWorkload: utiltestingapi.MakeWorkload("unit-test", "ns").Finalizers(kueue.ResourceInUseFinalizerName). - Condition(metav1.Condition{ - Type: "Finished", - Status: "True", - }). - ControllerReference(batchv1.SchemeGroupVersion.WithKind("Job"), "job", "test-uid"). - DeletionTimestamp(testStartTime). - Obj(), - }, "unadmitted workload with rejected checks gets deactivated": { workload: utiltestingapi.MakeWorkload("wl", "ns"). ControllerReference(batchv1.SchemeGroupVersion.WithKind("Job"), "ownername", "owneruid"). @@ -2336,6 +2310,7 @@ func TestReconcile(t *testing.T) { "shouldn't try to delete the workload (no event emitted) because it is already being deleted by kubernetes, object retention configured": { enableObjectRetentionPolicies: true, workload: utiltestingapi.MakeWorkload("wl", "ns"). + Finalizers(kueue.SafeDeleteFinalizerName, kueue.ResourceInUseFinalizerName). Condition(metav1.Condition{ Type: kueue.WorkloadFinished, Status: metav1.ConditionTrue, @@ -2344,7 +2319,6 @@ func TestReconcile(t *testing.T) { }, }). DeletionTimestamp(testStartTime). - Finalizers(kueue.ResourceInUseFinalizerName). Obj(), reconcilerOpts: []Option{ WithWorkloadRetention( @@ -2414,6 +2388,126 @@ func TestReconcile(t *testing.T) { }, wantError: nil, }, + "should clean up deleted workload because only safe-delete finalizer is set": { + cq: utiltestingapi.MakeClusterQueue("cq").Obj(), + lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), + workload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers(kueue.SafeDeleteFinalizerName). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantWorkload: nil, + wantError: nil, + wantWorkloadCached: false, + }, + "should clean up deleted workload because has both in-use and safe-delete finalizers but no owners": { + cq: utiltestingapi.MakeClusterQueue("cq").Obj(), + lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), + workload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers(kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantWorkload: nil, + wantError: nil, + wantWorkloadCached: false, + }, + "shouldn't clean up deleted workload because has in-use and safe-delete finalizers and still has owners": { + cq: utiltestingapi.MakeClusterQueue("cq").Obj(), + lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), + workload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers(kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName). + ControllerReference(batchv1.SchemeGroupVersion.WithKind("Job"), "ownername", "owneruid"). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantWorkload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers(kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName). + ControllerReference(batchv1.SchemeGroupVersion.WithKind("Job"), "ownername", "owneruid"). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantError: nil, + wantWorkloadCached: true, + }, + "should throw error because has in-use finalizer but not safe-delete finalizer": { + cq: utiltestingapi.MakeClusterQueue("cq").Obj(), + lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), + workload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers(kueue.ResourceInUseFinalizerName). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantWorkload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers(kueue.ResourceInUseFinalizerName). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantErrorMsg: "Illegal finalizer configuration for workload.", + wantWorkloadCached: true, + }, + "shouldn't clean up deleted workload because unknown finalizers are set": { + cq: utiltestingapi.MakeClusterQueue("cq").Obj(), + lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), + workload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers("unknown-finalizer"). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Obj(), + wantWorkload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers("unknown-finalizer"). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Obj(), + wantError: nil, + wantWorkloadCached: true, + }, } for name, tc := range cases { for _, enabled := range []bool{false, true} { @@ -2439,10 +2533,12 @@ func TestReconcile(t *testing.T) { cqCache := schdcache.New(cl) qManager := qcache.NewManager(cl, cqCache) reconciler := NewWorkloadReconciler(cl, qManager, cqCache, recorder, tc.reconcilerOpts...) + // use a fake clock with jitter = 0 to be able to assert on the requeueAt. reconciler.clock = fakeClock - ctxWithLogger, _ := utiltesting.ContextWithLog(t) + ctxWithLogger, log := utiltesting.ContextWithLog(t) + testLog := log.WithName("Test setup") ctx, ctxCancel := context.WithCancel(ctxWithLogger) defer ctxCancel() @@ -2452,6 +2548,9 @@ func TestReconcile(t *testing.T) { t.Errorf("couldn't create the cluster queue: %v", err) } if err := qManager.AddClusterQueue(ctx, testCq); err != nil { + t.Errorf("couldn't add the cluster queue to the queue cache manager: %v", err) + } + if err := cqCache.AddClusterQueue(ctx, testCq); err != nil { t.Errorf("couldn't add the cluster queue to the cache: %v", err) } } @@ -2485,6 +2584,11 @@ func TestReconcile(t *testing.T) { } } + if testWl != nil { + cqCache.AddOrUpdateWorkload(testLog, testWl) + qManager.AddOrUpdateWorkload(testWl) + } + gotResult, gotError := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(testWl)}) switch { @@ -2530,6 +2634,23 @@ func TestReconcile(t *testing.T) { t.Errorf("unexpected events (-want/+got):\n%s", diff) } + if tc.wantWorkloadCached { + wlKey := workload.Key(testWl) + cqRef := kueue.ClusterQueueReference(tc.cq.Name) + cacheCq := cqCache.GetClusterQueue(cqRef) + managerCq := qManager.GetClusterQueue(cqRef) + + _, wlInClusterQueue := cacheCq.Workloads[wlKey] + if !wlInClusterQueue { + t.Errorf("Expected workload %s to be in cluster queue cache %s.", wlKey, cqRef) + } + + wlInManagerQueue := managerCq.Info(wlKey) + if wlInManagerQueue == nil { + t.Errorf("Expected workload %s to be in cluster queue (manager) %s.", wlKey, cqRef) + } + } + // For DRA tests, verify that workloads are properly queued/cached if tc.enableDRAFeature && testWl != nil && len(testWl.Spec.PodSets) > 0 && diff --git a/pkg/util/testing/v1beta2/wrappers.go b/pkg/util/testing/v1beta2/wrappers.go index 6b96e7f381d..f1aa38239c7 100644 --- a/pkg/util/testing/v1beta2/wrappers.go +++ b/pkg/util/testing/v1beta2/wrappers.go @@ -372,6 +372,10 @@ func (w *WorkloadWrapper) DeletionTimestamp(t time.Time) *WorkloadWrapper { return w } +func (w *WorkloadWrapper) Delete() *WorkloadWrapper { + return w.DeletionTimestamp(time.Now()) +} + func (w *WorkloadWrapper) RequeueState(count *int32, requeueAt *metav1.Time) *WorkloadWrapper { if count == nil && requeueAt == nil { w.Status.RequeueState = nil From c20b6ac9daad3e6422c37c5e39a9460a4f01fc39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Mon, 3 Nov 2025 18:30:05 +0100 Subject: [PATCH 080/119] Remove remote client of insecurely setup cluster (#7486) --- .../multikueue/multikueuecluster.go | 1 + .../multikueue/multikueuecluster_test.go | 25 +++++ test/integration/multikueue/setup_test.go | 96 +++++++++++++++++++ 3 files changed, 122 insertions(+) diff --git a/pkg/controller/admissionchecks/multikueue/multikueuecluster.go b/pkg/controller/admissionchecks/multikueue/multikueuecluster.go index ba9b1c0e4f2..3c61c1ff8eb 100644 --- a/pkg/controller/admissionchecks/multikueue/multikueuecluster.go +++ b/pkg/controller/admissionchecks/multikueue/multikueuecluster.go @@ -424,6 +424,7 @@ func (c *clustersReconciler) Reconcile(ctx context.Context, req reconcile.Reques err = validateKubeconfig(kubeConfig) if err != nil { log.Error(err, "validating kubeconfig failed") + c.stopAndRemoveCluster(req.Name) if updateErr := c.updateStatus(ctx, cluster, false, "InsecureKubeConfig", fmt.Sprintf("insecure kubeconfig: %v", err)); updateErr != nil { return reconcile.Result{}, fmt.Errorf("failed to update MultiKueueCluster status: %w after detecting insecure kubeconfig: %w", updateErr, err) } diff --git a/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go b/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go index a78c1c9a631..8481b77aece 100644 --- a/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go +++ b/pkg/controller/admissionchecks/multikueue/multikueuecluster_test.go @@ -424,6 +424,31 @@ func TestUpdateConfig(t *testing.T) { }, wantErr: fmt.Errorf("validating kubeconfig failed: %w", errors.New("tokenFile is not allowed")), }, + "remove client with invalid kubeconfig": { + reconcileFor: "worker1", + clusters: []kueue.MultiKueueCluster{ + *utiltestingapi.MakeMultiKueueCluster("worker1"). + KubeConfig(kueue.SecretLocationType, "worker1"). + Generation(1). + Obj(), + }, + secrets: []corev1.Secret{ + makeTestSecret("worker1", testKubeconfigInsecure("worker1", ptr.To("/path/to/tokenfile"))), + }, + remoteClients: map[string]*remoteClient{ + "worker1": newTestClient(ctx, "worker1 old kubeconfig", cancelCalled), + }, + wantClusters: []kueue.MultiKueueCluster{ + *utiltestingapi.MakeMultiKueueCluster("worker1"). + KubeConfig(kueue.SecretLocationType, "worker1"). + Active(metav1.ConditionFalse, "InsecureKubeConfig", "insecure kubeconfig: tokenFile is not allowed", 1). + Generation(1). + Obj(), + }, + wantRemoteClients: map[string]*remoteClient{}, + wantCancelCalled: 1, + wantErr: fmt.Errorf("validating kubeconfig failed: %w", errors.New("tokenFile is not allowed")), + }, "skip insecure kubeconfig validation": { reconcileFor: "worker1", clusters: []kueue.MultiKueueCluster{ diff --git a/test/integration/multikueue/setup_test.go b/test/integration/multikueue/setup_test.go index d91929db2d5..a0cbca7aab5 100644 --- a/test/integration/multikueue/setup_test.go +++ b/test/integration/multikueue/setup_test.go @@ -27,15 +27,18 @@ import ( "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" ) @@ -722,4 +725,97 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, }) }) }) + + ginkgo.It("Should properly detect insecure kubeconfig of MultiKueueClusters and remove remote client", func() { + var w1KubeconfigInvalidBytes []byte + ginkgo.By("Create a kubeconfig with an invalid certificate authority path", func() { + cfg, err := worker1TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + w1KubeconfigInvalid, err := clientcmd.Load(cfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(w1KubeconfigInvalid).NotTo(gomega.BeNil()) + + w1KubeconfigInvalid.Clusters["default-cluster"].CertificateAuthority = "/some/random/path" + w1KubeconfigInvalidBytes, err = clientcmd.Write(*w1KubeconfigInvalid) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testing-secret", + Namespace: managersConfigNamespace.Name, + }, + Data: map[string][]byte{ + kueue.MultiKueueConfigSecretKey: w1KubeconfigInvalidBytes, + }, + } + + ginkgo.By("creating the secret, with insecure kubeconfig", func() { + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, secret)).Should(gomega.Succeed()) + ginkgo.DeferCleanup(func() error { return managerTestCluster.client.Delete(managerTestCluster.ctx, secret) }) + }) + + clusterKey := client.ObjectKeyFromObject(workerCluster1) + acKey := client.ObjectKeyFromObject(multiKueueAC) + + ginkgo.By("updating the cluster, the worker1 cluster becomes inactive", func() { + updatedCluster := kueue.MultiKueueCluster{} + ginkgo.By("updating the cluster spec", func() { + gomega.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, clusterKey, &updatedCluster)).To(gomega.Succeed()) + updatedCluster.Spec.KubeConfig.LocationType = kueue.SecretLocationType + updatedCluster.Spec.KubeConfig.Location = secret.Name + gomega.Expect(managerTestCluster.client.Update(managerTestCluster.ctx, &updatedCluster)).To(gomega.Succeed()) + }) + + ginkgo.By("wait for the status update", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, clusterKey, &updatedCluster)).To(gomega.Succeed()) + g.Expect(updatedCluster.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.MultiKueueClusterActive, + Status: metav1.ConditionFalse, + Reason: "InsecureKubeConfig", + Message: "insecure kubeconfig: certificate-authority file paths are not allowed, use certificate-authority-data for cluster default-cluster", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, multiKueueAC)).To(gomega.Succeed()) + g.Expect(multiKueueAC.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo(metav1.Condition{ + Type: kueue.AdmissionCheckActive, + Status: metav1.ConditionTrue, + Reason: "SomeActiveClusters", + Message: "Inactive clusters: [worker1]", + }, util.IgnoreConditionTimestampsAndObservedGeneration))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + + job := testingjob.MakeJob("job1", managerNs.Name). + Queue(kueue.LocalQueueName(managerLq.Name)). + Obj() + ginkgo.By("create a job and reserve quota, only existing remoteClients are part of the NominatedClusterNames", func() { + util.MustCreate(managerTestCluster.ctx, managerTestCluster.client, job) + wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job.Name, job.UID), Namespace: managerNs.Name} + + ginkgo.By("setting workload reservation in the management cluster", func() { + admission := utiltestingapi.MakeAdmission(managerCq.Name).Obj() + util.SetQuotaReservation(managerTestCluster.ctx, managerTestCluster.client, wlLookupKey, admission) + }) + + ginkgo.By("verify remote clients are managed correctly: worker1 was removed and worker2 is still active", func() { + managerWl := &kueue.Workload{} + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.NominatedClusterNames).NotTo(gomega.ContainElements(workerCluster1.Name)) + g.Expect(managerWl.Status.NominatedClusterNames).To(gomega.ContainElements(workerCluster2.Name)) + + createdWorkload := &kueue.Workload{} + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) + g.Expect(worker2TestCluster.client.Get(worker2TestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(createdWorkload.Spec).To(gomega.BeComparableTo(managerWl.Spec)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) }) From 5a83c4575826eb989a8801156ea1b1ca24b53b86 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Tue, 4 Nov 2025 00:12:06 +0530 Subject: [PATCH 081/119] Remove applyPreemption stub. (#7507) --- pkg/scheduler/preemption/preemption.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index 45284b7d8ce..44b8790a07b 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -59,9 +59,6 @@ type Preemptor struct { enableFairSharing bool fsStrategies []fairsharing.Strategy - // stubs - applyPreemption func(ctx context.Context, w *kueue.Workload, reason, message string) error - enabledAfs bool } @@ -93,14 +90,9 @@ func New( fsStrategies: parseStrategies(fs.PreemptionStrategies), enabledAfs: enabledAfs, } - p.applyPreemption = p.patchPreemption return p } -func (p *Preemptor) OverrideApply(f func(context.Context, *kueue.Workload, string, string) error) { - p.applyPreemption = f -} - type Target struct { WorkloadInfo *workload.Info Reason string @@ -177,7 +169,11 @@ func (p *Preemptor) IssuePreemptions(ctx context.Context, preemptor *workload.In target := targets[i] if !meta.IsStatusConditionTrue(target.WorkloadInfo.Obj.Status.Conditions, kueue.WorkloadEvicted) { message := preemptionMessage(preemptor.Obj, target.Reason) - err := p.applyPreemption(ctx, target.WorkloadInfo.Obj, target.Reason, message) + wlCopy := target.WorkloadInfo.Obj.DeepCopy() + err := workload.Evict(ctx, p.client, p.recorder, wlCopy, kueue.WorkloadEvictedByPreemption, message, "", p.clock, workload.WithCustomPrepare(func() (*kueue.Workload, error) { + workload.SetPreemptedCondition(wlCopy, p.clock.Now(), target.Reason, message) + return wlCopy, nil + })) if err != nil { errCh.SendErrorWithCancel(err, cancel) return @@ -194,14 +190,6 @@ func (p *Preemptor) IssuePreemptions(ctx context.Context, preemptor *workload.In return int(successfullyPreempted.Load()), errCh.ReceiveError() } -func (p *Preemptor) patchPreemption(ctx context.Context, w *kueue.Workload, reason, message string) error { - w = w.DeepCopy() - return workload.Evict(ctx, p.client, p.recorder, w, kueue.WorkloadEvictedByPreemption, message, "", p.clock, workload.WithCustomPrepare(func() (*kueue.Workload, error) { - workload.SetPreemptedCondition(w, p.clock.Now(), reason, message) - return w, nil - })) -} - type preemptionAttemptOpts struct { borrowing bool } From 07e442e6a6a071d76871711bdc8e7312551a3ff7 Mon Sep 17 00:00:00 2001 From: Obinna Odirionye Date: Tue, 4 Nov 2025 12:22:08 +0300 Subject: [PATCH 082/119] v1beta2: Remove deprecated PodIntegrationOptions API (#7406) * v1beta2: Remove deprecated PodIntegrationOptions API Remove the PodIntegrationOptions API from v1beta2 Configuration as it has been deprecated since v0.11. Users should migrate to using managedJobsNamespaceSelector for namespace-level pod filtering. Changes: - Remove PodIntegrationOptions type and podOptions field from v1beta2 Integrations - Update conversion functions to ignore PodOptions during v1beta1 to v1beta2 conversion - Update main.go to remove PodOptions integration setup - Update validation to use only managedJobsNamespaceSelector for pod integration - Update pod webhook to remove PodOptions handling - Regenerate deepcopy and conversion code Note: Some tests still need updates and will be addressed in follow-up commits. The API remains available in v1beta1 for backward compatibility. Addresses part of issue #7247 * v1beta2: Remove unnecessary comment about managedJobsNamespaceSelector in validation * v1beta2: Remove obsolete comment regarding PodOptions in SetupWebhook * Set IntegrationOptions to nil in TestProcessOptions --- .../v1beta1/configuration_conversion.go | 5 + .../config/v1beta1/zz_generated.conversion.go | 70 +++++--------- apis/config/v1beta2/configuration_types.go | 12 --- apis/config/v1beta2/zz_generated.deepcopy.go | 30 ------ cmd/kueue/main.go | 4 - pkg/config/config_test.go | 85 +---------------- pkg/config/validation.go | 14 +-- pkg/config/validation_test.go | 95 +------------------ .../jobframework/reconciler_test.go | 13 +-- pkg/controller/jobs/pod/pod_webhook.go | 24 ----- pkg/controller/jobs/pod/pod_webhook_test.go | 54 ----------- .../en/docs/reference/kueue-config.v1beta2.md | 41 -------- 12 files changed, 38 insertions(+), 409 deletions(-) diff --git a/apis/config/v1beta1/configuration_conversion.go b/apis/config/v1beta1/configuration_conversion.go index 249baf6ab7c..6f6449b0a2a 100644 --- a/apis/config/v1beta1/configuration_conversion.go +++ b/apis/config/v1beta1/configuration_conversion.go @@ -39,3 +39,8 @@ func (dst *Configuration) ConvertFrom(srcRaw conversion.Hub) error { func Convert_v1beta1_Configuration_To_v1beta2_Configuration(in *Configuration, out *v1beta2.Configuration, s conversionapi.Scope) error { return autoConvert_v1beta1_Configuration_To_v1beta2_Configuration(in, out, s) } + +// Convert_v1beta1_Integrations_To_v1beta2_Integrations is a conversion function that ignores deprecated PodOptions field. +func Convert_v1beta1_Integrations_To_v1beta2_Integrations(in *Integrations, out *v1beta2.Integrations, s conversionapi.Scope) error { + return autoConvert_v1beta1_Integrations_To_v1beta2_Integrations(in, out, s) +} diff --git a/apis/config/v1beta1/zz_generated.conversion.go b/apis/config/v1beta1/zz_generated.conversion.go index 9a64937ceb1..61e2ec2afbc 100644 --- a/apis/config/v1beta1/zz_generated.conversion.go +++ b/apis/config/v1beta1/zz_generated.conversion.go @@ -134,11 +134,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*Integrations)(nil), (*v1beta2.Integrations)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_Integrations_To_v1beta2_Integrations(a.(*Integrations), b.(*v1beta2.Integrations), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*v1beta2.Integrations)(nil), (*Integrations)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_Integrations_To_v1beta1_Integrations(a.(*v1beta2.Integrations), b.(*Integrations), scope) }); err != nil { @@ -184,16 +179,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*PodIntegrationOptions)(nil), (*v1beta2.PodIntegrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions(a.(*PodIntegrationOptions), b.(*v1beta2.PodIntegrationOptions), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta2.PodIntegrationOptions)(nil), (*PodIntegrationOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(a.(*v1beta2.PodIntegrationOptions), b.(*PodIntegrationOptions), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*RequeuingStrategy)(nil), (*v1beta2.RequeuingStrategy)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_RequeuingStrategy_To_v1beta2_RequeuingStrategy(a.(*RequeuingStrategy), b.(*v1beta2.RequeuingStrategy), scope) }); err != nil { @@ -249,6 +234,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*Integrations)(nil), (*v1beta2.Integrations)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_Integrations_To_v1beta2_Integrations(a.(*Integrations), b.(*v1beta2.Integrations), scope) + }); err != nil { + return err + } return nil } @@ -308,7 +298,15 @@ func autoConvert_v1beta1_Configuration_To_v1beta2_Configuration(in *Configuratio out.InternalCertManagement = (*v1beta2.InternalCertManagement)(unsafe.Pointer(in.InternalCertManagement)) out.WaitForPodsReady = (*v1beta2.WaitForPodsReady)(unsafe.Pointer(in.WaitForPodsReady)) out.ClientConnection = (*v1beta2.ClientConnection)(unsafe.Pointer(in.ClientConnection)) - out.Integrations = (*v1beta2.Integrations)(unsafe.Pointer(in.Integrations)) + if in.Integrations != nil { + in, out := &in.Integrations, &out.Integrations + *out = new(v1beta2.Integrations) + if err := Convert_v1beta1_Integrations_To_v1beta2_Integrations(*in, *out, s); err != nil { + return err + } + } else { + out.Integrations = nil + } // WARNING: in.QueueVisibility requires manual conversion: does not exist in peer-type out.MultiKueue = (*v1beta2.MultiKueue)(unsafe.Pointer(in.MultiKueue)) out.FairSharing = (*v1beta2.FairSharing)(unsafe.Pointer(in.FairSharing)) @@ -329,7 +327,15 @@ func autoConvert_v1beta2_Configuration_To_v1beta1_Configuration(in *v1beta2.Conf out.InternalCertManagement = (*InternalCertManagement)(unsafe.Pointer(in.InternalCertManagement)) out.WaitForPodsReady = (*WaitForPodsReady)(unsafe.Pointer(in.WaitForPodsReady)) out.ClientConnection = (*ClientConnection)(unsafe.Pointer(in.ClientConnection)) - out.Integrations = (*Integrations)(unsafe.Pointer(in.Integrations)) + if in.Integrations != nil { + in, out := &in.Integrations, &out.Integrations + *out = new(Integrations) + if err := Convert_v1beta2_Integrations_To_v1beta1_Integrations(*in, *out, s); err != nil { + return err + } + } else { + out.Integrations = nil + } out.MultiKueue = (*MultiKueue)(unsafe.Pointer(in.MultiKueue)) out.FairSharing = (*FairSharing)(unsafe.Pointer(in.FairSharing)) out.AdmissionFairSharing = (*AdmissionFairSharing)(unsafe.Pointer(in.AdmissionFairSharing)) @@ -525,20 +531,14 @@ func Convert_v1beta2_FairSharing_To_v1beta1_FairSharing(in *v1beta2.FairSharing, func autoConvert_v1beta1_Integrations_To_v1beta2_Integrations(in *Integrations, out *v1beta2.Integrations, s conversion.Scope) error { out.Frameworks = *(*[]string)(unsafe.Pointer(&in.Frameworks)) out.ExternalFrameworks = *(*[]string)(unsafe.Pointer(&in.ExternalFrameworks)) - out.PodOptions = (*v1beta2.PodIntegrationOptions)(unsafe.Pointer(in.PodOptions)) + // WARNING: in.PodOptions requires manual conversion: does not exist in peer-type out.LabelKeysToCopy = *(*[]string)(unsafe.Pointer(&in.LabelKeysToCopy)) return nil } -// Convert_v1beta1_Integrations_To_v1beta2_Integrations is an autogenerated conversion function. -func Convert_v1beta1_Integrations_To_v1beta2_Integrations(in *Integrations, out *v1beta2.Integrations, s conversion.Scope) error { - return autoConvert_v1beta1_Integrations_To_v1beta2_Integrations(in, out, s) -} - func autoConvert_v1beta2_Integrations_To_v1beta1_Integrations(in *v1beta2.Integrations, out *Integrations, s conversion.Scope) error { out.Frameworks = *(*[]string)(unsafe.Pointer(&in.Frameworks)) out.ExternalFrameworks = *(*[]string)(unsafe.Pointer(&in.ExternalFrameworks)) - out.PodOptions = (*PodIntegrationOptions)(unsafe.Pointer(in.PodOptions)) out.LabelKeysToCopy = *(*[]string)(unsafe.Pointer(&in.LabelKeysToCopy)) return nil } @@ -640,28 +640,6 @@ func Convert_v1beta2_ObjectRetentionPolicies_To_v1beta1_ObjectRetentionPolicies( return autoConvert_v1beta2_ObjectRetentionPolicies_To_v1beta1_ObjectRetentionPolicies(in, out, s) } -func autoConvert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions(in *PodIntegrationOptions, out *v1beta2.PodIntegrationOptions, s conversion.Scope) error { - out.NamespaceSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) - out.PodSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.PodSelector)) - return nil -} - -// Convert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions is an autogenerated conversion function. -func Convert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions(in *PodIntegrationOptions, out *v1beta2.PodIntegrationOptions, s conversion.Scope) error { - return autoConvert_v1beta1_PodIntegrationOptions_To_v1beta2_PodIntegrationOptions(in, out, s) -} - -func autoConvert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(in *v1beta2.PodIntegrationOptions, out *PodIntegrationOptions, s conversion.Scope) error { - out.NamespaceSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) - out.PodSelector = (*metav1.LabelSelector)(unsafe.Pointer(in.PodSelector)) - return nil -} - -// Convert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions is an autogenerated conversion function. -func Convert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(in *v1beta2.PodIntegrationOptions, out *PodIntegrationOptions, s conversion.Scope) error { - return autoConvert_v1beta2_PodIntegrationOptions_To_v1beta1_PodIntegrationOptions(in, out, s) -} - func autoConvert_v1beta1_RequeuingStrategy_To_v1beta2_RequeuingStrategy(in *RequeuingStrategy, out *v1beta2.RequeuingStrategy, s conversion.Scope) error { out.Timestamp = (*v1beta2.RequeuingTimestamp)(unsafe.Pointer(in.Timestamp)) out.BackoffLimitCount = (*int32)(unsafe.Pointer(in.BackoffLimitCount)) diff --git a/apis/config/v1beta2/configuration_types.go b/apis/config/v1beta2/configuration_types.go index 841d02e38a9..bf450a6a5c5 100644 --- a/apis/config/v1beta2/configuration_types.go +++ b/apis/config/v1beta2/configuration_types.go @@ -405,11 +405,6 @@ type Integrations struct { // List of GroupVersionKinds that are managed for Kueue by external controllers; // the expected format is `Kind.version.group.com`. ExternalFrameworks []string `json:"externalFrameworks,omitempty"` - // PodOptions defines kueue controller behaviour for pod objects - // Deprecated: This field will be removed on v1beta2, use ManagedJobsNamespaceSelector - // (https://kueue.sigs.k8s.io/docs/tasks/run/plain_pods/) - // instead. - PodOptions *PodIntegrationOptions `json:"podOptions,omitempty"` // labelKeysToCopy is a list of label keys that should be copied from the job into the // workload object. It is not required for the job to have all the labels from this @@ -423,13 +418,6 @@ type Integrations struct { LabelKeysToCopy []string `json:"labelKeysToCopy,omitempty"` } -type PodIntegrationOptions struct { - // NamespaceSelector can be used to omit some namespaces from pod reconciliation - NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` - // PodSelector can be used to choose what pods to reconcile - PodSelector *metav1.LabelSelector `json:"podSelector,omitempty"` -} - type Resources struct { // ExcludedResourcePrefixes defines which resources should be ignored by Kueue ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"` diff --git a/apis/config/v1beta2/zz_generated.deepcopy.go b/apis/config/v1beta2/zz_generated.deepcopy.go index ec28f195985..69cc90bd155 100644 --- a/apis/config/v1beta2/zz_generated.deepcopy.go +++ b/apis/config/v1beta2/zz_generated.deepcopy.go @@ -322,11 +322,6 @@ func (in *Integrations) DeepCopyInto(out *Integrations) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.PodOptions != nil { - in, out := &in.PodOptions, &out.PodOptions - *out = new(PodIntegrationOptions) - (*in).DeepCopyInto(*out) - } if in.LabelKeysToCopy != nil { in, out := &in.LabelKeysToCopy, &out.LabelKeysToCopy *out = make([]string, len(*in)) @@ -449,31 +444,6 @@ func (in *ObjectRetentionPolicies) DeepCopy() *ObjectRetentionPolicies { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *PodIntegrationOptions) DeepCopyInto(out *PodIntegrationOptions) { - *out = *in - if in.NamespaceSelector != nil { - in, out := &in.NamespaceSelector, &out.NamespaceSelector - *out = new(v1.LabelSelector) - (*in).DeepCopyInto(*out) - } - if in.PodSelector != nil { - in, out := &in.PodSelector, &out.PodSelector - *out = new(v1.LabelSelector) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodIntegrationOptions. -func (in *PodIntegrationOptions) DeepCopy() *PodIntegrationOptions { - if in == nil { - return nil - } - out := new(PodIntegrationOptions) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RequeuingStrategy) DeepCopyInto(out *RequeuingStrategy) { *out = *in diff --git a/cmd/kueue/main.go b/cmd/kueue/main.go index 94a63080c17..90cc6a2a882 100644 --- a/cmd/kueue/main.go +++ b/cmd/kueue/main.go @@ -28,7 +28,6 @@ import ( zaplog "go.uber.org/zap" "go.uber.org/zap/zapcore" - corev1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -407,9 +406,6 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, cCache *schdcache.C jobframework.WithQueues(queues), jobframework.WithObjectRetentionPolicies(cfg.ObjectRetentionPolicies), } - if cfg.Integrations.PodOptions != nil { - opts = append(opts, jobframework.WithIntegrationOptions(corev1.SchemeGroupVersion.WithKind("Pod").String(), cfg.Integrations.PodOptions)) - } nsSelector, err := metav1.LabelSelectorAsSelector(cfg.ManagedJobsNamespaceSelector) if err != nil { return fmt.Errorf("failed to parse managedJobsNamespaceSelector: %w", err) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1fc909c6c41..1456a102b8e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -282,28 +282,6 @@ integrations: t.Fatal(err) } - podIntegrationOptionsConfig := filepath.Join(tmpDir, "podIntegrationOptions.yaml") - if err := os.WriteFile(podIntegrationOptionsConfig, []byte(` -apiVersion: config.kueue.x-k8s.io/v1beta2 -kind: Configuration -integrations: - frameworks: - - pod - podOptions: - namespaceSelector: - matchExpressions: - - key: kubernetes.io/metadata.name - operator: NotIn - values: [ kube-system, kueue-system, prohibited-namespace ] - podSelector: - matchExpressions: - - key: kueue-job - operator: In - values: [ "true", "True", "yes" ] -`), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - multiKueueConfig := filepath.Join(tmpDir, "multiKueue.yaml") if err := os.WriteFile(multiKueueConfig, []byte(` apiVersion: config.kueue.x-k8s.io/v1beta2 @@ -804,68 +782,7 @@ objectRetentionPolicies: }, }, }, - { - name: "pod integration options config", - configFile: podIntegrationOptionsConfig, - wantConfiguration: configapi.Configuration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: configapi.GroupVersion.String(), - Kind: "Configuration", - }, - Namespace: ptr.To(configapi.DefaultNamespace), - ManageJobsWithoutQueueName: false, - InternalCertManagement: enableDefaultInternalCertManagement, - ClientConnection: defaultClientConnection, - Integrations: &configapi.Integrations{ - Frameworks: []string{ - "pod", - }, - PodOptions: &configapi.PodIntegrationOptions{ - NamespaceSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: corev1.LabelMetadataName, - Operator: metav1.LabelSelectorOpNotIn, - Values: []string{"kube-system", "kueue-system", "prohibited-namespace"}, - }, - }, - }, - PodSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "kueue-job", - Operator: metav1.LabelSelectorOpIn, - Values: []string{"true", "True", "yes"}, - }, - }, - }, - }, - }, - MultiKueue: defaultMultiKueue, - ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector, - WaitForPodsReady: defaultWaitForPodsReady, - }, - wantOptions: ctrl.Options{ - Cache: defaultControlCacheOptions(configapi.DefaultNamespace), - HealthProbeBindAddress: configapi.DefaultHealthProbeBindAddress, - Metrics: metricsserver.Options{ - BindAddress: configapi.DefaultMetricsBindAddress, - }, - LeaderElection: true, - LeaderElectionID: configapi.DefaultLeaderElectionID, - LeaderElectionResourceLock: resourcelock.LeasesResourceLock, - LeaderElectionReleaseOnCancel: true, - LeaseDuration: ptr.To(configapi.DefaultLeaderElectionLeaseDuration), - RenewDeadline: ptr.To(configapi.DefaultLeaderElectionRenewDeadline), - RetryPeriod: ptr.To(configapi.DefaultLeaderElectionRetryPeriod), - WebhookServer: &webhook.DefaultServer{ - Options: webhook.Options{ - Port: configapi.DefaultWebhookPort, - CertDir: configapi.DefaultWebhookCertDir, - }, - }, - }, - }, + { name: "multiKueue config", configFile: multiKueueConfig, diff --git a/pkg/config/validation.go b/pkg/config/validation.go index 7ef3c56513b..c3877ec6bf0 100644 --- a/pkg/config/validation.go +++ b/pkg/config/validation.go @@ -47,8 +47,6 @@ var ( integrationsPath = field.NewPath("integrations") integrationsFrameworksPath = integrationsPath.Child("frameworks") integrationsExternalFrameworkPath = integrationsPath.Child("externalFrameworks") - podOptionsPath = integrationsPath.Child("podOptions") - podOptionsNamespaceSelectorPath = podOptionsPath.Child("namespaceSelector") managedJobsNamespaceSelectorPath = field.NewPath("managedJobsNamespaceSelector") waitForPodsReadyPath = field.NewPath("waitForPodsReady") requeuingStrategyPath = waitForPodsReadyPath.Child("requeuingStrategy") @@ -255,19 +253,9 @@ func validatePodIntegrationOptions(c *configapi.Configuration) field.ErrorList { return allErrs } - // At least one namespace selector must be non-nil and enabled. - // It is ok for both to be non-nil; pods will only be managed if all non-nil selectors match - hasNamespaceSelector := false if c.ManagedJobsNamespaceSelector != nil { allErrs = validateNamespaceSelectorForPodIntegration(c, c.ManagedJobsNamespaceSelector, managedJobsNamespaceSelectorPath, allErrs) - hasNamespaceSelector = true - } - if c.Integrations.PodOptions != nil && c.Integrations.PodOptions.NamespaceSelector != nil { - allErrs = validateNamespaceSelectorForPodIntegration(c, c.Integrations.PodOptions.NamespaceSelector, podOptionsNamespaceSelectorPath, allErrs) - hasNamespaceSelector = true - } - - if !hasNamespaceSelector { + } else { allErrs = append(allErrs, field.Required(managedJobsNamespaceSelectorPath, "cannot be empty when pod integration is enabled")) } diff --git a/pkg/config/validation_test.go b/pkg/config/validation_test.go index ee1c87a6364..bb7e94a71d7 100644 --- a/pkg/config/validation_test.go +++ b/pkg/config/validation_test.go @@ -52,13 +52,8 @@ func TestValidate(t *testing.T) { }, }, } - defaultPodIntegrationOptions := &configapi.PodIntegrationOptions{ - NamespaceSelector: systemNamespacesSelector, - PodSelector: &metav1.LabelSelector{}, - } defaultIntegrations := &configapi.Integrations{ Frameworks: []string{"batch/job"}, - PodOptions: defaultPodIntegrationOptions, } testCases := map[string]struct { @@ -159,11 +154,10 @@ func TestValidate(t *testing.T) { }, }, }, - "nil PodIntegrationOptions and nil managedJobsNamespaceSelector": { + "nil managedJobsNamespaceSelector with pod framework": { cfg: &configapi.Configuration{ Integrations: &configapi.Integrations{ Frameworks: []string{"pod"}, - PodOptions: nil, }, }, wantErr: field.ErrorList{ @@ -173,27 +167,7 @@ func TestValidate(t *testing.T) { }, }, }, - "emptyLabelSelector": { - cfg: &configapi.Configuration{ - Namespace: ptr.To("kueue-system"), - Integrations: &configapi.Integrations{ - Frameworks: []string{"pod"}, - PodOptions: &configapi.PodIntegrationOptions{ - NamespaceSelector: &metav1.LabelSelector{}, - }, - }, - }, - wantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "integrations.podOptions.namespaceSelector", - }, - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "integrations.podOptions.namespaceSelector", - }, - }, - }, + "valid managedJobsNamespaceSelector ": { cfg: &configapi.Configuration{ ManagedJobsNamespaceSelector: systemNamespacesSelector, @@ -202,26 +176,6 @@ func TestValidate(t *testing.T) { wantErr: nil, }, - "prohibited namespace in MatchLabels": { - cfg: &configapi.Configuration{ - Integrations: &configapi.Integrations{ - Frameworks: []string{"pod"}, - PodOptions: &configapi.PodIntegrationOptions{ - NamespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - corev1.LabelMetadataName: "kube-system", - }, - }, - }, - }, - }, - wantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "integrations.podOptions.namespaceSelector", - }, - }, - }, "prohibited namespace in MatchLabels managedJobsNamespaceSelector": { cfg: &configapi.Configuration{ ManagedJobsNamespaceSelector: &metav1.LabelSelector{ @@ -238,30 +192,7 @@ func TestValidate(t *testing.T) { }, }, }, - "prohibited namespace in MatchExpressions with operator In": { - cfg: &configapi.Configuration{ - Integrations: &configapi.Integrations{ - Frameworks: []string{"pod"}, - PodOptions: &configapi.PodIntegrationOptions{ - NamespaceSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: corev1.LabelMetadataName, - Operator: metav1.LabelSelectorOpIn, - Values: []string{"kube-system"}, - }, - }, - }, - }, - }, - }, - wantErr: field.ErrorList{ - &field.Error{ - Type: field.ErrorTypeInvalid, - Field: "integrations.podOptions.namespaceSelector", - }, - }, - }, + "prohibited namespace in MatchExpressions with operator In managedJobsNamespaceSelector": { cfg: &configapi.Configuration{ ManagedJobsNamespaceSelector: &metav1.LabelSelector{ @@ -282,25 +213,7 @@ func TestValidate(t *testing.T) { }, }, }, - "prohibited namespace in MatchExpressions with operator NotIn": { - cfg: &configapi.Configuration{ - Integrations: &configapi.Integrations{ - Frameworks: []string{"pod"}, - PodOptions: &configapi.PodIntegrationOptions{ - NamespaceSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: corev1.LabelMetadataName, - Operator: metav1.LabelSelectorOpNotIn, - Values: []string{"kube-system", "kueue-system"}, - }, - }, - }, - }, - }, - }, - wantErr: nil, - }, + "prohibited namespace in MatchExpressions with operator NotIn managedJobsNamespaceSelector": { cfg: &configapi.Configuration{ ManagedJobsNamespaceSelector: &metav1.LabelSelector{ diff --git a/pkg/controller/jobframework/reconciler_test.go b/pkg/controller/jobframework/reconciler_test.go index a7aef3da6cb..b60cd1a5813 100644 --- a/pkg/controller/jobframework/reconciler_test.go +++ b/pkg/controller/jobframework/reconciler_test.go @@ -662,9 +662,6 @@ func TestProcessOptions(t *testing.T) { WithManageJobsWithoutQueueName(true), WithWaitForPodsReady(&configapi.WaitForPodsReady{Enable: true}), WithKubeServerVersion(&kubeversion.ServerVersionFetcher{}), - WithIntegrationOptions(corev1.SchemeGroupVersion.WithKind("Pod").String(), &configapi.PodIntegrationOptions{ - PodSelector: &metav1.LabelSelector{}, - }), WithLabelKeysToCopy([]string{"toCopyKey"}), WithClock(t, fakeClock), }, @@ -672,13 +669,9 @@ func TestProcessOptions(t *testing.T) { ManageJobsWithoutQueueName: true, WaitForPodsReady: true, KubeServerVersion: &kubeversion.ServerVersionFetcher{}, - IntegrationOptions: map[string]any{ - corev1.SchemeGroupVersion.WithKind("Pod").String(): &configapi.PodIntegrationOptions{ - PodSelector: &metav1.LabelSelector{}, - }, - }, - LabelKeysToCopy: []string{"toCopyKey"}, - Clock: fakeClock, + IntegrationOptions: nil, + LabelKeysToCopy: []string{"toCopyKey"}, + Clock: fakeClock, }, }, "a single option is passed": { diff --git a/pkg/controller/jobs/pod/pod_webhook.go b/pkg/controller/jobs/pod/pod_webhook.go index 82d5855a457..623e66d0160 100644 --- a/pkg/controller/jobs/pod/pod_webhook.go +++ b/pkg/controller/jobs/pod/pod_webhook.go @@ -18,7 +18,6 @@ package pod import ( "context" - "errors" "fmt" corev1 "k8s.io/api/core/v1" @@ -32,7 +31,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" "sigs.k8s.io/kueue/pkg/constants" @@ -53,8 +51,6 @@ var ( prebuiltWorkloadLabelPath = labelsPath.Key(ctrlconstants.PrebuiltWorkloadLabel) groupTotalCountAnnotationPath = annotationsPath.Key(podconstants.GroupTotalCountAnnotation) retriableInGroupAnnotationPath = annotationsPath.Key(podconstants.RetriableInGroupAnnotationKey) - - errPodOptsTypeAssertion = errors.New("options are not of type PodIntegrationOptions") ) type PodWebhook struct { @@ -69,20 +65,12 @@ type PodWebhook struct { // SetupWebhook configures the webhook for pods. func SetupWebhook(mgr ctrl.Manager, opts ...jobframework.Option) error { options := jobframework.ProcessOptions(opts...) - podOpts, err := getPodOptions(options.IntegrationOptions) - if err != nil { - return err - } wh := &PodWebhook{ client: mgr.GetClient(), queues: options.Queues, manageJobsWithoutQueueName: options.ManageJobsWithoutQueueName, managedJobsNamespaceSelector: options.ManagedJobsNamespaceSelector, } - if podOpts != nil { - wh.namespaceSelector = podOpts.NamespaceSelector - wh.podSelector = podOpts.PodSelector - } obj := &corev1.Pod{} return webhook.WebhookManagedBy(mgr). For(obj). @@ -91,18 +79,6 @@ func SetupWebhook(mgr ctrl.Manager, opts ...jobframework.Option) error { Complete() } -func getPodOptions(integrationOpts map[string]any) (*configapi.PodIntegrationOptions, error) { - opts, ok := integrationOpts[corev1.SchemeGroupVersion.WithKind("Pod").String()] - if !ok { - return nil, nil - } - podOpts, ok := opts.(*configapi.PodIntegrationOptions) - if !ok { - return nil, fmt.Errorf("%w, got %T", errPodOptsTypeAssertion, opts) - } - return podOpts, nil -} - // +kubebuilder:webhook:path=/mutate--v1-pod,mutating=true,failurePolicy=fail,sideEffects=None,groups="",resources=pods,verbs=create,versions=v1,name=mpod.kb.io,admissionReviewVersions=v1 // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch diff --git a/pkg/controller/jobs/pod/pod_webhook_test.go b/pkg/controller/jobs/pod/pod_webhook_test.go index 4497ea5e70f..95f41eb6697 100644 --- a/pkg/controller/jobs/pod/pod_webhook_test.go +++ b/pkg/controller/jobs/pod/pod_webhook_test.go @@ -35,7 +35,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" @@ -984,56 +983,3 @@ func TestValidateUpdate(t *testing.T) { }) } } - -func TestGetPodOptions(t *testing.T) { - cases := map[string]struct { - integrationOpts map[string]any - wantOpts *configapi.PodIntegrationOptions - wantError error - }{ - "proper podIntegrationOptions exists": { - integrationOpts: map[string]any{ - corev1.SchemeGroupVersion.WithKind("Pod").String(): &configapi.PodIntegrationOptions{ - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"podKey": "podValue"}, - }, - NamespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"nsKey": "nsValue"}, - }, - }, - batchv1.SchemeGroupVersion.WithKind("Job").String(): nil, - }, - wantOpts: &configapi.PodIntegrationOptions{ - PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"podKey": "podValue"}, - }, - NamespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"nsKey": "nsValue"}, - }, - }, - }, - "integrationOptions doesn't have podIntegrationOptions": { - integrationOpts: map[string]any{ - batchv1.SchemeGroupVersion.WithKind("Job").String(): nil, - }, - wantOpts: nil, - }, - "podIntegrationOptions isn't of type PodIntegrationOptions": { - integrationOpts: map[string]any{ - corev1.SchemeGroupVersion.WithKind("Pod").String(): &configapi.WaitForPodsReady{}, - }, - wantError: errPodOptsTypeAssertion, - }, - } - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - gotOpts, gotError := getPodOptions(tc.integrationOpts) - if diff := cmp.Diff(tc.wantError, gotError, cmpopts.EquateErrors()); len(diff) != 0 { - t.Errorf("Unexpected error from getPodOptions (-want,+got):\n%s", diff) - } - if diff := cmp.Diff(tc.wantOpts, gotOpts, cmpopts.EquateEmpty()); len(diff) != 0 { - t.Errorf("Unexpected podIntegrationOptions from gotPodOptions (-want,+got):\n%s", diff) - } - }) - } -} diff --git a/site/content/en/docs/reference/kueue-config.v1beta2.md b/site/content/en/docs/reference/kueue-config.v1beta2.md index e7a26995fb8..d08e8749483 100644 --- a/site/content/en/docs/reference/kueue-config.v1beta2.md +++ b/site/content/en/docs/reference/kueue-config.v1beta2.md @@ -455,16 +455,6 @@ Possible options:

    the expected format is Kind.version.group.com.

    -podOptions [Required]
    -PodIntegrationOptions - - -

    PodOptions defines kueue controller behaviour for pod objects -Deprecated: This field will be removed on v1beta2, use ManagedJobsNamespaceSelector -(https://kueue.sigs.k8s.io/docs/tasks/run/plain_pods/) -instead.

    - - labelKeysToCopy [Required]
    []string @@ -650,37 +640,6 @@ A nil value disables automatic deletion of Workloads.

    -## `PodIntegrationOptions` {#config-kueue-x-k8s-io-v1beta2-PodIntegrationOptions} - - -**Appears in:** - -- [Integrations](#config-kueue-x-k8s-io-v1beta2-Integrations) - - - - - - - - - - - - - - - -
    FieldDescription
    namespaceSelector [Required]
    -k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector -
    -

    NamespaceSelector can be used to omit some namespaces from pod reconciliation

    -
    podSelector [Required]
    -k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector -
    -

    PodSelector can be used to choose what pods to reconcile

    -
    - ## `PreemptionStrategy` {#config-kueue-x-k8s-io-v1beta2-PreemptionStrategy} (Alias of `string`) From f1347003bd2d0736f2c3fb4a75f189643abbb4a7 Mon Sep 17 00:00:00 2001 From: Omar Sayed Date: Tue, 4 Nov 2025 10:22:15 +0100 Subject: [PATCH 083/119] Switch Default TAS Placement Algorithm from BestFit to Mixed. (#7416) * Enable TAS Mixed Profile for default * adjust feature and kep * remove deprecation of current version for TASProfileMixed * cleanups * fix typo --- keps/2724-topology-aware-scheduling/README.md | 4 +-- pkg/cache/scheduler/tas_cache_test.go | 34 ++++++------------- pkg/features/kube_features.go | 2 +- 3 files changed, 13 insertions(+), 27 deletions(-) diff --git a/keps/2724-topology-aware-scheduling/README.md b/keps/2724-topology-aware-scheduling/README.md index e12a496e2d9..421c93cf75c 100644 --- a/keps/2724-topology-aware-scheduling/README.md +++ b/keps/2724-topology-aware-scheduling/README.md @@ -1072,11 +1072,11 @@ Since v0.15, the available feature gates are as follows: | feature gate / annotation | preferred | required | unconstrained | | ------------------------------------------ | ----------------- | ----------------- | ----------------- | -| None
    or TASProfileMixed (deprecated) | BestFit | BestFit | LeastFreeCapacity | +| None
    or TASProfileMixed (default) | BestFit | BestFit | LeastFreeCapacity | | TASProfileBestFit (deprecated) | BestFit | BestFit | BestFit | | TASProfileLeastFreeCapacity (deprecated) | LeastFreeCapacity | LeastFreeCapacity | LeastFreeCapacity | -Based on the user feedback, we decided to make `TASProfileMixed` default. (The corresponding feature gate is hence obsolete; we formally keep it "deprecated" for backwards compatibility). It differs from the previous default only in the `unconstrained` case - in which Kueue should prioritize minimizing fragmentation which is provided by the `LeastFreeCapacity` algorithm. +Based on the user feedback, we decided to make `TASProfileMixed` default starting in v0.15. (The corresponding feature gate is hence obsolete; we formally keep it "deprecated" for backwards compatibility). It differs from the previous default only in the `unconstrained` case - in which Kueue should prioritize minimizing fragmentation which is provided by the `LeastFreeCapacity` algorithm. For users still preferring the "always BestFit" profile, we introduce the `TASProfileBestFit` feature gate, marking it as deprecated. We will remove it in v0.17 if we see no report indicating a need for that configuration. diff --git a/pkg/cache/scheduler/tas_cache_test.go b/pkg/cache/scheduler/tas_cache_test.go index a4044c8d59d..f5ddda96341 100644 --- a/pkg/cache/scheduler/tas_cache_test.go +++ b/pkg/cache/scheduler/tas_cache_test.go @@ -658,18 +658,11 @@ func TestFindTopologyAssignments(t *testing.T) { wantAssignment: &kueue.TopologyAssignment{ Levels: defaultOneLevel, Domains: []kueue.TopologyDomainAssignment{ - { - Count: 4, - Values: []string{ - "x3", - }, - }, - { - Count: 2, - Values: []string{ - "x6", - }, - }, + {Count: 1, Values: []string{"x1"}}, + {Count: 1, Values: []string{"x3"}}, + {Count: 1, Values: []string{"x5"}}, + {Count: 1, Values: []string{"x2"}}, + {Count: 2, Values: []string{"x6"}}, }, }, }}, @@ -688,18 +681,11 @@ func TestFindTopologyAssignments(t *testing.T) { wantAssignment: &kueue.TopologyAssignment{ Levels: defaultOneLevel, Domains: []kueue.TopologyDomainAssignment{ - { - Count: 4, - Values: []string{ - "x3", - }, - }, - { - Count: 2, - Values: []string{ - "x6", - }, - }, + {Count: 1, Values: []string{"x1"}}, + {Count: 1, Values: []string{"x3"}}, + {Count: 1, Values: []string{"x5"}}, + {Count: 1, Values: []string{"x2"}}, + {Count: 2, Values: []string{"x6"}}, }, }, }}, diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 5cc93a14f54..9b99f27e2be 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -258,7 +258,7 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned }, TASProfileMixed: { {Version: version.MustParse("0.10"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("0.11"), Default: false, PreRelease: featuregate.Deprecated}, + {Version: version.MustParse("0.15"), Default: true, PreRelease: featuregate.Beta}, }, HierarchicalCohorts: { {Version: version.MustParse("0.11"), Default: true, PreRelease: featuregate.Beta}, From a1fb5ae13d7d3ef965f618ff0a6a41cc5a5e68ac Mon Sep 17 00:00:00 2001 From: Patryk Bundyra Date: Tue, 4 Nov 2025 14:12:10 +0100 Subject: [PATCH 084/119] Cleanup jobframework log (#7426) --- pkg/controller/jobframework/reconciler.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index dd417e7b7ee..6c9060a957f 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -486,7 +486,6 @@ func (r *JobReconciler) ReconcileGenericJob(ctx context.Context, req ctrl.Reques // 4. update reclaimable counts if implemented by the job if jobRecl, implementsReclaimable := job.(JobWithReclaimablePods); implementsReclaimable { - log.V(3).Info("update reclaimable counts if implemented by the job") reclPods, err := jobRecl.ReclaimablePods(ctx) if err != nil { log.Error(err, "Getting reclaimable pods") @@ -499,8 +498,10 @@ func (r *JobReconciler) ReconcileGenericJob(ctx context.Context, req ctrl.Reques log.Error(err, "Updating reclaimable pods") return ctrl.Result{}, err } + log.V(3).Info("updated reclaimable pods") return ctrl.Result{}, nil } + log.V(3).Info("reclaimable pods are up-to-date") } // 5. handle WaitForPodsReady only for a standalone job. From 05ba58b10ae762d9cfe8cff9a2d04fd96bb492bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Tue, 4 Nov 2025 15:00:12 +0100 Subject: [PATCH 085/119] Wrap with Eventually to avoid flake (#7523) --- test/integration/multikueue/jobs_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/multikueue/jobs_test.go b/test/integration/multikueue/jobs_test.go index cf1923732bd..48d071e44fd 100644 --- a/test/integration/multikueue/jobs_test.go +++ b/test/integration/multikueue/jobs_test.go @@ -1725,8 +1725,10 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, }) ginkgo.By("observe: job is no longer suspended in the manager cluster", func() { - getJob(manager.ctx, manager.client, job) - gomega.Expect(job.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(false))) + gomega.Eventually(func(g gomega.Gomega) { + getJob(manager.ctx, manager.client, job) + g.Expect(job.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(false))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) /* From b205f5d369c6fd0db6022a01bb8f19c98f9f9951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Tue, 4 Nov 2025 18:30:09 +0100 Subject: [PATCH 086/119] Flaky sticky workload - fix (#7528) --- test/util/util_scheduling.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/util/util_scheduling.go b/test/util/util_scheduling.go index d7d2e972201..3c6bc7f6158 100644 --- a/test/util/util_scheduling.go +++ b/test/util/util_scheduling.go @@ -45,7 +45,7 @@ func FinishRunningWorkloadsInCQ(ctx context.Context, k8sClient client.Client, cq gomega.ExpectWithOffset(1, finished).To(gomega.Equal(n), "Not enough workloads finished") } -func FinishEvictionsOfAnyWorkloadsInCq(ctx context.Context, k8sClient client.Client, cq *kueue.ClusterQueue) int { +func finishEvictionsOfAnyWorkloadsInCq(ctx context.Context, k8sClient client.Client, cq *kueue.ClusterQueue) sets.Set[types.UID] { var realClock = clock.RealClock{} finished := sets.New[types.UID]() var wList kueue.WorkloadList @@ -64,13 +64,13 @@ func FinishEvictionsOfAnyWorkloadsInCq(ctx context.Context, k8sClient client.Cli finished.Insert(wl.UID) } } - return finished.Len() + return finished } func FinishEvictionOfWorkloadsInCQ(ctx context.Context, k8sClient client.Client, cq *kueue.ClusterQueue, n int) { - finished := 0 + finished := sets.New[types.UID]() gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { - finished += FinishEvictionsOfAnyWorkloadsInCq(ctx, k8sClient, cq) - g.Expect(finished).Should(gomega.Equal(n), "Not enough workloads evicted") + finished.Insert(finishEvictionsOfAnyWorkloadsInCq(ctx, k8sClient, cq).UnsortedList()...) + g.Expect(finished.Len()).Should(gomega.Equal(n), "Not enough workloads evicted") }, Timeout, Interval).Should(gomega.Succeed()) } From 08319fa05936c97775083da32a7b01c71668a309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Tue, 4 Nov 2025 19:34:06 +0100 Subject: [PATCH 087/119] Use Equal instead of Equivalent for asserting Suspend (#7526) --- test/integration/multikueue/jobs_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration/multikueue/jobs_test.go b/test/integration/multikueue/jobs_test.go index 48d071e44fd..12aa2042212 100644 --- a/test/integration/multikueue/jobs_test.go +++ b/test/integration/multikueue/jobs_test.go @@ -1664,7 +1664,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, ginkgo.By("observe: the job is created in the manager cluster", func() { getJob(manager.ctx, manager.client, job) - gomega.Expect(job.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(true))) + gomega.Expect(job.Spec.Suspend).To(gomega.Equal(ptr.To(true))) }) ginkgo.By("observe: a new workload is created in the manager cluster") @@ -1709,7 +1709,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, gomega.Eventually(func(g gomega.Gomega) { remoteJob := job.DeepCopy() getJob(worker1.ctx, worker1.client, remoteJob) - g.Expect(remoteJob.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(false))) + g.Expect(remoteJob.Spec.Suspend).To(gomega.Equal(ptr.To(false))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1727,7 +1727,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, ginkgo.By("observe: job is no longer suspended in the manager cluster", func() { gomega.Eventually(func(g gomega.Gomega) { getJob(manager.ctx, manager.client, job) - g.Expect(job.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(false))) + g.Expect(job.Spec.Suspend).To(gomega.Equal(ptr.To(false))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1794,7 +1794,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, ginkgo.By("observe: the remote job is still active and has old parallelism count", func() { remoteJob := job.DeepCopy() getJob(worker1.ctx, worker1.client, remoteJob) - gomega.Expect(remoteJob.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(false))) + gomega.Expect(remoteJob.Spec.Suspend).To(gomega.Equal(ptr.To(false))) gomega.Expect(remoteJob.Spec.Parallelism).To(gomega.BeEquivalentTo(ptr.To(int32(1)))) }) @@ -1826,7 +1826,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, gomega.Eventually(func(g gomega.Gomega) { remoteJob := job.DeepCopy() getJob(worker1.ctx, worker1.client, remoteJob) - g.Expect(remoteJob.Spec.Suspend).To(gomega.BeEquivalentTo(ptr.To(false))) + g.Expect(remoteJob.Spec.Suspend).To(gomega.Equal(ptr.To(false))) g.Expect(remoteJob.Spec.Parallelism).To(gomega.BeEquivalentTo(ptr.To(int32(1)))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) From 933a22871745197aca1810af7cee71e562d166d4 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Wed, 5 Nov 2025 12:58:54 +0530 Subject: [PATCH 088/119] v1beta2: In FlavorFungibility API migrate Preempt/Borrow to MayStopSearch (#7527) * v1beta2: In FlavorFungibility API migrate Preempt/Borrow to MayStopSearch * Revert Convert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec changes. --- apis/kueue/v1beta1/clusterqueue_conversion.go | 8 +++++ apis/kueue/v1beta2/clusterqueue_types.go | 8 ++--- .../crd/kueue.x-k8s.io_clusterqueues.yaml | 4 --- .../bases/kueue.x-k8s.io_clusterqueues.yaml | 4 --- .../flavorassigner/flavorassigner_test.go | 36 +++++++++---------- .../en/docs/reference/kueue.v1beta2.md | 2 -- 6 files changed, 28 insertions(+), 34 deletions(-) diff --git a/apis/kueue/v1beta1/clusterqueue_conversion.go b/apis/kueue/v1beta1/clusterqueue_conversion.go index e316084b955..3b06b123f58 100644 --- a/apis/kueue/v1beta1/clusterqueue_conversion.go +++ b/apis/kueue/v1beta1/clusterqueue_conversion.go @@ -38,6 +38,14 @@ func (dst *ClusterQueue) ConvertFrom(srcRaw conversion.Hub) error { func Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in *ClusterQueueSpec, out *v1beta2.ClusterQueueSpec, s conversionapi.Scope) error { out.CohortName = v1beta2.CohortReference(in.Cohort) + if in.FlavorFungibility != nil && out.FlavorFungibility != nil { + if in.FlavorFungibility.WhenCanPreempt == Preempt { + out.FlavorFungibility.WhenCanPreempt = v1beta2.MayStopSearch + } + if in.FlavorFungibility.WhenCanBorrow == Borrow { + out.FlavorFungibility.WhenCanBorrow = v1beta2.MayStopSearch + } + } return autoConvert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in, out, s) } diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index 40ba8d10b46..735c851f878 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -364,8 +364,6 @@ const ( type FlavorFungibilityPolicy string const ( - Borrow FlavorFungibilityPolicy = "Borrow" - Preempt FlavorFungibilityPolicy = "Preempt" MayStopSearch FlavorFungibilityPolicy = "MayStopSearch" TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor" ) @@ -379,9 +377,8 @@ type FlavorFungibility struct { // - `MayStopSearch` (default): stop the search for candidate flavors if workload // fits or requires borrowing to fit. // - `TryNextFlavor`: try next flavor if workload requires borrowing to fit. - // - `Borrow` (deprecated): old name for `MayStopSearch`; please use new name. // - // +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor,Borrow} + // +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor} // +kubebuilder:default="MayStopSearch" WhenCanBorrow FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"` // whenCanPreempt determines whether a workload should try the next flavor @@ -391,9 +388,8 @@ type FlavorFungibility struct { // preemption to fit. // - `TryNextFlavor` (default): try next flavor if workload requires preemption // to fit in current flavor. - // - `Preempt` (deprecated): old name for `MayStopSearch`; please use new name. // - // +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor,Preempt} + // +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor} // +kubebuilder:default="TryNextFlavor" WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` } diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml index dd77c7d8d1c..5bbce9f2a12 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml @@ -957,11 +957,9 @@ spec: - `MayStopSearch` (default): stop the search for candidate flavors if workload fits or requires borrowing to fit. - `TryNextFlavor`: try next flavor if workload requires borrowing to fit. - - `Borrow` (deprecated): old name for `MayStopSearch`; please use new name. enum: - MayStopSearch - TryNextFlavor - - Borrow type: string whenCanPreempt: default: TryNextFlavor @@ -973,11 +971,9 @@ spec: preemption to fit. - `TryNextFlavor` (default): try next flavor if workload requires preemption to fit in current flavor. - - `Preempt` (deprecated): old name for `MayStopSearch`; please use new name. enum: - MayStopSearch - TryNextFlavor - - Preempt type: string type: object namespaceSelector: diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml index fed13a7bec1..a600387c593 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml @@ -945,11 +945,9 @@ spec: - `MayStopSearch` (default): stop the search for candidate flavors if workload fits or requires borrowing to fit. - `TryNextFlavor`: try next flavor if workload requires borrowing to fit. - - `Borrow` (deprecated): old name for `MayStopSearch`; please use new name. enum: - MayStopSearch - TryNextFlavor - - Borrow type: string whenCanPreempt: default: TryNextFlavor @@ -961,11 +959,9 @@ spec: preemption to fit. - `TryNextFlavor` (default): try next flavor if workload requires preemption to fit in current flavor. - - `Preempt` (deprecated): old name for `MayStopSearch`; please use new name. enum: - MayStopSearch - TryNextFlavor - - Preempt type: string type: object namespaceSelector: diff --git a/pkg/scheduler/flavorassigner/flavorassigner_test.go b/pkg/scheduler/flavorassigner/flavorassigner_test.go index 92015fc80ee..423f4081b62 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner_test.go +++ b/pkg/scheduler/flavorassigner/flavorassigner_test.go @@ -1443,14 +1443,14 @@ func TestAssignFlavors(t *testing.T) { }}, }, }, - "preempt before try next flavor; using deprecated WhenCanBorrow=Borrow,WhenCanPreempt=Preempt": { + "preempt before try next flavor; using WhenCanBorrow=MayStopSearch,WhenCanPreempt=MayStopSearch": { wlPods: []kueue.PodSet{ *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "9"). Obj(), }, clusterQueue: *utiltestingapi.MakeClusterQueue("test-clusterqueue"). - FlavorFungibility(kueue.FlavorFungibility{WhenCanBorrow: kueue.Borrow, WhenCanPreempt: kueue.Preempt}). + FlavorFungibility(kueue.FlavorFungibility{WhenCanBorrow: kueue.MayStopSearch, WhenCanPreempt: kueue.MayStopSearch}). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("one"). Resource(corev1.ResourcePods, "10"). @@ -1735,7 +1735,7 @@ func TestAssignFlavors(t *testing.T) { }}, }, }, - "when borrowing while preemption is needed for flavor one; WhenCanBorrow=Borrow,WhenCanPreempt=Preempt": { + "when borrowing while preemption is needed for flavor one; WhenCanBorrow=MayStopSearch,WhenCanPreempt=MayStopSearch": { wlPods: []kueue.PodSet{ *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "12"). @@ -1749,8 +1749,8 @@ func TestAssignFlavors(t *testing.T) { }, }). FlavorFungibility(kueue.FlavorFungibility{ - WhenCanBorrow: kueue.Borrow, - WhenCanPreempt: kueue.Preempt, + WhenCanBorrow: kueue.MayStopSearch, + WhenCanPreempt: kueue.MayStopSearch, }). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("one"). @@ -1851,7 +1851,7 @@ func TestAssignFlavors(t *testing.T) { }}, }, }, - "when borrowing while preemption is needed for flavor one, no borrowingLimit; WhenCanBorrow=Borrow,WhenCanPreempt=Preempt": { + "when borrowing while preemption is needed for flavor one, no borrowingLimit; WhenCanBorrow=MayStopSearch,WhenCanPreempt=MayStopSearch": { wlPods: []kueue.PodSet{ *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "12"). @@ -1865,8 +1865,8 @@ func TestAssignFlavors(t *testing.T) { }, }). FlavorFungibility(kueue.FlavorFungibility{ - WhenCanBorrow: kueue.Borrow, - WhenCanPreempt: kueue.Preempt, + WhenCanBorrow: kueue.MayStopSearch, + WhenCanPreempt: kueue.MayStopSearch, }). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("one"). @@ -1960,7 +1960,7 @@ func TestAssignFlavors(t *testing.T) { }}, }, }, - "when borrowing while preemption is needed for flavor one; WhenCanBorrow=TryNextFlavor,WhenCanPreempt=Preempt": { + "when borrowing while preemption is needed for flavor one; WhenCanBorrow=TryNextFlavor,WhenCanPreempt=MayStopSearch": { wlPods: []kueue.PodSet{ *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "12"). @@ -1975,7 +1975,7 @@ func TestAssignFlavors(t *testing.T) { }). FlavorFungibility(kueue.FlavorFungibility{ WhenCanBorrow: kueue.TryNextFlavor, - WhenCanPreempt: kueue.Preempt, + WhenCanPreempt: kueue.MayStopSearch, }). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("one"). @@ -2217,14 +2217,14 @@ func TestAssignFlavors(t *testing.T) { }}, }, }, - "cannot preempt in cohort (oracle returns None) for the first flavor, tries the second flavor (which fits); using deprecated WhenCanBorrow=Borrow,WhenCanPreempt=Preempt": { + "cannot preempt in cohort (oracle returns None) for the first flavor, tries the second flavor (which fits); using deprecated WhenCanBorrow=MayStopSearch,WhenCanPreempt=MayStopSearch": { wlPods: []kueue.PodSet{ *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "2"). Obj(), }, clusterQueue: *utiltestingapi.MakeClusterQueue("test-clusterqueue"). - FlavorFungibility(kueue.FlavorFungibility{WhenCanBorrow: kueue.Borrow, WhenCanPreempt: kueue.Preempt}). + FlavorFungibility(kueue.FlavorFungibility{WhenCanBorrow: kueue.MayStopSearch, WhenCanPreempt: kueue.MayStopSearch}). Preemption(kueue.ClusterQueuePreemption{ ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, BorrowWithinCohort: &kueue.BorrowWithinCohort{ @@ -2376,7 +2376,7 @@ func TestAssignFlavors(t *testing.T) { }}, }, }, - "when borrowing while preemption is needed for flavor one, fair sharing enabled, reclaimWithinCohort=Any; using deprecated WhenCanBorrow=Borrow,WhenCanPreempt=Preempt": { + "when borrowing while preemption is needed for flavor one, fair sharing enabled, reclaimWithinCohort=Any; using deprecated WhenCanBorrow=MayStopSearch,WhenCanPreempt=MayStopSearch": { enableFairSharing: true, wlPods: []kueue.PodSet{ *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). @@ -2385,7 +2385,7 @@ func TestAssignFlavors(t *testing.T) { }, clusterQueue: *utiltestingapi.MakeClusterQueue("test-clusterqueue"). Preemption(kueue.ClusterQueuePreemption{ReclaimWithinCohort: kueue.PreemptionPolicyAny}). - FlavorFungibility(kueue.FlavorFungibility{WhenCanBorrow: kueue.Borrow, WhenCanPreempt: kueue.Preempt}). + FlavorFungibility(kueue.FlavorFungibility{WhenCanBorrow: kueue.MayStopSearch, WhenCanPreempt: kueue.MayStopSearch}). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("one"). Resource(corev1.ResourceCPU, "0"). @@ -2476,7 +2476,7 @@ func TestAssignFlavors(t *testing.T) { }}, }, }, - "when borrowing while preemption is needed for flavor one, fair sharing enabled, reclaimWithinCohort=Never; using deprecated WhenCanBorrow=Borrow,WhenCanPreempt=Preempt": { + "when borrowing while preemption is needed for flavor one, fair sharing enabled, reclaimWithinCohort=Never; using deprecated WhenCanBorrow=MayStopSearch,WhenCanPreempt=MayStopSearch": { enableFairSharing: true, wlPods: []kueue.PodSet{ *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). @@ -2485,7 +2485,7 @@ func TestAssignFlavors(t *testing.T) { }, clusterQueue: *utiltestingapi.MakeClusterQueue("test-clusterqueue"). Preemption(kueue.ClusterQueuePreemption{ReclaimWithinCohort: kueue.PreemptionPolicyNever}). - FlavorFungibility(kueue.FlavorFungibility{WhenCanBorrow: kueue.Borrow, WhenCanPreempt: kueue.Preempt}). + FlavorFungibility(kueue.FlavorFungibility{WhenCanBorrow: kueue.MayStopSearch, WhenCanPreempt: kueue.MayStopSearch}). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("one"). Resource(corev1.ResourceCPU, "0"). @@ -2923,7 +2923,7 @@ func TestReclaimBeforePriorityPreemption(t *testing.T) { wantMode: Preempt, wantAssigment: rfMap{"gpu": "uno"}, }, - "Select first flavor when flavor fungibility is disabled; using deprecated WhenCanPreempt=Preempt": { + "Select first flavor when flavor fungibility is disabled; using deprecated WhenCanPreempt=MayStopSearch": { workloadRequests: utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1).Request("gpu", "10"), testClusterQueueUsage: resources.FlavorResourceQuantities{ {Flavor: "uno", Resource: "gpu"}: 1, @@ -2933,7 +2933,7 @@ func TestReclaimBeforePriorityPreemption(t *testing.T) { {Flavor: "tre", Resource: "gpu"}: 1, }, flavorFungibility: &kueue.FlavorFungibility{ - WhenCanPreempt: kueue.Preempt, + WhenCanPreempt: kueue.MayStopSearch, }, wantMode: Preempt, wantAssigment: rfMap{"gpu": "uno"}, diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md index ad144496515..b078a88d533 100644 --- a/site/content/en/docs/reference/kueue.v1beta2.md +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -1370,7 +1370,6 @@ before borrowing in current flavor. The possible values are:

  • MayStopSearch (default): stop the search for candidate flavors if workload fits or requires borrowing to fit.
  • TryNextFlavor: try next flavor if workload requires borrowing to fit.
  • -
  • Borrow (deprecated): old name for MayStopSearch; please use new name.
  • @@ -1385,7 +1384,6 @@ before borrowing in current flavor. The possible values are:

    preemption to fit.
  • TryNextFlavor (default): try next flavor if workload requires preemption to fit in current flavor.
  • -
  • Preempt (deprecated): old name for MayStopSearch; please use new name.
  • From 490d40d00fe1eca08c778c10ab2cfc3ee121ba1a Mon Sep 17 00:00:00 2001 From: Pannaga Rao Bhoja Ramamanohara Date: Wed, 5 Nov 2025 13:38:50 +0530 Subject: [PATCH 089/119] Graduate ManagedJobsNamespaceSelectorAlwaysRespected feature to Beta (#7493) * Graduate ManagedJobsNamespaceSelectorAlwaysRespected feature to Beta Signed-off-by: Pannaga Rao Bhoja Ramamanohara * Relax reconciler constraint for ManagedJobsNamespaceSelectorAlwaysRespected Signed-off-by: Pannaga Rao Bhoja Ramamanohara * Fix failing tests due to feature gate change for ManagedJobsNamespaceSelectorAlwaysRespected Signed-off-by: Pannaga Rao Bhoja Ramamanohara --------- Signed-off-by: Pannaga Rao Bhoja Ramamanohara --- pkg/controller/jobframework/reconciler.go | 9 ++++++++- pkg/controller/jobframework/reconciler_test.go | 2 ++ pkg/controller/jobs/pod/pod_controller_test.go | 6 +++++- .../jobs/raycluster/raycluster_controller_test.go | 4 +++- pkg/features/kube_features.go | 1 + site/content/en/docs/installation/_index.md | 3 ++- 6 files changed, 21 insertions(+), 4 deletions(-) diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index 6c9060a957f..36e168cdc0f 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -312,10 +312,17 @@ func (r *JobReconciler) ReconcileGenericJob(ctx context.Context, req ctrl.Reques if features.Enabled(features.ManagedJobsNamespaceSelectorAlwaysRespected) { ns := corev1.Namespace{} if err := r.client.Get(ctx, client.ObjectKey{Name: req.Namespace}, &ns); err != nil { + if apierrors.IsNotFound(err) { + log.V(2).Info("Namespace not found; skipping selector check", "namespace", req.Namespace) + return ctrl.Result{}, nil + } log.Error(err, "failed to get namespace for selector check") return ctrl.Result{}, err } - if !r.managedJobsNamespaceSelector.Matches(labels.Set(ns.GetLabels())) { + + if r.managedJobsNamespaceSelector == nil { + log.V(2).Info("ManagedJobsNamespaceSelector is nil; skipping selector enforcement") + } else if !r.managedJobsNamespaceSelector.Matches(labels.Set(ns.GetLabels())) { log.V(2).Info("Namespace not opted in for Kueue management", "namespace", ns.Name) return ctrl.Result{}, nil } diff --git a/pkg/controller/jobframework/reconciler_test.go b/pkg/controller/jobframework/reconciler_test.go index b60cd1a5813..26592bd7d72 100644 --- a/pkg/controller/jobframework/reconciler_test.go +++ b/pkg/controller/jobframework/reconciler_test.go @@ -162,6 +162,7 @@ func TestReconcileGenericJob(t *testing.T) { mgj.EXPECT().PodSets(gomock.Any()).Return(tc.podSets, nil).AnyTimes() cl := utiltesting.NewClientBuilder(batchv1.AddToScheme, kueue.AddToScheme). + WithObjects(utiltesting.MakeNamespace(tc.req.Namespace)). WithObjects(tc.objs...). WithObjects(tc.job). WithIndex(&kueue.Workload{}, indexer.OwnerReferenceIndexKey(testGVK), indexer.WorkloadOwnerIndexFunc(testGVK)). @@ -243,6 +244,7 @@ func TestReconcileGenericJobWithCustomWorkloadActivation(t *testing.T) { } cl := utiltesting.NewClientBuilder(batchv1.AddToScheme, kueue.AddToScheme). + WithObjects(utiltesting.MakeNamespace(testNS)). WithObjects(job, wl). WithIndex(&kueue.Workload{}, indexer.OwnerReferenceIndexKey(testGVK), indexer.WorkloadOwnerIndexFunc(testGVK)). Build() diff --git a/pkg/controller/jobs/pod/pod_controller_test.go b/pkg/controller/jobs/pod/pod_controller_test.go index 7af6a0eaf7f..69644911b0d 100644 --- a/pkg/controller/jobs/pod/pod_controller_test.go +++ b/pkg/controller/jobs/pod/pod_controller_test.go @@ -5595,7 +5595,9 @@ func TestReconciler(t *testing.T) { t.Fatalf("Could not setup indexes: %v", err) } - kcBuilder := clientBuilder.WithObjects(tc.initObjects...) + // Add namespace to prevent early return when ManagedJobsNamespaceSelectorAlwaysRespected is enabled + namespace := utiltesting.MakeNamespace("ns") + kcBuilder := clientBuilder.WithObjects(namespace).WithObjects(tc.initObjects...) for i := range tc.pods { kcBuilder = kcBuilder.WithObjects(&tc.pods[i]) } @@ -5710,6 +5712,7 @@ func TestReconciler_ErrorFinalizingPod(t *testing.T) { errMock := fmt.Errorf("connection refused: %w", syscall.ECONNREFUSED) kcBuilder := clientBuilder. + WithObjects(utiltesting.MakeNamespace("ns")). WithObjects(&pod). WithStatusSubresource(&wl). WithInterceptorFuncs(interceptor.Funcs{ @@ -5943,6 +5946,7 @@ func TestReconciler_DeletePodAfterTransientErrorsOnUpdateOrDeleteOps(t *testing. } kcBuilder := clientBuilder. + WithObjects(utiltesting.MakeNamespace("ns")). WithStatusSubresource(&wl). WithInterceptorFuncs(interceptor.Funcs{ Patch: func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { diff --git a/pkg/controller/jobs/raycluster/raycluster_controller_test.go b/pkg/controller/jobs/raycluster/raycluster_controller_test.go index 291abc4e2ff..59d5618756b 100644 --- a/pkg/controller/jobs/raycluster/raycluster_controller_test.go +++ b/pkg/controller/jobs/raycluster/raycluster_controller_test.go @@ -686,8 +686,10 @@ func TestReconciler(t *testing.T) { if err := SetupIndexes(ctx, indexer); err != nil { t.Fatalf("Could not setup indexes: %v", err) } + // Add namespace to prevent early return when ManagedJobsNamespaceSelectorAlwaysRespected is enabled + namespace := utiltesting.MakeNamespace("ns") objs := append(tc.priorityClasses, &tc.job) - kcBuilder := clientBuilder.WithObjects(objs...) + kcBuilder := clientBuilder.WithObjects(namespace).WithObjects(objs...) for i := range tc.workloads { kcBuilder = kcBuilder.WithStatusSubresource(&tc.workloads[i]) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 9b99f27e2be..90d0bd05f41 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -288,6 +288,7 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned }, ManagedJobsNamespaceSelectorAlwaysRespected: { {Version: version.MustParse("0.13"), Default: false, PreRelease: featuregate.Alpha}, + {Version: version.MustParse("0.15"), Default: true, PreRelease: featuregate.Beta}, }, FlavorFungibilityImplicitPreferenceDefault: { {Version: version.MustParse("0.13"), Default: false, PreRelease: featuregate.Alpha}, diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 7118340d929..4e6f8aa4859 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -294,7 +294,8 @@ spec: | `TASReplaceNodeOnPodTermination` | `false` | Alpha | 0.13 | 0.13 | | `TASReplaceNodeOnPodTermination` | `true` | Beta | 0.14 | | | `ElasticJobsViaWorkloadSlices` | `false` | Alpha | 0.13 | | -| `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | | +| `ManagedJobsNamespaceSelectorAlwaysRespected` | `false` | Alpha | 0.13 | 0.15 | +| `ManagedJobsNamespaceSelectorAlwaysRespected` | `true` | Beta | 0.15 | | | `FlavorFungibilityImplicitPreferenceDefault` | `false` | Alpha | 0.13 | | | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | | `SanitizePodSets` | `true` | Beta | 0.13 | | From ff7236f9c525017e9294cf6dfe11c80ebcce0357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Irving=20Mondrag=C3=B3n?= Date: Wed, 5 Nov 2025 09:08:57 +0100 Subject: [PATCH 090/119] Add Multikueue and ProvReq integration test (#7505) --- .../multikueue/provisioning_test.go | 283 ++++++++++++++++++ test/integration/multikueue/suite_test.go | 13 + 2 files changed, 296 insertions(+) create mode 100644 test/integration/multikueue/provisioning_test.go diff --git a/test/integration/multikueue/provisioning_test.go b/test/integration/multikueue/provisioning_test.go new file mode 100644 index 00000000000..db74ccdd0e8 --- /dev/null +++ b/test/integration/multikueue/provisioning_test.go @@ -0,0 +1,283 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multikueue + +import ( + "context" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + autoscaling "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + config "sigs.k8s.io/kueue/apis/config/v1beta2" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/pkg/controller/admissionchecks/provisioning" + workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" + "sigs.k8s.io/kueue/pkg/util/admissioncheck" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" + "sigs.k8s.io/kueue/test/util" +) + +var _ = ginkgo.Describe("MultiKueue with ProvisioningRequest", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { + var ( + managerNs *corev1.Namespace + worker1Ns *corev1.Namespace + + managerMultiKueueSecret1 *corev1.Secret + workerCluster1 *kueue.MultiKueueCluster + managerMultiKueueConfig *kueue.MultiKueueConfig + multiKueueAC *kueue.AdmissionCheck + + managerRf *kueue.ResourceFlavor + managerCq *kueue.ClusterQueue + managerLq *kueue.LocalQueue + + worker1ProvReqConfig *kueue.ProvisioningRequestConfig + worker1ProvReqAC *kueue.AdmissionCheck + worker1Rf *kueue.ResourceFlavor + worker1Cq *kueue.ClusterQueue + worker1Lq *kueue.LocalQueue + ) + + ginkgo.BeforeAll(func() { + managerTestCluster.fwk.StartManager(managerTestCluster.ctx, managerTestCluster.cfg, func(ctx context.Context, mgr manager.Manager) { + managerAndMultiKueueSetup(ctx, mgr, 2*time.Second, sets.New("batch/job"), config.MultiKueueDispatcherModeAllAtOnce) + }) + }) + + ginkgo.AfterAll(func() { + managerTestCluster.fwk.StopManager(managerTestCluster.ctx) + }) + + ginkgo.BeforeEach(func() { + managerNs = util.CreateNamespaceFromPrefixWithLog(managerTestCluster.ctx, managerTestCluster.client, "mk-prov-") + worker1Ns = util.CreateNamespaceWithLog(worker1TestCluster.ctx, worker1TestCluster.client, managerNs.Name) + + w1Kubeconfig, err := worker1TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Setup MultiKueue resources + managerMultiKueueSecret1 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multikueue-prov-secret", + Namespace: managersConfigNamespace.Name, + }, + Data: map[string][]byte{ + kueue.MultiKueueConfigSecretKey: w1Kubeconfig, + }, + } + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultiKueueSecret1)).To(gomega.Succeed()) + + workerCluster1 = utiltestingapi.MakeMultiKueueCluster("worker1-prov").KubeConfig(kueue.SecretLocationType, managerMultiKueueSecret1.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, workerCluster1)).To(gomega.Succeed()) + + managerMultiKueueConfig = utiltestingapi.MakeMultiKueueConfig("mk-prov-config").Clusters(workerCluster1.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultiKueueConfig)).Should(gomega.Succeed()) + + multiKueueAC = utiltestingapi.MakeAdmissionCheck("mk-ac"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", managerMultiKueueConfig.Name). + Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, multiKueueAC)).Should(gomega.Succeed()) + + ginkgo.By("wait for multikueue admission check to be active", func() { + gomega.Eventually(func(g gomega.Gomega) { + updatedMkAc := kueue.AdmissionCheck{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, client.ObjectKeyFromObject(multiKueueAC), &updatedMkAc)).To(gomega.Succeed()) + g.Expect(updatedMkAc.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.AdmissionCheckActive)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + managerRf = utiltestingapi.MakeResourceFlavor("manager-rf").NodeLabel("instance-type", "manager-node").Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerRf)).To(gomega.Succeed()) + + managerCq = utiltestingapi.MakeClusterQueue("cq-mk-prov"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas(managerRf.Name). + Resource(corev1.ResourceCPU, "5"). + Obj()). + AdmissionChecks(kueue.AdmissionCheckReference(multiKueueAC.Name)). + Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerCq)).Should(gomega.Succeed()) + util.ExpectClusterQueuesToBeActive(managerTestCluster.ctx, managerTestCluster.client, managerCq) + + managerLq = utiltestingapi.MakeLocalQueue(managerCq.Name, managerNs.Name).ClusterQueue(managerCq.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerLq)).Should(gomega.Succeed()) + util.ExpectLocalQueuesToBeActive(managerTestCluster.ctx, managerTestCluster.client, managerLq) + + worker1ProvReqConfig = utiltestingapi.MakeProvisioningRequestConfig("prov-config"). + ProvisioningClass("test-provisioning-class"). + Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1ProvReqConfig)).Should(gomega.Succeed()) + + worker1ProvReqAC = utiltestingapi.MakeAdmissionCheck("prov-ac"). + ControllerName(kueue.ProvisioningRequestControllerName). + Parameters(kueue.GroupVersion.Group, "ProvisioningRequestConfig", worker1ProvReqConfig.Name). + Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1ProvReqAC)).Should(gomega.Succeed()) + + ginkgo.By("wait for worker provisioning admission check to be active", func() { + gomega.Eventually(func(g gomega.Gomega) { + updatedProvAc := kueue.AdmissionCheck{} + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, client.ObjectKeyFromObject(worker1ProvReqAC), &updatedProvAc)).To(gomega.Succeed()) + g.Expect(updatedProvAc.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.AdmissionCheckActive)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + worker1Rf = utiltestingapi.MakeResourceFlavor("worker-rf").NodeLabel("instance-type", "worker-node").Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Rf)).To(gomega.Succeed()) + + worker1Cq = utiltestingapi.MakeClusterQueue("cq-mk-prov"). + ResourceGroup(*utiltestingapi.MakeFlavorQuotas(worker1Rf.Name). + Resource(corev1.ResourceCPU, "5"). + Obj()). + AdmissionChecks(kueue.AdmissionCheckReference(worker1ProvReqAC.Name)). + Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Cq)).Should(gomega.Succeed()) + util.ExpectClusterQueuesToBeActive(worker1TestCluster.ctx, worker1TestCluster.client, worker1Cq) + + worker1Lq = utiltestingapi.MakeLocalQueue(worker1Cq.Name, worker1Ns.Name).ClusterQueue(worker1Cq.Name).Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Lq)).Should(gomega.Succeed()) + util.ExpectLocalQueuesToBeActive(worker1TestCluster.ctx, worker1TestCluster.client, worker1Lq) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(managerTestCluster.ctx, managerTestCluster.client, managerNs)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(worker1TestCluster.ctx, worker1TestCluster.client, worker1Ns)).To(gomega.Succeed()) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerCq, true) + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1Cq, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerRf, true) + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1Rf, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, multiKueueAC, true) + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1ProvReqAC, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerMultiKueueConfig, true) + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1ProvReqConfig, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, workerCluster1, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerMultiKueueSecret1, true) + }) + + ginkgo.It("Should create workload on worker and provision resources", func() { + job := testingjob.MakeJob("test-job", managerNs.Name). + Queue(kueue.LocalQueueName(managerLq.Name)). + Request(corev1.ResourceCPU, "2"). + Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, job)).Should(gomega.Succeed()) + + managerWlKey := types.NamespacedName{ + Name: workloadjob.GetWorkloadNameForJob(job.Name, job.UID), + Namespace: managerNs.Name, + } + worker1WlKey := types.NamespacedName{ + Name: workloadjob.GetWorkloadNameForJob(job.Name, job.UID), + Namespace: worker1Ns.Name, + } + + ginkgo.By("setting quota reservation on manager cluster", func() { + admission := utiltestingapi.MakeAdmission(managerCq.Name). + PodSets( + utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, kueue.ResourceFlavorReference(managerRf.Name), "2"). + Obj(), + ). + Obj() + util.SetQuotaReservation(managerTestCluster.ctx, managerTestCluster.client, managerWlKey, admission) + }) + + ginkgo.By("verifying workload is created on worker cluster", func() { + gomega.Eventually(func(g gomega.Gomega) { + workerWl := &kueue.Workload{} + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, worker1WlKey, workerWl)).To(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("setting quota reservation on worker cluster", func() { + admission := utiltestingapi.MakeAdmission(worker1Cq.Name). + PodSets( + utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, kueue.ResourceFlavorReference(worker1Rf.Name), "2"). + Obj(), + ). + Obj() + util.SetQuotaReservation(worker1TestCluster.ctx, worker1TestCluster.client, worker1WlKey, admission) + }) + + provReqKey := types.NamespacedName{ + Namespace: worker1Ns.Name, + Name: provisioning.ProvisioningRequestName(worker1WlKey.Name, kueue.AdmissionCheckReference(worker1ProvReqAC.Name), 1), + } + + ginkgo.By("waiting for provisioning request and marking it provisioned", func() { + createdProvReq := &autoscaling.ProvisioningRequest{} + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, provReqKey, createdProvReq)).To(gomega.Succeed()) + apimeta.SetStatusCondition(&createdProvReq.Status.Conditions, metav1.Condition{ + Type: autoscaling.Provisioned, + Status: metav1.ConditionTrue, + Reason: autoscaling.Provisioned, + }) + g.Expect(worker1TestCluster.client.Status().Update(worker1TestCluster.ctx, createdProvReq)).To(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("checking the provisioning admission check is ready on worker", func() { + gomega.Eventually(func(g gomega.Gomega) { + workerWl := &kueue.Workload{} + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, worker1WlKey, workerWl)).To(gomega.Succeed()) + + provCheck := admissioncheck.FindAdmissionCheck(workerWl.Status.AdmissionChecks, kueue.AdmissionCheckReference(worker1ProvReqAC.Name)) + g.Expect(provCheck).NotTo(gomega.BeNil()) + g.Expect(provCheck.State).To(gomega.Equal(kueue.CheckStateReady)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("checking the multikueue admission check is ready on manager", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, managerWlKey, managerWl)).To(gomega.Succeed()) + + mkCheck := admissioncheck.FindAdmissionCheck(managerWl.Status.AdmissionChecks, kueue.AdmissionCheckReference(multiKueueAC.Name)) + g.Expect(mkCheck).NotTo(gomega.BeNil()) + g.Expect(mkCheck.State).To(gomega.Equal(kueue.CheckStateReady)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verifying the workload is admitted on worker", func() { + gomega.Eventually(func(g gomega.Gomega) { + workerWl := &kueue.Workload{} + g.Expect(worker1TestCluster.client.Get(worker1TestCluster.ctx, worker1WlKey, workerWl)).To(gomega.Succeed()) + g.Expect(workerWl.Status.Admission).NotTo(gomega.BeNil()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verifying the workload is admitted on manager", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, managerWlKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.Admission).NotTo(gomega.BeNil()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) +}) diff --git a/test/integration/multikueue/suite_test.go b/test/integration/multikueue/suite_test.go index f67267f413d..6f5345f9c9d 100644 --- a/test/integration/multikueue/suite_test.go +++ b/test/integration/multikueue/suite_test.go @@ -38,6 +38,7 @@ import ( schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue" + "sigs.k8s.io/kueue/pkg/controller/admissionchecks/provisioning" "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/controller/jobframework" @@ -108,6 +109,7 @@ func createCluster(setupFnc framework.ManagerSetup, apiFeatureGates ...string) c util.RayOperatorCrds, util.AppWrapperCrds, util.KfTrainerCrds, + util.AutoscalerCrds, }, APIServerFeatureGates: apiFeatureGates, } @@ -331,6 +333,17 @@ func managerSetup(ctx context.Context, mgr manager.Manager) { err = workloadtrainjob.SetupTrainJobWebhook(mgr, jobframework.WithCache(cCache), jobframework.WithQueues(queues)) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = provisioning.SetupIndexer(ctx, mgr.GetFieldIndexer()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + provReconciler, err := provisioning.NewController( + mgr.GetClient(), + mgr.GetEventRecorderFor("kueue-provisioning-request-controller")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = provReconciler.SetupWithManager(mgr) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) } func managerAndMultiKueueSetup( From a806a48af9d3aec627f30bddd9a1dd8c8ae93066 Mon Sep 17 00:00:00 2001 From: Patryk Bundyra Date: Wed, 5 Nov 2025 09:40:54 +0100 Subject: [PATCH 091/119] Add feature gate for reclaimable Pods (#7525) * Add reclaimable pods fg * Adjust KEP * Fix linter, adjust fg * Remove test focus * Adjust milestone * Add fg documentation and remove redundant fg from tests --- .../kep.yaml | 7 ++ pkg/controller/jobframework/reconciler.go | 2 +- pkg/features/kube_features.go | 8 ++ .../flavorassigner/flavorassigner_test.go | 49 +++++++- pkg/workload/workload.go | 3 + pkg/workload/workload_test.go | 89 +++++++++++++- site/content/en/docs/installation/_index.md | 5 +- .../core/clusterqueue_controller_test.go | 110 ++++++++++++++++++ .../singlecluster/scheduler/scheduler_test.go | 72 ++++++++++++ 9 files changed, 336 insertions(+), 9 deletions(-) diff --git a/keps/78-dynamically-reclaiming-resources/kep.yaml b/keps/78-dynamically-reclaiming-resources/kep.yaml index f1d3bd300e5..39e9a636392 100644 --- a/keps/78-dynamically-reclaiming-resources/kep.yaml +++ b/keps/78-dynamically-reclaiming-resources/kep.yaml @@ -14,3 +14,10 @@ status: implementable see-also: replaces: superseded-by: + +stage: beta + +latest-milestone: "v0.15" + +feature-gates: + - ReclaimablePods diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index 36e168cdc0f..ef7a5a6c8ab 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -492,7 +492,7 @@ func (r *JobReconciler) ReconcileGenericJob(ctx context.Context, req ctrl.Reques } // 4. update reclaimable counts if implemented by the job - if jobRecl, implementsReclaimable := job.(JobWithReclaimablePods); implementsReclaimable { + if jobRecl, implementsReclaimable := job.(JobWithReclaimablePods); implementsReclaimable && features.Enabled(features.ReclaimablePods) { reclPods, err := jobRecl.ReclaimablePods(ctx) if err != nil { log.Error(err, "Getting reclaimable pods") diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 90d0bd05f41..52a4794d2f5 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -196,6 +196,11 @@ const ( // Requires careful consideration as it may lead to security issues. // Deprecated: planned to be removed in 0.17 MultiKueueAllowInsecureKubeconfigs featuregate.Feature = "MultiKueueAllowInsecureKubeconfigs" + + // owner: @pbundyra + // + // Enables reclaimable pods counting towards quota. + ReclaimablePods featuregate.Feature = "ReclaimablePods" ) func init() { @@ -308,6 +313,9 @@ var defaultVersionedFeatureGates = map[featuregate.Feature]featuregate.Versioned MultiKueueAllowInsecureKubeconfigs: { {Version: version.MustParse("0.15"), Default: false, PreRelease: featuregate.Alpha}, }, + ReclaimablePods: { + {Version: version.MustParse("0.15"), Default: true, PreRelease: featuregate.Beta}, + }, } func SetFeatureGateDuringTest(tb testing.TB, f featuregate.Feature, value bool) { diff --git a/pkg/scheduler/flavorassigner/flavorassigner_test.go b/pkg/scheduler/flavorassigner/flavorassigner_test.go index 423f4081b62..dc33b997681 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner_test.go +++ b/pkg/scheduler/flavorassigner/flavorassigner_test.go @@ -26,6 +26,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/component-base/featuregate" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" @@ -109,6 +110,7 @@ func TestAssignFlavors(t *testing.T) { elasticJobsViaWorkloadSlicesEnabled bool preemptWorkloadSlice *workload.Info enableImplicitPreferenceDefault bool + featureGates map[featuregate.Feature]bool }{ "single flavor, fits": { wlPods: []kueue.PodSet{ @@ -1361,7 +1363,7 @@ func TestAssignFlavors(t *testing.T) { Usage: workload.Usage{Quota: resources.FlavorResourceQuantities{}}, }, }, - "with reclaimable pods": { + "with reclaimable pods; reclaimablePods on": { wlPods: []kueue.PodSet{ *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 5). Request(corev1.ResourceCPU, "1"). @@ -1401,6 +1403,48 @@ func TestAssignFlavors(t *testing.T) { }, wantRepMode: Fit, }, + "with reclaimable pods; reclaimablePods off": { + wlPods: []kueue.PodSet{ + *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 5). + Request(corev1.ResourceCPU, "1"). + Obj(), + }, + wlReclaimablePods: []kueue.ReclaimablePod{ + { + Name: kueue.DefaultPodSetName, + Count: 2, + }, + }, + clusterQueue: *utiltestingapi.MakeClusterQueue("test-clusterqueue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas("default"). + Resource(corev1.ResourcePods, "5"). + Resource(corev1.ResourceCPU, "10"). + Obj(), + ).Obj(), + wantAssignment: Assignment{ + PodSets: []PodSetAssignment{{ + Name: kueue.DefaultPodSetName, + Flavors: ResourceAssignment{ + + corev1.ResourceCPU: &FlavorAssignment{Name: "default", Mode: Fit, TriedFlavorIdx: -1}, + corev1.ResourcePods: &FlavorAssignment{Name: "default", Mode: Fit, TriedFlavorIdx: -1}, + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5"), + corev1.ResourcePods: resource.MustParse("5"), + }, + Count: 5, + }}, + Usage: workload.Usage{Quota: resources.FlavorResourceQuantities{ + {Flavor: "default", Resource: corev1.ResourcePods}: 5, + {Flavor: "default", Resource: corev1.ResourceCPU}: 5_000, + }}, + }, + wantRepMode: Fit, + featureGates: map[featuregate.Feature]bool{ + features.ReclaimablePods: false, + }}, "preempt before try next flavor": { wlPods: []kueue.PodSet{ *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). @@ -2800,6 +2844,9 @@ func TestAssignFlavors(t *testing.T) { if tc.enableImplicitPreferenceDefault { features.SetFeatureGateDuringTest(t, features.FlavorFungibilityImplicitPreferenceDefault, true) } + for fg, enabled := range tc.featureGates { + features.SetFeatureGateDuringTest(t, fg, enabled) + } wlInfo := workload.NewInfo(&kueue.Workload{ Spec: kueue.WorkloadSpec{ PodSets: tc.wlPods, diff --git a/pkg/workload/workload.go b/pkg/workload/workload.go index 8c6edb7b928..5ef39aa8924 100644 --- a/pkg/workload/workload.go +++ b/pkg/workload/workload.go @@ -461,6 +461,9 @@ func podSetsCounts(wl *kueue.Workload) map[kueue.PodSetReference]int32 { func podSetsCountsAfterReclaim(wl *kueue.Workload) map[kueue.PodSetReference]int32 { totalCounts := podSetsCounts(wl) + if !features.Enabled(features.ReclaimablePods) { + return totalCounts + } reclaimCounts := reclaimableCounts(wl) for podSetName := range totalCounts { if rc, found := reclaimCounts[podSetName]; found { diff --git a/pkg/workload/workload_test.go b/pkg/workload/workload_test.go index 25f0c89ca03..c5537913978 100644 --- a/pkg/workload/workload_test.go +++ b/pkg/workload/workload_test.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/component-base/featuregate" testingclock "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -45,9 +46,10 @@ import ( func TestNewInfo(t *testing.T) { cases := map[string]struct { - workload kueue.Workload - infoOptions []InfoOption - wantInfo Info + workload kueue.Workload + infoOptions []InfoOption + wantInfo Info + featureGates map[featuregate.Feature]bool }{ "pending": { workload: *utiltestingapi.MakeWorkload("", ""). @@ -67,7 +69,7 @@ func TestNewInfo(t *testing.T) { }, }, }, - "pending with reclaim": { + "pending with reclaim; reclaimablePods on": { workload: *utiltestingapi.MakeWorkload("", ""). PodSets( *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 5). @@ -95,6 +97,37 @@ func TestNewInfo(t *testing.T) { }, }, }, + "pending with reclaim; reclaimablePods off": { + workload: *utiltestingapi.MakeWorkload("", ""). + PodSets( + *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 5). + Request(corev1.ResourceCPU, "10m"). + Request(corev1.ResourceMemory, "512Ki"). + Obj(), + ). + ReclaimablePods( + kueue.ReclaimablePod{ + Name: kueue.DefaultPodSetName, + Count: 2, + }, + ). + Obj(), + wantInfo: Info{ + TotalRequests: []PodSetResources{ + { + Name: kueue.DefaultPodSetName, + Requests: resources.Requests{ + corev1.ResourceCPU: 5 * 10, + corev1.ResourceMemory: 5 * 512 * 1024, + }, + Count: 5, + }, + }, + }, + featureGates: map[featuregate.Feature]bool{ + features.ReclaimablePods: false, + }, + }, "admitted": { workload: *utiltestingapi.MakeWorkload("", ""). PodSets( @@ -159,7 +192,7 @@ func TestNewInfo(t *testing.T) { }, }, }, - "admitted with reclaim": { + "admitted with reclaim; reclaimablePods on": { workload: *utiltestingapi.MakeWorkload("", ""). PodSets( *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 5). @@ -200,6 +233,49 @@ func TestNewInfo(t *testing.T) { }, }, }, + "admitted with reclaim; reclaimablePods off": { + workload: *utiltestingapi.MakeWorkload("", ""). + PodSets( + *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 5). + Request(corev1.ResourceCPU, "10m"). + Request(corev1.ResourceMemory, "10Ki"). + Obj(), + ). + ReserveQuota( + utiltestingapi.MakeAdmission(""). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "f1", "50m"). + Assignment(corev1.ResourceMemory, "f1", "50Ki"). + Count(5). + Obj()). + Obj(), + ). + ReclaimablePods( + kueue.ReclaimablePod{ + Name: kueue.DefaultPodSetName, + Count: 2, + }, + ). + Obj(), + wantInfo: Info{ + TotalRequests: []PodSetResources{ + { + Name: kueue.DefaultPodSetName, + Flavors: map[corev1.ResourceName]kueue.ResourceFlavorReference{ + corev1.ResourceCPU: "f1", + corev1.ResourceMemory: "f1", + }, + Requests: resources.Requests{ + corev1.ResourceCPU: 5 * 10, + corev1.ResourceMemory: 5 * 10 * 1024, + }, + Count: 5, + }, + }, + }, + featureGates: map[featuregate.Feature]bool{ + features.ReclaimablePods: false, + }}, "admitted with reclaim and increased reclaim": { workload: *utiltestingapi.MakeWorkload("", ""). PodSets( @@ -363,6 +439,9 @@ func TestNewInfo(t *testing.T) { } for name, tc := range cases { t.Run(name, func(t *testing.T) { + for fg, enabled := range tc.featureGates { + features.SetFeatureGateDuringTest(t, fg, enabled) + } info := NewInfo(&tc.workload, tc.infoOptions...) if diff := cmp.Diff(info, &tc.wantInfo, cmpopts.IgnoreFields(Info{}, "Obj")); diff != "" { t.Errorf("NewInfo(_) = (-want,+got):\n%s", diff) diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 4e6f8aa4859..13dda163ea8 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -264,7 +264,7 @@ spec: ### Feature gates for alpha and beta features | Feature | Default | Stage | Since | Until | -|-----------------------------------------------|---------|-------|-------|-------| +| --------------------------------------------- | ------- | ----- | ----- | ----- | | `FlavorFungibility` | `true` | Beta | 0.5 | | | `MultiKueue` | `false` | Alpha | 0.6 | 0.8 | | `MultiKueue` | `true` | Beta | 0.9 | | @@ -300,6 +300,7 @@ spec: | `WorkloadRequestUseMergePatch` | `false` | Alpha | 0.14 | | | `SanitizePodSets` | `true` | Beta | 0.13 | | | `MultiKueueAllowInsecureKubeconfigs` | `false` | Alpha | 0.13 | | +| `ReclaimablePods` | `true` | Beta | 0.15 | | {{% alert title="Note" color="primary" %}} The SanitizePodSets and MultiKueueAllowInsecureKubeconfigs features are available starting from versions 0.13.8 and 0.14.3. @@ -308,7 +309,7 @@ The SanitizePodSets and MultiKueueAllowInsecureKubeconfigs features are availabl ### Feature gates for graduated or deprecated features | Feature | Default | Stage | Since | Until | -|---------------------------------------|---------|------------|-------|-------| +| ------------------------------------- | ------- | ---------- | ----- | ----- | | `ConfigurableResourceTransformations` | `false` | Alpha | 0.9 | 0.9 | | `ConfigurableResourceTransformations` | `true` | Beta | 0.10 | 0.13 | | `ConfigurableResourceTransformations` | `true` | GA | 0.14 | | diff --git a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go index cdc05a481fc..923374a7071 100644 --- a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go +++ b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go @@ -851,6 +851,116 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin }) }) + ginkgo.When("ReclaimablePods feature gate is off and clusterQueue usage status is reconciled", func() { + var ( + clusterQueue *kueue.ClusterQueue + localQueue *kueue.LocalQueue + modelAFlavor *kueue.ResourceFlavor + ac *kueue.AdmissionCheck + ) + + ginkgo.BeforeEach(func() { + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.ReclaimablePods, false) + + ac = utiltestingapi.MakeAdmissionCheck("ac").ControllerName("ac-controller").Obj() + util.MustCreate(ctx, k8sClient, ac) + util.SetAdmissionCheckActive(ctx, k8sClient, ac, metav1.ConditionTrue) + + clusterQueue = utiltestingapi.MakeClusterQueue("cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(flavorModelA). + Resource(resourceGPU, "5", "5").Obj(), + ). + Cohort("cohort"). + AdmissionChecks(kueue.AdmissionCheckReference(ac.Name)). + Obj() + util.MustCreate(ctx, k8sClient, clusterQueue) + localQueue = utiltestingapi.MakeLocalQueue("queue", ns.Name).ClusterQueue(clusterQueue.Name).Obj() + util.MustCreate(ctx, k8sClient, localQueue) + + modelAFlavor = utiltestingapi.MakeResourceFlavor(flavorModelA).NodeLabel(resourceGPU.String(), flavorModelA).Obj() + util.MustCreate(ctx, k8sClient, modelAFlavor) + }) + + ginkgo.AfterEach(func() { + util.ExpectObjectToBeDeleted(ctx, k8sClient, clusterQueue, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, modelAFlavor, true) + util.ExpectObjectToBeDeleted(ctx, k8sClient, ac, true) + }) + + ginkgo.It("Should ignore update status when workloads have reclaimable pods", framework.SlowSpec, func() { + wl := utiltestingapi.MakeWorkload("one", ns.Name). + Queue(kueue.LocalQueueName(localQueue.Name)). + PodSets( + *utiltestingapi.MakePodSet("workers", 5). + Request(resourceGPU, "1"). + Obj(), + ). + Obj() + ginkgo.By("Creating the workload", func() { + util.MustCreate(ctx, k8sClient, wl) + util.ExpectPendingWorkloadsMetric(clusterQueue, 1, 0) + util.ExpectLQPendingWorkloadsMetric(localQueue, 1, 0) + }) + + ginkgo.By("Admitting the workload", func() { + admission := utiltestingapi.MakeAdmission(clusterQueue.Name).PodSets( + kueue.PodSetAssignment{ + Name: "workers", + Flavors: map[corev1.ResourceName]kueue.ResourceFlavorReference{ + resourceGPU: "model-a", + }, + ResourceUsage: corev1.ResourceList{ + resourceGPU: resource.MustParse("5"), + }, + Count: ptr.To[int32](5), + }, + ).Obj() + util.SetQuotaReservation(ctx, k8sClient, client.ObjectKeyFromObject(wl), admission) + }) + + ginkgo.By("Validating CQ status has changed", func() { + util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 1) + util.ExpectLQReservingActiveWorkloadsMetric(localQueue, 1) + gomega.Eventually(func(g gomega.Gomega) { + var updatedCq kueue.ClusterQueue + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedCq)).To(gomega.Succeed()) + g.Expect(updatedCq.Status.FlavorsReservation).Should(gomega.BeComparableTo([]kueue.FlavorUsage{ + { + Name: flavorModelA, + Resources: []kueue.ResourceUsage{{ + Name: resourceGPU, + Total: resource.MustParse("5"), + }}, + }, + }, util.IgnoreConditionTimestamps)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Marking two workers as reclaimable", func() { + gomega.Expect(workload.UpdateReclaimablePods(ctx, k8sClient, wl, []kueue.ReclaimablePod{{Name: "workers", Count: 2}})).To(gomega.Succeed()) + }) + + ginkgo.By("Validating CQ status hasn't changed", func() { + util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 1) + util.ExpectLQReservingActiveWorkloadsMetric(localQueue, 1) + gomega.Eventually(func(g gomega.Gomega) { + var updatedCq kueue.ClusterQueue + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedCq)).To(gomega.Succeed()) + g.Expect(updatedCq.Status.FlavorsReservation).Should(gomega.BeComparableTo([]kueue.FlavorUsage{ + { + Name: flavorModelA, + Resources: []kueue.ResourceUsage{{ + Name: resourceGPU, + Total: resource.MustParse("5"), + }}, + }, + })) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) + ginkgo.When("Deleting clusterQueues", func() { var ( cq *kueue.ClusterQueue diff --git a/test/integration/singlecluster/scheduler/scheduler_test.go b/test/integration/singlecluster/scheduler/scheduler_test.go index 2630ae906cf..ba155bd8030 100644 --- a/test/integration/singlecluster/scheduler/scheduler_test.go +++ b/test/integration/singlecluster/scheduler/scheduler_test.go @@ -502,6 +502,78 @@ var _ = ginkgo.Describe("Scheduler", func() { }) }) + ginkgo.It("Should admit workloads when resources are gradually released", func() { + firstWl := utiltestingapi.MakeWorkload("first-wl", ns.Name).Queue(kueue.LocalQueueName(preemptionQueue.Name)). + PodSets( + *utiltestingapi.MakePodSet("first", 1).Request(corev1.ResourceCPU, "1").Obj(), + *utiltestingapi.MakePodSet("second", 1).Request(corev1.ResourceCPU, "1").Obj(), + *utiltestingapi.MakePodSet("third", 1).Request(corev1.ResourceCPU, "1").Obj(), + ). + Obj() + ginkgo.By("Creating first workload and admitting it", func() { + util.MustCreate(ctx, k8sClient, firstWl) + + util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, firstWl) + util.ExpectPendingWorkloadsMetric(preemptionClusterQ, 0, 0) + util.ExpectAdmittedWorkloadsTotalMetric(preemptionClusterQ, "", 1) + }) + + secondWl := utiltestingapi.MakeWorkload("second-wl", ns.Name).Queue(kueue.LocalQueueName(preemptionQueue.Name)). + PodSets( + *utiltestingapi.MakePodSet("first", 1).Request(corev1.ResourceCPU, "1").Obj(), + ). + Obj() + ginkgo.By("Creating the second workload which is pending", func() { + util.MustCreate(ctx, k8sClient, secondWl) + util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, firstWl) + util.ExpectPendingWorkloadsMetric(preemptionClusterQ, 0, 1) + util.ExpectAdmittedWorkloadsTotalMetric(preemptionClusterQ, "", 1) + }) + + ginkgo.By("Reclaim one pod from the first workload and admitting the second one", func() { + gomega.Expect(workload.UpdateReclaimablePods(ctx, k8sClient, firstWl, []kueue.ReclaimablePod{{Name: "third", Count: 1}})).To(gomega.Succeed()) + util.ExpectPendingWorkloadsMetric(preemptionClusterQ, 0, 0) + util.ExpectAdmittedWorkloadsTotalMetric(preemptionClusterQ, "", 2) + }) + }) + + ginkgo.It("Should not admit workloads when resources are dynamically reclaimed and the reclaimablePods feature gate is off", func() { + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.ReclaimablePods, false) + + firstWl := utiltestingapi.MakeWorkload("first-wl", ns.Name).Queue(kueue.LocalQueueName(preemptionQueue.Name)). + PodSets( + *utiltestingapi.MakePodSet("first", 1).Request(corev1.ResourceCPU, "1").Obj(), + *utiltestingapi.MakePodSet("second", 1).Request(corev1.ResourceCPU, "1").Obj(), + *utiltestingapi.MakePodSet("third", 1).Request(corev1.ResourceCPU, "1").Obj(), + ). + Obj() + ginkgo.By("Creating first workload and admitting it", func() { + util.MustCreate(ctx, k8sClient, firstWl) + util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, firstWl) + util.ExpectPendingWorkloadsMetric(preemptionClusterQ, 0, 0) + util.ExpectAdmittedWorkloadsTotalMetric(preemptionClusterQ, "", 1) + }) + + ginkgo.By("Creating the second workload which is pending", func() { + secondWl := utiltestingapi.MakeWorkload("second-wl", ns.Name).Queue(kueue.LocalQueueName(preemptionQueue.Name)). + PodSets( + *utiltestingapi.MakePodSet("first", 1).Request(corev1.ResourceCPU, "1").Obj(), + ). + Obj() + util.MustCreate(ctx, k8sClient, secondWl) + util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, firstWl) + util.ExpectPendingWorkloadsMetric(preemptionClusterQ, 0, 1) + util.ExpectAdmittedWorkloadsTotalMetric(preemptionClusterQ, "", 1) + }) + + ginkgo.By("Reclaim one pod from the first workload and admitting the second one", func() { + gomega.Expect(workload.UpdateReclaimablePods(ctx, k8sClient, firstWl, []kueue.ReclaimablePod{{Name: "third", Count: 1}})).To(gomega.Succeed()) + util.ExpectWorkloadsToBeAdmitted(ctx, k8sClient, firstWl) + util.ExpectPendingWorkloadsMetric(preemptionClusterQ, 0, 1) + util.ExpectAdmittedWorkloadsTotalMetric(preemptionClusterQ, "", 1) + }) + }) + ginkgo.It("Should admit workloads with admission checks", func() { wl1 := utiltestingapi.MakeWorkload("admission-check-wl1", ns.Name). Queue(kueue.LocalQueueName(admissionCheckQueue.Name)). From 0e7a10d0deba8c6f7719426273ba9fcf3a33e98a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Wed, 5 Nov 2025 13:46:52 +0100 Subject: [PATCH 092/119] Cleanup preemption message generation (#7541) --- pkg/scheduler/preemption/preemption.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index 44b8790a07b..f7cdfdc2614 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -17,6 +17,7 @@ limitations under the License. package preemption import ( + "cmp" "context" "fmt" "slices" @@ -142,19 +143,9 @@ var HumanReadablePreemptionReasons = map[string]string{ } func preemptionMessage(preemptor *kueue.Workload, reason string) string { - var wUID, jUID string - if preemptor.UID == "" { - wUID = "UNKNOWN" - } else { - wUID = string(preemptor.UID) - } + wUID := cmp.Or(string(preemptor.UID), "UNKNOWN") uid := preemptor.Labels[constants.JobUIDLabel] - if uid == "" { - jUID = "UNKNOWN" - } else { - jUID = uid - } - + jUID := cmp.Or(uid, "UNKNOWN") return fmt.Sprintf("Preempted to accommodate a workload (UID: %s, JobUID: %s) due to %s", wUID, jUID, HumanReadablePreemptionReasons[reason]) } From bd6e6fb279a0e508d604611908c42e1a8a9461c9 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Wed, 5 Nov 2025 18:44:53 +0530 Subject: [PATCH 093/119] Use wrappers in cluster_queue_test.go. (#7543) --- pkg/cache/queue/cluster_queue_test.go | 150 +++++++++----------------- 1 file changed, 52 insertions(+), 98 deletions(-) diff --git a/pkg/cache/queue/cluster_queue_test.go b/pkg/cache/queue/cluster_queue_test.go index 1c55073024b..5544fc3a912 100644 --- a/pkg/cache/queue/cluster_queue_test.go +++ b/pkg/cache/queue/cluster_queue_test.go @@ -726,96 +726,60 @@ func TestStrictFIFO(t *testing.T) { }{ { name: "w1.priority is higher than w2.priority", - w1: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w1", - CreationTimestamp: metav1.NewTime(t1), - }, - Spec: kueue.WorkloadSpec{ - PriorityClassName: "highPriority", - Priority: ptr.To(highPriority), - }, - }, - w2: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w2", - CreationTimestamp: metav1.NewTime(t2), - }, - Spec: kueue.WorkloadSpec{ - PriorityClassName: "lowPriority", - Priority: ptr.To(lowPriority), - }, - }, + w1: utiltestingapi.MakeWorkload("w1", ""). + Creation(t1). + PriorityClass("highPriority"). + Priority(highPriority). + Obj(), + w2: utiltestingapi.MakeWorkload("w2", ""). + Creation(t2). + PriorityClass("lowPriority"). + Priority(lowPriority). + Obj(), expected: "w1", }, { name: "w1.priority equals w2.priority and w1.create time is earlier than w2.create time", - w1: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w1", - CreationTimestamp: metav1.NewTime(t1), - }, - }, - w2: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w2", - CreationTimestamp: metav1.NewTime(t2), - }, - }, + w1: utiltestingapi.MakeWorkload("w1", ""). + Creation(t1). + Obj(), + w2: utiltestingapi.MakeWorkload("w2", ""). + Creation(t2). + Obj(), expected: "w1", }, { name: "w1.priority equals w2.priority and w1.create time is earlier than w2.create time but w1 was evicted", - w1: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w1", - CreationTimestamp: metav1.NewTime(t1), - }, - Status: kueue.WorkloadStatus{ - Conditions: []metav1.Condition{ - { - Type: kueue.WorkloadEvicted, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(t3), - Reason: kueue.WorkloadEvictedByPodsReadyTimeout, - Message: "by test", - }, - }, - }, - }, - w2: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w2", - CreationTimestamp: metav1.NewTime(t2), - }, - }, + w1: utiltestingapi.MakeWorkload("w1", ""). + Creation(t1). + Condition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(t3), + Reason: kueue.WorkloadEvictedByPodsReadyTimeout, + Message: "by test", + }). + Obj(), + w2: utiltestingapi.MakeWorkload("w2", ""). + Creation(t2). + Obj(), expected: "w2", }, { name: "w1.priority equals w2.priority and w1.create time is earlier than w2.create time and w1 was evicted but kueue is configured to always use the creation timestamp", - w1: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w1", - CreationTimestamp: metav1.NewTime(t1), - }, - Status: kueue.WorkloadStatus{ - Conditions: []metav1.Condition{ - { - Type: kueue.WorkloadEvicted, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(t3), - Reason: kueue.WorkloadEvictedByPodsReadyTimeout, - Message: "by test", - }, - }, - }, - }, - w2: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w2", - CreationTimestamp: metav1.NewTime(t2), - }, - }, + w1: utiltestingapi.MakeWorkload("w1", ""). + Creation(t1). + Condition(metav1.Condition{ + Type: kueue.WorkloadEvicted, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(t3), + Reason: kueue.WorkloadEvictedByPodsReadyTimeout, + Message: "by test", + }). + Obj(), + w2: utiltestingapi.MakeWorkload("w2", ""). + Creation(t2). + Obj(), workloadOrdering: &workload.Ordering{ PodsReadyRequeuingTimestamp: config.CreationTimestamp, }, @@ -823,26 +787,16 @@ func TestStrictFIFO(t *testing.T) { }, { name: "p1.priority is lower than p2.priority and w1.create time is earlier than w2.create time", - w1: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w1", - CreationTimestamp: metav1.NewTime(t1), - }, - Spec: kueue.WorkloadSpec{ - PriorityClassName: "lowPriority", - Priority: ptr.To(lowPriority), - }, - }, - w2: &kueue.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "w2", - CreationTimestamp: metav1.NewTime(t2), - }, - Spec: kueue.WorkloadSpec{ - PriorityClassName: "highPriority", - Priority: ptr.To(highPriority), - }, - }, + w1: utiltestingapi.MakeWorkload("w1", ""). + Creation(t1). + PriorityClass("lowPriority"). + Priority(lowPriority). + Obj(), + w2: utiltestingapi.MakeWorkload("w2", ""). + Creation(t2). + PriorityClass("highPriority"). + Priority(highPriority). + Obj(), expected: "w2", }, } { From e3fe657645c42e3aaae696c21bb8eb5070973a4b Mon Sep 17 00:00:00 2001 From: Singularity23x0 Date: Fri, 7 Nov 2025 16:11:11 +0000 Subject: [PATCH 094/119] Finalizer implementation finalized. --- pkg/controller/core/workload_controller.go | 59 ++-- .../core/workload_controller_test.go | 268 +++++++++++++++--- pkg/util/testing/v1beta2/wrappers.go | 2 +- .../core/workload_controller_test.go | 2 + 4 files changed, 262 insertions(+), 69 deletions(-) diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index 173c407013c..6ff97056904 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -19,7 +19,6 @@ package core import ( "cmp" "context" - "errors" "fmt" "slices" "time" @@ -167,33 +166,25 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if !wl.DeletionTimestamp.IsZero() { log = log.WithValues("deletionTimestamp", wl.DeletionTimestamp) - log.Info("Workload marked for deletion.") + log.Info("Attemtping to finalize workload.") - hasInUseFinalizer := controllerutil.ContainsFinalizer(&wl, kueue.ResourceInUseFinalizerName) - hasSafeDeleteFinlaizer := controllerutil.ContainsFinalizer(&wl, kueue.SafeDeleteFinalizerName) switch { - case hasInUseFinalizer && hasSafeDeleteFinlaizer: + case controllerutil.ContainsFinalizer(&wl, kueue.ResourceInUseFinalizerName): { log.Info("Manual deletion by a user detected.") if len(wl.OwnerReferences) == 0 { - return ctrl.Result{}, r.cleanUp(ctx, &wl, log) + return ctrl.Result{}, r.finalize(ctx, &wl, log) } else { - log.Info("Uable to delete. Workload still has owners.", "owners", wl.OwnerReferences) + log.Info("Uable to finalize: workload still has owners. Proceeding with reconcile.", "owners", wl.OwnerReferences) } } - case !hasInUseFinalizer && hasSafeDeleteFinlaizer: + case controllerutil.ContainsFinalizer(&wl, kueue.SafeDeleteFinalizerName): { - return ctrl.Result{}, r.cleanUp(ctx, &wl, log) - } - case hasInUseFinalizer && !hasSafeDeleteFinlaizer: - { - err := errors.New("Illegal finalizer configuration for workload.") - log.Error(err, "Illegal finalizer configuration for workload.") - return ctrl.Result{}, err + return ctrl.Result{}, r.finalize(ctx, &wl, log) } default: { - log.Info("Unknown finalizer(s) preventing workload deletion.") + log.Info("Unknown finalizer(s) present. Proceeding with reconcile.") } } } @@ -502,31 +493,36 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, nil } -func (r *WorkloadReconciler) cleanUp(ctx context.Context, wl *kueue.Workload, log logr.Logger) error { - log.Info("Finalizing workload before deletion.") +func (r *WorkloadReconciler) finalize(ctx context.Context, wl *kueue.Workload, log logr.Logger) error { + log.V(2).Info("Finalizing workload.") defer r.notifyWatchers(wl, nil) - var resultError error = nil if workload.HasQuotaReservation(wl) { + var err error = nil r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, wl, func() { - if err := r.cache.DeleteWorkload(log, wl); err != nil { - resultError = err - log.Error(err, "Failed to delete workload from cache") - } + err = r.cache.DeleteWorkload(log, wl) }) + if err != nil { + log.V(2).Error(err, "Failed to delete workload from cache.") + return err + } } else { r.queues.QueueAssociatedInadmissibleWorkloadsAfter(ctx, wl, func() { if err := r.cache.DeleteWorkload(log, wl); err != nil { - log.Info("Failed to delete workload from cache.", "Error", err, "Note", "this may be intended behavior") + log.V(2).Info("Failed to delete workload from cache.", "Error", err, "Note", "this may be intended behavior") } }) } r.queues.DeleteWorkload(wl) + controllerutil.RemoveFinalizer(wl, kueue.ResourceInUseFinalizerName) controllerutil.RemoveFinalizer(wl, kueue.SafeDeleteFinalizerName) - - return resultError + if err := r.client.Update(ctx, wl); err != nil { + return err + } + r.recorder.Eventf(wl, corev1.EventTypeNormal, "Finalized", "Workload %s has been finalized", workload.Key(wl)) + return nil } // isDisabledRequeuedByClusterQueueStopped returns true if the workload is unset requeued by cluster queue stopped. @@ -814,13 +810,12 @@ func (r *WorkloadReconciler) Create(e event.TypedCreateEvent[*kueue.Workload]) b } func (r *WorkloadReconciler) Delete(e event.TypedDeleteEvent[*kueue.Workload]) bool { - status := "unknown" - if !e.DeleteStateUnknown { - status = workload.Status(e.Object) + log := r.log.WithValues("workload", klog.KObj(e.Object), "queue", e.Object.Spec.QueueName, "status", workload.Status(e.Object)) + if e.DeleteStateUnknown { + log.V(2).Info("Workload delete event; delete status unknown") + } else { + log.V(2).Info("Workload delete event") } - log := r.log.WithValues("workload", klog.KObj(e.Object), "queue", e.Object.Spec.QueueName, "status", status) - log.V(2).Info("Workload delete event") - return true } diff --git a/pkg/controller/core/workload_controller_test.go b/pkg/controller/core/workload_controller_test.go index 601649a3cac..a94475ca9d6 100644 --- a/pkg/controller/core/workload_controller_test.go +++ b/pkg/controller/core/workload_controller_test.go @@ -37,11 +37,13 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" + "sigs.k8s.io/kueue/pkg/cache/scheduler" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" "sigs.k8s.io/kueue/pkg/dra" "sigs.k8s.io/kueue/pkg/features" @@ -2310,7 +2312,6 @@ func TestReconcile(t *testing.T) { "shouldn't try to delete the workload (no event emitted) because it is already being deleted by kubernetes, object retention configured": { enableObjectRetentionPolicies: true, workload: utiltestingapi.MakeWorkload("wl", "ns"). - Finalizers(kueue.SafeDeleteFinalizerName, kueue.ResourceInUseFinalizerName). Condition(metav1.Condition{ Type: kueue.WorkloadFinished, Status: metav1.ConditionTrue, @@ -2329,10 +2330,20 @@ func TestReconcile(t *testing.T) { }, wantWorkload: nil, wantError: nil, + wantEvents: []utiltesting.EventRecord{{ + Key: types.NamespacedName{ + Namespace: "ns", + Name: "wl", + }, + EventType: corev1.EventTypeNormal, + Reason: "Finalized", + Message: "Workload ns/wl has been finalized", + }}, }, "shouldn't try to delete the workload because the retention period hasn't elapsed yet, object retention configured": { enableObjectRetentionPolicies: true, workload: utiltestingapi.MakeWorkload("wl", "ns"). + Finalizers(). Condition(metav1.Condition{ Type: kueue.WorkloadFinished, Status: metav1.ConditionTrue, @@ -2350,6 +2361,7 @@ func TestReconcile(t *testing.T) { RequeueAfter: util.LongTimeout - util.Timeout, }, wantWorkload: utiltestingapi.MakeWorkload("wl", "ns"). + Finalizers(). Condition(metav1.Condition{ Type: kueue.WorkloadFinished, Status: metav1.ConditionTrue, @@ -2388,7 +2400,7 @@ func TestReconcile(t *testing.T) { }, wantError: nil, }, - "should clean up deleted workload because only safe-delete finalizer is set": { + "should finalize deleted workload because only safe-delete finalizer is set": { cq: utiltestingapi.MakeClusterQueue("cq").Obj(), lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), workload: utiltestingapi.MakeWorkload("wl", "ns"). @@ -2405,13 +2417,21 @@ func TestReconcile(t *testing.T) { wantWorkload: nil, wantError: nil, wantWorkloadCached: false, + wantEvents: []utiltesting.EventRecord{{ + Key: types.NamespacedName{ + Namespace: "ns", + Name: "wl", + }, + EventType: corev1.EventTypeNormal, + Reason: "Finalized", + Message: "Workload ns/wl has been finalized", + }}, }, - "should clean up deleted workload because has both in-use and safe-delete finalizers but no owners": { + "should finalize deleted workload because has in-use finalizer but no owners": { cq: utiltestingapi.MakeClusterQueue("cq").Obj(), lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), workload: utiltestingapi.MakeWorkload("wl", "ns"). Queue("lq"). - Finalizers(kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName). ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). Condition(metav1.Condition{ Type: kueue.WorkloadFinished, @@ -2423,13 +2443,21 @@ func TestReconcile(t *testing.T) { wantWorkload: nil, wantError: nil, wantWorkloadCached: false, + wantEvents: []utiltesting.EventRecord{{ + Key: types.NamespacedName{ + Namespace: "ns", + Name: "wl", + }, + EventType: corev1.EventTypeNormal, + Reason: "Finalized", + Message: "Workload ns/wl has been finalized", + }}, }, - "shouldn't clean up deleted workload because has in-use and safe-delete finalizers and still has owners": { + "shouldn't finalize deleted workload because has in-use finalizer and owners": { cq: utiltestingapi.MakeClusterQueue("cq").Obj(), lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), workload: utiltestingapi.MakeWorkload("wl", "ns"). Queue("lq"). - Finalizers(kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName). ControllerReference(batchv1.SchemeGroupVersion.WithKind("Job"), "ownername", "owneruid"). ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). Condition(metav1.Condition{ @@ -2441,7 +2469,6 @@ func TestReconcile(t *testing.T) { Obj(), wantWorkload: utiltestingapi.MakeWorkload("wl", "ns"). Queue("lq"). - Finalizers(kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName). ControllerReference(batchv1.SchemeGroupVersion.WithKind("Job"), "ownername", "owneruid"). ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). Condition(metav1.Condition{ @@ -2454,35 +2481,7 @@ func TestReconcile(t *testing.T) { wantError: nil, wantWorkloadCached: true, }, - "should throw error because has in-use finalizer but not safe-delete finalizer": { - cq: utiltestingapi.MakeClusterQueue("cq").Obj(), - lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), - workload: utiltestingapi.MakeWorkload("wl", "ns"). - Queue("lq"). - Finalizers(kueue.ResourceInUseFinalizerName). - ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). - Condition(metav1.Condition{ - Type: kueue.WorkloadFinished, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), - }). - Delete(). - Obj(), - wantWorkload: utiltestingapi.MakeWorkload("wl", "ns"). - Queue("lq"). - Finalizers(kueue.ResourceInUseFinalizerName). - ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). - Condition(metav1.Condition{ - Type: kueue.WorkloadFinished, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), - }). - Delete(). - Obj(), - wantErrorMsg: "Illegal finalizer configuration for workload.", - wantWorkloadCached: true, - }, - "shouldn't clean up deleted workload because unknown finalizers are set": { + "shouldn't finalize deleted workload because unknown finalizers are set": { cq: utiltestingapi.MakeClusterQueue("cq").Obj(), lq: utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj(), workload: utiltestingapi.MakeWorkload("wl", "ns"). @@ -2704,3 +2703,200 @@ func TestReconcile(t *testing.T) { } } } + +func TestFinalize(t *testing.T) { + testStartTime := time.Now().Truncate(time.Second) + fakeClock := testingclock.NewFakeClock(testStartTime) + cases := map[string]struct { + workload *kueue.Workload + wantError error + wantWorkloadCached bool + wantFinalizers []string + dontWantFinalizers []string + wantEvents []utiltesting.EventRecord + }{ + "clean up when both finalizers are set": { + workload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantError: nil, + wantWorkloadCached: false, + wantFinalizers: nil, + dontWantFinalizers: []string{kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName}, + wantEvents: []utiltesting.EventRecord{{ + Key: types.NamespacedName{ + Namespace: "ns", + Name: "wl", + }, + EventType: corev1.EventTypeNormal, + Reason: "Finalized", + Message: "Workload ns/wl has been finalized", + }}, + }, + "clean up when in-use finalzier is set": { + workload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers(kueue.ResourceInUseFinalizerName). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantError: nil, + wantWorkloadCached: false, + wantFinalizers: nil, + dontWantFinalizers: []string{kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName}, + wantEvents: []utiltesting.EventRecord{{ + Key: types.NamespacedName{ + Namespace: "ns", + Name: "wl", + }, + EventType: corev1.EventTypeNormal, + Reason: "Finalized", + Message: "Workload ns/wl has been finalized", + }}, + }, + "clean up when safe-delete finalzier is set": { + workload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + Finalizers(kueue.SafeDeleteFinalizerName). + ReserveQuota(utiltestingapi.MakeAdmission("cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantError: nil, + wantWorkloadCached: false, + wantFinalizers: nil, + dontWantFinalizers: []string{kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName}, + wantEvents: []utiltesting.EventRecord{{ + Key: types.NamespacedName{ + Namespace: "ns", + Name: "wl", + }, + EventType: corev1.EventTypeNormal, + Reason: "Finalized", + Message: "Workload ns/wl has been finalized", + }}, + }, + "throw error when unable to delete wl from cache": { + workload: utiltestingapi.MakeWorkload("wl", "ns"). + Queue("lq"). + ReserveQuota(utiltestingapi.MakeAdmission("unknown-cq").Obj()). + Condition(metav1.Condition{ + Type: kueue.WorkloadFinished, + Status: metav1.ConditionTrue, + LastTransitionTime: metav1.NewTime(testStartTime.Add(-2 * util.LongTimeout)), + }). + Delete(). + Obj(), + wantError: scheduler.ErrCqNotFound, + wantWorkloadCached: false, + wantFinalizers: []string{kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName}, + dontWantFinalizers: nil, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + testWl := tc.workload.DeepCopy() + objs := []client.Object{testWl} + + clientBuilder := utiltesting.NewClientBuilder().WithObjects(objs...).WithStatusSubresource(objs...).WithInterceptorFuncs(interceptor.Funcs{SubResourcePatch: utiltesting.TreatSSAAsStrategicMerge}) + cl := clientBuilder.Build() + recorder := &utiltesting.EventRecorder{} + + cqCache := schdcache.New(cl) + qManager := qcache.NewManager(cl, cqCache) + reconciler := NewWorkloadReconciler(cl, qManager, cqCache, recorder) + + // use a fake clock with jitter = 0 to be able to assert on the requeueAt. + reconciler.clock = fakeClock + + ctxWithLogger, log := utiltesting.ContextWithLog(t) + testLog := log.WithName("Test setup") + ctx, ctxCancel := context.WithCancel(ctxWithLogger) + defer ctxCancel() + + testCq := utiltestingapi.MakeClusterQueue("cq").Obj() + if err := cl.Create(ctx, testCq); err != nil { + t.Errorf("couldn't create the cluster queue: %v", err) + } + if err := qManager.AddClusterQueue(ctx, testCq); err != nil { + t.Errorf("couldn't add the cluster queue to the queue cache manager: %v", err) + } + if err := cqCache.AddClusterQueue(ctx, testCq); err != nil { + t.Errorf("couldn't add the cluster queue to the cache: %v", err) + } + + testLq := utiltestingapi.MakeLocalQueue("lq", "ns").ClusterQueue("cq").Obj() + if err := cl.Create(ctx, testLq); err != nil { + t.Errorf("couldn't create the local queue: %v", err) + } + if err := qManager.AddLocalQueue(ctx, testLq); err != nil { + t.Errorf("couldn't add the local queue to the cache: %v", err) + } + + cqCache.AddOrUpdateWorkload(testLog, testWl) + qManager.AddOrUpdateWorkload(testWl) + + gotError := reconciler.finalize(ctx, testWl, testLog) + + switch { + case tc.wantError != nil: + if gotError == nil { + t.Errorf("expected error %v, got nil", tc.wantError) + } else if !stderrors.Is(gotError, tc.wantError) { + t.Errorf("unexpected error type: want %v, got %v", tc.wantError, gotError) + } + case gotError != nil: + t.Errorf("unexpected error: %v", gotError) + } + + if diff := cmp.Diff(tc.wantEvents, recorder.RecordedEvents); diff != "" { + t.Errorf("unexpected events (-want/+got):\n%s", diff) + } + + if tc.wantWorkloadCached { + wlKey := workload.Key(testWl) + cqRef := kueue.ClusterQueueReference(testCq.Name) + cacheCq := cqCache.GetClusterQueue(cqRef) + managerCq := qManager.GetClusterQueue(cqRef) + + _, wlInClusterQueue := cacheCq.Workloads[wlKey] + if !wlInClusterQueue { + t.Errorf("Expected workload %s to be in cluster queue cache %s.", wlKey, cqRef) + } + + wlInManagerQueue := managerCq.Info(wlKey) + if wlInManagerQueue == nil { + t.Errorf("Expected workload %s to be in cluster queue (manager) %s.", wlKey, cqRef) + } + } + + for _, expectedFinalizer := range tc.wantFinalizers { + if !controllerutil.ContainsFinalizer(testWl, expectedFinalizer) { + t.Errorf("Expected finalizer missing: %s", expectedFinalizer) + } + } + + for _, expectedGone := range tc.dontWantFinalizers { + if controllerutil.ContainsFinalizer(testWl, expectedGone) { + t.Errorf("Expected %s finalizer to be gone", expectedGone) + } + } + }) + } +} diff --git a/pkg/util/testing/v1beta2/wrappers.go b/pkg/util/testing/v1beta2/wrappers.go index f1aa38239c7..76f94489ad5 100644 --- a/pkg/util/testing/v1beta2/wrappers.go +++ b/pkg/util/testing/v1beta2/wrappers.go @@ -65,7 +65,7 @@ func MakeWorkload(name, ns string) *WorkloadWrapper { ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: ns, - Finalizers: []string{kueue.SafeDeleteFinalizerName}, + Finalizers: []string{kueue.ResourceInUseFinalizerName, kueue.SafeDeleteFinalizerName}, }, Spec: kueue.WorkloadSpec{ PodSets: []kueue.PodSet{ diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go index db193caad22..7be8399167e 100644 --- a/test/integration/singlecluster/controller/core/workload_controller_test.go +++ b/test/integration/singlecluster/controller/core/workload_controller_test.go @@ -578,12 +578,14 @@ var _ = ginkgo.Describe("Workload controller with resource retention", ginkgo.Or ResourceGroup(*utiltestingapi.MakeFlavorQuotas(flavorOnDemand). Resource(corev1.ResourceCPU, "1").Obj()). Obj() + gomega.Expect(k8sClient.Create(ctx, clusterQueue)).To(gomega.Succeed()) localQueue = utiltestingapi.MakeLocalQueue("q", ns.Name).ClusterQueue("cq").Obj() gomega.Expect(k8sClient.Create(ctx, localQueue)).To(gomega.Succeed()) }) ginkgo.AfterEach(func() { gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + util.ExpectObjectToBeDeleted(ctx, k8sClient, localQueue, true) util.ExpectObjectToBeDeleted(ctx, k8sClient, clusterQueue, true) util.ExpectObjectToBeDeleted(ctx, k8sClient, flavor, true) }) From a77639e84cc569a65a07b8984a6d4f28e7e3291e Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 5 Nov 2025 14:48:54 -0500 Subject: [PATCH 095/119] enable optional, required and optionalandrequired linter checks (#7488) fix violations but do not convert values to pointers if suggested Assisted with Claude --- .golangci-kal.yml | 7 +- apis/kueue/v1beta2/admissioncheck_types.go | 23 ++-- apis/kueue/v1beta2/clusterqueue_types.go | 84 +++++++++----- apis/kueue/v1beta2/cohort_types.go | 11 +- apis/kueue/v1beta2/fairsharing_types.go | 8 +- apis/kueue/v1beta2/localqueue_types.go | 38 +++--- apis/kueue/v1beta2/multikueue_types.go | 24 ++-- .../provisioningrequestconfig_types.go | 13 ++- apis/kueue/v1beta2/resourceflavor_types.go | 6 +- apis/kueue/v1beta2/topology_types.go | 7 +- apis/kueue/v1beta2/workload_types.go | 88 ++++++++------ .../v1beta2/workloadpriorityclass_types.go | 6 +- .../crd/kueue.x-k8s.io_admissionchecks.yaml | 3 + .../crd/kueue.x-k8s.io_clusterqueues.yaml | 5 + .../kueue.x-k8s.io_multikueueclusters.yaml | 2 +- ...e.x-k8s.io_provisioningrequestconfigs.yaml | 1 + .../crd/kueue.x-k8s.io_workloads.yaml | 10 +- .../bases/kueue.x-k8s.io_admissionchecks.yaml | 3 + .../bases/kueue.x-k8s.io_clusterqueues.yaml | 5 + .../kueue.x-k8s.io_multikueueclusters.yaml | 2 +- ...e.x-k8s.io_provisioningrequestconfigs.yaml | 1 + .../crd/bases/kueue.x-k8s.io_workloads.yaml | 10 +- .../en/docs/reference/kueue.v1beta2.md | 108 +++++++++--------- 23 files changed, 276 insertions(+), 189 deletions(-) diff --git a/.golangci-kal.yml b/.golangci-kal.yml index f1c8e0e3537..344621545e8 100644 --- a/.golangci-kal.yml +++ b/.golangci-kal.yml @@ -26,10 +26,10 @@ linters: - "nomaps" # Ensure maps are not used. - "nonullable" # Ensure that types and fields do not have the nullable marker. - "notimestamp" # Prevents usage of 'Timestamp' fields - # - "optionalfields" # Ensure that all fields marked as optional adhere to being pointers and + - "optionalfields" # Ensure that all fields marked as optional adhere to being pointers and # having the `omitempty` value in their `json` tag where appropriate. - # - "optionalorrequired" # Every field should be marked as `+optional` or `+required`. - # - "requiredfields" # Required fields should not be pointers, and should not have `omitempty`. + - "optionalorrequired" # Every field should be marked as `+optional` or `+required`. + - "requiredfields" # Required fields should not be pointers, and should not have `omitempty`. - "ssatags" # Ensure array fields have the appropriate listType markers - "statusoptional" # Ensure all first children within status should be optional. - "statussubresource" # All root objects that have a `status` field should have a status subresource. @@ -70,3 +70,4 @@ linters: issues: max-same-issues: 0 + max-issues-per-linter: 0 diff --git a/apis/kueue/v1beta2/admissioncheck_types.go b/apis/kueue/v1beta2/admissioncheck_types.go index 00f2b1cca57..9ddb90f0679 100644 --- a/apis/kueue/v1beta2/admissioncheck_types.go +++ b/apis/kueue/v1beta2/admissioncheck_types.go @@ -48,11 +48,11 @@ const ( type AdmissionCheckSpec struct { // controllerName identifies the controller that processes the AdmissionCheck, // not necessarily a Kubernetes Pod or Deployment name. Cannot be empty. - // +kubebuilder:validation:Required + // +required // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable" // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:MinLength=1 - ControllerName string `json:"controllerName"` + ControllerName string `json:"controllerName"` //nolint:kubeapilinter // ignore omitempty // parameters identifies a configuration with additional parameters for the // check. @@ -62,17 +62,23 @@ type AdmissionCheckSpec struct { type AdmissionCheckParametersReference struct { // apiGroup is the group for the resource being referenced. + // +required // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" - APIGroup string `json:"apiGroup"` + APIGroup string `json:"apiGroup,omitempty"` // kind is the type of the resource being referenced. + // +required // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern="^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$" - Kind string `json:"kind"` + Kind string `json:"kind,omitempty"` // name is the name of the resource being referenced. + // +required // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" - Name string `json:"name"` + Name string `json:"name,omitempty"` } // AdmissionCheckStatus defines the observed state of AdmissionCheck @@ -105,13 +111,16 @@ const ( type AdmissionCheck struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the AdmissionCheck. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the AdmissionCheck. - Spec AdmissionCheckSpec `json:"spec,omitempty"` + // +optional + Spec AdmissionCheckSpec `json:"spec"` //nolint:kubeapilinter // spec should not be a pointer // status is the status of the AdmissionCheck. - Status AdmissionCheckStatus `json:"status,omitempty"` + // +optional + Status AdmissionCheckStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index 735c851f878..de57e82df28 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -62,6 +62,7 @@ type ClusterQueueSpec struct { // resourceGroups can be up to 16, with a max of 256 total flavors across all groups. // +listType=atomic // +kubebuilder:validation:MaxItems=16 + // +optional ResourceGroups []ResourceGroup `json:"resourceGroups,omitempty"` // cohortName that this ClusterQueue belongs to. CQs that belong to the @@ -76,7 +77,8 @@ type ClusterQueueSpec struct { // // A cohort is a name that links CQs together, but it doesn't reference any // object. - CohortName CohortReference `json:"cohortName,omitempty"` + // +optional + CohortName CohortReference `json:"cohortName,omitempty"` //nolint:kubeapilinter // should not be a pointer // queueingStrategy indicates the queueing strategy of the workloads // across the queues in this ClusterQueue. @@ -89,24 +91,28 @@ type ClusterQueueSpec struct { // however older workloads that can't be admitted will not block // admitting newer workloads that fit existing quota. // + // +optional // +kubebuilder:default=BestEffortFIFO // +kubebuilder:validation:Enum=StrictFIFO;BestEffortFIFO - QueueingStrategy QueueingStrategy `json:"queueingStrategy,omitempty"` + QueueingStrategy QueueingStrategy `json:"queueingStrategy,omitempty"` //nolint:kubeapilinter // should not be a pointer // namespaceSelector defines which namespaces are allowed to submit workloads to // this clusterQueue. Beyond this basic support for policy, a policy agent like // Gatekeeper should be used to enforce more advanced policies. // Defaults to null which is a nothing selector (no namespaces eligible). // If set to an empty selector `{}`, then all namespaces are eligible. + // +optional NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"` // flavorFungibility defines whether a workload should try the next flavor // before borrowing or preempting in the flavor being evaluated. + // +optional // +kubebuilder:default={} FlavorFungibility *FlavorFungibility `json:"flavorFungibility,omitempty"` // preemption defines the preemption policies. // +kubebuilder:default={} + // +optional Preemption *ClusterQueuePreemption `json:"preemption,omitempty"` // admissionChecks lists the AdmissionChecks required by this ClusterQueue. @@ -129,9 +135,9 @@ type ClusterQueueSpec struct { // - HoldAndDrain - Admitted workloads are evicted and Reserving workloads will cancel the reservation. // - Hold - Admitted workloads will run to completion and Reserving workloads will cancel the reservation. // - // +optional // +kubebuilder:validation:Enum=None;Hold;HoldAndDrain // +kubebuilder:default="None" + // +optional StopPolicy *StopPolicy `json:"stopPolicy,omitempty"` // fairSharing defines the properties of the ClusterQueue when @@ -150,6 +156,8 @@ type AdmissionChecksStrategy struct { // admissionChecks is a list of strategies for AdmissionChecks // +listType=map // +listMapKey=name + // +required + // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=64 AdmissionChecks []AdmissionCheckStrategyRule `json:"admissionChecks,omitempty"` } @@ -157,7 +165,8 @@ type AdmissionChecksStrategy struct { // AdmissionCheckStrategyRule defines rules for a single AdmissionCheck type AdmissionCheckStrategyRule struct { // name is an AdmissionCheck's name. - Name AdmissionCheckReference `json:"name"` + // +required + Name AdmissionCheckReference `json:"name,omitempty"` // onFlavors is a list of ResourceFlavors' names that this AdmissionCheck should run for. // If empty, the AdmissionCheck will run for all workloads submitted to the ClusterQueue. @@ -191,7 +200,8 @@ type ResourceGroup struct { // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=64 // +listType=atomic - CoveredResources []corev1.ResourceName `json:"coveredResources"` + // +required + CoveredResources []corev1.ResourceName `json:"coveredResources,omitempty"` // flavors is the list of flavors that provide the resources of this group. // Typically, different flavors represent different hardware models @@ -202,17 +212,19 @@ type ResourceGroup struct { // The list cannot be empty and it can contain up to 64 flavors, with a max of // 256 total flavors across all resource groups in the ClusterQueue. // +listType=map + // +required // +listMapKey=name // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=64 - Flavors []FlavorQuotas `json:"flavors"` + Flavors []FlavorQuotas `json:"flavors,omitempty"` } type FlavorQuotas struct { // name of this flavor. The name should match the .metadata.name of a // ResourceFlavor. If a matching ResourceFlavor does not exist, the // ClusterQueue will have an Active condition set to False. - Name ResourceFlavorReference `json:"name"` + // +required + Name ResourceFlavorReference `json:"name"` //nolint:kubeapilinter // should not be a pointer // resources is the list of quotas for this flavor per resource. // There could be up to 64 resources. @@ -220,12 +232,14 @@ type FlavorQuotas struct { // +listMapKey=name // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=64 - Resources []ResourceQuota `json:"resources"` + // +required + Resources []ResourceQuota `json:"resources,omitempty"` } type ResourceQuota struct { // name of this resource. - Name corev1.ResourceName `json:"name"` + // +required + Name corev1.ResourceName `json:"name,omitempty"` // nominalQuota is the quantity of this resource that is available for // Workloads admitted by this ClusterQueue at a point in time. @@ -239,7 +253,8 @@ type ResourceQuota struct { // If the ClusterQueue belongs to a cohort, the sum of the quotas for each // (flavor, resource) combination defines the maximum quantity that can be // allocated by a ClusterQueue in the cohort. - NominalQuota resource.Quantity `json:"nominalQuota"` + // +required + NominalQuota resource.Quantity `json:"nominalQuota,omitempty"` // borrowingLimit is the maximum amount of quota for the [flavor, resource] // combination that this ClusterQueue is allowed to borrow from the unused @@ -290,7 +305,7 @@ type ClusterQueueStatus struct { // +listMapKey=name // +kubebuilder:validation:MaxItems=64 // +optional - FlavorsReservation []FlavorUsage `json:"flavorsReservation"` + FlavorsReservation []FlavorUsage `json:"flavorsReservation,omitempty"` // flavorsUsage are the used quotas, by flavor, currently in use by the // workloads admitted in this ClusterQueue. @@ -298,22 +313,22 @@ type ClusterQueueStatus struct { // +listMapKey=name // +kubebuilder:validation:MaxItems=64 // +optional - FlavorsUsage []FlavorUsage `json:"flavorsUsage"` + FlavorsUsage []FlavorUsage `json:"flavorsUsage,omitempty"` // pendingWorkloads is the number of workloads currently waiting to be // admitted to this clusterQueue. // +optional - PendingWorkloads int32 `json:"pendingWorkloads"` + PendingWorkloads int32 `json:"pendingWorkloads"` //nolint:kubeapilinter // should not be a pointer // reservingWorkloads is the number of workloads currently reserving quota in this // clusterQueue. // +optional - ReservingWorkloads int32 `json:"reservingWorkloads"` + ReservingWorkloads int32 `json:"reservingWorkloads"` //nolint:kubeapilinter // should not be a pointer // admittedWorkloads is the number of workloads currently admitted to this // clusterQueue and haven't finished yet. // +optional - AdmittedWorkloads int32 `json:"admittedWorkloads"` + AdmittedWorkloads int32 `json:"admittedWorkloads"` //nolint:kubeapilinter // should not be a pointer // fairSharing contains the current state for this ClusterQueue // when participating in Fair Sharing. @@ -324,26 +339,31 @@ type ClusterQueueStatus struct { type FlavorUsage struct { // name of the flavor. - Name ResourceFlavorReference `json:"name"` + // +required + Name ResourceFlavorReference `json:"name,omitempty"` //nolint:kubeapilinter // should not be a pointer // resources lists the quota usage for the resources in this flavor. // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=64 - Resources []ResourceUsage `json:"resources"` + // +required + Resources []ResourceUsage `json:"resources,omitempty"` } type ResourceUsage struct { // name of the resource - Name corev1.ResourceName `json:"name"` + // +required + Name corev1.ResourceName `json:"name,omitempty"` // total is the total quantity of used quota, including the amount borrowed // from the cohort. - Total resource.Quantity `json:"total,omitempty"` + // +optional + Total resource.Quantity `json:"total,omitempty"` //nolint:kubeapilinter // should not be a pointer // borrowed is quantity of quota that is borrowed from the cohort. In other // words, it's the used quota that is over the nominalQuota. - Borrowed resource.Quantity `json:"borrowed,omitempty"` + // +optional + Borrowed resource.Quantity `json:"borrowed,omitempty"` //nolint:kubeapilinter // should not be a pointer } const ( @@ -380,7 +400,8 @@ type FlavorFungibility struct { // // +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor} // +kubebuilder:default="MayStopSearch" - WhenCanBorrow FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"` + // +optional + WhenCanBorrow FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"` //nolint:kubeapilinter // should not be a pointer // whenCanPreempt determines whether a workload should try the next flavor // before borrowing in current flavor. The possible values are: // @@ -391,7 +412,8 @@ type FlavorFungibility struct { // // +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor} // +kubebuilder:default="TryNextFlavor" - WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` + // +optional + WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` //nolint:kubeapilinter // should not be a pointer } // ClusterQueuePreemption contains policies to preempt Workloads from this @@ -432,14 +454,16 @@ type ClusterQueuePreemption struct { // cohort, irrespective of priority. **Fair Sharing** preempt Workloads // in the cohort that satisfy the Fair Sharing preemptionStrategies. // + // +optional // +kubebuilder:default=Never // +kubebuilder:validation:Enum=Never;LowerPriority;Any - ReclaimWithinCohort PreemptionPolicy `json:"reclaimWithinCohort,omitempty"` + ReclaimWithinCohort PreemptionPolicy `json:"reclaimWithinCohort,omitempty"` //nolint:kubeapilinter // should not be a pointer // borrowWithinCohort determines whether a pending Workload can preempt // Workloads from other ClusterQueues in the cohort if the workload requires borrowing. // May only be configured with Classical Preemption, and __not__ with Fair Sharing. // +kubebuilder:default={} + // +optional BorrowWithinCohort *BorrowWithinCohort `json:"borrowWithinCohort,omitempty"` // withinClusterQueue determines whether a pending Workload that doesn't fit @@ -455,7 +479,8 @@ type ClusterQueuePreemption struct { // // +kubebuilder:default=Never // +kubebuilder:validation:Enum=Never;LowerPriority;LowerOrNewerEqualPriority - WithinClusterQueue PreemptionPolicy `json:"withinClusterQueue,omitempty"` + // +optional + WithinClusterQueue PreemptionPolicy `json:"withinClusterQueue,omitempty"` //nolint:kubeapilinter // should not be a pointer } type BorrowWithinCohortPolicy string @@ -479,8 +504,8 @@ type BorrowWithinCohort struct { // // +kubebuilder:default=Never // +kubebuilder:validation:Enum=Never;LowerPriority - Policy BorrowWithinCohortPolicy `json:"policy,omitempty"` - + // +optional + Policy BorrowWithinCohortPolicy `json:"policy,omitempty"` //nolint:kubeapilinter // should not be a pointer // maxPriorityThreshold allows to restrict the set of workloads which // might be preempted by a borrowing workload, to only workloads with // priority less than or equal to the specified threshold priority. @@ -505,12 +530,15 @@ type BorrowWithinCohort struct { type ClusterQueue struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the ClusterQueue. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the ClusterQueue. - Spec ClusterQueueSpec `json:"spec,omitempty"` + // +optional + Spec ClusterQueueSpec `json:"spec,omitempty"` //nolint:kubeapilinter // spec should not be a pointer // status is the status of the ClusterQueue. - Status ClusterQueueStatus `json:"status,omitempty"` + // +optional + Status ClusterQueueStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/cohort_types.go b/apis/kueue/v1beta2/cohort_types.go index 8e2a6d34b95..e53c824f396 100644 --- a/apis/kueue/v1beta2/cohort_types.go +++ b/apis/kueue/v1beta2/cohort_types.go @@ -32,7 +32,8 @@ type CohortSpec struct { // Cohort, including ClusterQueues, until the cycle is // removed. We prevent further admission while the cycle // exists. - ParentName CohortReference `json:"parentName,omitempty"` + // +optional + ParentName CohortReference `json:"parentName,omitempty"` //nolint:kubeapilinter // should not be a pointer // resourceGroups describes groupings of Resources and // Flavors. Each ResourceGroup defines a list of Resources @@ -60,6 +61,7 @@ type CohortSpec struct { // // +listType=atomic // +kubebuilder:validation:MaxItems=16 + // +optional ResourceGroups []ResourceGroup `json:"resourceGroups,omitempty"` // fairSharing defines the properties of the Cohort when @@ -91,12 +93,15 @@ type CohortStatus struct { type Cohort struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the Cohort. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the Cohort. - Spec CohortSpec `json:"spec,omitempty"` + // +optional + Spec CohortSpec `json:"spec"` //nolint:kubeapilinter // spec should not be a pointer // status is the status of the Cohort. - Status CohortStatus `json:"status,omitempty"` + // +optional + Status CohortStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/fairsharing_types.go b/apis/kueue/v1beta2/fairsharing_types.go index b0fb0445ba2..fbcb0bf97bc 100644 --- a/apis/kueue/v1beta2/fairsharing_types.go +++ b/apis/kueue/v1beta2/fairsharing_types.go @@ -41,6 +41,7 @@ type FairSharing struct { // disadvantage against other ClusterQueues and Cohorts. // When not 0, Weight must be greater than 10^-9. // +kubebuilder:default=1 + // +optional Weight *resource.Quantity `json:"weight,omitempty"` } @@ -53,7 +54,8 @@ type FairSharingStatus struct { // the Node is below the nominal quota. If the Node has a // weight of zero and is borrowing, this will return // 9223372036854775807, the maximum possible share value. - WeightedShare int64 `json:"weightedShare"` + // +required + WeightedShare int64 `json:"weightedShare"` //nolint:kubeapilinter // do not add omitempty // admissionFairSharingStatus represents information relevant to the Admission Fair Sharing // +optional @@ -69,7 +71,7 @@ type AdmissionFairSharingStatus struct { // lastUpdate is the time when share and consumed resources were updated. // +required - LastUpdate metav1.Time `json:"lastUpdate"` + LastUpdate metav1.Time `json:"lastUpdate"` //nolint:kubeapilinter // exclude omitempty } type AdmissionScope struct { @@ -80,7 +82,7 @@ type AdmissionScope struct { // // +kubebuilder:validation:Enum=UsageBasedAdmissionFairSharing;NoAdmissionFairSharing // +required - AdmissionMode AdmissionMode `json:"admissionMode"` + AdmissionMode AdmissionMode `json:"admissionMode,omitempty"` } type AdmissionMode string diff --git a/apis/kueue/v1beta2/localqueue_types.go b/apis/kueue/v1beta2/localqueue_types.go index fc964c1a4a1..b6386d08bb0 100644 --- a/apis/kueue/v1beta2/localqueue_types.go +++ b/apis/kueue/v1beta2/localqueue_types.go @@ -33,7 +33,8 @@ type LocalQueueName string type LocalQueueSpec struct { // clusterQueue is a reference to a clusterQueue that backs this localQueue. // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable" - ClusterQueue ClusterQueueReference `json:"clusterQueue,omitempty"` + // +optional + ClusterQueue ClusterQueueReference `json:"clusterQueue,omitempty"` //nolint:kubeapilinter // should not be a pointer // stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, // no new reservation being made. @@ -60,18 +61,16 @@ type TopologyInfo struct { // name is the name of the topology. // // +required - // +kubebuilder:validation:Required - Name TopologyReference `json:"name"` + Name TopologyReference `json:"name"` //nolint:kubeapilinter // should not be a pointer // levels define the levels of topology. // // +required // +listType=atomic - // +kubebuilder:validation:Required // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 // +kubebuilder:validation:items:MaxLength=317 - Levels []string `json:"levels"` + Levels []string `json:"levels,omitempty"` } // LocalQueueStatus defines the observed state of LocalQueue @@ -89,17 +88,17 @@ type LocalQueueStatus struct { // pendingWorkloads is the number of Workloads in the LocalQueue not yet admitted to a ClusterQueue // +optional - PendingWorkloads int32 `json:"pendingWorkloads"` + PendingWorkloads int32 `json:"pendingWorkloads"` //nolint:kubeapilinter // should not be a pointer // reservingWorkloads is the number of workloads in this LocalQueue // reserving quota in a ClusterQueue and that haven't finished yet. // +optional - ReservingWorkloads int32 `json:"reservingWorkloads"` + ReservingWorkloads int32 `json:"reservingWorkloads"` //nolint:kubeapilinter // should not be a pointer // admittedWorkloads is the number of workloads in this LocalQueue // admitted to a ClusterQueue and that haven't finished yet. // +optional - AdmittedWorkloads int32 `json:"admittedWorkloads"` + AdmittedWorkloads int32 `json:"admittedWorkloads"` //nolint:kubeapilinter // should not be a pointer // flavorsReservation are the reserved quotas, by flavor currently in use by the // workloads assigned to this LocalQueue. @@ -107,7 +106,7 @@ type LocalQueueStatus struct { // +listMapKey=name // +kubebuilder:validation:MaxItems=16 // +optional - FlavorsReservation []LocalQueueFlavorUsage `json:"flavorsReservation"` + FlavorsReservation []LocalQueueFlavorUsage `json:"flavorsReservation,omitempty"` // flavorsUsage are the used quotas, by flavor currently in use by the // workloads assigned to this LocalQueue. @@ -115,7 +114,7 @@ type LocalQueueStatus struct { // +listMapKey=name // +kubebuilder:validation:MaxItems=16 // +optional - FlavorsUsage []LocalQueueFlavorUsage `json:"flavorsUsage"` + FlavorsUsage []LocalQueueFlavorUsage `json:"flavorsUsage,omitempty"` // fairSharing contains the information about the current status of fair sharing. // +optional @@ -130,21 +129,25 @@ const ( type LocalQueueFlavorUsage struct { // name of the flavor. - Name ResourceFlavorReference `json:"name"` + // +required + Name ResourceFlavorReference `json:"name"` //nolint:kubeapilinter // should not be a pointer // resources lists the quota usage for the resources in this flavor. // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=16 - Resources []LocalQueueResourceUsage `json:"resources"` + // +required + Resources []LocalQueueResourceUsage `json:"resources,omitempty"` } type LocalQueueResourceUsage struct { // name of the resource. - Name corev1.ResourceName `json:"name"` + // +required + Name corev1.ResourceName `json:"name,omitempty"` // total is the total quantity of used quota. - Total resource.Quantity `json:"total,omitempty"` + // +optional + Total resource.Quantity `json:"total,omitempty"` //nolint:kubeapilinter // should not be a pointer } // +genclient @@ -159,12 +162,15 @@ type LocalQueueResourceUsage struct { type LocalQueue struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the LocalQueue. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the LocalQueue. - Spec LocalQueueSpec `json:"spec,omitempty"` + // +optional + Spec LocalQueueSpec `json:"spec"` //nolint:kubeapilinter // should not be a pointer // status is the status of the LocalQueue. - Status LocalQueueStatus `json:"status,omitempty"` + // +optional + Status LocalQueueStatus `json:"status,omitempty"` //nolint:kubeapilinter // should not be a pointer } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/multikueue_types.go b/apis/kueue/v1beta2/multikueue_types.go index 38ea931f7a6..629558cb385 100644 --- a/apis/kueue/v1beta2/multikueue_types.go +++ b/apis/kueue/v1beta2/multikueue_types.go @@ -50,18 +50,22 @@ type KubeConfig struct { // If LocationType is Secret then Location is the name of the secret inside the namespace in // which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. // +kubebuilder:validation:MaxLength=256 - Location string `json:"location"` + // +kubebuilder:validation:MinLength=1 + // +required + Location string `json:"location,omitempty"` // locationType of the KubeConfig. // // +kubebuilder:default=Secret + // +optional // +kubebuilder:validation:Enum=Secret;Path - LocationType LocationType `json:"locationType"` + LocationType LocationType `json:"locationType"` //nolint:kubeapilinter // should not be a pointer } type MultiKueueClusterSpec struct { // kubeConfig is information on how to connect to the cluster. - KubeConfig KubeConfig `json:"kubeConfig"` + // +required + KubeConfig KubeConfig `json:"kubeConfig,omitempty,omitzero"` } type MultiKueueClusterStatus struct { @@ -89,13 +93,16 @@ type MultiKueueClusterStatus struct { type MultiKueueCluster struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the MultiKueueCluster. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the MultiKueueCluster. - Spec MultiKueueClusterSpec `json:"spec,omitempty"` + // +optional + Spec MultiKueueClusterSpec `json:"spec,omitempty,omitzero"` //nolint:kubeapilinter // spec should not be a pointer // status is the status of the MultiKueueCluster. - Status MultiKueueClusterStatus `json:"status,omitempty"` + // +optional + Status MultiKueueClusterStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer } // +kubebuilder:object:root=true @@ -115,7 +122,8 @@ type MultiKueueConfigSpec struct { // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=10 // +kubebuilder:validation:items:MaxLength=256 - Clusters []string `json:"clusters"` + // +required + Clusters []string `json:"clusters,omitempty,omitzero"` } // +genclient @@ -127,10 +135,12 @@ type MultiKueueConfigSpec struct { type MultiKueueConfig struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the MultiKueueConfig. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the MultiKueueConfig. - Spec MultiKueueConfigSpec `json:"spec,omitempty"` + // +optional + Spec MultiKueueConfigSpec `json:"spec,omitempty"` //nolint:kubeapilinter // spec should not be a pointer } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/provisioningrequestconfig_types.go b/apis/kueue/v1beta2/provisioningrequestconfig_types.go index ff29f7efd4f..2e3f75c1db5 100644 --- a/apis/kueue/v1beta2/provisioningrequestconfig_types.go +++ b/apis/kueue/v1beta2/provisioningrequestconfig_types.go @@ -37,10 +37,11 @@ type ProvisioningRequestConfigSpec struct { // provisioningClassName describes the different modes of provisioning the resources. // Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. // - // +kubebuilder:validation:Required + // +required // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` // +kubebuilder:validation:MaxLength=253 - ProvisioningClassName string `json:"provisioningClassName"` + // +kubebuilder:validation:MinLength=1 + ProvisioningClassName string `json:"provisioningClassName,omitempty"` // parameters contains all other parameters classes may require. // @@ -107,7 +108,7 @@ type ProvisioningRequestPodSetUpdatesNodeSelector struct { // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=317 // +kubebuilder:validation:Pattern=`^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$` - Key string `json:"key"` + Key string `json:"key,omitempty"` // valueFromProvisioningClassDetail specifies the key of the // ProvisioningRequest.status.provisioningClassDetails from which the value @@ -116,7 +117,7 @@ type ProvisioningRequestPodSetUpdatesNodeSelector struct { // +required // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=32768 - ValueFromProvisioningClassDetail string `json:"valueFromProvisioningClassDetail"` + ValueFromProvisioningClassDetail string `json:"valueFromProvisioningClassDetail,omitempty"` } type ProvisioningRequestRetryStrategy struct { @@ -165,10 +166,12 @@ type Parameter string type ProvisioningRequestConfig struct { metav1.TypeMeta `json:",inline"` // metadata is the standard object metadata. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the ProvisioningRequestConfig. - Spec ProvisioningRequestConfigSpec `json:"spec,omitempty"` + // +optional + Spec ProvisioningRequestConfigSpec `json:"spec"` //nolint:kubeapilinter // should not be a pointer } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/resourceflavor_types.go b/apis/kueue/v1beta2/resourceflavor_types.go index 09c2cd4eff4..e1dc741963a 100644 --- a/apis/kueue/v1beta2/resourceflavor_types.go +++ b/apis/kueue/v1beta2/resourceflavor_types.go @@ -30,10 +30,12 @@ import ( type ResourceFlavor struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the ResourceFlavor. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the ResourceFlavor. - Spec ResourceFlavorSpec `json:"spec,omitempty"` + // +optional + Spec ResourceFlavorSpec `json:"spec,omitempty"` //nolint:kubeapilinter // spec should not be a pointer } // TopologyReference is the name of the Topology. @@ -103,7 +105,7 @@ type ResourceFlavorSpec struct { // nodes matching to the Resource Flavor node labels. // // +optional - TopologyName *TopologyReference `json:"topologyName,omitempty"` + TopologyName *TopologyReference `json:"topologyName,omitempty"` //nolint:kubeapilinter // should be a pointer } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/topology_types.go b/apis/kueue/v1beta2/topology_types.go index 4ecbd596f8c..e29b7350a09 100644 --- a/apis/kueue/v1beta2/topology_types.go +++ b/apis/kueue/v1beta2/topology_types.go @@ -116,11 +116,10 @@ type TopologyLevel struct { // - cloud.provider.com/topology-rack // // +required - // +kubebuilder:validation:Required // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=316 // +kubebuilder:validation:Pattern=`^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$` - NodeLabel string `json:"nodeLabel"` + NodeLabel string `json:"nodeLabel,omitempty"` } // +genclient @@ -132,10 +131,12 @@ type TopologyLevel struct { type Topology struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the Topology. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the Topology. - Spec TopologySpec `json:"spec,omitempty"` + // +optional + Spec TopologySpec `json:"spec"` //nolint:kubeapilinter // spec should not be a pointer } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index 431dd7ea69d..c2e9077cbe3 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -35,11 +35,13 @@ type WorkloadSpec struct { // +listMapKey=name // +kubebuilder:validation:MaxItems=8 // +kubebuilder:validation:MinItems=1 - PodSets []PodSet `json:"podSets"` + // +optional + PodSets []PodSet `json:"podSets"` //nolint:kubeapilinter // do not add omitempty tag // queueName is the name of the LocalQueue the Workload is associated with. // queueName cannot be changed while .status.admission is not null. - QueueName LocalQueueName `json:"queueName,omitempty"` + // +optional + QueueName LocalQueueName `json:"queueName,omitempty"` //nolint:kubeapilinter // should not be a pointer // priorityClassName is the name of the PriorityClass the Workload is associated with. // If specified, indicates the workload's priority. @@ -49,14 +51,17 @@ type WorkloadSpec struct { // PriorityClass object with that name. If not specified, the workload // priority will be default or zero if there is no default. // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" - PriorityClassName string `json:"priorityClassName,omitempty"` + // +optional + PriorityClassName string `json:"priorityClassName,omitempty"` //nolint:kubeapilinter // should not be a pointer // priority determines the order of access to the resources managed by the // ClusterQueue where the workload is queued. // The priority value is populated from PriorityClassName. // The higher the value, the higher the priority. // If priorityClassName is specified, priority must not be null. + // +optional Priority *int32 `json:"priority,omitempty"` // priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. @@ -64,7 +69,8 @@ type WorkloadSpec struct { // When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. // +kubebuilder:default="" // +kubebuilder:validation:Enum=kueue.x-k8s.io/workloadpriorityclass;scheduling.k8s.io/priorityclass;"" - PriorityClassSource string `json:"priorityClassSource,omitempty"` + // +optional + PriorityClassSource string `json:"priorityClassSource,omitempty"` //nolint:kubeapilinter // should not be a pointer // active determines if a workload can be admitted into a queue. // Changing active from true to false will evict any running workloads. @@ -122,16 +128,19 @@ type PodSetTopologyRequest struct { // - Kubeflow: training.kubeflow.org/replica-index // This is limited to 317 characters. // +kubebuilder:validation:MaxLength=317 + // +optional PodIndexLabel *string `json:"podIndexLabel,omitempty"` // subGroupIndexLabel indicates the name of the label indexing the instances of replicated Jobs (groups) // within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.io/job-index. // This is limited to 317 characters. // +kubebuilder:validation:MaxLength=317 + // +optional SubGroupIndexLabel *string `json:"subGroupIndexLabel,omitempty"` // subGroupCount indicates the count of replicated Jobs (groups) within a PodSet. // For example, in the context of JobSet this value is read from jobset.sigs.k8s.io/replicatedjob-replicas. + // +optional SubGroupCount *int32 `json:"subGroupCount,omitempty"` // podSetGroupName indicates the name of the group of PodSets to which this PodSet belongs to. @@ -159,13 +168,15 @@ type PodSetTopologyRequest struct { type Admission struct { // clusterQueue is the name of the ClusterQueue that admitted this workload. - ClusterQueue ClusterQueueReference `json:"clusterQueue"` + // +optional + ClusterQueue ClusterQueueReference `json:"clusterQueue"` //nolint:kubeapilinter // should not be a pointer // podSetAssignments hold the admission results for each of the .spec.podSets entries. // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=8 - PodSetAssignments []PodSetAssignment `json:"podSetAssignments"` + // +optional + PodSetAssignments []PodSetAssignment `json:"podSetAssignments"` //nolint:kubeapilinter // do not add omitempty } // PodSetReference is the name of a PodSet. @@ -180,9 +191,11 @@ func NewPodSetReference(name string) PodSetReference { type PodSetAssignment struct { // name is the name of the podSet. It should match one of the names in .spec.podSets. // +kubebuilder:default=main - Name PodSetReference `json:"name"` + // +optional + Name PodSetReference `json:"name"` //nolint:kubeapilinter // should not be a pointer // flavors are the flavors assigned to the workload for each resource. + // +optional Flavors map[corev1.ResourceName]ResourceFlavorReference `json:"flavors,omitempty"` // resourceUsage keeps track of the total resources all the pods in the podset need to run. @@ -190,6 +203,7 @@ type PodSetAssignment struct { // Beside what is provided in podSet's specs, this calculation takes into account // the LimitRange defaults and RuntimeClass overheads at the moment of admission. // This field will not change in case of quota reclaim. + // +optional ResourceUsage corev1.ResourceList `json:"resourceUsage,omitempty"` //nolint:kubeapilinter // map type is required for standard Kubernetes ResourceList // count is the number of pods taken into account at admission time. @@ -284,7 +298,7 @@ type TopologyAssignment struct { // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 // +kubebuilder:validation:items:MaxLength=317 - Levels []string `json:"levels"` + Levels []string `json:"levels,omitempty"` // domains is a list of topology assignments split by topology domains at // the lowest level of the topology. @@ -292,7 +306,7 @@ type TopologyAssignment struct { // +required // +listType=atomic // +kubebuilder:validation:MaxItems=100000 - Domains []TopologyDomainAssignment `json:"domains"` + Domains []TopologyDomainAssignment `json:"domains,omitempty"` } type TopologyDomainAssignment struct { @@ -305,21 +319,22 @@ type TopologyDomainAssignment struct { // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=16 // +kubebuilder:validation:items:MaxLength=63 - Values []string `json:"values"` + Values []string `json:"values,omitempty"` // count indicates the number of Pods to be scheduled in the topology // domain indicated by the values field. // // +required // +kubebuilder:validation:Minimum=1 - Count int32 `json:"count"` + Count int32 `json:"count,omitempty"` } // +kubebuilder:validation:XValidation:rule="has(self.minCount) ? self.minCount <= self.count : true", message="minCount should be positive and less or equal to count" type PodSet struct { // name is the PodSet name. // +kubebuilder:default=main - Name PodSetReference `json:"name,omitempty"` + // +optional + Name PodSetReference `json:"name,omitempty"` //nolint:kubeapilinter // should not be a pointer // template is the Pod template. // @@ -334,12 +349,14 @@ type PodSet struct { // the keys in the nodeLabels from the ResourceFlavors considered for this // Workload are used to filter the ResourceFlavors that can be assigned to // this podSet. - Template corev1.PodTemplateSpec `json:"template"` + // +required + Template corev1.PodTemplateSpec `json:"template,omitempty"` // count is the number of pods for the spec. // +kubebuilder:default=1 // +kubebuilder:validation:Minimum=0 - Count int32 `json:"count"` + // +optional + Count int32 `json:"count"` //nolint:kubeapilinter // should not be a pointer // minCount is the minimum number of pods for the spec acceptable // if the workload supports partial admission. @@ -491,32 +508,28 @@ type WorkloadSchedulingStatsEviction struct { // reason specifies the programmatic identifier for the eviction cause. // // +required - // +kubebuilder:validation:Required // +kubebuilder:validation:MaxLength=316 - Reason string `json:"reason"` + Reason string `json:"reason"` //nolint:kubeapilinter // should not be a pointer // underlyingCause specifies a finer-grained explanation that complements the eviction reason. // This may be an empty string. // // +required - // +kubebuilder:validation:Required - UnderlyingCause EvictionUnderlyingCause `json:"underlyingCause"` + UnderlyingCause EvictionUnderlyingCause `json:"underlyingCause"` //nolint:kubeapilinter // should not be a pointer // count tracks the number of evictions for this reason and detailed reason. // // +required - // +kubebuilder:validation:Required // +kubebuilder:validation:Minimum=0 - Count int32 `json:"count"` + Count int32 `json:"count"` //nolint:kubeapilinter // should not be a pointer } type UnhealthyNode struct { // name is the name of the unhealthy node. // // +required - // +kubebuilder:validation:Required // +kubebuilder:validation:MaxLength=63 - Name string `json:"name"` + Name string `json:"name"` //nolint:kubeapilinter // should not be a pointer } type RequeueState struct { @@ -538,31 +551,28 @@ type RequeueState struct { // AdmissionCheckReference is the name of an AdmissionCheck. // +kubebuilder:validation:MaxLength=316 +// +kubebuilder:validation:MinLength=1 type AdmissionCheckReference string type AdmissionCheckState struct { // name identifies the admission check. // +required - // +kubebuilder:validation:Required - Name AdmissionCheckReference `json:"name"` + Name AdmissionCheckReference `json:"name,omitempty"` // state of the admissionCheck, one of Pending, Ready, Retry, Rejected // +required - // +kubebuilder:validation:Required // +kubebuilder:validation:Enum=Pending;Ready;Retry;Rejected - State CheckState `json:"state"` + State CheckState `json:"state,omitempty"` // lastTransitionTime is the last time the condition transitioned from one status to another. // This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. // +required - // +kubebuilder:validation:Required // +kubebuilder:validation:Type=string // +kubebuilder:validation:Format=date-time - LastTransitionTime metav1.Time `json:"lastTransitionTime"` + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty,omitzero"` // message is a human readable message indicating details about the transition. // This may be an empty string. // +required - // +kubebuilder:validation:Required // +kubebuilder:validation:MaxLength=32768 - Message string `json:"message" protobuf:"bytes,6,opt,name=message"` + Message string `json:"message" protobuf:"bytes,6,opt,name=message"` //nolint:kubeapilinter // disable should be a pointer rule // podSetUpdates contains a list of pod set modifications suggested by AdmissionChecks. // +optional @@ -578,8 +588,7 @@ type AdmissionCheckState struct { type PodSetUpdate struct { // name of the PodSet to modify. Should match to one of the Workload's PodSets. // +required - // +kubebuilder:validation:Required - Name PodSetReference `json:"name"` + Name PodSetReference `json:"name"` //nolint:kubeapilinter // duplicatemarkers seems to say these are duplicates // labels of the PodSet to modify. // +optional @@ -608,18 +617,18 @@ type PodSetUpdate struct { type ReclaimablePod struct { // name is the PodSet name. // +required - Name PodSetReference `json:"name"` + Name PodSetReference `json:"name"` //nolint:kubeapilinter // duplicatemarkers seems to say these are duplicates // count is the number of pods for which the requested resources are no longer needed. // +kubebuilder:validation:Minimum=0 - Count int32 `json:"count"` + // +required + Count int32 `json:"count"` //nolint:kubeapilinter // should not be a pointer } type PodSetRequest struct { // name is the name of the podSet. It should match one of the names in .spec.podSets. - // +kubebuilder:validation:Required // +required - Name PodSetReference `json:"name"` + Name PodSetReference `json:"name"` //nolint:kubeapilinter // duplicatemarkers seems to say these are duplicates // resources is the total resources all the pods in the podset need to run. // @@ -798,12 +807,15 @@ const ( type Workload struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the Workload. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec is the specification of the Workload. - Spec WorkloadSpec `json:"spec,omitempty"` + // +optional + Spec WorkloadSpec `json:"spec"` //nolint:kubeapilinter // spec should not be a pointer // status is the status of the Workload. - Status WorkloadStatus `json:"status,omitempty"` + // +optional + Status WorkloadStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/workloadpriorityclass_types.go b/apis/kueue/v1beta2/workloadpriorityclass_types.go index fc48d7d1a5e..c6727a659ec 100644 --- a/apis/kueue/v1beta2/workloadpriorityclass_types.go +++ b/apis/kueue/v1beta2/workloadpriorityclass_types.go @@ -30,19 +30,21 @@ import ( type WorkloadPriorityClass struct { metav1.TypeMeta `json:",inline"` // metadata is the metadata of the WorkloadPriorityClass. + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // value represents the integer value of this workloadPriorityClass. This is the actual priority that workloads // receive when jobs have the name of this class in their workloadPriorityClass label. // Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. - Value int32 `json:"value"` + // +required + Value int32 `json:"value"` //nolint:kubeapilinter // disabling should be a pointer // description is an arbitrary string that usually provides guidelines on // when this workloadPriorityClass should be used. // The description is limited to a maximum of 2048 characters. // +optional // +kubebuilder:validation:MaxLength=2048 - Description string `json:"description,omitempty"` + Description string `json:"description,omitempty"` //nolint:kubeapilinter // disabling should not be a pointer } // +kubebuilder:object:root=true diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml index d486a0f8774..a7b8fb70323 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_admissionchecks.yaml @@ -219,16 +219,19 @@ spec: apiGroup: description: apiGroup is the group for the resource being referenced. maxLength: 253 + minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string kind: description: kind is the type of the resource being referenced. maxLength: 63 + minLength: 1 pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ type: string name: description: name is the name of the resource being referenced. maxLength: 63 + minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string required: diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml index 5bbce9f2a12..6ce0fc1a46b 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml @@ -846,6 +846,7 @@ spec: items: description: AdmissionCheckReference is the name of an AdmissionCheck. maxLength: 316 + minLength: 1 type: string type: array admissionChecksStrategy: @@ -861,6 +862,7 @@ spec: name: description: name is an AdmissionCheck's name. maxLength: 316 + minLength: 1 type: string onFlavors: description: |- @@ -878,10 +880,13 @@ spec: - name type: object maxItems: 64 + minItems: 1 type: array x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + required: + - admissionChecks type: object admissionScope: description: admissionScope indicates whether ClusterQueue uses the Admission Fair Sharing diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml index efdd0b52730..b5175084c59 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_multikueueclusters.yaml @@ -208,6 +208,7 @@ spec: If LocationType is Secret then Location is the name of the secret inside the namespace in which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. maxLength: 256 + minLength: 1 type: string locationType: default: Secret @@ -218,7 +219,6 @@ spec: type: string required: - location - - locationType type: object required: - kubeConfig diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml index a7d3a0c63f3..52f57f11f5a 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_provisioningrequestconfigs.yaml @@ -283,6 +283,7 @@ spec: provisioningClassName describes the different modes of provisioning the resources. Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. maxLength: 253 + minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string retryStrategy: diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index 43336aa3d68..4659b61c8ca 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -17104,7 +17104,6 @@ spec: type: boolean type: object required: - - count - template type: object x-kubernetes-validations: @@ -17135,6 +17134,7 @@ spec: PriorityClass object with that name. If not specified, the workload priority will be default or zero if there is no default. maxLength: 253 + minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string priorityClassSource: @@ -17155,8 +17155,6 @@ spec: maxLength: 253 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - required: - - podSets type: object x-kubernetes-validations: - message: priority should not be nil when priorityClassName is set @@ -17327,17 +17325,12 @@ spec: - domains - levels type: object - required: - - name type: object maxItems: 8 type: array x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map - required: - - clusterQueue - - podSetAssignments type: object admissionChecks: description: admissionChecks list all the admission checks required by the workload and the current status @@ -17358,6 +17351,7 @@ spec: name: description: name identifies the admission check. maxLength: 316 + minLength: 1 type: string podSetUpdates: description: podSetUpdates contains a list of pod set modifications suggested by AdmissionChecks. diff --git a/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml b/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml index dd5b207380c..a78a59fdea5 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_admissionchecks.yaml @@ -198,16 +198,19 @@ spec: apiGroup: description: apiGroup is the group for the resource being referenced. maxLength: 253 + minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string kind: description: kind is the type of the resource being referenced. maxLength: 63 + minLength: 1 pattern: ^(?i)[a-z]([-a-z0-9]*[a-z0-9])?$ type: string name: description: name is the name of the resource being referenced. maxLength: 63 + minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ type: string required: diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml index a600387c593..5e83724747e 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml @@ -831,6 +831,7 @@ spec: items: description: AdmissionCheckReference is the name of an AdmissionCheck. maxLength: 316 + minLength: 1 type: string type: array admissionChecksStrategy: @@ -847,6 +848,7 @@ spec: name: description: name is an AdmissionCheck's name. maxLength: 316 + minLength: 1 type: string onFlavors: description: |- @@ -865,10 +867,13 @@ spec: - name type: object maxItems: 64 + minItems: 1 type: array x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + required: + - admissionChecks type: object admissionScope: description: admissionScope indicates whether ClusterQueue uses the diff --git a/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml b/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml index 22943efa928..b99bd85fb47 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_multikueueclusters.yaml @@ -187,6 +187,7 @@ spec: If LocationType is Secret then Location is the name of the secret inside the namespace in which the kueue controller manager is running. The config should be stored in the "kubeconfig" key. maxLength: 256 + minLength: 1 type: string locationType: default: Secret @@ -197,7 +198,6 @@ spec: type: string required: - location - - locationType type: object required: - kubeConfig diff --git a/config/components/crd/bases/kueue.x-k8s.io_provisioningrequestconfigs.yaml b/config/components/crd/bases/kueue.x-k8s.io_provisioningrequestconfigs.yaml index 822a4aa1423..5d11037c7fe 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_provisioningrequestconfigs.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_provisioningrequestconfigs.yaml @@ -269,6 +269,7 @@ spec: provisioningClassName describes the different modes of provisioning the resources. Check autoscaling.x-k8s.io ProvisioningRequestSpec.ProvisioningClassName for details. maxLength: 253 + minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string retryStrategy: diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index f57543ccd4a..cd69125907f 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -18093,7 +18093,6 @@ spec: type: boolean type: object required: - - count - template type: object x-kubernetes-validations: @@ -18124,6 +18123,7 @@ spec: PriorityClass object with that name. If not specified, the workload priority will be default or zero if there is no default. maxLength: 253 + minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string priorityClassSource: @@ -18144,8 +18144,6 @@ spec: maxLength: 253 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - required: - - podSets type: object x-kubernetes-validations: - message: priority should not be nil when priorityClassName is set @@ -18321,17 +18319,12 @@ spec: - domains - levels type: object - required: - - name type: object maxItems: 8 type: array x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map - required: - - clusterQueue - - podSetAssignments type: object admissionChecks: description: admissionChecks list all the admission checks required @@ -18353,6 +18346,7 @@ spec: name: description: name identifies the admission check. maxLength: 316 + minLength: 1 type: string podSetUpdates: description: podSetUpdates contains a list of pod set modifications diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md index b078a88d533..2a721d37202 100644 --- a/site/content/en/docs/reference/kueue.v1beta2.md +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -41,14 +41,14 @@ description: Generated API reference documentation for kueue.x-k8s.io/v1beta2. kind
    stringAdmissionCheck -spec [Required]
    +spec
    AdmissionCheckSpec

    spec is the specification of the AdmissionCheck.

    -status [Required]
    +status
    AdmissionCheckStatus @@ -76,14 +76,14 @@ description: Generated API reference documentation for kueue.x-k8s.io/v1beta2. kind
    stringClusterQueue -spec [Required]
    +spec
    ClusterQueueSpec

    spec is the specification of the ClusterQueue.

    -status [Required]
    +status
    ClusterQueueStatus @@ -114,14 +114,14 @@ V0.9 and V0.10 is unsupported, and results in undefined behavior.

    kind
    stringCohort -spec [Required]
    +spec
    CohortSpec

    spec is the specification of the Cohort.

    -status [Required]
    +status
    CohortStatus @@ -149,14 +149,14 @@ V0.9 and V0.10 is unsupported, and results in undefined behavior.

    kind
    stringLocalQueue -spec [Required]
    +spec
    LocalQueueSpec

    spec is the specification of the LocalQueue.

    -status [Required]
    +status
    LocalQueueStatus @@ -184,14 +184,14 @@ V0.9 and V0.10 is unsupported, and results in undefined behavior.

    kind
    stringMultiKueueCluster -spec [Required]
    +spec,omitempty,omitzero
    MultiKueueClusterSpec

    spec is the specification of the MultiKueueCluster.

    -status [Required]
    +status
    MultiKueueClusterStatus @@ -219,7 +219,7 @@ V0.9 and V0.10 is unsupported, and results in undefined behavior.

    kind
    stringMultiKueueConfig -spec [Required]
    +spec
    MultiKueueConfigSpec @@ -247,7 +247,7 @@ V0.9 and V0.10 is unsupported, and results in undefined behavior.

    kind
    stringProvisioningRequestConfig -spec [Required]
    +spec
    ProvisioningRequestConfigSpec @@ -275,7 +275,7 @@ V0.9 and V0.10 is unsupported, and results in undefined behavior.

    kind
    stringResourceFlavor -spec [Required]
    +spec
    ResourceFlavorSpec @@ -303,7 +303,7 @@ V0.9 and V0.10 is unsupported, and results in undefined behavior.

    kind
    stringTopology -spec [Required]
    +spec
    TopologySpec @@ -331,14 +331,14 @@ V0.9 and V0.10 is unsupported, and results in undefined behavior.

    kind
    stringWorkload -spec [Required]
    +spec
    WorkloadSpec

    spec is the specification of the Workload.

    -status [Required]
    +status
    WorkloadStatus @@ -401,14 +401,14 @@ The description is limited to a maximum of 2048 characters.

    -clusterQueue [Required]
    +clusterQueue
    ClusterQueueReference

    clusterQueue is the name of the ClusterQueue that admitted this workload.

    -podSetAssignments [Required]
    +podSetAssignments
    []PodSetAssignment @@ -537,7 +537,7 @@ check.

    state of the admissionCheck, one of Pending, Ready, Retry, Rejected

    -lastTransitionTime [Required]
    +lastTransitionTime,omitempty,omitzero [Required]
    k8s.io/apimachinery/pkg/apis/meta/v1.Time @@ -743,7 +743,7 @@ within cohort while borrowing. It only works with Classical Preemption, -policy [Required]
    +policy
    BorrowWithinCohortPolicy @@ -830,7 +830,7 @@ lower priority first.

    -reclaimWithinCohort [Required]
    +reclaimWithinCohort
    PreemptionPolicy @@ -852,7 +852,7 @@ in the cohort that satisfy the Fair Sharing preemptionStrategies. -borrowWithinCohort [Required]
    +borrowWithinCohort
    BorrowWithinCohort @@ -861,7 +861,7 @@ Workloads from other ClusterQueues in the cohort if the workload requires borrow May only be configured with Classical Preemption, and not with Fair Sharing.

    -withinClusterQueue [Required]
    +withinClusterQueue
    PreemptionPolicy @@ -914,7 +914,7 @@ It must be a DNS (RFC 1123) and has the maximum length of 253 characters.

    -resourceGroups [Required]
    +resourceGroups
    []ResourceGroup @@ -925,7 +925,7 @@ Each resource and each flavor can only form part of one resource group. resourceGroups can be up to 16, with a max of 256 total flavors across all groups.

    -cohortName [Required]
    +cohortName
    CohortReference @@ -941,7 +941,7 @@ vice versa.

    object.

    -queueingStrategy [Required]
    +queueingStrategy
    QueueingStrategy @@ -958,7 +958,7 @@ admitting newer workloads that fit existing quota. -namespaceSelector [Required]
    +namespaceSelector
    k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector @@ -969,7 +969,7 @@ Defaults to null which is a nothing selector (no namespaces eligible). If set to an empty selector {}, then all namespaces are eligible.

    -flavorFungibility [Required]
    +flavorFungibility
    FlavorFungibility @@ -977,7 +977,7 @@ If set to an empty selector {}, then all namespaces are eligible. -preemption [Required]
    +preemption
    ClusterQueuePreemption @@ -1146,7 +1146,7 @@ subdomain in DNS (RFC 1123).

    -parentName [Required]
    +parentName
    CohortReference @@ -1163,7 +1163,7 @@ removed. We prevent further admission while the cycle exists.

    -resourceGroups [Required]
    +resourceGroups
    []ResourceGroup @@ -1280,7 +1280,7 @@ V0.9 and V0.10 is unsupported, and results in undefined behavior.

    -weight [Required]
    +weight
    k8s.io/apimachinery/pkg/api/resource.Quantity @@ -1360,7 +1360,7 @@ before borrowing or preempting in current flavor.

    -whenCanBorrow [Required]
    +whenCanBorrow
    FlavorFungibilityPolicy @@ -1373,7 +1373,7 @@ fits or requires borrowing to fit. -whenCanPreempt [Required]
    +whenCanPreempt
    FlavorFungibilityPolicy @@ -1490,7 +1490,7 @@ There could be up to 64 resources.

    which the kueue controller manager is running. The config should be stored in the "kubeconfig" key.

    -locationType [Required]
    +locationType
    LocationType @@ -1567,7 +1567,7 @@ It must be a DNS (RFC 1123) and has the maximum length of 253 characters.

    name of the resource.

    -total [Required]
    +total
    k8s.io/apimachinery/pkg/api/resource.Quantity @@ -1593,7 +1593,7 @@ It must be a DNS (RFC 1123) and has the maximum length of 253 characters.

    -clusterQueue [Required]
    +clusterQueue
    ClusterQueueReference @@ -1726,7 +1726,7 @@ workloads assigned to this LocalQueue.

    -kubeConfig [Required]
    +kubeConfig,omitempty,omitzero [Required]
    KubeConfig @@ -1778,7 +1778,7 @@ conditions are limited to 16 elements.

    -clusters [Required]
    +clusters,omitempty,omitzero [Required]
    []string @@ -1816,7 +1816,7 @@ conditions are limited to 16 elements.

    -name [Required]
    +name
    PodSetReference @@ -1839,7 +1839,7 @@ Workload are used to filter the ResourceFlavors that can be assigned to this podSet.

    -count [Required]
    +count
    int32 @@ -1882,21 +1882,21 @@ enabled.

    -name [Required]
    +name
    PodSetReference

    name is the name of the podSet. It should match one of the names in .spec.podSets.

    -flavors [Required]
    +flavors
    map[ResourceName]ResourceFlavorReference

    flavors are the flavors assigned to the workload for each resource.

    -resourceUsage [Required]
    +resourceUsage
    k8s.io/api/core/v1.ResourceList @@ -2083,7 +2083,7 @@ the entire available capacity, without constraints on the compactness of the pla This is indicated by the kueue.x-k8s.io/podset-unconstrained-topology PodSet annotation.

    -podIndexLabel [Required]
    +podIndexLabel
    string @@ -2097,7 +2097,7 @@ This is limited to 317 characters. -subGroupIndexLabel [Required]
    +subGroupIndexLabel
    string @@ -2106,7 +2106,7 @@ within a PodSet. For example, in the context of JobSet this is jobset.sigs.k8s.i This is limited to 317 characters.

    -subGroupCount [Required]
    +subGroupCount
    int32 @@ -2715,7 +2715,7 @@ This field is in beta stage and is enabled by default.

    name of the resource

    -total [Required]
    +total
    k8s.io/apimachinery/pkg/api/resource.Quantity @@ -2723,7 +2723,7 @@ This field is in beta stage and is enabled by default.

    from the cohort.

    -borrowed [Required]
    +borrowed
    k8s.io/apimachinery/pkg/api/resource.Quantity @@ -2992,7 +2992,7 @@ This may be an empty string.

    -podSets [Required]
    +podSets
    []PodSet @@ -3002,7 +3002,7 @@ There must be at least one element and at most 8. podSets cannot be changed.

    -queueName [Required]
    +queueName
    LocalQueueName @@ -3010,7 +3010,7 @@ podSets cannot be changed.

    queueName cannot be changed while .status.admission is not null.

    -priorityClassName [Required]
    +priorityClassName
    string @@ -3023,7 +3023,7 @@ PriorityClass object with that name. If not specified, the workload priority will be default or zero if there is no default.

    -priority [Required]
    +priority
    int32 @@ -3034,7 +3034,7 @@ The higher the value, the higher the priority. If priorityClassName is specified, priority must not be null.

    -priorityClassSource [Required]
    +priorityClassSource
    string From 1edfeafd4c340f43fc7d6779772d5f8bcef81104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Wed, 5 Nov 2025 21:32:51 +0100 Subject: [PATCH 096/119] Add preemptor and preemptee path to the Preemption message (#7522) --- .../preemption/fairsharing/target.go | 5 + pkg/scheduler/preemption/preemption.go | 34 +++- .../preemption_hierarchical_test.go | 78 ++++---- pkg/scheduler/preemption/preemption_test.go | 170 +++++++++--------- pkg/scheduler/scheduler.go | 2 +- pkg/scheduler/scheduler_tas_test.go | 64 +++---- pkg/scheduler/scheduler_test.go | 108 +++++------ .../scheduler/preemption_test.go | 18 +- test/util/util.go | 4 +- 9 files changed, 261 insertions(+), 222 deletions(-) diff --git a/pkg/scheduler/preemption/fairsharing/target.go b/pkg/scheduler/preemption/fairsharing/target.go index c07ff068623..a958323f7e7 100644 --- a/pkg/scheduler/preemption/fairsharing/target.go +++ b/pkg/scheduler/preemption/fairsharing/target.go @@ -71,3 +71,8 @@ func (t *TargetClusterQueue) ComputeTargetShareAfterRemoval(wl *workload.Info) T _, almostLCA := getAlmostLCAs(t) return TargetNewShare(almostLCA.DominantResourceShare()) } + +// GetTargetCq returns the target ClusterQueue snapshot. +func (t *TargetClusterQueue) GetTargetCq() *schdcache.ClusterQueueSnapshot { + return t.targetCq +} diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index f7cdfdc2614..96e6aca6eb5 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "slices" + "strings" "sync/atomic" "github.com/go-logr/logr" @@ -97,6 +98,7 @@ func New( type Target struct { WorkloadInfo *workload.Info Reason string + WorkloadCq *schdcache.ClusterQueueSnapshot } // ensures that Target implements ObjectRefProvider interface at compile time @@ -142,15 +144,17 @@ var HumanReadablePreemptionReasons = map[string]string{ "": "UNKNOWN", } -func preemptionMessage(preemptor *kueue.Workload, reason string) string { +func preemptionMessage(preemptor *kueue.Workload, reason, preemptorPath, preempteePath string) string { wUID := cmp.Or(string(preemptor.UID), "UNKNOWN") uid := preemptor.Labels[constants.JobUIDLabel] jUID := cmp.Or(uid, "UNKNOWN") - return fmt.Sprintf("Preempted to accommodate a workload (UID: %s, JobUID: %s) due to %s", wUID, jUID, HumanReadablePreemptionReasons[reason]) + preemptorMsgPath := cmp.Or(preemptorPath, "UNKNOWN") + preempteeMsgPath := cmp.Or(preempteePath, "UNKNOWN") + return fmt.Sprintf("Preempted to accommodate a workload (UID: %s, JobUID: %s) due to %s; preemptor path: %s; preemptee path: %s", wUID, jUID, HumanReadablePreemptionReasons[reason], preemptorMsgPath, preempteeMsgPath) } // IssuePreemptions marks the target workloads as evicted. -func (p *Preemptor) IssuePreemptions(ctx context.Context, preemptor *workload.Info, targets []*Target) (int, error) { +func (p *Preemptor) IssuePreemptions(ctx context.Context, preemptor *workload.Info, targets []*Target, snap *schdcache.ClusterQueueSnapshot) (int, error) { log := ctrl.LoggerFrom(ctx) errCh := routine.NewErrorChannel() ctx, cancel := context.WithCancel(ctx) @@ -159,7 +163,10 @@ func (p *Preemptor) IssuePreemptions(ctx context.Context, preemptor *workload.In workqueue.ParallelizeUntil(ctx, parallelPreemptions, len(targets), func(i int) { target := targets[i] if !meta.IsStatusConditionTrue(target.WorkloadInfo.Obj.Status.Conditions, kueue.WorkloadEvicted) { - message := preemptionMessage(preemptor.Obj, target.Reason) + preemptorPath := buildCQPath(string(preemptor.ClusterQueue), snap) + preempteePath := buildCQPath(string(target.WorkloadInfo.ClusterQueue), target.WorkloadCq) + + message := preemptionMessage(preemptor.Obj, target.Reason, preemptorPath, preempteePath) wlCopy := target.WorkloadInfo.Obj.DeepCopy() err := workload.Evict(ctx, p.client, p.recorder, wlCopy, kueue.WorkloadEvictedByPreemption, message, "", p.clock, workload.WithCustomPrepare(func() (*kueue.Workload, error) { workload.SetPreemptedCondition(wlCopy, p.clock.Now(), target.Reason, message) @@ -170,7 +177,9 @@ func (p *Preemptor) IssuePreemptions(ctx context.Context, preemptor *workload.In return } - log.V(3).Info("Preempted", "targetWorkload", klog.KObj(target.WorkloadInfo.Obj), "preemptingWorkload", klog.KObj(preemptor.Obj), "preemptorUID", string(preemptor.Obj.UID), "preemptorJobUID", preemptor.Obj.Labels[constants.JobUIDLabel], "reason", target.Reason, "message", message, "targetClusterQueue", klog.KRef("", string(target.WorkloadInfo.ClusterQueue))) + log.V(3).Info("Preempted", "targetWorkload", klog.KObj(target.WorkloadInfo.Obj), "preemptingWorkload", klog.KObj(preemptor.Obj), "preemptorUID", string(preemptor.Obj.UID), + "preemptorJobUID", preemptor.Obj.Labels[constants.JobUIDLabel], "reason", target.Reason, "message", message, "targetClusterQueue", klog.KRef("", string(target.WorkloadInfo.ClusterQueue)), + "preemptorPath", preemptorPath, "preempteePath", preempteePath) p.recorder.Eventf(target.WorkloadInfo.Obj, corev1.EventTypeNormal, "Preempted", message) workload.ReportPreemption(preemptor.ClusterQueue, target.Reason, target.WorkloadInfo.ClusterQueue) } else { @@ -237,6 +246,7 @@ func (p *Preemptor) classicalPreemptions(preemptionCtx *preemptionCtx) []*Target targets = append(targets, &Target{ WorkloadInfo: candidate, Reason: reason, + WorkloadCq: preemptionCtx.snapshot.ClusterQueue(candidate.ClusterQueue), }) if workloadFits(preemptionCtx, attemptOpts.borrowing) { targets = fillBackWorkloads(preemptionCtx, targets, attemptOpts.borrowing) @@ -304,6 +314,7 @@ func runFirstFsStrategy(preemptionCtx *preemptionCtx, candidates []*workload.Inf targets = append(targets, &Target{ WorkloadInfo: candWl, Reason: kueue.InClusterQueueReason, + WorkloadCq: candCQ.GetTargetCq(), }) if workloadFitsForFairSharing(preemptionCtx) { return true, targets, nil @@ -322,6 +333,7 @@ func runFirstFsStrategy(preemptionCtx *preemptionCtx, candidates []*workload.Inf targets = append(targets, &Target{ WorkloadInfo: candWl, Reason: reason, + WorkloadCq: candCQ.GetTargetCq(), }) if workloadFitsForFairSharing(preemptionCtx) { return true, targets, nil @@ -351,6 +363,7 @@ func runSecondFsStrategy(retryCandidates []*workload.Info, preemptionCtx *preemp targets = append(targets, &Target{ WorkloadInfo: candWl, Reason: kueue.InCohortFairSharingReason, + WorkloadCq: candCQ.GetTargetCq(), }) if workloadFitsForFairSharing(preemptionCtx) { return true, targets @@ -512,3 +525,14 @@ func queueUnderNominalInResourcesNeedingPreemption(preemptionCtx *preemptionCtx) } return true } + +// buildCQPath constructs a path like "/parent/.../cq" for a given ClusterQueue snapshot. +func buildCQPath(cqName string, cqSnap *schdcache.ClusterQueueSnapshot) string { + parts := []string{cqName} + for ancestor := range cqSnap.PathParentToRoot() { + parts = append(parts, string(ancestor.GetName())) + } + // Reverse the slice since we want parent first + slices.Reverse(parts) + return "/" + strings.Join(parts, "/") +} diff --git a/pkg/scheduler/preemption/preemption_hierarchical_test.go b/pkg/scheduler/preemption/preemption_hierarchical_test.go index 358c3d9ecc2..412920d88d2 100644 --- a/pkg/scheduler/preemption/preemption_hierarchical_test.go +++ b/pkg/scheduler/preemption/preemption_hierarchical_test.go @@ -122,14 +122,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -221,14 +221,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -305,14 +305,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -327,14 +327,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -429,14 +429,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /r/c/q; preemptee path: /r/c/q", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /r/c/q; preemptee path: /r/c/q", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -451,14 +451,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /r/c/q; preemptee path: /r/c/q_same_cohort", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclaimWhileBorrowing", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /r/c/q; preemptee path: /r/c/q_same_cohort", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -535,14 +535,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -646,14 +646,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /r/c/q; preemptee path: /r/c/q_same_cohort", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclaimWhileBorrowing", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /r/c/q; preemptee path: /r/c/q_same_cohort", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -738,14 +738,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -760,14 +760,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/c/q_same_cohort", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/c/q_same_cohort", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -865,14 +865,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -894,14 +894,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /r/c/q; preemptee path: /r/c/q_same_cohort", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclaimWhileBorrowing", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /r/c/q; preemptee path: /r/c/q_same_cohort", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -999,14 +999,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1028,14 +1028,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /r/c/q; preemptee path: /r/c/q", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /r/c/q; preemptee path: /r/c/q", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1141,14 +1141,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1362,14 +1362,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1548,14 +1548,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c/q; preemptee path: /r/q_borrowing", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1734,14 +1734,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c11/c21/c31/q5; preemptee path: /r/c12/q1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c11/c21/c31/q5; preemptee path: /r/c12/q1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1770,14 +1770,14 @@ func TestHierarchicalPreemptions(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c11/c21/c31/q5; preemptee path: /r/c11/c23/q2", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /r/c11/c21/c31/q5; preemptee path: /r/c11/c23/q2", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1857,7 +1857,7 @@ func TestHierarchicalPreemptions(t *testing.T) { wlInfo := workload.NewInfo(tc.incoming) wlInfo.ClusterQueue = tc.targetCQ targets := preemptor.GetTargets(log, *wlInfo, tc.assignment, snapshotWorkingCopy) - preempted, err := preemptor.IssuePreemptions(ctx, wlInfo, targets) + preempted, err := preemptor.IssuePreemptions(ctx, wlInfo, targets, snapshotWorkingCopy.ClusterQueue(wlInfo.ClusterQueue)) if err != nil { t.Fatalf("Failed doing preemption") } diff --git a/pkg/scheduler/preemption/preemption_test.go b/pkg/scheduler/preemption/preemption_test.go index 53e1fe50ac2..65bdcc025a5 100644 --- a/pkg/scheduler/preemption/preemption_test.go +++ b/pkg/scheduler/preemption/preemption_test.go @@ -367,14 +367,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -471,14 +471,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -497,14 +497,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -717,14 +717,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -832,14 +832,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -941,14 +941,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1045,14 +1045,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort/c1; preemptee path: /cohort/c2", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort/c1; preemptee path: /cohort/c2", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1114,14 +1114,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort/c1; preemptee path: /cohort/c2", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort/c1; preemptee path: /cohort/c2", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1357,14 +1357,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort/c1; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort/c1; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1463,14 +1463,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort/c1; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort/c1; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1569,14 +1569,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-no-limits/d1; preemptee path: /cohort-no-limits/d1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-no-limits/d1; preemptee path: /cohort-no-limits/d1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1692,14 +1692,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort/c1; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort/c1; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1810,14 +1810,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /legion/l1; preemptee path: /legion/l1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /legion/l1; preemptee path: /legion/l1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1971,14 +1971,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort/c2; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort/c2; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2085,14 +2085,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort/c1; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort/c1; preemptee path: /cohort/c1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2123,14 +2123,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort/c1; preemptee path: /cohort/c2", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort/c1; preemptee path: /cohort/c2", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2268,14 +2268,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2295,14 +2295,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /standalone; preemptee path: /standalone", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2403,14 +2403,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /preventStarvation; preemptee path: /preventStarvation", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /preventStarvation; preemptee path: /preventStarvation", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2485,14 +2485,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /with_shared_cq/a_standard; preemptee path: /with_shared_cq/a_best_effort", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclaimWhileBorrowing", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /with_shared_cq/a_standard; preemptee path: /with_shared_cq/a_best_effort", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2600,14 +2600,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /with_shared_cq/a_standard; preemptee path: /with_shared_cq/b_standard", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /with_shared_cq/a_standard; preemptee path: /with_shared_cq/b_standard", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2760,14 +2760,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /with_shared_cq/b_standard; preemptee path: /with_shared_cq/b_standard", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /with_shared_cq/b_standard; preemptee path: /with_shared_cq/b_standard", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2878,14 +2878,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /with_shared_cq/b_standard; preemptee path: /with_shared_cq/a_best_effort", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclaimWhileBorrowing", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort while borrowing; preemptor path: /with_shared_cq/b_standard; preemptee path: /with_shared_cq/a_best_effort", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2917,14 +2917,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /with_shared_cq/b_standard; preemptee path: /with_shared_cq/b_standard", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /with_shared_cq/b_standard; preemptee path: /with_shared_cq/b_standard", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3021,14 +3021,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort-lend/lend1; preemptee path: /cohort-lend/lend2", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort-lend/lend1; preemptee path: /cohort-lend/lend2", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3112,14 +3112,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-lend/lend1; preemptee path: /cohort-lend/lend1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-lend/lend1; preemptee path: /cohort-lend/lend1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3150,14 +3150,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort-lend/lend1; preemptee path: /cohort-lend/lend2", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort-lend/lend1; preemptee path: /cohort-lend/lend2", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3320,14 +3320,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3347,14 +3347,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3532,14 +3532,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3561,14 +3561,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3740,14 +3740,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3767,14 +3767,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3932,14 +3932,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to prioritization in the ClusterQueue; preemptor path: /cohort-three/a; preemptee path: /cohort-three/a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4007,14 +4007,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort-three/a; preemptee path: /cohort-three/b", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /cohort-three/a; preemptee path: /cohort-three/b", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4083,14 +4083,14 @@ func TestPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /root/cohort-left/cq-left; preemptee path: /root/cohort-right/cq-right", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-in, JobUID: job-in) due to reclamation within the cohort; preemptor path: /root/cohort-left/cq-left; preemptee path: /root/cohort-right/cq-right", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4145,7 +4145,7 @@ func TestPreemption(t *testing.T) { wlInfo := workload.NewInfo(tc.incoming) wlInfo.ClusterQueue = tc.targetCQ targets := preemptor.GetTargets(log, *wlInfo, tc.assignment, snapshotWorkingCopy) - preempted, err := preemptor.IssuePreemptions(ctx, wlInfo, targets) + preempted, err := preemptor.IssuePreemptions(ctx, wlInfo, targets, snapshotWorkingCopy.ClusterQueue(wlInfo.ClusterQueue)) if err != nil { t.Fatalf("Failed doing preemption") } @@ -4308,30 +4308,34 @@ func singlePodSetAssignment(assignments flavorassigner.ResourceAssignment) flavo func TestPreemptionMessage(t *testing.T) { cases := []struct { - preemptor *kueue.Workload - reason string - want string + preemptor *kueue.Workload + reason string + preemptorPath string + preempteePath string + want string }{ { preemptor: &kueue.Workload{}, - want: "Preempted to accommodate a workload (UID: UNKNOWN, JobUID: UNKNOWN) due to UNKNOWN", + want: "Preempted to accommodate a workload (UID: UNKNOWN, JobUID: UNKNOWN) due to UNKNOWN; preemptor path: UNKNOWN; preemptee path: UNKNOWN", }, { preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid"}}, - want: "Preempted to accommodate a workload (UID: uid, JobUID: UNKNOWN) due to UNKNOWN", + want: "Preempted to accommodate a workload (UID: uid, JobUID: UNKNOWN) due to UNKNOWN; preemptor path: UNKNOWN; preemptee path: UNKNOWN", }, { preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid", Labels: map[string]string{controllerconstants.JobUIDLabel: "juid"}}}, - want: "Preempted to accommodate a workload (UID: uid, JobUID: juid) due to UNKNOWN", + want: "Preempted to accommodate a workload (UID: uid, JobUID: juid) due to UNKNOWN; preemptor path: UNKNOWN; preemptee path: UNKNOWN", }, { - preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid", Labels: map[string]string{controllerconstants.JobUIDLabel: "juid"}}}, - reason: kueue.InClusterQueueReason, - want: "Preempted to accommodate a workload (UID: uid, JobUID: juid) due to prioritization in the ClusterQueue", + preemptor: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{UID: "uid", Labels: map[string]string{controllerconstants.JobUIDLabel: "juid"}}}, + reason: kueue.InClusterQueueReason, + preemptorPath: "/a", + preempteePath: "/b", + want: "Preempted to accommodate a workload (UID: uid, JobUID: juid) due to prioritization in the ClusterQueue; preemptor path: /a; preemptee path: /b", }, } for _, tc := range cases { - got := preemptionMessage(tc.preemptor, tc.reason) + got := preemptionMessage(tc.preemptor, tc.reason, tc.preemptorPath, tc.preempteePath) if got != tc.want { t.Errorf("preemptionMessage(preemptor=kueue.Workload{UID:%v, Labels:%v}, reason=%q) returned %q, want %q", tc.preemptor.UID, tc.preemptor.Labels, tc.reason, got, tc.want) } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 92057a10132..f2ab75e6a07 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -295,7 +295,7 @@ func (s *Scheduler) schedule(ctx context.Context) wait.SpeedSignal { if e.assignment.RepresentativeMode() == flavorassigner.Preempt { // If preemptions are issued, the next attempt should try all the flavors. e.LastAssignment = nil - preempted, err := s.preemptor.IssuePreemptions(ctx, &e.Info, preemptionTargets) + preempted, err := s.preemptor.IssuePreemptions(ctx, &e.Info, preemptionTargets, e.clusterQueueSnapshot) if err != nil { log.Error(err, "Failed to preempt workloads") } diff --git a/pkg/scheduler/scheduler_tas_test.go b/pkg/scheduler/scheduler_tas_test.go index 5f469cdcc86..5b0b29e31e0 100644 --- a/pkg/scheduler/scheduler_tas_test.go +++ b/pkg/scheduler/scheduler_tas_test.go @@ -2664,14 +2664,14 @@ func TestScheduleForTASPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2682,10 +2682,10 @@ func TestScheduleForTASPreemption(t *testing.T) { }, wantEvents: []utiltesting.EventRecord{ utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "foo", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: insufficient unused quota for cpu in flavor tas-default, 5 more needed. Pending the preemption of 1 workload(s)`). @@ -2778,14 +2778,14 @@ func TestScheduleForTASPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2796,10 +2796,10 @@ func TestScheduleForTASPreemption(t *testing.T) { }, wantEvents: []utiltesting.EventRecord{ utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "foo", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: topology "tas-single-level" doesn't allow to fit any of 1 pod(s). Pending the preemption of 1 workload(s)`). @@ -2896,14 +2896,14 @@ func TestScheduleForTASPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2914,10 +2914,10 @@ func TestScheduleForTASPreemption(t *testing.T) { }, wantEvents: []utiltesting.EventRecord{ utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-high-priority-waiting, JobUID: job-high-priority-waiting) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "high-priority-waiting", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: topology "tas-single-level" doesn't allow to fit any of 1 pod(s). Pending the preemption of 1 workload(s)`). @@ -3030,14 +3030,14 @@ func TestScheduleForTASPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3068,10 +3068,10 @@ func TestScheduleForTASPreemption(t *testing.T) { }, wantEvents: []utiltesting.EventRecord{ utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "foo", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: topology "tas-single-level" doesn't allow to fit any of 1 pod(s). Pending the preemption of 1 workload(s)`). @@ -3186,14 +3186,14 @@ func TestScheduleForTASPreemption(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3224,10 +3224,10 @@ func TestScheduleForTASPreemption(t *testing.T) { }, wantEvents: []utiltesting.EventRecord{ utiltesting.MakeEventRecord("default", "low-priority-admitted", "EvictedDueToPreempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "low-priority-admitted", "Preempted", "Normal"). - Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue"). + Message("Preempted to accommodate a workload (UID: wl-foo, JobUID: job-foo) due to prioritization in the ClusterQueue; preemptor path: /tas-main; preemptee path: /tas-main"). Obj(), utiltesting.MakeEventRecord("default", "foo", "Pending", "Warning"). Message(`couldn't assign flavors to pod set one: topology "tas-single-level" allows to fit only 1 out of 2 pod(s). Pending the preemption of 1 workload(s)`). @@ -3686,14 +3686,14 @@ func TestScheduleForTASCohorts(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -3838,14 +3838,14 @@ func TestScheduleForTASCohorts(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4036,14 +4036,14 @@ func TestScheduleForTASCohorts(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4221,14 +4221,14 @@ func TestScheduleForTASCohorts(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4256,14 +4256,14 @@ func TestScheduleForTASCohorts(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-b1, JobUID: job-b1) due to reclamation within the cohort; preemptor path: /tas-cohort-main/tas-cq-b; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4942,14 +4942,14 @@ func TestScheduleForTASCohorts(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue; preemptor path: /tas-cohort-main/tas-cq-a; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue; preemptor path: /tas-cohort-main/tas-cq-a; preemptee path: /tas-cohort-main/tas-cq-a", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index dc709757451..4f93d61c55e 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -1285,14 +1285,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1326,14 +1326,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-beta", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-beta", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1463,14 +1463,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort; preemptor path: /other/other-beta; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort; preemptor path: /other/other-beta; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -1946,14 +1946,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-beta", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-beta", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -2028,14 +2028,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-beta", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to prioritization in the ClusterQueue; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-beta", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4001,14 +4001,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4026,14 +4026,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-incoming, JobUID: job-incoming) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4299,14 +4299,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortFairSharing", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4346,14 +4346,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-gamma", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortFairSharing", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /eng/eng-beta; preemptee path: /eng/eng-gamma", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4500,14 +4500,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4545,14 +4545,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-beta; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-beta; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4696,14 +4696,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-beta; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-beta; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -4845,14 +4845,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -5039,14 +5039,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -5082,14 +5082,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-beta; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-beta; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -5125,14 +5125,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-gamma; preemptee path: /other/other-gamma", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-gamma; preemptee path: /other/other-gamma", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -5295,14 +5295,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -5369,14 +5369,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /other/other-alpha; preemptee path: /other/other-gamma", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortFairSharing", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /other/other-alpha; preemptee path: /other/other-gamma", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -5661,14 +5661,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort; preemptor path: /other/other-alpha; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to reclamation within the cohort; preemptor path: /other/other-alpha; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -5778,14 +5778,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /other/other-alpha; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortFairSharing", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /other/other-alpha; preemptee path: /other/other-beta", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -5872,14 +5872,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -6005,14 +6005,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to prioritization in the ClusterQueue; preemptor path: /other/other-alpha; preemptee path: /other/other-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -6216,14 +6216,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-WL2, JobUID: job-WL2) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-WL2, JobUID: job-WL2) due to reclamation within the cohort; preemptor path: /other/CQ2; preemptee path: /other/CQ3", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-WL2, JobUID: job-WL2) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-WL2, JobUID: job-WL2) due to reclamation within the cohort; preemptor path: /other/CQ2; preemptee path: /other/CQ3", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -7211,14 +7211,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue; preemptor path: /root-cohort/child-cohort/queue1; preemptee path: /root-cohort/child-cohort/queue1", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InClusterQueue", - Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue", + Message: "Preempted to accommodate a workload (UID: wl-a2, JobUID: job-a2) due to prioritization in the ClusterQueue; preemptor path: /root-cohort/child-cohort/queue1; preemptee path: /root-cohort/child-cohort/queue1", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -7356,14 +7356,14 @@ func TestSchedule(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /root-cohort/child-cohort/queue1; preemptee path: /root-cohort/child-cohort/queue2", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortFairSharing", - Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-preemptor, JobUID: job-preemptor) due to Fair Sharing within the cohort; preemptor path: /root-cohort/child-cohort/queue1; preemptee path: /root-cohort/child-cohort/queue2", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -8355,14 +8355,14 @@ func TestLastSchedulingContext(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort; preemptor path: /cohort/eng-cohort-theta; preemptee path: /cohort/eng-cohort-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort; preemptor path: /cohort/eng-cohort-theta; preemptee path: /cohort/eng-cohort-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). @@ -8483,14 +8483,14 @@ func TestLastSchedulingContext(t *testing.T) { Type: kueue.WorkloadEvicted, Status: metav1.ConditionTrue, Reason: "Preempted", - Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort; preemptor path: /cohort/eng-cohort-beta; preemptee path: /cohort/eng-cohort-alpha", LastTransitionTime: metav1.NewTime(now), }). SetOrReplaceCondition(metav1.Condition{ Type: kueue.WorkloadPreempted, Status: metav1.ConditionTrue, Reason: "InCohortReclamation", - Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort", + Message: "Preempted to accommodate a workload (UID: wl-new, JobUID: job-new) due to reclamation within the cohort; preemptor path: /cohort/eng-cohort-beta; preemptee path: /cohort/eng-cohort-alpha", LastTransitionTime: metav1.NewTime(now), }). SchedulingStatsEviction(kueue.WorkloadSchedulingStatsEviction{Reason: "Preempted", Count: 1}). diff --git a/test/integration/singlecluster/scheduler/preemption_test.go b/test/integration/singlecluster/scheduler/preemption_test.go index 93325336f7f..4ddf02cc71d 100644 --- a/test/integration/singlecluster/scheduler/preemption_test.go +++ b/test/integration/singlecluster/scheduler/preemption_test.go @@ -209,7 +209,8 @@ var _ = ginkgo.Describe("Preemption", func() { Obj() util.MustCreate(ctx, k8sClient, highWl) - util.ExpectPreemptedCondition(ctx, k8sClient, kueue.InClusterQueueReason, metav1.ConditionTrue, lowWl, highWl, string(highWl.UID), "job-uid") + cQPath := "/" + cq.Name + util.ExpectPreemptedCondition(ctx, k8sClient, kueue.InClusterQueueReason, metav1.ConditionTrue, lowWl, highWl, string(highWl.UID), "job-uid", cQPath, cQPath) }) }) @@ -301,9 +302,12 @@ var _ = ginkgo.Describe("Preemption", func() { util.ExpectWorkloadsToBePending(ctx, k8sClient, alphaLowWl, betaMidWl) conditionCmpOpts := cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime") + alphaCqPath := "/" + string(alphaCQ.Spec.CohortName) + "/" + alphaCQ.Name + betaCqPath := "/" + string(betaCQ.Spec.CohortName) + "/" + betaCQ.Name + ginkgo.By("Verify the Preempted condition", func() { - util.ExpectPreemptedCondition(ctx, k8sClient, kueue.InClusterQueueReason, metav1.ConditionTrue, alphaLowWl, alphaMidWl, string(alphaMidWl.UID), "UNKNOWN") - util.ExpectPreemptedCondition(ctx, k8sClient, kueue.InCohortReclamationReason, metav1.ConditionTrue, betaMidWl, alphaMidWl, string(alphaMidWl.UID), "UNKNOWN") + util.ExpectPreemptedCondition(ctx, k8sClient, kueue.InClusterQueueReason, metav1.ConditionTrue, alphaLowWl, alphaMidWl, string(alphaMidWl.UID), "UNKNOWN", alphaCqPath, alphaCqPath) + util.ExpectPreemptedCondition(ctx, k8sClient, kueue.InCohortReclamationReason, metav1.ConditionTrue, betaMidWl, alphaMidWl, string(alphaMidWl.UID), "UNKNOWN", alphaCqPath, betaCqPath) util.ExpectPreemptedWorkloadsTotalMetric(alphaCQ.Name, kueue.InClusterQueueReason, 1) util.ExpectPreemptedWorkloadsTotalMetric(alphaCQ.Name, kueue.InCohortReclamationReason, 1) util.ExpectPreemptedWorkloadsTotalMetric(betaCQ.Name, kueue.InClusterQueueReason, 0) @@ -322,7 +326,7 @@ var _ = ginkgo.Describe("Preemption", func() { Status: metav1.ConditionFalse, ObservedGeneration: alphaLowWl.Generation, Reason: "QuotaReserved", - Message: fmt.Sprintf("Previously: Preempted to accommodate a workload (UID: %s, JobUID: UNKNOWN) due to %s", alphaMidWl.UID, preemption.HumanReadablePreemptionReasons[kueue.InClusterQueueReason]), + Message: fmt.Sprintf("Previously: Preempted to accommodate a workload (UID: %s, JobUID: UNKNOWN) due to %s; preemptor path: %s; preemptee path: %s", alphaMidWl.UID, preemption.HumanReadablePreemptionReasons[kueue.InClusterQueueReason], alphaCqPath, alphaCqPath), }, conditionCmpOpts)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -333,7 +337,7 @@ var _ = ginkgo.Describe("Preemption", func() { Status: metav1.ConditionFalse, ObservedGeneration: betaMidWl.Generation, Reason: "QuotaReserved", - Message: fmt.Sprintf("Previously: Preempted to accommodate a workload (UID: %s, JobUID: UNKNOWN) due to %s", alphaMidWl.UID, preemption.HumanReadablePreemptionReasons[kueue.InCohortReclamationReason]), + Message: fmt.Sprintf("Previously: Preempted to accommodate a workload (UID: %s, JobUID: UNKNOWN) due to %s; preemptor path: %s; preemptee path: %s", alphaMidWl.UID, preemption.HumanReadablePreemptionReasons[kueue.InCohortReclamationReason], alphaCqPath, betaCqPath), }, conditionCmpOpts)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -715,7 +719,9 @@ var _ = ginkgo.Describe("Preemption", func() { Obj() util.MustCreate(ctx, k8sClient, aStandardVeryHighWl) - util.ExpectPreemptedCondition(ctx, k8sClient, kueue.InCohortReclaimWhileBorrowingReason, metav1.ConditionTrue, aBestEffortLowWl, aStandardVeryHighWl, string(aStandardVeryHighWl.UID), "UNKNOWN") + aStandardCQPath := "/" + string(aStandardCQ.Spec.CohortName) + "/" + aStandardCQ.Name + aBestEffortCQPath := "/" + string(aBestEffortCQ.Spec.CohortName) + "/" + aBestEffortCQ.Name + util.ExpectPreemptedCondition(ctx, k8sClient, kueue.InCohortReclaimWhileBorrowingReason, metav1.ConditionTrue, aBestEffortLowWl, aStandardVeryHighWl, string(aStandardVeryHighWl.UID), "UNKNOWN", aStandardCQPath, aBestEffortCQPath) util.ExpectPreemptedWorkloadsTotalMetric(aStandardCQ.Name, kueue.InCohortReclaimWhileBorrowingReason, 1) util.ExpectPreemptedWorkloadsTotalMetric(aBestEffortCQ.Name, kueue.InCohortReclaimWhileBorrowingReason, 0) diff --git a/test/util/util.go b/test/util/util.go index 28498eb77e8..06d1cc22130 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -989,7 +989,7 @@ readCh: gomega.ExpectWithOffset(1, gotObjs).To(gomega.Equal(objs)) } -func ExpectPreemptedCondition(ctx context.Context, k8sClient client.Client, reason string, status metav1.ConditionStatus, preemptedWl, preempteeWl *kueue.Workload, preemteeWorkloadUID, preempteeJobUID string) { +func ExpectPreemptedCondition(ctx context.Context, k8sClient client.Client, reason string, status metav1.ConditionStatus, preemptedWl, preempteeWl *kueue.Workload, preemteeWorkloadUID, preempteeJobUID, preemptorPath, preempteePath string) { conditionCmpOpts := cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime", "ObservedGeneration") gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(preemptedWl), preemptedWl)).To(gomega.Succeed()) @@ -997,7 +997,7 @@ func ExpectPreemptedCondition(ctx context.Context, k8sClient client.Client, reas Type: kueue.WorkloadPreempted, Status: status, Reason: reason, - Message: fmt.Sprintf("Preempted to accommodate a workload (UID: %s, JobUID: %s) due to %s", preemteeWorkloadUID, preempteeJobUID, preemption.HumanReadablePreemptionReasons[reason]), + Message: fmt.Sprintf("Preempted to accommodate a workload (UID: %s, JobUID: %s) due to %s; preemptor path: %s; preemptee path: %s", preemteeWorkloadUID, preempteeJobUID, preemption.HumanReadablePreemptionReasons[reason], preemptorPath, preempteePath), }, conditionCmpOpts))) }, Timeout, Interval).Should(gomega.Succeed()) } From 2544a73d80c6b1d6724a55856de859022d8e77ff Mon Sep 17 00:00:00 2001 From: Harshal Patil <12152047+harche@users.noreply.github.com> Date: Wed, 5 Nov 2025 15:32:58 -0500 Subject: [PATCH 097/119] Refactor DRA validation to use field.ErrorList (#7529) Signed-off-by: Harshal Patil <12152047+harche@users.noreply.github.com> --- pkg/controller/core/workload_controller.go | 5 +- .../core/workload_controller_test.go | 10 +-- pkg/dra/claims.go | 80 +++++++++++-------- pkg/dra/claims_test.go | 71 +++++++++------- 4 files changed, 96 insertions(+), 70 deletions(-) diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index 6ff97056904..aedf6b33000 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -232,8 +232,9 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } log.V(3).Info("Processing DRA resources for workload") - draResources, err := dra.GetResourceRequestsForResourceClaimTemplates(ctx, r.client, &wl) - if err != nil { + draResources, fieldErrs := dra.GetResourceRequestsForResourceClaimTemplates(ctx, r.client, &wl) + if len(fieldErrs) > 0 { + err := fieldErrs.ToAggregate() log.Error(err, "Failed to process DRA resources for workload") if workload.UnsetQuotaReservationWithCondition(&wl, kueue.WorkloadInadmissible, err.Error(), r.clock.Now()) { workload.SetRequeuedCondition(&wl, kueue.WorkloadInadmissible, err.Error(), false) diff --git a/pkg/controller/core/workload_controller_test.go b/pkg/controller/core/workload_controller_test.go index a94475ca9d6..6224507220c 100644 --- a/pkg/controller/core/workload_controller_test.go +++ b/pkg/controller/core/workload_controller_test.go @@ -563,13 +563,13 @@ func TestReconcile(t *testing.T) { Type: kueue.WorkloadQuotaReserved, Status: metav1.ConditionFalse, Reason: kueue.WorkloadInadmissible, - Message: "DeviceClass unmapped.example.com is not mapped in DRA configuration for workload wlUnmappedDRA podset main: DeviceClass is not mapped in DRA configuration", + Message: "spec.podSets[0].template.spec.resourceClaims[0].resourceClaimTemplateName: Not found: \"DeviceClass unmapped.example.com is not mapped in DRA configuration for podset main\"", }). Condition(metav1.Condition{ Type: kueue.WorkloadRequeued, Status: metav1.ConditionFalse, Reason: kueue.WorkloadInadmissible, - Message: "DeviceClass unmapped.example.com is not mapped in DRA configuration for workload wlUnmappedDRA podset main: DeviceClass is not mapped in DRA configuration", + Message: "spec.podSets[0].template.spec.resourceClaims[0].resourceClaimTemplateName: Not found: \"DeviceClass unmapped.example.com is not mapped in DRA configuration for podset main\"", }). Obj() wl.Spec.PodSets[0].Template.Spec.ResourceClaims = []corev1.PodResourceClaim{{ @@ -580,7 +580,7 @@ func TestReconcile(t *testing.T) { } return wl }(), - wantErrorMsg: "DeviceClass is not mapped in DRA configuration", + wantErrorMsg: "not mapped in DRA configuration", wantEvents: nil, }, "reconcile DRA ResourceClaimTemplate not found should return error": { @@ -606,13 +606,13 @@ func TestReconcile(t *testing.T) { Type: kueue.WorkloadQuotaReserved, Status: metav1.ConditionFalse, Reason: kueue.WorkloadInadmissible, - Message: `failed to get claim spec for ResourceClaimTemplate missing-template in workload wlMissingTemplate podset main: failed to get claim spec: resourceclaimtemplates.resource.k8s.io "missing-template" not found`, + Message: `spec.podSets[0].template.spec.resourceClaims[0]: Internal error: failed to get claim spec for ResourceClaimTemplate missing-template in podset main: resourceclaimtemplates.resource.k8s.io "missing-template" not found`, }). Condition(metav1.Condition{ Type: kueue.WorkloadRequeued, Status: metav1.ConditionFalse, Reason: kueue.WorkloadInadmissible, - Message: `failed to get claim spec for ResourceClaimTemplate missing-template in workload wlMissingTemplate podset main: failed to get claim spec: resourceclaimtemplates.resource.k8s.io "missing-template" not found`, + Message: `spec.podSets[0].template.spec.resourceClaims[0]: Internal error: failed to get claim spec for ResourceClaimTemplate missing-template in podset main: resourceclaimtemplates.resource.k8s.io "missing-template" not found`, }). Obj(), wantErrorMsg: "failed to get claim spec", diff --git a/pkg/dra/claims.go b/pkg/dra/claims.go index cfdac49331e..5d0b327bad6 100644 --- a/pkg/dra/claims.go +++ b/pkg/dra/claims.go @@ -18,13 +18,13 @@ package dra import ( "context" - "errors" "fmt" "strconv" corev1 "k8s.io/api/core/v1" resourcev1 "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" @@ -32,61 +32,56 @@ import ( utilresource "sigs.k8s.io/kueue/pkg/util/resource" ) -var ( - errDeviceClassNotMapped = errors.New("DeviceClass is not mapped in DRA configuration") - errClaimSpecNotFound = errors.New("failed to get claim spec") - errUnsupportedDRAFeature = errors.New("unsupported DRA feature") - - // Specific unsupported DRA feature errors - errUnsupportedDRADeviceConstraints = errors.New("device constraints (MatchAttribute) are not supported") - errUnsupportedDRADeviceConfig = errors.New("device config is not supported") - errUnsupportedDRAFirstAvailable = errors.New("FirstAvailable device selection is not supported") - errUnsupportedDRACELSelectors = errors.New("CEL selectors are not supported") - errUnsupportedDRAAdminAccess = errors.New("AdminAccess is not supported") - errUnsupportedDRAAllocationModeAll = errors.New("AllocationMode 'All' is not supported") -) - // countDevicesPerClass returns a resources.Requests representing the // total number of devices requested for each DeviceClass inside the provided // ResourceClaimSpec. It validates that only supported DRA features are used -// and returns an error if unsupported features are detected. -func countDevicesPerClass(claimSpec *resourcev1.ResourceClaimSpec) (resources.Requests, error) { +// and returns field errors if unsupported features are detected. +func countDevicesPerClass(claimSpec *resourcev1.ResourceClaimSpec) (resources.Requests, field.ErrorList) { out := resources.Requests{} if claimSpec == nil { return out, nil } + var allErrs field.ErrorList + // Check for unsupported device constraints if len(claimSpec.Devices.Constraints) > 0 { - return nil, errUnsupportedDRADeviceConstraints + allErrs = append(allErrs, field.Invalid(field.NewPath("devices", "constraints"), nil, "device constraints (MatchAttribute) are not supported")) + return nil, allErrs } // Check for unsupported device config if len(claimSpec.Devices.Config) > 0 { - return nil, errUnsupportedDRADeviceConfig + allErrs = append(allErrs, field.Invalid(field.NewPath("devices", "config"), nil, "device config is not supported")) + return nil, allErrs } - for _, req := range claimSpec.Devices.Requests { + for i, req := range claimSpec.Devices.Requests { // v1 DeviceRequest has Exactly or FirstAvailable. For Step 1, we // preserve existing semantics by only supporting Exactly with Count. var dcName string var q int64 if req.FirstAvailable != nil { - return nil, errUnsupportedDRAFirstAvailable + allErrs = append(allErrs, field.Invalid(field.NewPath("devices", "requests").Index(i), nil, "FirstAvailable device selection is not supported")) + return nil, allErrs } switch { case len(req.Exactly.Selectors) > 0: - return nil, errUnsupportedDRACELSelectors + allErrs = append(allErrs, field.Invalid(field.NewPath("devices", "requests").Index(i).Child("exactly", "selectors"), nil, "CEL selectors are not supported")) + return nil, allErrs case req.Exactly.AdminAccess != nil && *req.Exactly.AdminAccess: - return nil, errUnsupportedDRAAdminAccess + allErrs = append(allErrs, field.Invalid(field.NewPath("devices", "requests").Index(i).Child("exactly", "adminAccess"), nil, "AdminAccess is not supported")) + return nil, allErrs case req.Exactly.AllocationMode == resourcev1.DeviceAllocationModeAll: - return nil, errUnsupportedDRAAllocationModeAll + allErrs = append(allErrs, field.Invalid(field.NewPath("devices", "requests").Index(i).Child("exactly", "allocationMode"), resourcev1.DeviceAllocationModeAll, "AllocationMode 'All' is not supported")) + return nil, allErrs case req.Exactly.AllocationMode == resourcev1.DeviceAllocationModeExactCount: dcName = req.Exactly.DeviceClassName q = req.Exactly.Count default: - return nil, fmt.Errorf("%w: unsupported allocation mode: %s", errUnsupportedDRAFeature, req.Exactly.AllocationMode) + allErrs = append(allErrs, field.Invalid(field.NewPath("devices", "requests").Index(i).Child("exactly", "allocationMode"), req.Exactly.AllocationMode, fmt.Sprintf("unsupported allocation mode: %s", req.Exactly.AllocationMode))) + return nil, allErrs } dc := corev1.ResourceName(dcName) @@ -125,37 +120,56 @@ func getClaimSpec(ctx context.Context, cl client.Client, namespace string, prc c // returns the aggregated quantities per PodSet. // // If at least one DeviceClass is not present in the DRA configuration or if unsupported DRA -// features are detected, the function returns an error. +// features are detected, the function returns field errors. func GetResourceRequestsForResourceClaimTemplates( ctx context.Context, cl client.Client, - wl *kueue.Workload) (map[kueue.PodSetReference]corev1.ResourceList, error) { + wl *kueue.Workload) (map[kueue.PodSetReference]corev1.ResourceList, field.ErrorList) { perPodSet := make(map[kueue.PodSetReference]corev1.ResourceList) + var allErrs field.ErrorList + for i := range wl.Spec.PodSets { ps := &wl.Spec.PodSets[i] aggregated := corev1.ResourceList{} - for _, prc := range ps.Template.Spec.ResourceClaims { + for j, prc := range ps.Template.Spec.ResourceClaims { if prc.ResourceClaimTemplateName == nil { continue } spec, err := getClaimSpec(ctx, cl, wl.Namespace, prc) if err != nil { - return nil, fmt.Errorf("failed to get claim spec for ResourceClaimTemplate %s in workload %s podset %s: %w", *prc.ResourceClaimTemplateName, wl.Name, ps.Name, fmt.Errorf("%w: %v", errClaimSpecNotFound, err)) + allErrs = append(allErrs, field.InternalError( + field.NewPath("spec", "podSets").Index(i).Child("template", "spec", "resourceClaims").Index(j), + fmt.Errorf("failed to get claim spec for ResourceClaimTemplate %s in podset %s: %w", *prc.ResourceClaimTemplateName, ps.Name, err), + )) + return nil, allErrs } if spec == nil { continue } - deviceCounts, err := countDevicesPerClass(spec) - if err != nil { - return nil, fmt.Errorf("unsupported DRA feature in ResourceClaimTemplate %s in workload %s podset %s: %w", *prc.ResourceClaimTemplateName, wl.Name, ps.Name, err) + deviceCounts, fieldErrs := countDevicesPerClass(spec) + if len(fieldErrs) > 0 { + // Prefix the field paths with the podset and resource claim context + for _, fieldErr := range fieldErrs { + allErrs = append(allErrs, &field.Error{ + Type: fieldErr.Type, + Field: field.NewPath("spec", "podSets").Index(i).Child("template", "spec", "resourceClaims").Index(j).String() + "." + fieldErr.Field, + BadValue: fieldErr.BadValue, + Detail: fmt.Sprintf("ResourceClaimTemplate %s: %s", *prc.ResourceClaimTemplateName, fieldErr.Detail), + }) + } + return nil, allErrs } for dc, qty := range deviceCounts { logical, found := Mapper().lookup(dc) if !found { - return nil, fmt.Errorf("DeviceClass %s is not mapped in DRA configuration for workload %s podset %s: %w", dc, wl.Name, ps.Name, errDeviceClassNotMapped) + allErrs = append(allErrs, field.NotFound( + field.NewPath("spec", "podSets").Index(i).Child("template", "spec", "resourceClaims").Index(j).Child("resourceClaimTemplateName"), + fmt.Sprintf("DeviceClass %s is not mapped in DRA configuration for podset %s", dc, ps.Name), + )) + return nil, allErrs } aggregated = utilresource.MergeResourceListKeepSum(aggregated, corev1.ResourceList{logical: resource.MustParse(strconv.FormatInt(qty, 10))}) } diff --git a/pkg/dra/claims_test.go b/pkg/dra/claims_test.go index 382c005b915..2aa728af989 100644 --- a/pkg/dra/claims_test.go +++ b/pkg/dra/claims_test.go @@ -17,14 +17,16 @@ limitations under the License. package dra import ( - "errors" "reflect" "testing" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -79,7 +81,7 @@ func Test_GetResourceRequests(t *testing.T) { extraObjects []runtime.Object lookup func(corev1.ResourceName) (corev1.ResourceName, bool) want map[kueue.PodSetReference]corev1.ResourceList - wantErr error + wantErr field.ErrorList }{ { name: "Single claim template with single device", @@ -98,9 +100,11 @@ func Test_GetResourceRequests(t *testing.T) { }, }, { - name: "Unmapped DeviceClass returns error", - lookup: noLookup, - wantErr: errDeviceClassNotMapped, + name: "Unmapped DeviceClass returns error", + lookup: noLookup, + wantErr: field.ErrorList{ + field.NotFound(field.NewPath("spec", "podSets").Index(0).Child("template", "spec", "resourceClaims").Index(0).Child("resourceClaimTemplateName"), ""), + }, }, { name: "Two containers each using different claim templates", @@ -229,8 +233,10 @@ func Test_GetResourceRequests(t *testing.T) { {Name: "req-all", ResourceClaimTemplateName: ptr.To("claim-tmpl-all")}, } }, - lookup: defaultLookup, - wantErr: errUnsupportedDRAAllocationModeAll, + lookup: defaultLookup, + wantErr: field.ErrorList{ + field.Invalid(field.NewPath("spec", "podSets").Index(0).Child("template", "spec", "resourceClaims").Index(0).Child("devices", "requests").Index(0).Child("exactly", "allocationMode"), "", ""), + }, }, { name: "CEL selectors returns error", @@ -245,8 +251,10 @@ func Test_GetResourceRequests(t *testing.T) { {Name: "req-cel", ResourceClaimTemplateName: ptr.To("claim-tmpl-cel")}, } }, - lookup: defaultLookup, - wantErr: errUnsupportedDRACELSelectors, + lookup: defaultLookup, + wantErr: field.ErrorList{ + field.Invalid(field.NewPath("spec", "podSets").Index(0).Child("template", "spec", "resourceClaims").Index(0).Child("devices", "requests").Index(0).Child("exactly", "selectors"), "", ""), + }, }, { name: "Device constraints returns error", @@ -262,8 +270,10 @@ func Test_GetResourceRequests(t *testing.T) { {Name: "req-constraints", ResourceClaimTemplateName: ptr.To("claim-tmpl-constraints")}, } }, - lookup: defaultLookup, - wantErr: errUnsupportedDRADeviceConstraints, + lookup: defaultLookup, + wantErr: field.ErrorList{ + field.Invalid(field.NewPath("spec", "podSets").Index(0).Child("template", "spec", "resourceClaims").Index(0).Child("devices", "constraints"), "", ""), + }, }, { name: "FirstAvailable returns error", @@ -277,8 +287,10 @@ func Test_GetResourceRequests(t *testing.T) { {Name: "req-first", ResourceClaimTemplateName: ptr.To("claim-tmpl-first")}, } }, - lookup: defaultLookup, - wantErr: errUnsupportedDRAFirstAvailable, + lookup: defaultLookup, + wantErr: field.ErrorList{ + field.Invalid(field.NewPath("spec", "podSets").Index(0).Child("template", "spec", "resourceClaims").Index(0).Child("devices", "requests").Index(0), "", ""), + }, }, { name: "AdminAccess returns error", @@ -293,8 +305,10 @@ func Test_GetResourceRequests(t *testing.T) { {Name: "req-admin", ResourceClaimTemplateName: ptr.To("claim-tmpl-admin")}, } }, - lookup: defaultLookup, - wantErr: errUnsupportedDRAAdminAccess, + lookup: defaultLookup, + wantErr: field.ErrorList{ + field.Invalid(field.NewPath("spec", "podSets").Index(0).Child("template", "spec", "resourceClaims").Index(0).Child("devices", "requests").Index(0).Child("exactly", "adminAccess"), "", ""), + }, }, { name: "Device config returns error", @@ -309,8 +323,10 @@ func Test_GetResourceRequests(t *testing.T) { {Name: "req-config", ResourceClaimTemplateName: ptr.To("claim-tmpl-config")}, } }, - lookup: defaultLookup, - wantErr: errUnsupportedDRADeviceConfig, + lookup: defaultLookup, + wantErr: field.ErrorList{ + field.Invalid(field.NewPath("spec", "podSets").Index(0).Child("template", "spec", "resourceClaims").Index(0).Child("devices", "config"), "", ""), + }, }, } @@ -351,20 +367,15 @@ func Test_GetResourceRequests(t *testing.T) { ctx, _ := utiltesting.ContextWithLog(t) got, err := GetResourceRequestsForResourceClaimTemplates(ctx, baseClient, wlCopy) - if tc.wantErr != nil { - if err == nil { - t.Fatalf("expected error but got none") - } - if !errors.Is(err, tc.wantErr) { - t.Fatalf("unexpected error: got=%v, want=%v", err, tc.wantErr) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) + + if diff := cmp.Diff(tc.wantErr, err, cmpopts.IgnoreFields(field.Error{}, "Detail", "BadValue")); diff != "" { + t.Errorf("GetResourceRequestsForResourceClaimTemplates() error mismatch (-want +got):\n%s", diff) } - if !reflect.DeepEqual(got, tc.want) { - t.Fatalf("unexpected result; got=%v want=%v", got, tc.want) + + if err == nil { + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("unexpected result; got=%v want=%v", got, tc.want) + } } }) } From daaa5c255ffdee7b6fdfbcb21be5c7f6df9dd7c4 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 6 Nov 2025 09:54:50 +0530 Subject: [PATCH 098/119] Remove redundant type conversions. (#7545) --- pkg/cache/scheduler/tas_cache_test.go | 2 +- pkg/config/config_test.go | 2 +- pkg/controller/jobs/job/job_controller_test.go | 12 ++++++------ pkg/util/routine/wrapper.go | 4 ++-- pkg/webhooks/clusterqueue_webhook_test.go | 2 +- test/e2e/customconfigs/waitforpodsready_test.go | 4 ++-- test/integration/multikueue/external_job_test.go | 4 ++-- test/integration/multikueue/jobs_test.go | 2 +- .../jobs/raycluster/raycluster_controller_test.go | 6 +++--- test/integration/singlecluster/tas/tas_test.go | 2 +- .../singlecluster/webhook/core/clusterqueue_test.go | 4 ++-- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pkg/cache/scheduler/tas_cache_test.go b/pkg/cache/scheduler/tas_cache_test.go index f5ddda96341..8e003b54662 100644 --- a/pkg/cache/scheduler/tas_cache_test.go +++ b/pkg/cache/scheduler/tas_cache_test.go @@ -3027,7 +3027,7 @@ func TestFindTopologyAssignments(t *testing.T) { levels: defaultThreeLevels, podSets: []PodSetTestCase{{ topologyRequest: &kueue.PodSetTopologyRequest{ - Required: ptr.To(string(tasBlockLabel)), + Required: ptr.To(tasBlockLabel), PodSetSliceRequiredTopology: ptr.To("not-existing-topology-level"), PodSetSliceSize: ptr.To(int32(1)), }, diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1456a102b8e..c961e942124 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -991,7 +991,7 @@ func TestEncode(t *testing.T) { "gcInterval": "1m0s", "origin": "multikueue", "workerLostTimeout": "15m0s", - "dispatcherName": string(configapi.MultiKueueDispatcherModeAllAtOnce), + "dispatcherName": configapi.MultiKueueDispatcherModeAllAtOnce, }, "waitForPodsReady": map[string]any{}, }, diff --git a/pkg/controller/jobs/job/job_controller_test.go b/pkg/controller/jobs/job/job_controller_test.go index c734bcf3484..fc02ee2ba98 100644 --- a/pkg/controller/jobs/job/job_controller_test.go +++ b/pkg/controller/jobs/job/job_controller_test.go @@ -978,7 +978,7 @@ func TestReconciler(t *testing.T) { Key: types.NamespacedName{Name: "job", Namespace: "ns"}, EventType: "Normal", Reason: "CreatedWorkload", - Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, types.UID("test-uid")), + Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, "test-uid"), }, }, }, @@ -1013,7 +1013,7 @@ func TestReconciler(t *testing.T) { Key: types.NamespacedName{Name: "job", Namespace: "ns"}, EventType: "Normal", Reason: "CreatedWorkload", - Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, types.UID("test-uid")), + Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, "test-uid"), }, }, }, @@ -2629,7 +2629,7 @@ func TestReconciler(t *testing.T) { Key: types.NamespacedName{Name: "job", Namespace: "ns"}, EventType: "Normal", Reason: "CreatedWorkload", - Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, types.UID("test-uid")), + Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, "test-uid"), }, }, }, @@ -3204,7 +3204,7 @@ func TestReconciler(t *testing.T) { Key: types.NamespacedName{Name: "job", Namespace: "ns"}, EventType: "Normal", Reason: "CreatedWorkload", - Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, types.UID("test-uid")), + Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, "test-uid"), }, }, }, @@ -3249,7 +3249,7 @@ func TestReconciler(t *testing.T) { Key: types.NamespacedName{Name: "job", Namespace: "ns"}, EventType: "Normal", Reason: "CreatedWorkload", - Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, types.UID("test-uid")), + Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, "test-uid"), }, }, }, @@ -3296,7 +3296,7 @@ func TestReconciler(t *testing.T) { Key: types.NamespacedName{Name: "job", Namespace: "ns"}, EventType: "Normal", Reason: "CreatedWorkload", - Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, types.UID("test-uid")), + Message: "Created Workload: ns/" + GetWorkloadNameForJob(baseJobWrapper.Name, "test-uid"), }, }, }, diff --git a/pkg/util/routine/wrapper.go b/pkg/util/routine/wrapper.go index 8ca14949b57..85c3c0bd4ff 100644 --- a/pkg/util/routine/wrapper.go +++ b/pkg/util/routine/wrapper.go @@ -23,9 +23,9 @@ type Wrapper interface { var _ Wrapper = &wrapper{} -var DefaultWrapper Wrapper = NewWrapper(nil, nil) +var DefaultWrapper = NewWrapper(nil, nil) -// wrapper implement the Wrapper interface. +// wrapper implements the Wrapper interface. // before() will be executed before the function starts, and after() // will be executed after the function ends. type wrapper struct { diff --git a/pkg/webhooks/clusterqueue_webhook_test.go b/pkg/webhooks/clusterqueue_webhook_test.go index 95117bdde1f..84d5716f801 100644 --- a/pkg/webhooks/clusterqueue_webhook_test.go +++ b/pkg/webhooks/clusterqueue_webhook_test.go @@ -426,7 +426,7 @@ func makeCoveredResources(n int) []kueue.ResourceGroup { return []kueue.ResourceGroup{{ CoveredResources: resources, Flavors: []kueue.FlavorQuotas{{ - Name: kueue.ResourceFlavorReference("default"), + Name: "default", Resources: quotas, }}, }} diff --git a/test/e2e/customconfigs/waitforpodsready_test.go b/test/e2e/customconfigs/waitforpodsready_test.go index 2f04416c18d..863d1796c88 100644 --- a/test/e2e/customconfigs/waitforpodsready_test.go +++ b/test/e2e/customconfigs/waitforpodsready_test.go @@ -170,7 +170,7 @@ var _ = ginkgo.Describe("WaitForPodsReady with tiny Timeout and no RecoveryTimeo ginkgo.By("verifying that the metric is updated", func() { util.ExpectMetricsToBeAvailable(ctx, cfg, restClient, curlPod.Name, curlContainerName, [][]string{ - {"kueue_evicted_workloads_once_total", cq.Name, string(kueue.WorkloadEvictedByPodsReadyTimeout), kueue.WorkloadWaitForStart, "1"}, + {"kueue_evicted_workloads_once_total", cq.Name, kueue.WorkloadEvictedByPodsReadyTimeout, kueue.WorkloadWaitForStart, "1"}, }) }) @@ -337,7 +337,7 @@ var _ = ginkgo.Describe("WaitForPodsReady with default Timeout and a tiny Recove ginkgo.By("verifying that the metric is updated", func() { util.ExpectMetricsToBeAvailable(ctx, cfg, restClient, curlPod.Name, curlContainerName, [][]string{ - {"kueue_evicted_workloads_once_total", cq.Name, string(kueue.WorkloadEvictedByPodsReadyTimeout), kueue.WorkloadWaitForRecovery, "1"}, + {"kueue_evicted_workloads_once_total", cq.Name, kueue.WorkloadEvictedByPodsReadyTimeout, kueue.WorkloadWaitForRecovery, "1"}, }) }) }) diff --git a/test/integration/multikueue/external_job_test.go b/test/integration/multikueue/external_job_test.go index 2745e12d4e1..d1b3cf65ea6 100644 --- a/test/integration/multikueue/external_job_test.go +++ b/test/integration/multikueue/external_job_test.go @@ -132,11 +132,11 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, multikueue.WithWorkerLostTimeout(testingWorkerLostTimeout), multikueue.WithEventsBatchPeriod(100*time.Millisecond), multikueue.WithAdapters(adapters), - multikueue.WithDispatcherName(string(config.MultiKueueDispatcherModeAllAtOnce)), + multikueue.WithDispatcherName(config.MultiKueueDispatcherModeAllAtOnce), ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - _, err = dispatcher.SetupControllers(mgr, configuration, string(config.MultiKueueDispatcherModeAllAtOnce)) + _, err = dispatcher.SetupControllers(mgr, configuration, config.MultiKueueDispatcherModeAllAtOnce) gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) }) diff --git a/test/integration/multikueue/jobs_test.go b/test/integration/multikueue/jobs_test.go index 12aa2042212..1505dd1f436 100644 --- a/test/integration/multikueue/jobs_test.go +++ b/test/integration/multikueue/jobs_test.go @@ -77,7 +77,7 @@ import ( "sigs.k8s.io/kueue/test/util" ) -var defaultEnabledIntegrations sets.Set[string] = sets.New( +var defaultEnabledIntegrations = sets.New( "batch/job", "kubeflow.org/mpijob", "ray.io/rayjob", "ray.io/raycluster", "jobset.x-k8s.io/jobset", "kubeflow.org/paddlejob", "kubeflow.org/pytorchjob", "kubeflow.org/tfjob", "kubeflow.org/xgboostjob", "kubeflow.org/jaxjob", diff --git a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go index 99fa3afd456..4348d7e92d0 100644 --- a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go @@ -737,7 +737,7 @@ var _ = ginkgo.Describe("RayCluster with elastic jobs via workload-slices suppor ginkgo.It("Should support raycluster scale-down and scale-up", func() { testRayCluster := testingraycluster.MakeCluster("foo", ns.Name). SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). - Queue(string(kueue.LocalQueueName(localQueue.Name))). + Queue(localQueue.Name). Request(rayv1.HeadNode, corev1.ResourceCPU, "1"). RequestWorkerGroup(corev1.ResourceCPU, "1"). WithEnableAutoscaling(ptr.To(true)). @@ -872,7 +872,7 @@ var _ = ginkgo.Describe("RayCluster with elastic jobs via workload-slices suppor testRayClusterA := testingraycluster.MakeCluster("raycluster-a", ns.Name). SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). - Queue(string(kueue.LocalQueueName(localQueue.Name))). + Queue(localQueue.Name). Request(rayv1.HeadNode, corev1.ResourceCPU, "1"). RequestWorkerGroup(corev1.ResourceCPU, "1"). WithEnableAutoscaling(ptr.To(true)). @@ -899,7 +899,7 @@ var _ = ginkgo.Describe("RayCluster with elastic jobs via workload-slices suppor testRayClusterB := testingraycluster.MakeCluster("raycluster-b", ns.Name). SetAnnotation(workloadslicing.EnabledAnnotationKey, workloadslicing.EnabledAnnotationValue). - Queue(string(kueue.LocalQueueName(localQueue.Name))). + Queue(localQueue.Name). Request(rayv1.HeadNode, corev1.ResourceCPU, "1"). RequestWorkerGroup(corev1.ResourceCPU, "1"). WithEnableAutoscaling(ptr.To(true)). diff --git a/test/integration/singlecluster/tas/tas_test.go b/test/integration/singlecluster/tas/tas_test.go index 78dee73f868..12e69bea4a0 100644 --- a/test/integration/singlecluster/tas/tas_test.go +++ b/test/integration/singlecluster/tas/tas_test.go @@ -2992,7 +2992,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { gomega.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(1)) gomega.Expect(wl.Status.Admission.PodSetAssignments[0].Flavors).Should(gomega.Equal( map[corev1.ResourceName]kueue.ResourceFlavorReference{ - "nvidia.com/gpu": kueue.ResourceFlavorReference("non-tas-gpu-flavor"), + "nvidia.com/gpu": "non-tas-gpu-flavor", }, )) // Without TAS, there should be no TopologyAssignment diff --git a/test/integration/singlecluster/webhook/core/clusterqueue_test.go b/test/integration/singlecluster/webhook/core/clusterqueue_test.go index 284b7e4a6b9..956d7132fab 100644 --- a/test/integration/singlecluster/webhook/core/clusterqueue_test.go +++ b/test/integration/singlecluster/webhook/core/clusterqueue_test.go @@ -282,10 +282,10 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }).Obj(), testing.BeForbiddenError()), ginkgo.Entry("Should forbid to create clusterQueue with unknown clusterQueueingStrategy", - utiltestingapi.MakeClusterQueue("cluster-queue").QueueingStrategy(kueue.QueueingStrategy("unknown")).Obj(), + utiltestingapi.MakeClusterQueue("cluster-queue").QueueingStrategy("unknown").Obj(), testing.BeInvalidError()), ginkgo.Entry("Should allow to create clusterQueue with empty clusterQueueingStrategy", - utiltestingapi.MakeClusterQueue("cluster-queue").QueueingStrategy(kueue.QueueingStrategy("")).Obj(), + utiltestingapi.MakeClusterQueue("cluster-queue").QueueingStrategy("").Obj(), gomega.Succeed()), ginkgo.Entry("Should allow to create clusterQueue with empty preemption", utiltestingapi.MakeClusterQueue("cluster-queue").Preemption(kueue.ClusterQueuePreemption{}).Obj(), From 2c0904f0ec77531479156a0fae3dd152b5cfc8c8 Mon Sep 17 00:00:00 2001 From: brejman Date: Thu, 6 Nov 2025 09:08:52 +0100 Subject: [PATCH 099/119] Ensure roundtrip success for Quantities (#7430) --- pkg/resources/requests.go | 24 +++- pkg/resources/requests_test.go | 85 +++++++++++++ .../provisioning/provisioning_test.go | 118 ++++++++++++++++++ 3 files changed, 225 insertions(+), 2 deletions(-) diff --git a/pkg/resources/requests.go b/pkg/resources/requests.go index 3762ef07ecd..d244b4ebba9 100644 --- a/pkg/resources/requests.go +++ b/pkg/resources/requests.go @@ -108,15 +108,35 @@ func ResourceQuantity(name corev1.ResourceName, v int64) resource.Quantity { case corev1.ResourceCPU: return *resource.NewMilliQuantity(v, resource.DecimalSI) case corev1.ResourceMemory, corev1.ResourceEphemeralStorage: - return *resource.NewQuantity(v, resource.BinarySI) + return newCanonicalQuantity(v, resource.BinarySI) default: if strings.HasPrefix(string(name), corev1.ResourceHugePagesPrefix) { - return *resource.NewQuantity(v, resource.BinarySI) + return newCanonicalQuantity(v, resource.BinarySI) } return *resource.NewQuantity(v, resource.DecimalSI) } } +// newCanonicalQuantity returns a Quantity that will successfully round-trip. +// +// This means the returned quantity can be serialized then deserialized back to +// an identical quantity. +// +// If the value can round-trip using the preferred format, that one will be used. +// Otherwise, the format will be automatically determined. +// +// For example, if preferred format is BinarySI, 128000 will use BinarySI format +// (because it can be represented as 125Ki), but 100000 will use DecimalSI format. +func newCanonicalQuantity(v int64, preferredFormat resource.Format) resource.Quantity { + preferred := *resource.NewQuantity(v, preferredFormat) + final, err := resource.ParseQuantity(preferred.String()) + if err != nil { + // Should never happen + return preferred + } + return final +} + func ResourceQuantityString(name corev1.ResourceName, v int64) string { rq := ResourceQuantity(name, v) return rq.String() diff --git a/pkg/resources/requests_test.go b/pkg/resources/requests_test.go index 7a39ab7d463..1aed0565a98 100644 --- a/pkg/resources/requests_test.go +++ b/pkg/resources/requests_test.go @@ -17,10 +17,12 @@ limitations under the License. package resources import ( + "encoding/json" "math" "testing" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" ) func TestCountIn(t *testing.T) { @@ -121,3 +123,86 @@ func TestCountIn(t *testing.T) { }) } } + +func TestResourceQuantityRoundTrips(t *testing.T) { + cases := map[string]struct { + resource corev1.ResourceName + value int64 + expected string + }{ + "1": { + resource: corev1.ResourceMemory, + value: 1, + expected: "1", + }, + "1k": { + resource: corev1.ResourceMemory, + value: 1000, + expected: "1k", + }, + "100k": { + resource: corev1.ResourceMemory, + value: 100000, + expected: "100k", + }, + "1M": { + resource: corev1.ResourceMemory, + value: 1000000, + expected: "1M", + }, + "1500k (1.5M)": { + resource: corev1.ResourceMemory, + value: 1500000, + expected: "1500k", + }, + "1Ki": { + resource: corev1.ResourceMemory, + value: 1024, + expected: "1Ki", + }, + "125Ki (128k)": { + resource: corev1.ResourceMemory, + value: 128000, + expected: "125Ki", + }, + "1Mi": { + resource: corev1.ResourceMemory, + value: 1024 * 1024, + expected: "1Mi", + }, + "1536Ki (1.5Mi)": { + resource: corev1.ResourceMemory, + value: 1024 * 1024 * 1.5, + expected: "1536Ki", + }, + "1Gi": { + resource: corev1.ResourceMemory, + value: 1024 * 1024 * 1024, + expected: "1Gi", + }, + "976562500Ki (10G)": { + resource: corev1.ResourceMemory, + value: 10000000000, + expected: "9765625Ki", + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + quantity := ResourceQuantity(tc.resource, tc.value) + initial := quantity.String() + + if initial != tc.expected { + t.Errorf("unexpected result, want=%s, got=%s", tc.expected, initial) + } + + serialized, _ := json.Marshal(quantity) + var deserialized resource.Quantity + _ = json.Unmarshal(serialized, &deserialized) + roundtrip := deserialized.String() + + if roundtrip != tc.expected { + t.Errorf("unexpected result after roundtrip, want=%s, got=%s", tc.expected, roundtrip) + } + }) + } +} diff --git a/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go b/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go index 6c9d338d7f8..40de32aa317 100644 --- a/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go +++ b/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go @@ -1886,5 +1886,123 @@ var _ = ginkgo.Describe("Provisioning with scheduling", ginkgo.Ordered, ginkgo.C }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) + + ginkgo.It("Should be successfully re-admitted on another flavor with a decimal memory request", func() { + ginkgo.By("Set up ClusterQueue and LocalQueue", func() { + cq = utiltestingapi.MakeClusterQueue("cluster-queue"). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + }). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(rf1.Name). + Resource(corev1.ResourceCPU, "0.75"). + Resource(corev1.ResourceMemory, "5G"). + Obj(), + *utiltestingapi.MakeFlavorQuotas(rf2.Name). + Resource(corev1.ResourceCPU, "0.5"). + Resource(corev1.ResourceMemory, "5G"). + Obj(), + ). + AdmissionCheckStrategy(kueue.AdmissionCheckStrategyRule{ + Name: ac1Ref, + OnFlavors: []kueue.ResourceFlavorReference{flavor1Ref}, + }). + Obj() + util.MustCreate(ctx, k8sClient, cq) + util.ExpectClusterQueuesToBeActive(ctx, k8sClient, cq) + + lq = utiltestingapi.MakeLocalQueue("queue", ns.Name).ClusterQueue(cq.Name).Obj() + util.MustCreate(ctx, k8sClient, lq) + util.ExpectLocalQueuesToBeActive(ctx, k8sClient, lq) + }) + + ginkgo.By("submit the Job", func() { + job1 := testingjob.MakeJob("job1", ns.Name). + Queue(kueue.LocalQueueName(lq.Name)). + Request(corev1.ResourceCPU, "500m"). + Request(corev1.ResourceMemory, "220M"). + Obj() + util.MustCreate(ctx, k8sClient, job1) + ginkgo.DeferCleanup(func() { + util.ExpectObjectToBeDeleted(ctx, k8sClient, job1, true) + }) + wl1Key = types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job1.Name, job1.UID), Namespace: ns.Name} + }) + + ginkgo.By("await for wl1 to be created", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, wl1Key, &wlObj)).Should(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("await for wl1 to have QuotaReserved on flavor-1", func() { + gomega.Eventually(func(g gomega.Gomega) { + gomega.Expect(k8sClient.Get(ctx, wl1Key, &wlObj)).Should(gomega.Succeed()) + g.Expect(workload.Status(&wlObj)).To(gomega.Equal(workload.StatusQuotaReserved)) + psa := wlObj.Status.Admission.PodSetAssignments + g.Expect(psa).Should(gomega.HaveLen(1)) + g.Expect(psa[0].Flavors).To(gomega.Equal(map[corev1.ResourceName]kueue.ResourceFlavorReference{ + corev1.ResourceCPU: flavor1Ref, + corev1.ResourceMemory: flavor1Ref, + })) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("await for the ProvisioningRequest on flavor-1 to be created", func() { + provReqKey = types.NamespacedName{ + Namespace: wl1Key.Namespace, + Name: provisioning.ProvisioningRequestName(wl1Key.Name, ac1Ref, 1), + } + + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("set the ProvisioningRequest on flavor-1 as Provisioned", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(gomega.Succeed()) + apimeta.SetStatusCondition(&createdRequest.Status.Conditions, metav1.Condition{ + Type: autoscaling.Provisioned, + Status: metav1.ConditionTrue, + Reason: autoscaling.Provisioned, + }) + g.Expect(k8sClient.Status().Update(ctx, &createdRequest)).Should(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("await for wl1 to be Admitted", func() { + gomega.Eventually(func(g gomega.Gomega) { + gomega.Expect(k8sClient.Get(ctx, wl1Key, &wlObj)).Should(gomega.Succeed()) + g.Expect(workload.Status(&wlObj)).To(gomega.Equal(workload.StatusAdmitted)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("submit a high-priority job2", func() { + job2 := testingjob.MakeJob("job2", ns.Name). + Queue(kueue.LocalQueueName(lq.Name)). + WorkloadPriorityClass(priorityClassName). + Request(corev1.ResourceCPU, "750m"). + Obj() + util.MustCreate(ctx, k8sClient, job2) + ginkgo.DeferCleanup(func() { + util.ExpectObjectToBeDeleted(ctx, k8sClient, job2, true) + }) + wl2Key = types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job2.Name, job2.UID), Namespace: ns.Name} + }) + + ginkgo.By("await for wl1 to be Admitted on flavor-2", func() { + gomega.Eventually(func(g gomega.Gomega) { + gomega.Expect(k8sClient.Get(ctx, wl1Key, &wlObj)).Should(gomega.Succeed()) + g.Expect(workload.Status(&wlObj)).To(gomega.Equal(workload.StatusAdmitted)) + psa := wlObj.Status.Admission.PodSetAssignments + g.Expect(psa).Should(gomega.HaveLen(1)) + g.Expect(psa[0].Flavors).To(gomega.Equal(map[corev1.ResourceName]kueue.ResourceFlavorReference{ + corev1.ResourceCPU: flavor2Ref, + corev1.ResourceMemory: flavor2Ref, + })) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) }) }) From 62db2e8f9045c80ab7a106265a98ef2882303a53 Mon Sep 17 00:00:00 2001 From: Obinna Odirionye Date: Thu, 6 Nov 2025 12:44:54 +0300 Subject: [PATCH 100/119] Remove deprecated AdmissionChecks field from v1beta2 ClusterQueue API (#7490) The deprecated .spec.admissionChecks field has been removed from the v1beta2 ClusterQueue API as part of the v1beta2 stabilization effort. All usage has been migrated to .spec.admissionChecksStrategy. Changes: - Removed AdmissionChecks field from v1beta2 ClusterQueueSpec - Added bidirectional conversion logic between v1beta1 and v1beta2 - Updated controllers to use AdmissionChecksStrategy exclusively - Removed webhook validation for the deprecated field - Updated integration tests to use the new field - Regenerated CRDs, client-go, and documentation ```release-note Remove deprecated .spec.admissionChecks field from v1beta2 ClusterQueue API. Use .spec.admissionChecksStrategy instead. ``` --- apis/kueue/v1beta1/clusterqueue_conversion.go | 14 +++++++++ apis/kueue/v1beta1/zz_generated.conversion.go | 3 +- apis/kueue/v1beta2/clusterqueue_types.go | 7 ----- apis/kueue/v1beta2/zz_generated.deepcopy.go | 5 --- .../crd/kueue.x-k8s.io_clusterqueues.yaml | 15 +-------- .../kueue/v1beta2/clusterqueuespec.go | 11 ------- .../bases/kueue.x-k8s.io_clusterqueues.yaml | 16 ++-------- .../core/admissioncheck_controller.go | 31 ++++++++++++++----- pkg/controller/core/workload_controller.go | 1 - pkg/util/admissioncheck/admissioncheck.go | 7 ++--- pkg/util/testing/v1beta2/wrappers.go | 14 +++++++-- pkg/webhooks/clusterqueue_webhook.go | 10 ------ pkg/webhooks/clusterqueue_webhook_test.go | 9 ------ .../en/docs/reference/kueue.v1beta2.md | 14 +-------- .../core/admissioncheck_controller_test.go | 6 +++- .../core/clusterqueue_controller_test.go | 8 +++-- .../core/workload_controller_test.go | 7 ++++- 17 files changed, 72 insertions(+), 106 deletions(-) diff --git a/apis/kueue/v1beta1/clusterqueue_conversion.go b/apis/kueue/v1beta1/clusterqueue_conversion.go index 3b06b123f58..99935586974 100644 --- a/apis/kueue/v1beta1/clusterqueue_conversion.go +++ b/apis/kueue/v1beta1/clusterqueue_conversion.go @@ -38,6 +38,19 @@ func (dst *ClusterQueue) ConvertFrom(srcRaw conversion.Hub) error { func Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in *ClusterQueueSpec, out *v1beta2.ClusterQueueSpec, s conversionapi.Scope) error { out.CohortName = v1beta2.CohortReference(in.Cohort) + + // Convert AdmissionChecks to AdmissionChecksStrategy before autoConvert + if len(in.AdmissionChecks) > 0 { + out.AdmissionChecksStrategy = &v1beta2.AdmissionChecksStrategy{ + AdmissionChecks: make([]v1beta2.AdmissionCheckStrategyRule, len(in.AdmissionChecks)), + } + for i, checkRef := range in.AdmissionChecks { + out.AdmissionChecksStrategy.AdmissionChecks[i] = v1beta2.AdmissionCheckStrategyRule{ + Name: v1beta2.AdmissionCheckReference(checkRef), + } + } + } + if in.FlavorFungibility != nil && out.FlavorFungibility != nil { if in.FlavorFungibility.WhenCanPreempt == Preempt { out.FlavorFungibility.WhenCanPreempt = v1beta2.MayStopSearch @@ -46,6 +59,7 @@ func Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in *ClusterQue out.FlavorFungibility.WhenCanBorrow = v1beta2.MayStopSearch } } + return autoConvert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in, out, s) } diff --git a/apis/kueue/v1beta1/zz_generated.conversion.go b/apis/kueue/v1beta1/zz_generated.conversion.go index 12d5833795e..a95e8f2194e 100644 --- a/apis/kueue/v1beta1/zz_generated.conversion.go +++ b/apis/kueue/v1beta1/zz_generated.conversion.go @@ -1188,7 +1188,7 @@ func autoConvert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in *Cluste out.NamespaceSelector = (*v1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) out.FlavorFungibility = (*v1beta2.FlavorFungibility)(unsafe.Pointer(in.FlavorFungibility)) out.Preemption = (*v1beta2.ClusterQueuePreemption)(unsafe.Pointer(in.Preemption)) - out.AdmissionChecks = *(*[]v1beta2.AdmissionCheckReference)(unsafe.Pointer(&in.AdmissionChecks)) + // WARNING: in.AdmissionChecks requires manual conversion: does not exist in peer-type out.AdmissionChecksStrategy = (*v1beta2.AdmissionChecksStrategy)(unsafe.Pointer(in.AdmissionChecksStrategy)) out.StopPolicy = (*v1beta2.StopPolicy)(unsafe.Pointer(in.StopPolicy)) out.FairSharing = (*v1beta2.FairSharing)(unsafe.Pointer(in.FairSharing)) @@ -1203,7 +1203,6 @@ func autoConvert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec(in *v1beta out.NamespaceSelector = (*v1.LabelSelector)(unsafe.Pointer(in.NamespaceSelector)) out.FlavorFungibility = (*FlavorFungibility)(unsafe.Pointer(in.FlavorFungibility)) out.Preemption = (*ClusterQueuePreemption)(unsafe.Pointer(in.Preemption)) - out.AdmissionChecks = *(*[]AdmissionCheckReference)(unsafe.Pointer(&in.AdmissionChecks)) out.AdmissionChecksStrategy = (*AdmissionChecksStrategy)(unsafe.Pointer(in.AdmissionChecksStrategy)) out.StopPolicy = (*StopPolicy)(unsafe.Pointer(in.StopPolicy)) out.FairSharing = (*FairSharing)(unsafe.Pointer(in.FairSharing)) diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index de57e82df28..195497331e4 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -115,14 +115,7 @@ type ClusterQueueSpec struct { // +optional Preemption *ClusterQueuePreemption `json:"preemption,omitempty"` - // admissionChecks lists the AdmissionChecks required by this ClusterQueue. - // Cannot be used along with AdmissionCheckStrategy. - // Admission checks are limited to at most 64 items. - // +optional - AdmissionChecks []AdmissionCheckReference `json:"admissionChecks,omitempty"` //nolint:kubeapilinter // field is being removed - // admissionChecksStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. - // This property cannot be used in conjunction with the 'admissionChecks' property. // +optional AdmissionChecksStrategy *AdmissionChecksStrategy `json:"admissionChecksStrategy,omitempty"` diff --git a/apis/kueue/v1beta2/zz_generated.deepcopy.go b/apis/kueue/v1beta2/zz_generated.deepcopy.go index 40980e392f7..b641161baff 100644 --- a/apis/kueue/v1beta2/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta2/zz_generated.deepcopy.go @@ -391,11 +391,6 @@ func (in *ClusterQueueSpec) DeepCopyInto(out *ClusterQueueSpec) { *out = new(ClusterQueuePreemption) (*in).DeepCopyInto(*out) } - if in.AdmissionChecks != nil { - in, out := &in.AdmissionChecks, &out.AdmissionChecks - *out = make([]AdmissionCheckReference, len(*in)) - copy(*out, *in) - } if in.AdmissionChecksStrategy != nil { in, out := &in.AdmissionChecksStrategy, &out.AdmissionChecksStrategy *out = new(AdmissionChecksStrategy) diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml index 6ce0fc1a46b..8bc12f60d28 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml @@ -838,21 +838,8 @@ spec: spec: description: spec is the specification of the ClusterQueue. properties: - admissionChecks: - description: |- - admissionChecks lists the AdmissionChecks required by this ClusterQueue. - Cannot be used along with AdmissionCheckStrategy. - Admission checks are limited to at most 64 items. - items: - description: AdmissionCheckReference is the name of an AdmissionCheck. - maxLength: 316 - minLength: 1 - type: string - type: array admissionChecksStrategy: - description: |- - admissionChecksStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. - This property cannot be used in conjunction with the 'admissionChecks' property. + description: admissionChecksStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. properties: admissionChecks: description: admissionChecks is a list of strategies for AdmissionChecks diff --git a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuespec.go b/client-go/applyconfiguration/kueue/v1beta2/clusterqueuespec.go index b76cd0f2426..50ab20740a1 100644 --- a/client-go/applyconfiguration/kueue/v1beta2/clusterqueuespec.go +++ b/client-go/applyconfiguration/kueue/v1beta2/clusterqueuespec.go @@ -31,7 +31,6 @@ type ClusterQueueSpecApplyConfiguration struct { NamespaceSelector *v1.LabelSelectorApplyConfiguration `json:"namespaceSelector,omitempty"` FlavorFungibility *FlavorFungibilityApplyConfiguration `json:"flavorFungibility,omitempty"` Preemption *ClusterQueuePreemptionApplyConfiguration `json:"preemption,omitempty"` - AdmissionChecks []kueuev1beta2.AdmissionCheckReference `json:"admissionChecks,omitempty"` AdmissionChecksStrategy *AdmissionChecksStrategyApplyConfiguration `json:"admissionChecksStrategy,omitempty"` StopPolicy *kueuev1beta2.StopPolicy `json:"stopPolicy,omitempty"` FairSharing *FairSharingApplyConfiguration `json:"fairSharing,omitempty"` @@ -97,16 +96,6 @@ func (b *ClusterQueueSpecApplyConfiguration) WithPreemption(value *ClusterQueueP return b } -// WithAdmissionChecks adds the given value to the AdmissionChecks field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the AdmissionChecks field. -func (b *ClusterQueueSpecApplyConfiguration) WithAdmissionChecks(values ...kueuev1beta2.AdmissionCheckReference) *ClusterQueueSpecApplyConfiguration { - for i := range values { - b.AdmissionChecks = append(b.AdmissionChecks, values[i]) - } - return b -} - // WithAdmissionChecksStrategy sets the AdmissionChecksStrategy field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the AdmissionChecksStrategy field is set to the value of the last call. diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml index 5e83724747e..35fc0bdd914 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml @@ -823,21 +823,9 @@ spec: spec: description: spec is the specification of the ClusterQueue. properties: - admissionChecks: - description: |- - admissionChecks lists the AdmissionChecks required by this ClusterQueue. - Cannot be used along with AdmissionCheckStrategy. - Admission checks are limited to at most 64 items. - items: - description: AdmissionCheckReference is the name of an AdmissionCheck. - maxLength: 316 - minLength: 1 - type: string - type: array admissionChecksStrategy: - description: |- - admissionChecksStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. - This property cannot be used in conjunction with the 'admissionChecks' property. + description: admissionChecksStrategy defines a list of strategies + to determine which ResourceFlavors require AdmissionChecks. properties: admissionChecks: description: admissionChecks is a list of strategies for AdmissionChecks diff --git a/pkg/controller/core/admissioncheck_controller.go b/pkg/controller/core/admissioncheck_controller.go index 2f6ee429772..386f923180d 100644 --- a/pkg/controller/core/admissioncheck_controller.go +++ b/pkg/controller/core/admissioncheck_controller.go @@ -164,7 +164,20 @@ func (r *AdmissionCheckReconciler) Generic(e event.TypedGenericEvent[*kueue.Admi func (r *AdmissionCheckReconciler) NotifyClusterQueueUpdate(oldCq *kueue.ClusterQueue, newCq *kueue.ClusterQueue) { log := r.log.WithValues("oldClusterQueue", klog.KObj(oldCq), "newClusterQueue", klog.KObj(newCq)) log.V(5).Info("Cluster queue notification") - noChange := newCq != nil && oldCq != nil && slices.CmpNoOrder(oldCq.Spec.AdmissionChecks, newCq.Spec.AdmissionChecks) + + // Helper to extract admission check names from strategy + getAcNames := func(cq *kueue.ClusterQueue) []kueue.AdmissionCheckReference { + if cq.Spec.AdmissionChecksStrategy == nil { + return nil + } + names := make([]kueue.AdmissionCheckReference, len(cq.Spec.AdmissionChecksStrategy.AdmissionChecks)) + for i, ac := range cq.Spec.AdmissionChecksStrategy.AdmissionChecks { + names[i] = ac.Name + } + return names + } + + noChange := newCq != nil && oldCq != nil && slices.CmpNoOrder(getAcNames(oldCq), getAcNames(newCq)) if noChange { return } @@ -196,14 +209,16 @@ func (h *acCqHandler) Generic(ctx context.Context, e event.GenericEvent, q workq log := log.FromContext(ctx).WithValues("clusterQueue", klog.KObj(cq)) log.V(6).Info("Cluster queue generic event") - for _, ac := range cq.Spec.AdmissionChecks { - if cqs := h.cache.ClusterQueuesUsingAdmissionCheck(ac); len(cqs) == 0 { - req := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Name: string(ac), - }, + if cq.Spec.AdmissionChecksStrategy != nil { + for _, ac := range cq.Spec.AdmissionChecksStrategy.AdmissionChecks { + if cqs := h.cache.ClusterQueuesUsingAdmissionCheck(ac.Name); len(cqs) == 0 { + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: string(ac.Name), + }, + } + q.Add(req) } - q.Add(req) } } } diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index aedf6b33000..c2ff65cc25d 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -1143,7 +1143,6 @@ func (w *workloadQueueHandler) Update(ctx context.Context, ev event.UpdateEvent, log.V(5).Info("Workload cluster queue update event") if !newCq.DeletionTimestamp.IsZero() || - !utilslices.CmpNoOrder(oldCq.Spec.AdmissionChecks, newCq.Spec.AdmissionChecks) || !gocmp.Equal(oldCq.Spec.AdmissionChecksStrategy, newCq.Spec.AdmissionChecksStrategy) || !ptr.Equal(oldCq.Spec.StopPolicy, newCq.Spec.StopPolicy) { w.queueReconcileForWorkloadsOfClusterQueue(ctx, newCq.Name, wq) diff --git a/pkg/util/admissioncheck/admissioncheck.go b/pkg/util/admissioncheck/admissioncheck.go index fe227968223..04d5e371493 100644 --- a/pkg/util/admissioncheck/admissioncheck.go +++ b/pkg/util/admissioncheck/admissioncheck.go @@ -163,7 +163,7 @@ func FilterProvReqAnnotations(annotations map[string]string) map[string]string { return res } -// NewAdmissionChecks aggregates AdmissionChecks from .spec.AdmissionChecks and .spec.AdmissionChecksStrategy +// NewAdmissionChecks aggregates AdmissionChecks from .spec.AdmissionChecksStrategy func NewAdmissionChecks(cq *kueue.ClusterQueue) map[kueue.AdmissionCheckReference]sets.Set[kueue.ResourceFlavorReference] { var checks map[kueue.AdmissionCheckReference]sets.Set[kueue.ResourceFlavorReference] if cq.Spec.AdmissionChecksStrategy != nil { @@ -176,10 +176,7 @@ func NewAdmissionChecks(cq *kueue.ClusterQueue) map[kueue.AdmissionCheckReferenc } } } else { - checks = make(map[kueue.AdmissionCheckReference]sets.Set[kueue.ResourceFlavorReference], len(cq.Spec.AdmissionChecks)) - for _, checkName := range cq.Spec.AdmissionChecks { - checks[checkName] = allFlavors(cq) - } + checks = make(map[kueue.AdmissionCheckReference]sets.Set[kueue.ResourceFlavorReference], 0) } return checks } diff --git a/pkg/util/testing/v1beta2/wrappers.go b/pkg/util/testing/v1beta2/wrappers.go index 76f94489ad5..08c484e7ff9 100644 --- a/pkg/util/testing/v1beta2/wrappers.go +++ b/pkg/util/testing/v1beta2/wrappers.go @@ -943,9 +943,19 @@ func (c *ClusterQueueWrapper) ResourceGroup(flavors ...kueue.FlavorQuotas) *Clus return c } -// AdmissionChecks replaces the queue additional checks +// AdmissionChecks replaces the queue additional checks. +// This is a convenience wrapper that converts to the AdmissionChecksStrategy format. func (c *ClusterQueueWrapper) AdmissionChecks(checks ...kueue.AdmissionCheckReference) *ClusterQueueWrapper { - c.Spec.AdmissionChecks = checks + // Convert simple admission check references to the strategy format + acs := make([]kueue.AdmissionCheckStrategyRule, len(checks)) + for i, check := range checks { + acs[i] = kueue.AdmissionCheckStrategyRule{ + Name: check, + } + } + c.Spec.AdmissionChecksStrategy = &kueue.AdmissionChecksStrategy{ + AdmissionChecks: acs, + } return c } diff --git a/pkg/webhooks/clusterqueue_webhook.go b/pkg/webhooks/clusterqueue_webhook.go index 1824a1354c3..c23c5a0a85a 100644 --- a/pkg/webhooks/clusterqueue_webhook.go +++ b/pkg/webhooks/clusterqueue_webhook.go @@ -105,7 +105,6 @@ func ValidateClusterQueue(cq *kueue.ClusterQueue) field.ErrorList { allErrs = append(allErrs, validateResourceGroups(cq.Spec.ResourceGroups, config, path.Child("resourceGroups"), false)...) allErrs = append(allErrs, validation.ValidateLabelSelector(cq.Spec.NamespaceSelector, validation.LabelSelectorValidationOptions{}, path.Child("namespaceSelector"))...) - allErrs = append(allErrs, validateCQAdmissionChecks(&cq.Spec, path)...) if cq.Spec.Preemption != nil { allErrs = append(allErrs, validatePreemption(cq.Spec.Preemption, path.Child("preemption"))...) } @@ -174,15 +173,6 @@ func validatePreemption(preemption *kueue.ClusterQueuePreemption, path *field.Pa return allErrs } -func validateCQAdmissionChecks(spec *kueue.ClusterQueueSpec, path *field.Path) field.ErrorList { - var allErrs field.ErrorList - if spec.AdmissionChecksStrategy != nil && len(spec.AdmissionChecks) != 0 { - allErrs = append(allErrs, field.Invalid(path, spec, "Either AdmissionChecks or AdmissionCheckStrategy can be set, but not both")) - } - - return allErrs -} - func validateResourceGroups(resourceGroups []kueue.ResourceGroup, config validationConfig, path *field.Path, isCohort bool) field.ErrorList { var allErrs field.ErrorList seenResources := sets.New[corev1.ResourceName]() diff --git a/pkg/webhooks/clusterqueue_webhook_test.go b/pkg/webhooks/clusterqueue_webhook_test.go index 84d5716f801..3ab4a85c83b 100644 --- a/pkg/webhooks/clusterqueue_webhook_test.go +++ b/pkg/webhooks/clusterqueue_webhook_test.go @@ -71,15 +71,6 @@ func TestValidateClusterQueue(t *testing.T) { *utiltestingapi.MakeAdmissionCheckStrategyRule("ac1", "flavor1").Obj(), ).Obj(), }, - { - name: "both admissionChecks and admissionCheckStrategy is defined", - clusterQueue: utiltestingapi.MakeClusterQueue("cluster-queue"). - AdmissionChecks("ac1"). - AdmissionCheckStrategy().Obj(), - wantErr: field.ErrorList{ - field.Invalid(specPath, "spec", "Either AdmissionChecks or AdmissionCheckStrategy can be set, but not both"), - }, - }, { name: "in cohort", clusterQueue: utiltestingapi.MakeClusterQueue("cluster-queue").Cohort("prod").Obj(), diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md index 2a721d37202..dc6b1ef176e 100644 --- a/site/content/en/docs/reference/kueue.v1beta2.md +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -466,8 +466,6 @@ The description is limited to a maximum of 2048 characters.

    - [AdmissionCheckStrategyRule](#kueue-x-k8s-io-v1beta2-AdmissionCheckStrategyRule) -- [ClusterQueueSpec](#kueue-x-k8s-io-v1beta2-ClusterQueueSpec) -

    AdmissionCheckReference is the name of an AdmissionCheck.

    @@ -984,21 +982,11 @@ before borrowing or preempting in the flavor being evaluated.

    preemption defines the preemption policies.

    -admissionChecks
    -[]AdmissionCheckReference - - -

    admissionChecks lists the AdmissionChecks required by this ClusterQueue. -Cannot be used along with AdmissionCheckStrategy. -Admission checks are limited to at most 64 items.

    - - admissionChecksStrategy
    AdmissionChecksStrategy -

    admissionChecksStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks. -This property cannot be used in conjunction with the 'admissionChecks' property.

    +

    admissionChecksStrategy defines a list of strategies to determine which ResourceFlavors require AdmissionChecks.

    stopPolicy
    diff --git a/test/integration/singlecluster/controller/core/admissioncheck_controller_test.go b/test/integration/singlecluster/controller/core/admissioncheck_controller_test.go index 95e1a611918..dbef554d070 100644 --- a/test/integration/singlecluster/controller/core/admissioncheck_controller_test.go +++ b/test/integration/singlecluster/controller/core/admissioncheck_controller_test.go @@ -101,7 +101,11 @@ var _ = ginkgo.Describe("AdmissionCheck controller", ginkgo.Ordered, ginkgo.Cont ginkgo.By("Change clusterQueue's checks") gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &cq)).Should(gomega.Succeed()) - cq.Spec.AdmissionChecks = []kueue.AdmissionCheckReference{"check2"} + cq.Spec.AdmissionChecksStrategy = &kueue.AdmissionChecksStrategy{ + AdmissionChecks: []kueue.AdmissionCheckStrategyRule{ + {Name: "check2"}, + }, + } g.Expect(k8sClient.Update(ctx, &cq)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) diff --git a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go index 923374a7071..6e0c6364ec3 100644 --- a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go +++ b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go @@ -799,7 +799,6 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin gomega.Eventually(func(g gomega.Gomega) { updatedCq := &kueue.ClusterQueue{} g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cq), updatedCq)).Should(gomega.Succeed()) - updatedCq.Spec.AdmissionChecks = nil updatedCq.Spec.AdmissionChecksStrategy = &kueue.AdmissionChecksStrategy{ AdmissionChecks: []kueue.AdmissionCheckStrategyRule{ *utiltestingapi.MakeAdmissionCheckStrategyRule("check1", flavorCPUArchA).Obj(), @@ -831,8 +830,11 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin gomega.Eventually(func(g gomega.Gomega) { var updatedCq kueue.ClusterQueue g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cq), &updatedCq)).Should(gomega.Succeed()) - updatedCq.Spec.AdmissionChecks = []kueue.AdmissionCheckReference{"check1"} - updatedCq.Spec.AdmissionChecksStrategy = nil + updatedCq.Spec.AdmissionChecksStrategy = &kueue.AdmissionChecksStrategy{ + AdmissionChecks: []kueue.AdmissionCheckStrategyRule{ + {Name: "check1"}, + }, + } g.Expect(k8sClient.Update(ctx, &updatedCq)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go index 7be8399167e..9d697b5e7bd 100644 --- a/test/integration/singlecluster/controller/core/workload_controller_test.go +++ b/test/integration/singlecluster/controller/core/workload_controller_test.go @@ -246,7 +246,12 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn queueKey := client.ObjectKeyFromObject(clusterQueue) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, queueKey, &createdQueue)).To(gomega.Succeed()) - createdQueue.Spec.AdmissionChecks = []kueue.AdmissionCheckReference{"check2", "check3"} + createdQueue.Spec.AdmissionChecksStrategy = &kueue.AdmissionChecksStrategy{ + AdmissionChecks: []kueue.AdmissionCheckStrategyRule{ + {Name: "check2"}, + {Name: "check3"}, + }, + } g.Expect(k8sClient.Update(ctx, &createdQueue)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) From 50c1fb0806a095d3e79044a071e6c39f7335b6f0 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 6 Nov 2025 15:54:52 +0530 Subject: [PATCH 101/119] Use GomegaMatcher instead of OmegaMatcher. (#7552) --- .../webhook/core/workload_test.go | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/test/integration/singlecluster/webhook/core/workload_test.go b/test/integration/singlecluster/webhook/core/workload_test.go index ac439501706..045dca16ae2 100644 --- a/test/integration/singlecluster/webhook/core/workload_test.go +++ b/test/integration/singlecluster/webhook/core/workload_test.go @@ -23,6 +23,7 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + gomegatypes "github.com/onsi/gomega/types" corev1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -152,13 +153,8 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.Entry("valid podSet", 3, 3, false), ) - ginkgo.DescribeTable("Should have valid values when creating", func(w func() *kueue.Workload, errorType gomega.OmegaMatcher) { - err := k8sClient.Create(ctx, w()) - if errorType != nil { - gomega.Expect(err).Should(errorType) - } else { - gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - } + ginkgo.DescribeTable("Should have valid values when creating", func(w func() *kueue.Workload, matcher gomegatypes.GomegaMatcher) { + gomega.Expect(k8sClient.Create(ctx, w())).Should(matcher) }, ginkgo.Entry("valid workload", func() *kueue.Workload { @@ -167,7 +163,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { *utiltestingapi.MakePodSet("workers", 100).Obj(), ).Obj() }, - nil), + gomega.Succeed()), ginkgo.Entry("invalid podSet name", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name).PodSets( @@ -188,7 +184,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { return utiltestingapi.MakeWorkload(workloadName, ns.Name). Obj() }, - nil), + gomega.Succeed()), ginkgo.Entry("priority should not be nil when priorityClassName is set", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -229,7 +225,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { AdmissionChecks(kueue.AdmissionCheckState{}). Obj() }, - nil), + gomega.Succeed()), ginkgo.Entry("matched names in podSetUpdates with names in podSets", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -275,7 +271,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ). Obj() }, - nil), + gomega.Succeed()), ginkgo.Entry("invalid podSet minCount (negative)", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -317,10 +313,10 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { MaximumExecutionTimeSeconds(1). Obj() }, - nil), + gomega.Succeed()), ) - ginkgo.DescribeTable("Should have valid values when setting Admission", func(w func() *kueue.Workload, a *kueue.Admission, errorType gomega.OmegaMatcher) { + ginkgo.DescribeTable("Should have valid values when setting Admission", func(w func() *kueue.Workload, a *kueue.Admission, matcher gomegatypes.GomegaMatcher) { wl := w() util.MustCreate(ctx, k8sClient, wl) @@ -329,11 +325,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { err := workload.PatchAdmissionStatus(ctx, k8sClient, wl, clock.RealClock{}, func() (*kueue.Workload, bool, error) { return wl, workload.SetQuotaReservation(wl, a, clock.RealClock{}), nil }) - if errorType != nil { - g.Expect(err).Should(errorType) - } else { - g.Expect(err).ShouldNot(gomega.HaveOccurred()) - } + g.Expect(err).Should(matcher) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }, ginkgo.Entry("invalid clusterQueue name", @@ -473,7 +465,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }) ginkgo.DescribeTable("Validate Workload on update", - func(w func() *kueue.Workload, setQuotaReservation bool, updateWl func(newWL *kueue.Workload), matcher gomega.OmegaMatcher) { + func(w func() *kueue.Workload, setQuotaReservation bool, updateWl func(newWL *kueue.Workload), matcher gomegatypes.GomegaMatcher) { ginkgo.By("Creating a new Workload") workload := w() util.MustCreate(ctx, k8sClient, workload) @@ -1272,7 +1264,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al }) ginkgo.DescribeTable("Validate Workload status on update", - func(setupWlStatus func(w *kueue.Workload), updateWlStatus func(w *kueue.Workload), setupMatcher, updateMatcher gomega.OmegaMatcher) { + func(setupWlStatus func(w *kueue.Workload), updateWlStatus func(w *kueue.Workload), setupMatcher, updateMatcher gomegatypes.GomegaMatcher) { ginkgo.By("Creating a new Workload") workload := utiltestingapi.MakeWorkloadWithGeneratedName(workloadName, ns.Name).Obj() util.MustCreate(ctx, k8sClient, workload) @@ -1382,7 +1374,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher In }) ginkgo.DescribeTable("Validate Workload status on update", - func(setupWlStatus func(w *kueue.Workload), updateWlStatus func(w *kueue.Workload), setupMatcher, updateMatcher gomega.OmegaMatcher) { + func(setupWlStatus func(w *kueue.Workload), updateWlStatus func(w *kueue.Workload), setupMatcher, updateMatcher gomegatypes.GomegaMatcher) { ginkgo.By("Creating a new Workload") workload := utiltestingapi.MakeWorkloadWithGeneratedName(workloadName, ns.Name).Obj() util.MustCreate(ctx, k8sClient, workload) From c759957ebf33ec39f122d847b423c6209860d95e Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 6 Nov 2025 16:22:56 +0530 Subject: [PATCH 102/119] Use ExpectWorkloadsWithWorkloadPriority and ExpectWorkloadsWithPodPriority. (#7548) --- test/e2e/singlecluster/e2e_test.go | 12 ++- .../e2e/singlecluster/leaderworkerset_test.go | 38 +++++---- test/e2e/singlecluster/statefulset_test.go | 14 ++-- .../appwrapper/appwrapper_controller_test.go | 6 +- .../jobs/job/job_controller_test.go | 84 +++++++++++-------- .../jobs/jobset/jobset_controller_test.go | 6 +- .../controller/jobs/kubeflow/kubeflowjob.go | 7 +- .../jobs/mpijob/mpijob_controller_test.go | 6 +- .../raycluster/raycluster_controller_test.go | 6 +- .../jobs/rayjob/rayjob_controller_test.go | 6 +- test/util/util.go | 30 +++++-- 11 files changed, 127 insertions(+), 88 deletions(-) diff --git a/test/e2e/singlecluster/e2e_test.go b/test/e2e/singlecluster/e2e_test.go index 563e34e8faa..c673cb2e544 100644 --- a/test/e2e/singlecluster/e2e_test.go +++ b/test/e2e/singlecluster/e2e_test.go @@ -613,19 +613,25 @@ var _ = ginkgo.Describe("Kueue", func() { util.MustCreate(ctx, k8sClient, sampleJob) }) - ginkgo.By("Verify workload is created and not admitted", func() { + ginkgo.By("Verify job is created and suspended", func() { createdJob := &batchv1.Job{} jobKey = client.ObjectKeyFromObject(sampleJob) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, jobKey, createdJob)).Should(gomega.Succeed()) g.Expect(*createdJob.Spec.Suspend).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(sampleJob.Name, sampleJob.UID), Namespace: ns.Name} + + ginkgo.By("Verify workload is created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, samplePriorityClass.Name, samplePriorityClass.Value, wlLookupKey) + }) - wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(sampleJob.Name, sampleJob.UID), Namespace: ns.Name} + ginkgo.By("Verify workload is admitted", func() { createdWorkload := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(samplePriority)) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeFalse()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/e2e/singlecluster/leaderworkerset_test.go b/test/e2e/singlecluster/leaderworkerset_test.go index 091db4951b8..d740860d14c 100644 --- a/test/e2e/singlecluster/leaderworkerset_test.go +++ b/test/e2e/singlecluster/leaderworkerset_test.go @@ -28,7 +28,6 @@ import ( leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/constants" ctrlconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/leaderworkerset" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" @@ -737,11 +736,14 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { Name: leaderworkerset.GetWorkloadName(lowPriorityLWS.UID, lowPriorityLWS.Name, "0"), Namespace: ns.Name, } - ginkgo.By("Check the low priority Workload is created and admitted", func() { + + ginkgo.By("Verify workload is created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, lowPriorityWPC.Name, lowPriorityWPC.Value, lowPriorityWlLookupKey) + }) + + ginkgo.By("Check the low priority Workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowPriorityWlLookupKey, createdLowPriorityWl)).To(gomega.Succeed()) - g.Expect(createdLowPriorityWl.Spec.PriorityClassSource).To(gomega.Equal(constants.WorkloadPriorityClassSource)) - g.Expect(createdLowPriorityWl.Spec.PriorityClassName).To(gomega.Equal(lowPriorityWPC.Name)) g.Expect(createdLowPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -783,12 +785,15 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { Name: leaderworkerset.GetWorkloadName(highPriorityLWS.UID, highPriorityLWS.Name, "0"), Namespace: ns.Name, } + + ginkgo.By("Verify the high priority Workload created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, highPriorityWPC.Name, highPriorityWPC.Value, highPriorityWlLookupKey) + }) + ginkgo.By("Await for the high priority Workload to be admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, highPriorityWlLookupKey, createdHighPriorityWl)).To(gomega.Succeed()) g.Expect(createdHighPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) - g.Expect(createdHighPriorityWl.Spec.PriorityClassSource).To(gomega.Equal(constants.WorkloadPriorityClassSource)) - g.Expect(createdHighPriorityWl.Spec.PriorityClassName).To(gomega.Equal(highPriorityLWS.Name)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -835,11 +840,14 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { Name: leaderworkerset.GetWorkloadName(lowPriorityLWS.UID, lowPriorityLWS.Name, "0"), Namespace: ns.Name, } - ginkgo.By("Check the low priority Workload is created and admitted", func() { + + ginkgo.By("Verify the low priority Workload created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, lowPriorityWPC.Name, lowPriorityWPC.Value, lowPriorityWlLookupKey) + }) + + ginkgo.By("Check the low priority Workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowPriorityWlLookupKey, createdLowPriorityWl)).To(gomega.Succeed()) - g.Expect(createdLowPriorityWl.Spec.PriorityClassSource).To(gomega.Equal(constants.WorkloadPriorityClassSource)) - g.Expect(createdLowPriorityWl.Spec.PriorityClassName).To(gomega.Equal(lowPriorityWPC.Name)) g.Expect(createdLowPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -863,11 +871,8 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { Namespace: ns.Name, } - ginkgo.By("Check the new Workload is created with low priority", func() { - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, updatablePriorityWlLookupKey, createdUpdatablePriorityWl)).To(gomega.Succeed()) - g.Expect(createdUpdatablePriorityWl.Spec.PriorityClassName).To(gomega.Equal(lowPriorityWPC.Name)) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) + ginkgo.By("Verify another low priority Workload created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, lowPriorityWPC.Name, lowPriorityWPC.Value, updatablePriorityWlLookupKey) }) ginkgo.By("Updating priority of second LeaderWorkerSet to high-priority", func() { @@ -880,10 +885,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { }) ginkgo.By("Checking the Workload priority is updated", func() { - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, updatablePriorityWlLookupKey, createdUpdatablePriorityWl)).To(gomega.Succeed()) - g.Expect(createdUpdatablePriorityWl.Spec.PriorityClassName).To(gomega.Equal(highPriorityWPC.Name)) - }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, highPriorityWPC.Name, highPriorityWPC.Value, updatablePriorityWlLookupKey) }) ginkgo.By("Await for the low-priority LeaderWorkerSet to be preempted", func() { diff --git a/test/e2e/singlecluster/statefulset_test.go b/test/e2e/singlecluster/statefulset_test.go index 4a72d7df116..d2d24c3c81b 100644 --- a/test/e2e/singlecluster/statefulset_test.go +++ b/test/e2e/singlecluster/statefulset_test.go @@ -28,7 +28,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/constants" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/statefulset" "sigs.k8s.io/kueue/pkg/util/testing" @@ -482,11 +481,14 @@ var _ = ginkgo.Describe("StatefulSet integration", func() { Name: statefulset.GetWorkloadName(lowPrioritySTS.Name), Namespace: ns.Name, } + + ginkgo.By("Verify the low-priority Workload created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, lowPriorityWPC.Name, lowPriorityWPC.Value, lowPriorityWlKey) + }) + ginkgo.By("Check the low-priority Workload is created and admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowPriorityWlKey, createdLowPriorityWl)).To(gomega.Succeed()) - g.Expect(createdLowPriorityWl.Spec.PriorityClassSource).To(gomega.Equal(constants.WorkloadPriorityClassSource)) - g.Expect(createdLowPriorityWl.Spec.PriorityClassName).To(gomega.Equal(lowPriorityWPC.Name)) g.Expect(createdLowPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -525,11 +527,13 @@ var _ = ginkgo.Describe("StatefulSet integration", func() { Name: statefulset.GetWorkloadName(highPrioritySTS.Name), Namespace: ns.Name, } + ginkgo.By("Verify the high-priority Workload created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, highPriorityWPC.Name, highPriorityWPC.Value, highPriorityWlKey) + }) + ginkgo.By("Await for the high-priority Workload to be admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, highPriorityWlKey, createdHighPriorityWl)).To(gomega.Succeed()) - g.Expect(createdHighPriorityWl.Spec.PriorityClassSource).To(gomega.Equal(constants.WorkloadPriorityClassSource)) - g.Expect(createdHighPriorityWl.Spec.PriorityClassName).To(gomega.Equal(highPriorityWPC.Name)) g.Expect(createdHighPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go index 785bdc88e97..287e04d6854 100644 --- a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go @@ -142,9 +142,9 @@ var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.Continue gomega.Expect(createdWorkload.Spec.QueueName).Should(gomega.Equal(kueue.LocalQueueName("")), "The Workload shouldn't have .spec.queueName set") gomega.Expect(metav1.IsControlledBy(createdWorkload, createdAppWrapper)).To(gomega.BeTrue(), "The Workload should be owned by the AppWrapper") - ginkgo.By("checking the workload is created with priority and priorityName") - gomega.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) - gomega.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(priorityValue)) + ginkgo.By("checking the workload is created with workload priority class", func() { + util.ExpectWorkloadsWithPodPriority(ctx, k8sClient, priorityClassName, priorityValue, wlLookupKey) + }) ginkgo.By("checking the workload is updated with queue name when the AppWrapper does") awQueueName := "test-queue" diff --git a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go index d50563d4041..94f7d7f06bf 100644 --- a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go @@ -57,15 +57,14 @@ import ( ) const ( - parallelism = 4 - jobName = "test-job" - instanceKey = "cloud.provider.com/instance" - priorityClassName = "test-priority-class" - priorityValue = 10 - highPriorityClassName = "high-priority-class" - highPriorityValue = 20 - parentJobName = jobName + "-parent" - childJobName = jobName + "-child" + parallelism = 4 + jobName = "test-job" + instanceKey = "cloud.provider.com/instance" + priorityClassName = "test-priority-class" + priorityValue = 10 + highPriorityValue = 20 + parentJobName = jobName + "-parent" + childJobName = jobName + "-child" ) var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { @@ -132,10 +131,13 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu createdTime := createdWorkload.CreationTimestamp - ginkgo.By("checking the workload is created with priority, priorityName, and ProvisioningRequest annotations") - gomega.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) - gomega.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(int32(priorityValue))) - gomega.Expect(createdWorkload.Annotations).Should(gomega.Equal(map[string]string{"provreq.kueue.x-k8s.io/ValidUntilSeconds": "0"})) + ginkgo.By("checking the workload is created with workload priority", func() { + util.ExpectWorkloadsWithPodPriority(ctx, k8sClient, priorityClassName, priorityValue, wlLookupKey) + }) + + ginkgo.By("checking the workload is created with ProvisioningRequest annotations", func() { + gomega.Expect(createdWorkload.Annotations).Should(gomega.Equal(map[string]string{"provreq.kueue.x-k8s.io/ValidUntilSeconds": "0"})) + }) gomega.Expect(createdWorkload.Labels["toCopyKey"]).Should(gomega.Equal("toCopyValue")) gomega.Expect(createdWorkload.Labels).ShouldNot(gomega.ContainElement("doNotCopyValue")) @@ -1365,11 +1367,13 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con createdWorkload1 := &kueue.Workload{} wlLookupKey1 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job1.Name, job1.UID), Namespace: ns.Name} - ginkgo.By("checking that the first workload is created with a priority and admitted", func() { + ginkgo.By("verify the first workload is created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, highWorkloadPriorityClass.Name, highWorkloadPriorityClass.Value, wlLookupKey1) + }) + + ginkgo.By("checking that the first workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey1, createdWorkload1)).Should(gomega.Succeed()) - g.Expect(createdWorkload1.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) - g.Expect(*createdWorkload1.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) g.Expect(createdWorkload1.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1386,7 +1390,11 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con createdWorkload2 := &kueue.Workload{} wlLookupKey2 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job2.Name, job2.UID), Namespace: ns.Name} - ginkgo.By("checking that the second workload is created with a priority and admitted", func() { + ginkgo.By("verify the second workload is created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, highWorkloadPriorityClass.Name, highWorkloadPriorityClass.Value, wlLookupKey2) + }) + + ginkgo.By("checking that the second workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) @@ -1407,11 +1415,13 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con createdWorkload3 := &kueue.Workload{} wlLookupKey3 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job3.Name, job3.UID), Namespace: ns.Name} - ginkgo.By("checking that the third workload is created with a priority and not admitted", func() { + ginkgo.By("verify the third workload is created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, highWorkloadPriorityClass.Name, highWorkloadPriorityClass.Value, wlLookupKey3) + }) + + ginkgo.By("checking that the third workload is not admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) - g.Expect(createdWorkload3.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) - g.Expect(*createdWorkload3.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1426,11 +1436,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con }) ginkgo.By("checking that the second workload’s WorkloadPriorityClass is updated when the job’s priority label changes", func() { - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) - g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(lowWorkloadPriorityClass.Name)) - g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(lowWorkloadPriorityClass.Value)) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, lowWorkloadPriorityClass.Name, lowWorkloadPriorityClass.Value, wlLookupKey2) }) ginkgo.By("checking that the third workload is admitted", func() { @@ -1462,11 +1468,13 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con createdWorkload1 := &kueue.Workload{} wlLookupKey1 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job1.Name, job1.UID), Namespace: ns.Name} - ginkgo.By("checking that the first workload is created with a priority and admitted", func() { + ginkgo.By("verify the first workload is created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, highWorkloadPriorityClass.Name, highWorkloadPriorityClass.Value, wlLookupKey1) + }) + + ginkgo.By("checking that the first workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey1, createdWorkload1)).Should(gomega.Succeed()) - g.Expect(createdWorkload1.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) - g.Expect(*createdWorkload1.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) g.Expect(createdWorkload1.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1483,7 +1491,11 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con createdWorkload2 := &kueue.Workload{} wlLookupKey2 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job2.Name, job2.UID), Namespace: ns.Name} - ginkgo.By("checking that the second workload is created with a priority and admitted", func() { + ginkgo.By("verify the second workload is created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, highWorkloadPriorityClass.Name, highWorkloadPriorityClass.Value, wlLookupKey2) + }) + + ginkgo.By("checking that the second workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) @@ -1504,11 +1516,13 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con createdWorkload3 := &kueue.Workload{} wlLookupKey3 := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job3.Name, job3.UID), Namespace: ns.Name} - ginkgo.By("checking that the third workload is created with a priority and not admitted", func() { + ginkgo.By("verify the third workload is created with workload priority class", func() { + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, lowWorkloadPriorityClass.Name, lowWorkloadPriorityClass.Value, wlLookupKey3) + }) + + ginkgo.By("checking that the third workload is not admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) - g.Expect(createdWorkload3.Spec.PriorityClassName).Should(gomega.Equal(lowWorkloadPriorityClass.Name)) - g.Expect(*createdWorkload3.Spec.Priority).Should(gomega.Equal(lowWorkloadPriorityClass.Value)) g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1523,11 +1537,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con }) ginkgo.By("checking that the second workload’s WorkloadPriorityClass is updated when the job’s priority label changes", func() { - gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) - g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(lowWorkloadPriorityClass.Name)) - g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(lowWorkloadPriorityClass.Value)) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) + util.ExpectWorkloadsWithWorkloadPriority(ctx, k8sClient, lowWorkloadPriorityClass.Name, lowWorkloadPriorityClass.Value, wlLookupKey2) }) ginkgo.By("checking that the third workload is not admitted and the second workload is not evicted", func() { diff --git a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go index ccb16d6c734..921bf9edeae 100644 --- a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go @@ -146,9 +146,9 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa gomega.Expect(createdWorkload.Spec.QueueName).Should(gomega.Equal(kueue.LocalQueueName("")), "The Workload shouldn't have .spec.queueName set") gomega.Expect(metav1.IsControlledBy(createdWorkload, createdJobSet)).To(gomega.BeTrue(), "The Workload should be owned by the JobSet") - ginkgo.By("checking the workload is created with priority and priorityName") - gomega.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) - gomega.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(priorityValue)) + ginkgo.By("checking the workload is created with workload priority class", func() { + util.ExpectWorkloadsWithPodPriority(ctx, k8sClient, priorityClassName, priorityValue, wlLookupKey) + }) ginkgo.By("checking the workload is updated with queue name when the JobSet does") var jobSetQueueName kueue.LocalQueueName = "test-queue" diff --git a/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go b/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go index 859cdf7f0c0..4638c4c2b8a 100644 --- a/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go +++ b/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go @@ -83,12 +83,11 @@ func ShouldReconcileJob(ctx context.Context, k8sClient client.Client, job, creat ginkgo.By("checking the workload is created without queue assigned") createdWorkload := util.AwaitAndVerifyCreatedWorkload(ctx, k8sClient, wlLookupKey, createdJob.Object()) - util.VerifyWorkloadPriority(createdWorkload, priorityClassName, priorityValue) gomega.Expect(createdWorkload.Spec.QueueName).Should(gomega.Equal(kueue.LocalQueueName("")), "The Workload shouldn't have .spec.queueName set") - ginkgo.By("checking the workload is created with priority and priorityName") - gomega.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) - gomega.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(int32(priorityValue))) + ginkgo.By("checking the workload is created with workload priority class", func() { + util.ExpectWorkloadsWithPodPriority(ctx, k8sClient, priorityClassName, priorityValue, wlLookupKey) + }) ginkgo.By("checking the workload is updated with queue name when the job does") createdJob.Object().SetLabels(map[string]string{constants.QueueLabel: string(jobQueueName)}) diff --git a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go index c016a46b056..dbc7d526a18 100644 --- a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go @@ -104,9 +104,9 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Expect(createdWorkload.Spec.QueueName).Should(gomega.Equal(kueue.LocalQueueName("")), "The Workload shouldn't have .spec.queueName set") gomega.Expect(metav1.IsControlledBy(createdWorkload, createdJob)).To(gomega.BeTrue(), "The Workload should be owned by the Job") - ginkgo.By("checking the workload is created with priority and priorityName") - gomega.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) - gomega.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(int32(priorityValue))) + ginkgo.By("checking the workload is created with workload priority class", func() { + util.ExpectWorkloadsWithPodPriority(ctx, k8sClient, priorityClassName, priorityValue, wlLookupKey) + }) ginkgo.By("checking the workload is updated with queue name when the job does") var jobQueueName kueue.LocalQueueName = "test-queue" diff --git a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go index 4348d7e92d0..be28e4fe9ba 100644 --- a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go @@ -107,9 +107,9 @@ var _ = ginkgo.Describe("RayCluster controller", ginkgo.Ordered, ginkgo.Continue gomega.Expect(createdWorkload.Spec.QueueName).Should(gomega.Equal(kueue.LocalQueueName("")), "The Workload shouldn't have .spec.queueName set") gomega.Expect(metav1.IsControlledBy(createdWorkload, createdJob)).To(gomega.BeTrue(), "The Workload should be owned by the Job") - ginkgo.By("checking the workload is created with priority and priorityName") - gomega.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) - gomega.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(priorityValue)) + ginkgo.By("checking the workload is created with workload priority class", func() { + util.ExpectWorkloadsWithPodPriority(ctx, k8sClient, priorityClassName, priorityValue, wlLookupKey) + }) ginkgo.By("checking the workload is updated with queue name when the job does") var jobQueueName kueue.LocalQueueName = "test-queue" diff --git a/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go b/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go index ef321691037..526f43f53da 100644 --- a/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go @@ -112,9 +112,9 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Expect(createdWorkload.Spec.QueueName).Should(gomega.Equal(kueue.LocalQueueName("")), "The Workload shouldn't have .spec.queueName set") gomega.Expect(metav1.IsControlledBy(createdWorkload, createdJob)).To(gomega.BeTrue(), "The Workload should be owned by the Job") - ginkgo.By("checking the workload is created with priority and priorityName") - gomega.Expect(createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) - gomega.Expect(*createdWorkload.Spec.Priority).Should(gomega.Equal(priorityValue)) + ginkgo.By("checking the workload is created with workload priority class", func() { + util.ExpectWorkloadsWithPodPriority(ctx, k8sClient, priorityClassName, priorityValue, wlLookupKey) + }) ginkgo.By("checking the workload is updated with queue name when the job does") var jobQueueName kueue.LocalQueueName = "test-queue" diff --git a/test/util/util.go b/test/util/util.go index 06d1cc22130..68ab01fb0d2 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -67,6 +67,7 @@ import ( leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/pkg/constants" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/metrics" "sigs.k8s.io/kueue/pkg/scheduler/preemption" @@ -367,6 +368,29 @@ func expectWorkloadsToBeAdmittedCountWithOffset(ctx context.Context, offset int, }, Timeout, Interval).Should(gomega.Succeed()) } +func ExpectWorkloadsWithWorkloadPriority(ctx context.Context, c client.Client, name string, value int32, wlKeys ...client.ObjectKey) { + ginkgo.GinkgoHelper() + expectWorkloadsWithPriority(ctx, c, constants.WorkloadPriorityClassSource, name, value, wlKeys...) +} + +func ExpectWorkloadsWithPodPriority(ctx context.Context, c client.Client, name string, value int32, wlKeys ...client.ObjectKey) { + ginkgo.GinkgoHelper() + expectWorkloadsWithPriority(ctx, c, constants.PodPriorityClassSource, name, value, wlKeys...) +} + +func expectWorkloadsWithPriority(ctx context.Context, c client.Client, priorityClassSource, name string, value int32, wlKeys ...client.ObjectKey) { + ginkgo.GinkgoHelper() + createdWl := &kueue.Workload{} + gomega.Eventually(func(g gomega.Gomega) { + for _, wlKey := range wlKeys { + g.Expect(c.Get(ctx, wlKey, createdWl)).To(gomega.Succeed()) + g.Expect(createdWl.Spec.PriorityClassSource).To(gomega.Equal(priorityClassSource)) + g.Expect(createdWl.Spec.PriorityClassName).To(gomega.Equal(name)) + g.Expect(createdWl.Spec.Priority).To(gomega.Equal(&value)) + } + }, Timeout, Interval).Should(gomega.Succeed()) +} + func ExpectWorkloadToFinish(ctx context.Context, k8sClient client.Client, wlKey client.ObjectKey) { gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { var wl kueue.Workload @@ -890,12 +914,6 @@ func AwaitAndVerifyCreatedWorkload(ctx context.Context, client client.Client, wl return createdWorkload } -func VerifyWorkloadPriority(createdWorkload *kueue.Workload, priorityClassName string, priorityValue int32) { - ginkgo.By("checking the workload is created with priority and priorityName") - gomega.ExpectWithOffset(1, createdWorkload.Spec.PriorityClassName).Should(gomega.Equal(priorityClassName)) - gomega.ExpectWithOffset(1, *createdWorkload.Spec.Priority).Should(gomega.Equal(priorityValue)) -} - func SetPodsPhase(ctx context.Context, k8sClient client.Client, phase corev1.PodPhase, pods ...*corev1.Pod) { for _, p := range pods { updatedPod := corev1.Pod{} From f8df1e434090b7b4c9001a867b13b304ae7ab207 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Thu, 6 Nov 2025 17:44:54 +0530 Subject: [PATCH 103/119] Fix test case to check creation workload with empty priorityClassName. (#7553) --- test/integration/singlecluster/webhook/core/workload_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/singlecluster/webhook/core/workload_test.go b/test/integration/singlecluster/webhook/core/workload_test.go index 045dca16ae2..632c6be0e85 100644 --- a/test/integration/singlecluster/webhook/core/workload_test.go +++ b/test/integration/singlecluster/webhook/core/workload_test.go @@ -181,8 +181,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { testing.BeInvalidError()), ginkgo.Entry("empty priorityClassName is valid", func() *kueue.Workload { - return utiltestingapi.MakeWorkload(workloadName, ns.Name). - Obj() + return utiltestingapi.MakeWorkload(workloadName, ns.Name).Priority(0).Obj() }, gomega.Succeed()), ginkgo.Entry("priority should not be nil when priorityClassName is set", From e1a8d180b7ee0b850eab5054eb031519d43cd1ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Thu, 6 Nov 2025 13:44:55 +0100 Subject: [PATCH 104/119] Update wrappers to use utiltesting alias (#7561) --- pkg/util/testingjobs/appwrapper/wrappers.go | 4 ++-- pkg/util/testingjobs/job/wrappers.go | 4 ++-- pkg/util/testingjobs/jobset/wrappers.go | 4 ++-- pkg/util/testingjobs/mpijob/wrappers_mpijob.go | 4 ++-- pkg/util/testingjobs/pod/wrappers.go | 4 ++-- pkg/util/testingjobs/raycluster/wrappers.go | 4 ++-- pkg/util/testingjobs/rayjob/wrappers.go | 4 ++-- pkg/util/testingjobs/statefulset/wrappers.go | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/util/testingjobs/appwrapper/wrappers.go b/pkg/util/testingjobs/appwrapper/wrappers.go index 97da447d2c9..edc51cd28cf 100644 --- a/pkg/util/testingjobs/appwrapper/wrappers.go +++ b/pkg/util/testingjobs/appwrapper/wrappers.go @@ -29,7 +29,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/kueue/pkg/controller/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) // AppWrapperWrapper wraps an AppWrapper. @@ -98,7 +98,7 @@ func (aw *AppWrapperWrapper) UID(uid string) *AppWrapperWrapper { // OwnerReference adds a ownerReference to the default container. func (aw *AppWrapperWrapper) OwnerReference(ownerName string, ownerGVK schema.GroupVersionKind) *AppWrapperWrapper { - testing.AppendOwnerReference(&aw.AppWrapper, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) + utiltesting.AppendOwnerReference(&aw.AppWrapper, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) return aw } diff --git a/pkg/util/testingjobs/job/wrappers.go b/pkg/util/testingjobs/job/wrappers.go index 403dd5c57cf..5c0acf10b51 100644 --- a/pkg/util/testingjobs/job/wrappers.go +++ b/pkg/util/testingjobs/job/wrappers.go @@ -29,7 +29,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) // JobWrapper wraps a Job. @@ -212,7 +212,7 @@ func (j *JobWrapper) Image(image string, args []string) *JobWrapper { // OwnerReference adds a ownerReference to the default container. func (j *JobWrapper) OwnerReference(ownerName string, ownerGVK schema.GroupVersionKind) *JobWrapper { - testing.AppendOwnerReference(&j.Job, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) + utiltesting.AppendOwnerReference(&j.Job, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) return j } diff --git a/pkg/util/testingjobs/jobset/wrappers.go b/pkg/util/testingjobs/jobset/wrappers.go index 22f61add1f7..8bcf1918015 100644 --- a/pkg/util/testingjobs/jobset/wrappers.go +++ b/pkg/util/testingjobs/jobset/wrappers.go @@ -28,7 +28,7 @@ import ( jobsetutil "sigs.k8s.io/jobset/pkg/util/testing" "sigs.k8s.io/kueue/pkg/controller/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) // JobSetWrapper wraps a JobSet. @@ -206,6 +206,6 @@ func (j *JobSetWrapper) ManagedBy(c string) *JobSetWrapper { // OwnerReference adds a ownerReference to the default container. func (j *JobSetWrapper) OwnerReference(ownerName string, ownerGVK schema.GroupVersionKind) *JobSetWrapper { - testing.AppendOwnerReference(j, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) + utiltesting.AppendOwnerReference(j, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) return j } diff --git a/pkg/util/testingjobs/mpijob/wrappers_mpijob.go b/pkg/util/testingjobs/mpijob/wrappers_mpijob.go index 17d2d6f326c..9732d534d2a 100644 --- a/pkg/util/testingjobs/mpijob/wrappers_mpijob.go +++ b/pkg/util/testingjobs/mpijob/wrappers_mpijob.go @@ -26,7 +26,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/kueue/pkg/controller/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) // MPIJobWrapper wraps a Job. @@ -204,7 +204,7 @@ func (j *MPIJobWrapper) UID(uid string) *MPIJobWrapper { // OwnerReference adds a ownerReference to the default container. func (j *MPIJobWrapper) OwnerReference(ownerName string, ownerGVK schema.GroupVersionKind) *MPIJobWrapper { - testing.AppendOwnerReference(&j.MPIJob, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) + utiltesting.AppendOwnerReference(&j.MPIJob, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) return j } diff --git a/pkg/util/testingjobs/pod/wrappers.go b/pkg/util/testingjobs/pod/wrappers.go index 4610bc726ab..28fb6169188 100644 --- a/pkg/util/testingjobs/pod/wrappers.go +++ b/pkg/util/testingjobs/pod/wrappers.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) // PodWrapper wraps a Pod. @@ -256,7 +256,7 @@ func (p *PodWrapper) Limit(r corev1.ResourceName, v string) *PodWrapper { // OwnerReference adds a ownerReference to the default container. func (p *PodWrapper) OwnerReference(ownerName string, ownerGVK schema.GroupVersionKind) *PodWrapper { - testing.AppendOwnerReference(&p.Pod, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) + utiltesting.AppendOwnerReference(&p.Pod, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) return p } diff --git a/pkg/util/testingjobs/raycluster/wrappers.go b/pkg/util/testingjobs/raycluster/wrappers.go index 85bf74c309d..af9a9b91610 100644 --- a/pkg/util/testingjobs/raycluster/wrappers.go +++ b/pkg/util/testingjobs/raycluster/wrappers.go @@ -24,7 +24,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/kueue/pkg/controller/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) // ClusterWrapper wraps a RayCluster. @@ -39,7 +39,7 @@ func MakeCluster(name, ns string) *ClusterWrapper { Annotations: make(map[string]string, 1), }, Spec: rayv1.RayClusterSpec{ - RayVersion: testing.TestRayVersion(), + RayVersion: utiltesting.TestRayVersion(), HeadGroupSpec: rayv1.HeadGroupSpec{ RayStartParams: map[string]string{}, Template: corev1.PodTemplateSpec{ diff --git a/pkg/util/testingjobs/rayjob/wrappers.go b/pkg/util/testingjobs/rayjob/wrappers.go index 2d552a9f408..a4dd55fe89d 100644 --- a/pkg/util/testingjobs/rayjob/wrappers.go +++ b/pkg/util/testingjobs/rayjob/wrappers.go @@ -24,7 +24,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/kueue/pkg/controller/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) // JobWrapper wraps a RayJob. @@ -41,7 +41,7 @@ func MakeJob(name, ns string) *JobWrapper { Spec: rayv1.RayJobSpec{ ShutdownAfterJobFinishes: true, RayClusterSpec: &rayv1.RayClusterSpec{ - RayVersion: testing.TestRayVersion(), + RayVersion: utiltesting.TestRayVersion(), HeadGroupSpec: rayv1.HeadGroupSpec{ RayStartParams: map[string]string{}, Template: corev1.PodTemplateSpec{ diff --git a/pkg/util/testingjobs/statefulset/wrappers.go b/pkg/util/testingjobs/statefulset/wrappers.go index 869ac204d30..5386efd9fcf 100644 --- a/pkg/util/testingjobs/statefulset/wrappers.go +++ b/pkg/util/testingjobs/statefulset/wrappers.go @@ -32,7 +32,7 @@ import ( controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" ) // StatefulSetWrapper wraps a StatefulSet. @@ -108,7 +108,7 @@ func (ss *StatefulSetWrapper) UID(uid string) *StatefulSetWrapper { // OwnerReference adds a ownerReference to the StatefulSet. func (ss *StatefulSetWrapper) OwnerReference(ownerName string, ownerGVK schema.GroupVersionKind) *StatefulSetWrapper { - testing.AppendOwnerReference(&ss.StatefulSet, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) + utiltesting.AppendOwnerReference(&ss.StatefulSet, ownerGVK, ownerName, ownerName, ptr.To(true), ptr.To(true)) return ss } From fcf5ea328b3b84795d6efcfa3072794f971ae05b Mon Sep 17 00:00:00 2001 From: Yuki Iwai Date: Thu, 6 Nov 2025 23:16:55 +0900 Subject: [PATCH 105/119] Add CHANGELOG for v0.13.9 (#7562) Signed-off-by: Yuki Iwai --- CHANGELOG/CHANGELOG-0.13.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG/CHANGELOG-0.13.md b/CHANGELOG/CHANGELOG-0.13.md index c6c83eba315..492e76204fb 100644 --- a/CHANGELOG/CHANGELOG-0.13.md +++ b/CHANGELOG/CHANGELOG-0.13.md @@ -1,3 +1,20 @@ +## v0.13.9 + +Changes since `v0.13.8`: + +## Changes by Kind + +### Feature + +- `ReclaimablePods` feature gate is introduced to enable users switching on and off the reclaimable Pods feature (#7536, @PBundyra) + +### Bug or Regression + +- Fix eviction of jobs with memory requests in decimal format (#7557, @brejman) +- Fix the bug for the StatefulSet integration that the scale up could get stuck if + triggered immediately after scale down to zero. (#7499, @IrvingMg) +- MultiKueue: Remove remoteClient from clusterReconciler when kubeconfig is detected as invalid or insecure, preventing workloads from being admitted to misconfigured clusters. (#7516, @mszadkow) + ## v0.13.8 Changes since `v0.13.7`: From e8c47b8214b6a72f6e8ff335f9c8f11910e616af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Thu, 6 Nov 2025 15:17:03 +0100 Subject: [PATCH 106/119] Update intergation tests to use utiltesting alias (#7563) --- .../provisioning/provisioning_test.go | 24 +- .../core/clusterqueue_controller_test.go | 4 +- .../core/localqueue_controller_test.go | 4 +- .../core/workload_controller_test.go | 6 +- .../appwrapper/appwrapper_controller_test.go | 16 +- .../jobs/jaxjob/jaxjob_controller_test.go | 18 +- .../jobs/job/job_controller_test.go | 56 ++--- .../jobs/jobset/jobset_controller_test.go | 34 +-- .../controller/jobs/kubeflow/kubeflowjob.go | 14 +- .../jobs/mpijob/mpijob_controller_test.go | 34 +-- .../paddlejob/paddlejob_controller_test.go | 20 +- .../jobs/pod/pod_controller_test.go | 62 ++--- .../pytorchjob/pytorchjob_controller_test.go | 22 +- .../raycluster/raycluster_controller_test.go | 24 +- .../jobs/rayjob/rayjob_controller_test.go | 28 +-- .../jobs/tfjob/tfjob_controller_test.go | 26 +- .../jobs/trainjob/trainjob_controller_test.go | 28 +-- .../xgboostjob/xgboostjob_controller_test.go | 20 +- .../singlecluster/kueuectl/create_test.go | 4 +- .../kueuectl/passthrough_test.go | 4 +- .../singlecluster/scheduler/scheduler_test.go | 6 +- .../integration/singlecluster/tas/tas_test.go | 228 +++++++++--------- .../webhook/core/admissioncheck_test.go | 20 +- .../webhook/core/clusterqueue_test.go | 64 ++--- .../singlecluster/webhook/core/cohort_test.go | 48 ++-- .../webhook/core/localqueue_test.go | 6 +- .../webhook/core/resourceflavor_test.go | 12 +- .../webhook/core/workload_test.go | 102 ++++---- .../webhook/jobs/appwrapper_webhook_test.go | 4 +- .../webhook/jobs/deployment_webhook_test.go | 6 +- .../webhook/jobs/job_webhook_test.go | 6 +- .../webhook/jobs/jobset_webhook_test.go | 10 +- .../webhook/jobs/mpijob_webhook_test.go | 4 +- .../webhook/jobs/paddlejob_webhook_test.go | 4 +- .../webhook/jobs/pytorchjob_webhook_test.go | 4 +- .../webhook/jobs/raycluster_webhook_test.go | 4 +- .../webhook/jobs/rayjob_webhook_test.go | 10 +- .../webhook/jobs/tfjob_webhook_test.go | 4 +- .../webhook/jobs/xgboostjob_webhook_test.go | 4 +- 39 files changed, 497 insertions(+), 497 deletions(-) diff --git a/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go b/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go index 40de32aa317..cc2af7ede00 100644 --- a/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go +++ b/test/integration/singlecluster/controller/admissionchecks/provisioning/provisioning_test.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/admissionchecks/provisioning" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/admissioncheck" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workload" @@ -171,7 +171,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure ginkgo.It("Should not create provisioning requests before quota is reserved", framework.SlowSpec, func() { ginkgo.By("Checking no provision request is created", func() { gomega.Consistently(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(utiltesting.BeNotFoundError()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) }) @@ -347,7 +347,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure cmpopts.IgnoreFields(kueue.AdmissionCheckState{}, "LastTransitionTime", "PodSetUpdates")))) g.Expect(workload.IsActive(&updatedWl)).To(gomega.BeFalse()) - ok, err := testing.HasEventAppeared(ctx, k8sClient, corev1.Event{ + ok, err := utiltesting.HasEventAppeared(ctx, k8sClient, corev1.Event{ Reason: "AdmissionCheckRejected", Type: corev1.EventTypeWarning, Message: fmt.Sprintf("Deactivating workload because AdmissionCheck for %v was Rejected: ", ac.Name), @@ -408,7 +408,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure cmpopts.IgnoreFields(kueue.AdmissionCheckState{}, "LastTransitionTime", "PodSetUpdates")))) g.Expect(workload.IsActive(&updatedWl)).To(gomega.BeFalse()) - ok, err := testing.HasEventAppeared(ctx, k8sClient, corev1.Event{ + ok, err := utiltesting.HasEventAppeared(ctx, k8sClient, corev1.Event{ Reason: "AdmissionCheckRejected", Type: corev1.EventTypeWarning, Message: fmt.Sprintf("Deactivating workload because AdmissionCheck for %v was Rejected: ", ac.Name), @@ -457,7 +457,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure cmpopts.IgnoreFields(kueue.AdmissionCheckState{}, "LastTransitionTime", "PodSetUpdates")))) g.Expect(workload.IsActive(&updatedWl)).To(gomega.BeFalse()) - ok, err := testing.HasEventAppeared(ctx, k8sClient, corev1.Event{ + ok, err := utiltesting.HasEventAppeared(ctx, k8sClient, corev1.Event{ Reason: "AdmissionCheckRejected", Type: corev1.EventTypeWarning, Message: fmt.Sprintf("Deactivating workload because AdmissionCheck for %v was Rejected: ", ac.Name), @@ -677,7 +677,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure ginkgo.By("Checking no provision request is deleted", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -753,7 +753,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure ginkgo.By("Checking provision request is deleted", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -770,7 +770,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure ginkgo.By("Checking the provisioning request remains deleted", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, provReqKey, &createdRequest)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -948,7 +948,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, &updatedWl)).To(gomega.Succeed()) - g.Expect(updatedWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadRequeued)) + g.Expect(updatedWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadRequeued)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1045,7 +1045,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, &updatedWl)).To(gomega.Succeed()) - g.Expect(updatedWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadRequeued)) + g.Expect(updatedWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadRequeued)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1093,7 +1093,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure cmpopts.IgnoreFields(kueue.AdmissionCheckState{}, "LastTransitionTime", "PodSetUpdates")))) g.Expect(workload.IsActive(&updatedWl)).To(gomega.BeFalse()) - ok, err := testing.HasEventAppeared(ctx, k8sClient, corev1.Event{ + ok, err := utiltesting.HasEventAppeared(ctx, k8sClient, corev1.Event{ Reason: "AdmissionCheckRejected", Type: corev1.EventTypeWarning, Message: fmt.Sprintf("Deactivating workload because AdmissionCheck for %v was Rejected: ", ac.Name), @@ -1163,7 +1163,7 @@ var _ = ginkgo.Describe("Provisioning", ginkgo.Ordered, ginkgo.ContinueOnFailure }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, &updatedWl)).To(gomega.Succeed()) - g.Expect(updatedWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadRequeued)) + g.Expect(updatedWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadRequeued)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go index 6e0c6364ec3..872377d1738 100644 --- a/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go +++ b/test/integration/singlecluster/controller/core/clusterqueue_controller_test.go @@ -28,7 +28,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" @@ -1002,7 +1002,7 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin gomega.Eventually(func(g gomega.Gomega) { updatedWl := &kueue.Workload{} g.Expect(k8sClient.Get(ctx, key, updatedWl)).To(gomega.Succeed()) - g.Expect(updatedWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(updatedWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("Delete clusterQueue") diff --git a/test/integration/singlecluster/controller/core/localqueue_controller_test.go b/test/integration/singlecluster/controller/core/localqueue_controller_test.go index 1d33cd7995b..61386465718 100644 --- a/test/integration/singlecluster/controller/core/localqueue_controller_test.go +++ b/test/integration/singlecluster/controller/core/localqueue_controller_test.go @@ -26,7 +26,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/features" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/integration/framework" "sigs.k8s.io/kueue/test/util" @@ -524,7 +524,7 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedClusterQueue)).To(gomega.Succeed()) updatedClusterQueue.Finalizers = nil g.Expect(k8sClient.Update(ctx, &updatedClusterQueue)).To(gomega.Succeed()) - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedClusterQueue)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedClusterQueue)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go index 9d697b5e7bd..c58ca2a2424 100644 --- a/test/integration/singlecluster/controller/core/workload_controller_test.go +++ b/test/integration/singlecluster/controller/core/workload_controller_test.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" "sigs.k8s.io/kueue/pkg/util/slices" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/util" @@ -303,7 +303,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, updatedWl)).To(gomega.Succeed()) g.Expect(workload.IsActive(updatedWl)).To(gomega.BeFalse()) - ok, err := testing.HasEventAppeared(ctx, k8sClient, corev1.Event{ + ok, err := utiltesting.HasEventAppeared(ctx, k8sClient, corev1.Event{ Reason: "AdmissionCheckRejected", Type: corev1.EventTypeWarning, Message: fmt.Sprintf("Deactivating workload because AdmissionCheck for %v was Rejected: %s", "check1", "check rejected"), @@ -384,7 +384,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, updatedWl)).To(gomega.Succeed()) g.Expect(workload.IsActive(updatedWl)).To(gomega.BeFalse()) - ok, err := testing.HasEventAppeared(ctx, k8sClient, corev1.Event{ + ok, err := utiltesting.HasEventAppeared(ctx, k8sClient, corev1.Event{ Reason: "AdmissionCheckRejected", Type: corev1.EventTypeWarning, Message: fmt.Sprintf("Deactivating workload because AdmissionCheck for %v was Rejected: %s", "check1", "check rejected"), diff --git a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go index 287e04d6854..8ed59b7f9c2 100644 --- a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadaw "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingaw "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" utiltestingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" @@ -62,7 +62,7 @@ var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.Continue ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) ginkgo.AfterAll(func() { @@ -111,7 +111,7 @@ var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.Continue ginkgo.It("Should reconcile AppWrappers", func() { ginkgo.By("checking the AppWrapper gets suspended when created unsuspended") - priorityClass := testing.MakePriorityClass(priorityClassName). + priorityClass := utiltesting.MakePriorityClass(priorityClassName). PriorityValue(priorityValue).Obj() util.MustCreate(ctx, k8sClient, priorityClass) @@ -169,7 +169,7 @@ var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.Continue key := types.NamespacedName{Name: secondWl.Name, Namespace: secondWl.Namespace} gomega.Eventually(func(g gomega.Gomega) { wl := &kueue.Workload{} - g.Expect(k8sClient.Get(ctx, key, wl)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, key, wl)).Should(utiltesting.BeNotFoundError()) // check the original wl is still there g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -196,7 +196,7 @@ var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.Continue gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lookupKey, createdAppWrapper)).Should(gomega.Succeed()) g.Expect(createdAppWrapper.Spec.Suspend, false).Should(gomega.BeFalse()) - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Expect(createdAppWrapper.Spec.Components[0].PodSetInfos[0].NodeSelector).Should(gomega.Equal(map[string]string{instanceKey: onDemandFlavor.Name})) @@ -229,7 +229,7 @@ var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.Continue gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: awName, Namespace: aw.Namespace}, createdAppWrapper)).Should(gomega.Succeed()) g.Expect(createdAppWrapper.Spec.Suspend, false).Should(gomega.BeFalse()) - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -299,7 +299,7 @@ var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.Continue ginkgo.By("the workload should stay admitted", func() { gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -548,7 +548,7 @@ var _ = ginkgo.Describe("AppWrapper controller for workloads when only jobs with createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadaw.GetWorkloadNameForAppWrapper(aw.Name, aw.UID), Namespace: ns.Name} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the workload is created when queue name is set") diff --git a/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go b/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go index 3c26c88ffc8..3de5e2c5235 100644 --- a/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjaxjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/jaxjob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjaxjob "sigs.k8s.io/kueue/pkg/util/testingjobs/jaxjob" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" @@ -54,7 +54,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) @@ -125,7 +125,7 @@ var _ = ginkgo.Describe("Job controller for workloads when only jobs with queue createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadjaxjob.GetWorkloadNameForJAXJob(job.Name, job.UID), Namespace: ns.Name} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the workload is created when queue name is set") @@ -589,8 +589,8 @@ var _ = ginkgo.Describe("JAXJob controller with TopologyAwareScheduling", ginkgo nodes = []corev1.Node{ *testingnode.MakeNode("b1r1"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -636,7 +636,7 @@ var _ = ginkgo.Describe("JAXJob controller with TopologyAwareScheduling", ginkgo ReplicaType: kftraining.JAXJobReplicaTypeWorker, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, ). @@ -658,7 +658,7 @@ var _ = ginkgo.Describe("JAXJob controller with TopologyAwareScheduling", ginkgo Name: kueue.NewPodSetReference(string(kftraining.JAXJobReplicaTypeWorker)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultBlockTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultBlockTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -678,13 +678,13 @@ var _ = ginkgo.Describe("JAXJob controller with TopologyAwareScheduling", ginkgo g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(1)) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) diff --git a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go index 94f7d7f06bf..55e6ec82736 100644 --- a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go @@ -46,7 +46,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" @@ -76,7 +76,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")), jobframework.WithLabelKeysToCopy([]string{"toCopyKey"}), )) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) ginkgo.AfterAll(func() { @@ -99,7 +99,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.It("Should reconcile workload and job for all jobs", framework.SlowSpec, func() { ginkgo.By("checking the job gets suspended when created unsuspended") - priorityClass := testing.MakePriorityClass(priorityClassName). + priorityClass := utiltesting.MakePriorityClass(priorityClassName). PriorityValue(int32(priorityValue)).Obj() util.MustCreate(ctx, k8sClient, priorityClass) ginkgo.DeferCleanup(func() { @@ -181,7 +181,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", workload.Key(secondWl)), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", workload.Key(secondWl)), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -211,7 +211,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu g.Expect(createdJob.Spec.Suspend).Should(gomega.Equal(ptr.To(false))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Expect(createdJob.Spec.Template.Spec.NodeSelector).Should(gomega.HaveLen(1)) @@ -238,7 +238,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu g.Expect(createdJob.Spec.Template.Spec.NodeSelector).Should(gomega.BeEmpty()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -294,7 +294,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -323,7 +323,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lookupKey, job)).Should(gomega.Succeed()) g.Expect(job.Spec.Suspend).Should(gomega.Equal(ptr.To(false))) - g.Expect(k8sClient.Get(ctx, childWlLookupKey, childWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, childWlLookupKey, childWorkload)).Should(utiltesting.BeNotFoundError()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -382,7 +382,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu childWorkload := &kueue.Workload{} childWlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(childJob.Name, childJob.UID), Namespace: ns.Name} gomega.Consistently(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, childWlLookupKey, childWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, childWlLookupKey, childWorkload)).Should(utiltesting.BeNotFoundError()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -535,7 +535,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &createdWl)).To(gomega.Succeed()) util.MustHaveOwnerReference(g, createdWl.OwnerReferences, job, k8sClient.Scheme()) // The workload is not marked as finished. - g.Expect(createdWl.Status.Conditions).ShouldNot(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWl.Status.Conditions).ShouldNot(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -654,7 +654,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.By("the workload should stay admitted", func() { gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, wl)).To(gomega.Succeed()) - g.Expect(wl.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(wl.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -1374,7 +1374,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con ginkgo.By("checking that the first workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey1, createdWorkload1)).Should(gomega.Succeed()) - g.Expect(createdWorkload1.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload1.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1399,7 +1399,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) - g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload2.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1422,7 +1422,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con ginkgo.By("checking that the third workload is not admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) - g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) + g.Expect(createdWorkload3.Status.Conditions).Should(utiltesting.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1442,15 +1442,15 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con ginkgo.By("checking that the third workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) - g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload3.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.By("checking that the second workload is evicted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) - g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadEvicted)) - g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadPreempted)) + g.Expect(createdWorkload2.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadEvicted)) + g.Expect(createdWorkload2.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadPreempted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -1475,7 +1475,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con ginkgo.By("checking that the first workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey1, createdWorkload1)).Should(gomega.Succeed()) - g.Expect(createdWorkload1.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload1.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1500,7 +1500,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) - g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload2.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1523,7 +1523,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con ginkgo.By("checking that the third workload is not admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) - g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) + g.Expect(createdWorkload3.Status.Conditions).Should(utiltesting.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1543,11 +1543,11 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con ginkgo.By("checking that the third workload is not admitted and the second workload is not evicted", func() { gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) - g.Expect(createdWorkload2.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) - g.Expect(createdWorkload2.Status.Conditions).ShouldNot(testing.HaveConditionStatusTrue(kueue.WorkloadEvicted)) - g.Expect(createdWorkload2.Status.Conditions).ShouldNot(testing.HaveConditionStatusTrue(kueue.WorkloadPreempted)) + g.Expect(createdWorkload2.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload2.Status.Conditions).ShouldNot(utiltesting.HaveConditionStatusTrue(kueue.WorkloadEvicted)) + g.Expect(createdWorkload2.Status.Conditions).ShouldNot(utiltesting.HaveConditionStatusTrue(kueue.WorkloadPreempted)) g.Expect(k8sClient.Get(ctx, wlLookupKey3, createdWorkload3)).Should(gomega.Succeed()) - g.Expect(createdWorkload3.Status.Conditions).Should(testing.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) + g.Expect(createdWorkload3.Status.Conditions).Should(utiltesting.HaveConditionStatusFalseAndReason(kueue.WorkloadQuotaReserved, "Pending")) }, util.ConsistentDuration, util.ConsistentDuration).Should(gomega.Succeed()) }) }) @@ -1893,7 +1893,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con }) ginkgo.It("Should readmit preempted Job with priorityClass in alternative flavor", func() { - highPriorityClass := testing.MakePriorityClass("high").PriorityValue(100).Obj() + highPriorityClass := utiltesting.MakePriorityClass("high").PriorityValue(100).Obj() util.MustCreate(ctx, k8sClient, highPriorityClass) ginkgo.DeferCleanup(func() { gomega.Expect(k8sClient.Delete(ctx, highPriorityClass)).To(gomega.Succeed()) @@ -2210,7 +2210,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con // Workload should stay pending g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wll), wll)).Should(gomega.Succeed()) // Should have Evicted condition - g.Expect(wll.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadEvicted)) + g.Expect(wll.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadEvicted)) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) ginkgo.By("checking the first job becomes unsuspended after we update the Active field back to true") @@ -3835,13 +3835,13 @@ var _ = ginkgo.Describe("Job reconciliation", ginkgo.Ordered, func() { ), ) - managedNs = testing.MakeNamespaceWrapper(""). + managedNs = utiltesting.MakeNamespaceWrapper(""). GenerateName("managed-ns-"). Label("managed-by-kueue", "true"). Obj() util.MustCreate(ctx, k8sClient, managedNs) - unmanagedNs = testing.MakeNamespaceWrapper(""). + unmanagedNs = utiltesting.MakeNamespaceWrapper(""). GenerateName("unmanaged-ns-"). Obj() util.MustCreate(ctx, k8sClient, unmanagedNs) diff --git a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go index 921bf9edeae..251b06b0fcd 100644 --- a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go @@ -39,7 +39,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjobset "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" @@ -60,7 +60,7 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) ginkgo.AfterAll(func() { @@ -109,7 +109,7 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa ginkgo.It("Should reconcile JobSets", func() { ginkgo.By("checking the JobSet gets suspended when created unsuspended") - priorityClass := testing.MakePriorityClass(priorityClassName). + priorityClass := utiltesting.MakePriorityClass(priorityClassName). PriorityValue(priorityValue).Obj() util.MustCreate(ctx, k8sClient, priorityClass) @@ -173,7 +173,7 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa key := types.NamespacedName{Name: secondWl.Name, Namespace: secondWl.Namespace} gomega.Eventually(func(g gomega.Gomega) { wl := &kueue.Workload{} - g.Expect(k8sClient.Get(ctx, key, wl)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, key, wl)).Should(utiltesting.BeNotFoundError()) // check the original wl is still there g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -200,7 +200,7 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lookupKey, createdJobSet)).Should(gomega.Succeed()) g.Expect(ptr.Deref(createdJobSet.Spec.Suspend, false)).Should(gomega.BeFalse()) - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Expect(createdJobSet.Spec.ReplicatedJobs[0].Template.Spec.Template.Spec.NodeSelector).Should(gomega.Equal(map[string]string{instanceKey: onDemandFlavor.Name})) @@ -218,7 +218,7 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa g.Expect(jobSet.Spec.ReplicatedJobs[0].Template.Spec.Template.Spec.NodeSelector).Should(gomega.BeEmpty()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -298,7 +298,7 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: jobSetName, Namespace: jobSet.Namespace}, createdJobSet)).Should(gomega.Succeed()) g.Expect(ptr.Deref(createdJobSet.Spec.Suspend, false)).Should(gomega.BeFalse()) - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -368,7 +368,7 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa ginkgo.By("the workload should stay admitted", func() { gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -726,7 +726,7 @@ var _ = ginkgo.Describe("JobSet controller for workloads when only jobs with que createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadjobset.GetWorkloadNameForJobSet(jobSet.Name, jobSet.UID), Namespace: ns.Name} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the workload is created when queue name is set") @@ -1165,8 +1165,8 @@ var _ = ginkgo.Describe("JobSet controller with TopologyAwareScheduling", ginkgo nodes = []corev1.Node{ *testingnode.MakeNode("b1r1"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -1215,7 +1215,7 @@ var _ = ginkgo.Describe("JobSet controller with TopologyAwareScheduling", ginkgo Parallelism: 1, Completions: 1, PodAnnotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, Image: util.GetAgnHostImage(), Args: util.BehaviorExitFast, @@ -1226,7 +1226,7 @@ var _ = ginkgo.Describe("JobSet controller with TopologyAwareScheduling", ginkgo Parallelism: 1, Completions: 1, PodAnnotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, Image: util.GetAgnHostImage(), Args: util.BehaviorExitFast, @@ -1253,7 +1253,7 @@ var _ = ginkgo.Describe("JobSet controller with TopologyAwareScheduling", ginkgo Name: "rj1", Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultBlockTopologyLevel), + Required: ptr.To(utiltesting.DefaultBlockTopologyLevel), PodIndexLabel: ptr.To(batchv1.JobCompletionIndexAnnotation), SubGroupIndexLabel: ptr.To(jobsetapi.JobIndexKey), SubGroupCount: ptr.To[int32](1), @@ -1263,7 +1263,7 @@ var _ = ginkgo.Describe("JobSet controller with TopologyAwareScheduling", ginkgo Name: "rj2", Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultRackTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultRackTopologyLevel), PodIndexLabel: ptr.To(batchv1.JobCompletionIndexAnnotation), SubGroupIndexLabel: ptr.To(jobsetapi.JobIndexKey), SubGroupCount: ptr.To[int32](1), @@ -1285,13 +1285,13 @@ var _ = ginkgo.Describe("JobSet controller with TopologyAwareScheduling", ginkgo g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(2)) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) g.Expect(wl.Status.Admission.PodSetAssignments[1].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) diff --git a/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go b/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go index 4638c4c2b8a..d0bec901205 100644 --- a/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go +++ b/test/integration/singlecluster/controller/jobs/kubeflow/kubeflowjob.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -59,7 +59,7 @@ var ReplicaTypeWorker = kftraining.ReplicaType("Worker") func ShouldReconcileJob(ctx context.Context, k8sClient client.Client, job, createdJob kubeflowjob.KubeflowJob, podSetsResources []PodSetsResource) { ginkgo.By("checking the job gets suspended when created unsuspended") - priorityClass := testing.MakePriorityClass(priorityClassName). + priorityClass := utiltesting.MakePriorityClass(priorityClassName). PriorityValue(int32(priorityValue)).Obj() util.MustCreate(ctx, k8sClient, priorityClass) @@ -108,7 +108,7 @@ func ShouldReconcileJob(ctx context.Context, k8sClient client.Client, job, creat util.MustCreate(ctx, k8sClient, secondWl) gomega.Eventually(func(g gomega.Gomega) { wl := &kueue.Workload{} - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(secondWl), wl)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(secondWl), wl)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) // check the original wl is still there gomega.Eventually(func(g gomega.Gomega) { @@ -137,7 +137,7 @@ func ShouldReconcileJob(ctx context.Context, k8sClient client.Client, job, creat g.Expect(createdJob.IsSuspended()).Should(gomega.BeFalse()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) for _, psr := range podSetsResources { @@ -160,7 +160,7 @@ func ShouldReconcileJob(ctx context.Context, k8sClient client.Client, job, creat g.Expect(createdJob.KFJobControl.ReplicaSpecs()[ReplicaTypeWorker].Template.Spec.NodeSelector).Should(gomega.BeEmpty()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -199,7 +199,7 @@ func ShouldReconcileJob(ctx context.Context, k8sClient client.Client, job, creat gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(createdWorkload.Status.Conditions).ShouldNot(gomega.HaveLen(2)) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) } @@ -216,7 +216,7 @@ func ShouldNotReconcileUnmanagedJob(ctx context.Context, k8sClient client.Client gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lookupKey, createdJob.Object())).Should(gomega.Succeed()) g.Expect(createdJob.IsSuspended()).Should(gomega.BeFalse()) - g.Expect(k8sClient.Get(ctx, wlLookupKey, workload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, workload)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) } diff --git a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go index dbc7d526a18..22af84fd500 100644 --- a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go @@ -38,7 +38,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadmpijob "sigs.k8s.io/kueue/pkg/controller/jobs/mpijob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingmpijob "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" @@ -60,7 +60,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(false, jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) ginkgo.AfterAll(func() { @@ -79,7 +79,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.It("Should reconcile MPIJobs", func() { ginkgo.By("checking the job gets suspended when created unsuspended") - priorityClass := testing.MakePriorityClass(priorityClassName). + priorityClass := utiltesting.MakePriorityClass(priorityClassName). PriorityValue(int32(priorityValue)).Obj() util.MustCreate(ctx, k8sClient, priorityClass) @@ -131,7 +131,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu util.MustCreate(ctx, k8sClient, secondWl) gomega.Eventually(func(g gomega.Gomega) { wl := &kueue.Workload{} - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(secondWl), wl)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(secondWl), wl)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) // check the original wl is still there gomega.Eventually(func(g gomega.Gomega) { @@ -177,7 +177,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu g.Expect(createdJob.Spec.RunPolicy.Suspend).Should(gomega.Equal(ptr.To(false))) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Expect(createdJob.Spec.MPIReplicaSpecs[kfmpi.MPIReplicaTypeLauncher].Template.Spec.NodeSelector).Should(gomega.HaveLen(1)) @@ -200,7 +200,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu g.Expect(createdJob.Spec.MPIReplicaSpecs[kfmpi.MPIReplicaTypeWorker].Template.Spec.NodeSelector).Should(gomega.BeEmpty()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) util.SyncAdmittedConditionForWorkloads(ctx, k8sClient, createdWorkload) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -257,7 +257,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(createdWorkload.Status.Conditions).ShouldNot(gomega.HaveLen(2)) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -275,7 +275,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lookupKey, job)).Should(gomega.Succeed()) g.Expect(job.Spec.RunPolicy.Suspend).Should(gomega.Equal(ptr.To(false))) - g.Expect(k8sClient.Get(ctx, childWlLookupKey, childWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, childWlLookupKey, childWorkload)).Should(utiltesting.BeNotFoundError()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -499,7 +499,7 @@ var _ = ginkgo.Describe("Job controller for workloads when only jobs with queue createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadmpijob.GetWorkloadNameForMPIJob(job.Name, job.UID), Namespace: ns.Name} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the workload is created when queue name is set") @@ -921,8 +921,8 @@ var _ = ginkgo.Describe("MPIJob controller with TopologyAwareScheduling", ginkgo nodes = []corev1.Node{ *testingnode.MakeNode("b1r1"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -965,8 +965,8 @@ var _ = ginkgo.Describe("MPIJob controller with TopologyAwareScheduling", ginkgo mpiJob := testingmpijob.MakeMPIJob(jobName, ns.Name). Queue(localQueue.Name). GenericLauncherAndWorker(). - PodAnnotation(kfmpi.MPIReplicaTypeLauncher, kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel). - PodAnnotation(kfmpi.MPIReplicaTypeWorker, kueue.PodSetPreferredTopologyAnnotation, testing.DefaultRackTopologyLevel). + PodAnnotation(kfmpi.MPIReplicaTypeLauncher, kueue.PodSetRequiredTopologyAnnotation, utiltesting.DefaultBlockTopologyLevel). + PodAnnotation(kfmpi.MPIReplicaTypeWorker, kueue.PodSetPreferredTopologyAnnotation, utiltesting.DefaultRackTopologyLevel). Request(kfmpi.MPIReplicaTypeLauncher, corev1.ResourceCPU, "100m"). Request(kfmpi.MPIReplicaTypeWorker, corev1.ResourceCPU, "100m"). Obj() @@ -988,7 +988,7 @@ var _ = ginkgo.Describe("MPIJob controller with TopologyAwareScheduling", ginkgo Name: kueue.NewPodSetReference(string(kfmpi.MPIReplicaTypeLauncher)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultBlockTopologyLevel), + Required: ptr.To(utiltesting.DefaultBlockTopologyLevel), PodIndexLabel: ptr.To(kfmpi.ReplicaIndexLabel), }, }, @@ -996,7 +996,7 @@ var _ = ginkgo.Describe("MPIJob controller with TopologyAwareScheduling", ginkgo Name: kueue.NewPodSetReference(string(kfmpi.MPIReplicaTypeWorker)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultRackTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultRackTopologyLevel), PodIndexLabel: ptr.To(kfmpi.ReplicaIndexLabel), }, }, @@ -1016,13 +1016,13 @@ var _ = ginkgo.Describe("MPIJob controller with TopologyAwareScheduling", ginkgo g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(2)) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) g.Expect(wl.Status.Admission.PodSetAssignments[1].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) diff --git a/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go b/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go index 122e181d018..b554f245f60 100644 --- a/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/paddlejob/paddlejob_controller_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadpaddlejob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/paddlejob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingpaddlejob "sigs.k8s.io/kueue/pkg/util/testingjobs/paddlejob" @@ -50,7 +50,7 @@ var _ = ginkgo.Describe("Job controller", framework.RedundantSpec, ginkgo.Ordere ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) @@ -311,8 +311,8 @@ var _ = ginkgo.Describe("PaddleJob controller with TopologyAwareScheduling", fra nodes = []corev1.Node{ *testingnode.MakeNode("b1r1"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -358,14 +358,14 @@ var _ = ginkgo.Describe("PaddleJob controller with TopologyAwareScheduling", fra ReplicaType: kftraining.PaddleJobReplicaTypeMaster, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, }, testingpaddlejob.PaddleReplicaSpecRequirement{ ReplicaType: kftraining.PaddleJobReplicaTypeWorker, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, ). @@ -388,7 +388,7 @@ var _ = ginkgo.Describe("PaddleJob controller with TopologyAwareScheduling", fra Name: kueue.NewPodSetReference(string(kftraining.PaddleJobReplicaTypeMaster)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -396,7 +396,7 @@ var _ = ginkgo.Describe("PaddleJob controller with TopologyAwareScheduling", fra Name: kueue.NewPodSetReference(string(kftraining.PaddleJobReplicaTypeWorker)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultBlockTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultBlockTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -416,13 +416,13 @@ var _ = ginkgo.Describe("PaddleJob controller with TopologyAwareScheduling", fra g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(2)) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) g.Expect(wl.Status.Admission.PodSetAssignments[1].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) diff --git a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go index 29dfc6fb854..d617e41fe93 100644 --- a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go @@ -40,7 +40,7 @@ import ( podcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/pod" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/features" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" @@ -192,7 +192,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu util.SyncAdmittedConditionForWorkloads(ctx, k8sClient, createdWorkload) gomega.Eventually(func(g gomega.Gomega) { - ok, err := testing.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) + ok, err := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) g.Expect(err).NotTo(gomega.HaveOccurred()) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -212,7 +212,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu util.SetPodsPhase(ctx, k8sClient, corev1.PodSucceeded, pod) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPodsJustFinalized(ctx, k8sClient, lookupKey) @@ -259,7 +259,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.By("checking that the finalizer is removed when the Pod is deleted early") gomega.Expect(k8sClient.Delete(ctx, createdPod)).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, lookupKey, createdPod)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, lookupKey, createdPod)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -281,7 +281,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu g.Expect(k8sClient.Update(ctx, pod)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, lookupKey, &corev1.Pod{})).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, lookupKey, &corev1.Pod{})).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -386,7 +386,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.By(fmt.Sprintf("checking that workload '%s' is not created", wlLookupKey)) createdWorkload := &kueue.Workload{} - gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(testing.BeNotFoundError()) + gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) }) ginkgo.It("Should skip the pod with MultiKueueBatchJobWithManagedBy on", func() { features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.MultiKueueBatchJobWithManagedBy, true) @@ -419,7 +419,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.By(fmt.Sprintf("checking that workload '%s' is not created", wlLookupKey)) createdWorkload := &kueue.Workload{} - gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(testing.BeNotFoundError()) + gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) }) }) @@ -582,9 +582,9 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.By("Workload should not be finished", func() { gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, wl)).Should(gomega.Succeed()) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) - g.Expect(wl.Status.Conditions).ToNot(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(wl.Status.Conditions).ToNot(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -600,7 +600,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu createdWorkload := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -691,7 +691,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPodsJustFinalized(ctx, k8sClient, pod1LookupKey, pod2LookupKey) @@ -761,7 +761,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPodsJustFinalized(ctx, k8sClient, pod1LookupKey, pod2LookupKey) @@ -1068,7 +1068,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) g.Expect(createdWorkload.UID).To(gomega.Equal(wlUID)) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1178,7 +1178,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectPodsJustFinalized(ctx, k8sClient, pod2LookupKey, replacementPod2LookupKey) @@ -1326,7 +1326,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { for i := range pods { key := types.NamespacedName{Namespace: ns.Name, Name: fmt.Sprintf("test-pod-%d", i)} - g.Expect(k8sClient.Get(ctx, key, &corev1.Pod{})).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, key, &corev1.Pod{})).To(utiltesting.BeNotFoundError()) } }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1586,7 +1586,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.By("await for pod2 to be deleted", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pod2), createdPod2)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pod2), createdPod2)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1675,9 +1675,9 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.By("Workload should not be finished", func() { gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, wl)).Should(gomega.Succeed()) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) - g.Expect(wl.Status.Conditions).ToNot(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(wl.Status.Conditions).ToNot(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -1709,7 +1709,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu createdWorkload := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1959,7 +1959,7 @@ var _ = ginkgo.Describe("Pod controller interacting with scheduler", ginkgo.Orde ginkgo.By("checking if pods are deleted", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, lookupKey, createdPod)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, lookupKey, createdPod)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -2161,7 +2161,7 @@ var _ = ginkgo.Describe("Pod controller interacting with Workload controller whe Message: "The workload is deactivated due to exceeding the maximum number of re-queuing retries", }, util.IgnoreConditionTimestampsAndObservedGeneration), )) - g.Expect(wl.Status.Conditions).ShouldNot(testing.HaveCondition(kueue.WorkloadDeactivationTarget)) + g.Expect(wl.Status.Conditions).ShouldNot(utiltesting.HaveCondition(kueue.WorkloadDeactivationTarget)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -2393,8 +2393,8 @@ var _ = ginkgo.Describe("Pod controller with TASReplaceNodeOnPodTermination", gi nodes = []corev1.Node{ *testingnode.MakeNode("x1"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("2"), @@ -2405,8 +2405,8 @@ var _ = ginkgo.Describe("Pod controller with TASReplaceNodeOnPodTermination", gi Obj(), *testingnode.MakeNode("x2"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x2"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -2417,8 +2417,8 @@ var _ = ginkgo.Describe("Pod controller with TASReplaceNodeOnPodTermination", gi Obj(), *testingnode.MakeNode("x3"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x3"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("2"), @@ -2462,7 +2462,7 @@ var _ = ginkgo.Describe("Pod controller with TASReplaceNodeOnPodTermination", gi var nodeName string podgroup := testingpod.MakePod(podGroupName, ns.Name). Queue(localQueue.Name). - Annotation(kueue.PodSetPreferredTopologyAnnotation, testing.DefaultRackTopologyLevel). + Annotation(kueue.PodSetPreferredTopologyAnnotation, utiltesting.DefaultRackTopologyLevel). Request(corev1.ResourceCPU, "100m"). MakeIndexedGroup(2) ginkgo.By("Creating the Pod group", func() { @@ -2544,7 +2544,7 @@ var _ = ginkgo.Describe("Pod controller with TASReplaceNodeOnPodTermination", gi var nodeName string podgroup := testingpod.MakePod(podGroupName, ns.Name). Queue(localQueue.Name). - Annotation(kueue.PodSetPreferredTopologyAnnotation, testing.DefaultRackTopologyLevel). + Annotation(kueue.PodSetPreferredTopologyAnnotation, utiltesting.DefaultRackTopologyLevel). Request(corev1.ResourceCPU, "100m"). MakeIndexedGroup(2) ginkgo.By("Creating the Pod group", func() { diff --git a/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go b/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go index 15ce2c835f0..d00a656eacf 100644 --- a/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadpytorchjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/pytorchjob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingpytorchjob "sigs.k8s.io/kueue/pkg/util/testingjobs/pytorchjob" @@ -54,7 +54,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) @@ -129,7 +129,7 @@ var _ = ginkgo.Describe("Job controller for workloads when only jobs with queue createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadpytorchjob.GetWorkloadNameForPyTorchJob(job.Name, job.UID), Namespace: ns.Name} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the workload is created when queue name is set") @@ -610,8 +610,8 @@ var _ = ginkgo.Describe("PyTorchJob controller with TopologyAwareScheduling", gi nodes = []corev1.Node{ *testingnode.MakeNode("b1r1"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -657,14 +657,14 @@ var _ = ginkgo.Describe("PyTorchJob controller with TopologyAwareScheduling", gi ReplicaType: kftraining.PyTorchJobReplicaTypeMaster, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, }, testingpytorchjob.PyTorchReplicaSpecRequirement{ ReplicaType: kftraining.PyTorchJobReplicaTypeWorker, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, ). @@ -687,7 +687,7 @@ var _ = ginkgo.Describe("PyTorchJob controller with TopologyAwareScheduling", gi Name: kueue.NewPodSetReference(string(kftraining.PyTorchJobReplicaTypeMaster)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -695,7 +695,7 @@ var _ = ginkgo.Describe("PyTorchJob controller with TopologyAwareScheduling", gi Name: kueue.NewPodSetReference(string(kftraining.PyTorchJobReplicaTypeWorker)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultBlockTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultBlockTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -715,13 +715,13 @@ var _ = ginkgo.Describe("PyTorchJob controller with TopologyAwareScheduling", gi g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(2)) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) g.Expect(wl.Status.Admission.PodSetAssignments[1].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) diff --git a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go index be28e4fe9ba..9f443f459b4 100644 --- a/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/raycluster/raycluster_controller_test.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadraycluster "sigs.k8s.io/kueue/pkg/controller/jobs/raycluster" "sigs.k8s.io/kueue/pkg/features" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingraycluster "sigs.k8s.io/kueue/pkg/util/testingjobs/raycluster" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" @@ -79,7 +79,7 @@ var _ = ginkgo.Describe("RayCluster controller", ginkgo.Ordered, ginkgo.Continue ginkgo.It("Should reconcile RayClusters", func() { ginkgo.By("checking the job gets suspended when created unsuspended") - priorityClass := testing.MakePriorityClass(priorityClassName). + priorityClass := utiltesting.MakePriorityClass(priorityClassName). PriorityValue(priorityValue).Obj() util.MustCreate(ctx, k8sClient, priorityClass) defer func() { @@ -135,7 +135,7 @@ var _ = ginkgo.Describe("RayCluster controller", ginkgo.Ordered, ginkgo.Continue util.MustCreate(ctx, k8sClient, secondWl) gomega.Eventually(func(g gomega.Gomega) { wl := &kueue.Workload{} - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(secondWl), wl)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(secondWl), wl)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) // check the original wl is still there gomega.Eventually(func(g gomega.Gomega) { @@ -179,7 +179,7 @@ var _ = ginkgo.Describe("RayCluster controller", ginkgo.Ordered, ginkgo.Continue }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Expect(createdJob.Spec.HeadGroupSpec.Template.Spec.NodeSelector).Should(gomega.HaveLen(1)) @@ -188,7 +188,7 @@ var _ = ginkgo.Describe("RayCluster controller", ginkgo.Ordered, ginkgo.Continue gomega.Expect(createdJob.Spec.WorkerGroupSpecs[0].Template.Spec.NodeSelector[instanceKey]).Should(gomega.Equal(spotFlavor.Name)) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the job gets suspended when parallelism changes and the added node selectors are removed") @@ -202,7 +202,7 @@ var _ = ginkgo.Describe("RayCluster controller", ginkgo.Ordered, ginkgo.Continue g.Expect(createdJob.Spec.WorkerGroupSpecs[0].Template.Spec.NodeSelector).Should(gomega.BeEmpty()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -226,7 +226,7 @@ var _ = ginkgo.Describe("RayCluster controller", ginkgo.Ordered, ginkgo.Continue gomega.Expect(createdJob.Spec.WorkerGroupSpecs[0].Template.Spec.NodeSelector[instanceKey]).Should(gomega.Equal(spotFlavor.Name)) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -260,7 +260,7 @@ var _ = ginkgo.Describe("Job controller RayCluster for workloads when only jobs createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadraycluster.GetWorkloadNameForRayCluster(job.Name, job.UID), Namespace: ns.Name} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the workload is created when queue name is set") @@ -551,7 +551,7 @@ var _ = ginkgo.Describe("RayCluster Job controller interacting with scheduler", ginkgo.By("deleting the job", func() { gomega.Expect(k8sClient.Delete(ctx, job)).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(job), job)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(job), job)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -613,7 +613,7 @@ var _ = ginkgo.Describe("Job controller with preemption enabled", ginkgo.Ordered util.MustCreate(ctx, k8sClient, localQueue) ginkgo.By("creating priority") - priorityClass = testing.MakePriorityClass(priorityClassName). + priorityClass = utiltesting.MakePriorityClass(priorityClassName). PriorityValue(priorityValue).Obj() util.MustCreate(ctx, k8sClient, priorityClass) }) @@ -653,7 +653,7 @@ var _ = ginkgo.Describe("Job controller with preemption enabled", ginkgo.Ordered gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, highPriorityLookupKey, highPriorityWL)).Should(gomega.Succeed()) - g.Expect(highPriorityWL.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(highPriorityWL.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("Low priority workload should not be admitted") @@ -681,7 +681,7 @@ var _ = ginkgo.Describe("Job controller with preemption enabled", ginkgo.Ordered createdWorkload = &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowPriorityLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("Low priority RayCluster should be unsuspended") diff --git a/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go b/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go index 526f43f53da..6c091f3300e 100644 --- a/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/rayjob/rayjob_controller_test.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadrayjob "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" "sigs.k8s.io/kueue/test/util" @@ -63,7 +63,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) @@ -83,7 +83,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.It("Should reconcile RayJobs", func() { ginkgo.By("checking the job gets suspended when created unsuspended") - priorityClass := testing.MakePriorityClass(priorityClassName). + priorityClass := utiltesting.MakePriorityClass(priorityClassName). PriorityValue(priorityValue).Obj() util.MustCreate(ctx, k8sClient, priorityClass) defer func() { @@ -140,7 +140,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Eventually(func(g gomega.Gomega) { wl := &kueue.Workload{} key := types.NamespacedName{Name: secondWl.Name, Namespace: secondWl.Namespace} - g.Expect(k8sClient.Get(ctx, key, wl)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, key, wl)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) // check the original wl is still there gomega.Eventually(func(g gomega.Gomega) { @@ -187,7 +187,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu g.Expect(createdJob.Spec.Suspend).Should(gomega.BeFalse()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Expect(createdJob.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.NodeSelector).Should(gomega.HaveLen(1)) @@ -196,7 +196,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Expect(createdJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Template.Spec.NodeSelector[instanceKey]).Should(gomega.Equal(spotFlavor.Name)) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the job gets suspended when parallelism changes and the added node selectors are removed") @@ -210,7 +210,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu g.Expect(createdJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Template.Spec.NodeSelector).Should(gomega.BeEmpty()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "DeletedWorkload", corev1.EventTypeNormal, fmt.Sprintf("Deleted not matching Workload: %v", wlLookupKey.String()), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -234,7 +234,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Expect(createdJob.Spec.RayClusterSpec.WorkerGroupSpecs[0].Template.Spec.NodeSelector[instanceKey]).Should(gomega.Equal(spotFlavor.Name)) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the workload is finished when job is completed") @@ -245,7 +245,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Expect(k8sClient.Status().Update(ctx, createdJob)).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -262,7 +262,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: job.Name, Namespace: job.Namespace}, createdJob)).Should(gomega.Succeed()) g.Expect(createdJob.Spec.Suspend).Should(gomega.BeFalse()) - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) }) @@ -298,7 +298,7 @@ var _ = ginkgo.Describe("Job controller for workloads when only jobs with queue createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadrayjob.GetWorkloadNameForRayJob(job.Name, job.UID), Namespace: ns.Name} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the workload is created when queue name is set") @@ -602,7 +602,7 @@ var _ = ginkgo.Describe("Job controller with preemption enabled", ginkgo.Ordered util.MustCreate(ctx, k8sClient, localQueue) ginkgo.By("creating priority") - priorityClass = testing.MakePriorityClass(priorityClassName). + priorityClass = utiltesting.MakePriorityClass(priorityClassName). PriorityValue(priorityValue).Obj() util.MustCreate(ctx, k8sClient, priorityClass) }) @@ -644,7 +644,7 @@ var _ = ginkgo.Describe("Job controller with preemption enabled", ginkgo.Ordered gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, highPriorityLookupKey, highPriorityWL)).To(gomega.Succeed()) - g.Expect(highPriorityWL.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(highPriorityWL.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("Low priority workload should not be admitted") @@ -672,7 +672,7 @@ var _ = ginkgo.Describe("Job controller with preemption enabled", ginkgo.Ordered createdWorkload = &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowPriorityLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("Low priority rayJob should be unsuspended") diff --git a/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go b/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go index 9c0cc2a4c7a..bfa13849ec3 100644 --- a/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/tfjob/tfjob_controller_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadtfjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/tfjob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingtfjob "sigs.k8s.io/kueue/pkg/util/testingjobs/tfjob" @@ -50,7 +50,7 @@ var _ = ginkgo.Describe("Job controller", framework.RedundantSpec, ginkgo.Ordere ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) @@ -325,8 +325,8 @@ var _ = ginkgo.Describe("TFJob controller with TopologyAwareScheduling", framewo nodes = []corev1.Node{ *testingnode.MakeNode("b1r1"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -372,21 +372,21 @@ var _ = ginkgo.Describe("TFJob controller with TopologyAwareScheduling", framewo ReplicaType: kftraining.TFJobReplicaTypeChief, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, }, testingtfjob.TFReplicaSpecRequirement{ ReplicaType: kftraining.TFJobReplicaTypePS, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, }, testingtfjob.TFReplicaSpecRequirement{ ReplicaType: kftraining.TFJobReplicaTypeWorker, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, ). @@ -410,7 +410,7 @@ var _ = ginkgo.Describe("TFJob controller with TopologyAwareScheduling", framewo Name: kueue.NewPodSetReference(string(kftraining.TFJobReplicaTypeChief)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -418,7 +418,7 @@ var _ = ginkgo.Describe("TFJob controller with TopologyAwareScheduling", framewo Name: kueue.NewPodSetReference(string(kftraining.TFJobReplicaTypePS)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -426,7 +426,7 @@ var _ = ginkgo.Describe("TFJob controller with TopologyAwareScheduling", framewo Name: kueue.NewPodSetReference(string(kftraining.TFJobReplicaTypeWorker)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultBlockTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultBlockTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -446,19 +446,19 @@ var _ = ginkgo.Describe("TFJob controller with TopologyAwareScheduling", framewo g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(3)) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) g.Expect(wl.Status.Admission.PodSetAssignments[1].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) diff --git a/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go b/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go index bcb55822838..db477799ab1 100644 --- a/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go @@ -38,7 +38,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadtrainjob "sigs.k8s.io/kueue/pkg/controller/jobs/trainjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" @@ -55,7 +55,7 @@ var _ = ginkgo.Describe("Trainjob controller", ginkgo.Ordered, ginkgo.ContinueOn ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) ginkgo.AfterAll(func() { @@ -168,7 +168,7 @@ var _ = ginkgo.Describe("Trainjob controller", ginkgo.Ordered, ginkgo.ContinueOn gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lookupKey, &createdTrainJob)).Should(gomega.Succeed()) g.Expect(ptr.Deref(createdTrainJob.Spec.Suspend, false)).Should(gomega.BeFalse()) - ok, _ := testing.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) + ok, _ := utiltesting.CheckEventRecordedFor(ctx, k8sClient, "Started", corev1.EventTypeNormal, fmt.Sprintf("Admitted by clusterQueue %v", clusterQueue.Name), lookupKey) g.Expect(ok).Should(gomega.BeTrue()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.ExpectWorkloadsToHaveQuotaReservation(ctx, k8sClient, clusterQueue.Name, createdWorkload) @@ -203,7 +203,7 @@ var _ = ginkgo.Describe("Trainjob controller", ginkgo.Ordered, ginkgo.ContinueOn gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: trainJob.Name, Namespace: trainJob.Namespace}, createdTrainJob)).Should(gomega.Succeed()) g.Expect(ptr.Deref(createdTrainJob.Spec.Suspend, false)).Should(gomega.BeFalse()) - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) }) @@ -273,7 +273,7 @@ var _ = ginkgo.Describe("Trainjob controller", ginkgo.Ordered, ginkgo.ContinueOn ginkgo.By("the workload should stay admitted", func() { gomega.Consistently(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -337,7 +337,7 @@ var _ = ginkgo.Describe("TrainJob controller for workloads when only jobs with q createdWorkload := &kueue.Workload{} wlLookupKey := types.NamespacedName{Name: workloadtrainjob.GetWorkloadNameForTrainJob(trainJob.Name, trainJob.UID), Namespace: ns.Name} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) ginkgo.By("checking the workload is created when queue name is set") @@ -589,8 +589,8 @@ var _ = ginkgo.Describe("TrainJob controller with TopologyAwareScheduling", gink nodes = []corev1.Node{ *testingnode.MakeNode("b1r1"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -637,7 +637,7 @@ var _ = ginkgo.Describe("TrainJob controller with TopologyAwareScheduling", gink Parallelism: 1, Completions: 1, PodAnnotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, testingjobset.ReplicatedJobRequirements{ Name: "node-2", @@ -645,7 +645,7 @@ var _ = ginkgo.Describe("TrainJob controller with TopologyAwareScheduling", gink Parallelism: 1, Completions: 1, PodAnnotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, }, ). @@ -681,7 +681,7 @@ var _ = ginkgo.Describe("TrainJob controller with TopologyAwareScheduling", gink Name: "node-1", Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultBlockTopologyLevel), + Required: ptr.To(utiltesting.DefaultBlockTopologyLevel), PodIndexLabel: ptr.To(batchv1.JobCompletionIndexAnnotation), SubGroupIndexLabel: ptr.To(jobsetapi.JobIndexKey), SubGroupCount: ptr.To[int32](1), @@ -691,7 +691,7 @@ var _ = ginkgo.Describe("TrainJob controller with TopologyAwareScheduling", gink Name: "node-2", Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultRackTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultRackTopologyLevel), PodIndexLabel: ptr.To(batchv1.JobCompletionIndexAnnotation), SubGroupIndexLabel: ptr.To(jobsetapi.JobIndexKey), SubGroupCount: ptr.To[int32](1), @@ -713,13 +713,13 @@ var _ = ginkgo.Describe("TrainJob controller with TopologyAwareScheduling", gink g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(2)) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) g.Expect(wl.Status.Admission.PodSetAssignments[1].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) diff --git a/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go b/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go index 68aa61668dd..3a4b04ad9c0 100644 --- a/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/xgboostjob/xgboostjob_controller_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadxgboostjob "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/xgboostjob" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/kubeflowjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" testingxgboostjob "sigs.k8s.io/kueue/pkg/util/testingjobs/xgboostjob" @@ -50,7 +50,7 @@ var _ = ginkgo.Describe("Job controller", framework.RedundantSpec, ginkgo.Ordere ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) @@ -308,8 +308,8 @@ var _ = ginkgo.Describe("XGBoostJob controller with TopologyAwareScheduling", fr nodes = []corev1.Node{ *testingnode.MakeNode("b1r1"). Label(nodeGroupLabel, "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -355,14 +355,14 @@ var _ = ginkgo.Describe("XGBoostJob controller with TopologyAwareScheduling", fr ReplicaType: kftraining.XGBoostJobReplicaTypeMaster, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, }, testingxgboostjob.XGBReplicaSpecRequirement{ ReplicaType: kftraining.XGBoostJobReplicaTypeWorker, ReplicaCount: 1, Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, ). @@ -385,7 +385,7 @@ var _ = ginkgo.Describe("XGBoostJob controller with TopologyAwareScheduling", fr Name: kueue.NewPodSetReference(string(kftraining.XGBoostJobReplicaTypeMaster)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -393,7 +393,7 @@ var _ = ginkgo.Describe("XGBoostJob controller with TopologyAwareScheduling", fr Name: kueue.NewPodSetReference(string(kftraining.XGBoostJobReplicaTypeWorker)), Count: 1, TopologyRequest: &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultBlockTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultBlockTopologyLevel), PodIndexLabel: ptr.To(kftraining.ReplicaIndexLabel), }, }, @@ -413,13 +413,13 @@ var _ = ginkgo.Describe("XGBoostJob controller with TopologyAwareScheduling", fr g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(2)) g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) g.Expect(wl.Status.Admission.PodSetAssignments[1].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"b1", "r1"}}}, }, )) diff --git a/test/integration/singlecluster/kueuectl/create_test.go b/test/integration/singlecluster/kueuectl/create_test.go index 4e7f9a09e90..7916a5d3654 100644 --- a/test/integration/singlecluster/kueuectl/create_test.go +++ b/test/integration/singlecluster/kueuectl/create_test.go @@ -33,7 +33,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/cmd/kueuectl/app" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -548,7 +548,7 @@ var _ = ginkgo.Describe("Kueuectl Create", ginkgo.Ordered, ginkgo.ContinueOnFail var resourceFlavor kueue.ResourceFlavor gomega.Eventually(func(g gomega.Gomega) { rfKey := types.NamespacedName{Name: rfName, Namespace: ns.Name} - g.Expect(k8sClient.Get(ctx, rfKey, &resourceFlavor)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, rfKey, &resourceFlavor)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/integration/singlecluster/kueuectl/passthrough_test.go b/test/integration/singlecluster/kueuectl/passthrough_test.go index 99a638a19f0..0143af1d921 100644 --- a/test/integration/singlecluster/kueuectl/passthrough_test.go +++ b/test/integration/singlecluster/kueuectl/passthrough_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/utils/set" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -121,7 +121,7 @@ var _ = ginkgo.Describe("Kueuectl Pass-through", ginkgo.Ordered, ginkgo.Continue gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%q", string(out)) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, key, obj)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, key, obj)).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }, diff --git a/test/integration/singlecluster/scheduler/scheduler_test.go b/test/integration/singlecluster/scheduler/scheduler_test.go index ba155bd8030..b0578485b82 100644 --- a/test/integration/singlecluster/scheduler/scheduler_test.go +++ b/test/integration/singlecluster/scheduler/scheduler_test.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/core" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" @@ -1743,7 +1743,7 @@ var _ = ginkgo.Describe("Scheduler", func() { Cohort(chName). Obj() util.MustCreate(ctx, k8sClient, strictFIFOClusterQ) - matchingNS = util.CreateNamespaceFromObjectWithLog(ctx, k8sClient, testing.MakeNamespaceWrapper("").GenerateName("foo-").Label("dep", "eng").Obj()) + matchingNS = util.CreateNamespaceFromObjectWithLog(ctx, k8sClient, utiltesting.MakeNamespaceWrapper("").GenerateName("foo-").Label("dep", "eng").Obj()) }) ginkgo.AfterEach(func() { @@ -1961,7 +1961,7 @@ var _ = ginkgo.Describe("Scheduler", func() { } ginkgo.DescribeTable("", func(tp testParams) { - lrBuilder := testing.MakeLimitRange("limit", ns.Name) + lrBuilder := utiltesting.MakeLimitRange("limit", ns.Name) if tp.limitType != "" { lrBuilder.WithType(tp.limitType) } diff --git a/test/integration/singlecluster/tas/tas_test.go b/test/integration/singlecluster/tas/tas_test.go index 12e69bea4a0..a3d8fd9d975 100644 --- a/test/integration/singlecluster/tas/tas_test.go +++ b/test/integration/singlecluster/tas/tas_test.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" utiltas "sigs.k8s.io/kueue/pkg/util/tas" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" "sigs.k8s.io/kueue/pkg/workload" @@ -280,8 +280,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("b1-r1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -291,8 +291,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("b1-r2"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r2"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r2"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -302,8 +302,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("b2-r1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b2"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b2"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -313,8 +313,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("b2-r2"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b2"). - Label(testing.DefaultRackTopologyLevel, "r2"). + Label(utiltesting.DefaultBlockTopologyLevel, "b2"). + Label(utiltesting.DefaultRackTopologyLevel, "r2"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -360,7 +360,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl1 := utiltestingapi.MakeWorkload("wl1-inadmissible", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "2").Obj() wl1.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), } util.MustCreate(ctx, k8sClient, wl1) }) @@ -377,7 +377,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl1.Spec.PodSets[0].Count = 2 wl1.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultBlockTopologyLevel), + Required: ptr.To(utiltesting.DefaultBlockTopologyLevel), } util.MustCreate(ctx, k8sClient, wl1) }) @@ -392,8 +392,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { gomega.Expect(wl1.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ Levels: []string{ - testing.DefaultBlockTopologyLevel, - testing.DefaultRackTopologyLevel, + utiltesting.DefaultBlockTopologyLevel, + utiltesting.DefaultRackTopologyLevel, }, Domains: []kueue.TopologyDomainAssignment{ { @@ -419,7 +419,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl2 = utiltestingapi.MakeWorkload("wl2", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl2.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), } util.MustCreate(ctx, k8sClient, wl2) }) @@ -434,8 +434,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { gomega.Expect(wl2.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ Levels: []string{ - testing.DefaultBlockTopologyLevel, - testing.DefaultRackTopologyLevel, + utiltesting.DefaultBlockTopologyLevel, + utiltesting.DefaultRackTopologyLevel, }, Domains: []kueue.TopologyDomainAssignment{ { @@ -454,7 +454,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl3 = utiltestingapi.MakeWorkload("wl3", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl3.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), } util.MustCreate(ctx, k8sClient, wl3) }) @@ -469,8 +469,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { gomega.Expect(wl3.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ Levels: []string{ - testing.DefaultBlockTopologyLevel, - testing.DefaultRackTopologyLevel, + utiltesting.DefaultBlockTopologyLevel, + utiltesting.DefaultRackTopologyLevel, }, Domains: []kueue.TopologyDomainAssignment{ { @@ -489,7 +489,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl4 = utiltestingapi.MakeWorkload("wl4", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl4.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), } util.MustCreate(ctx, k8sClient, wl4) util.ExpectWorkloadsToBePending(ctx, k8sClient, wl4) @@ -510,8 +510,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { gomega.Expect(wl4.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ Levels: []string{ - testing.DefaultBlockTopologyLevel, - testing.DefaultRackTopologyLevel, + utiltesting.DefaultBlockTopologyLevel, + utiltesting.DefaultRackTopologyLevel, }, Domains: []kueue.TopologyDomainAssignment{ { @@ -534,7 +534,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl1.Spec.PodSets[0].Count = 4 wl1.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultBlockTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultBlockTopologyLevel), } util.MustCreate(ctx, k8sClient, wl1) }) @@ -548,7 +548,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl2 = utiltestingapi.MakeWorkload("wl2", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl2.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), } util.MustCreate(ctx, k8sClient, wl2) }) @@ -577,7 +577,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl3 = utiltestingapi.MakeWorkload("wl3", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl3.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Preferred: ptr.To(testing.DefaultRackTopologyLevel), + Preferred: ptr.To(utiltesting.DefaultRackTopologyLevel), } util.MustCreate(ctx, k8sClient, wl3) }) @@ -632,7 +632,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload which requires block and can fit", func() { wl = utiltestingapi.MakeWorkload("wl", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).PodSets(*utiltestingapi.MakePodSet("worker", 2). - RequiredTopologyRequest(testing.DefaultBlockTopologyLevel). + RequiredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl) }) @@ -656,7 +656,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), wl)).To(gomega.Succeed()) gomega.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{ {Count: 1, Values: []string{"b1", "r1"}}, {Count: 1, Values: []string{"b1", "r2"}}, @@ -699,7 +699,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload which requires block and can fit", func() { wl = utiltestingapi.MakeWorkload("wl", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).PodSets(*utiltestingapi.MakePodSet("worker", 2). - RequiredTopologyRequest(testing.DefaultBlockTopologyLevel). + RequiredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl) }) @@ -725,7 +725,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), wl)).To(gomega.Succeed()) gomega.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ - Levels: []string{testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel}, + Levels: []string{utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel}, Domains: []kueue.TopologyDomainAssignment{ {Count: 1, Values: []string{"b1", "r1"}}, {Count: 1, Values: []string{"b1", "r2"}}, @@ -749,8 +749,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("x3"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x3"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -761,8 +761,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("x1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r2"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r2"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -773,8 +773,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("x4"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b2"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b2"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x4"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -785,8 +785,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("x2"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b2"). - Label(testing.DefaultRackTopologyLevel, "r2"). + Label(utiltesting.DefaultBlockTopologyLevel, "b2"). + Label(utiltesting.DefaultRackTopologyLevel, "r2"). Label(corev1.LabelHostname, "x2"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -832,7 +832,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload which requires rack, but does not fit in any", func() { wl1 := utiltestingapi.MakeWorkload("wl1-inadmissible", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 4). - PreferredTopologyRequest(testing.DefaultRackTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultRackTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "2").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -848,7 +848,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload which can fit", func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 1). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -862,7 +862,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating second a workload which cannot fit", func() { wl2 = utiltestingapi.MakeWorkload("wl2", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker-2", 4). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl2) @@ -923,7 +923,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload", func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -972,7 +972,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload", func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -1027,7 +1027,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). // requiring the same block makes sure that no replacement is possible - RequiredTopologyRequest(testing.DefaultBlockTopologyLevel). + RequiredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -1091,7 +1091,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload", func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). - RequiredTopologyRequest(testing.DefaultBlockTopologyLevel). + RequiredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -1145,7 +1145,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload", func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -1168,7 +1168,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a second workload", func() { wl2 = utiltestingapi.MakeWorkload("wl2", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl2) @@ -1242,8 +1242,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload", func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). - NodeSelector(map[string]string{testing.DefaultBlockTopologyLevel: "b1"}). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + NodeSelector(map[string]string{utiltesting.DefaultBlockTopologyLevel: "b1"}). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -1293,7 +1293,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload", func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -1356,8 +1356,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("x3"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x3"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -1368,8 +1368,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("x1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -1380,8 +1380,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("x4"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r2"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r2"). Label(corev1.LabelHostname, "x4"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -1392,8 +1392,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("x2"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r2"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r2"). Label(corev1.LabelHostname, "x2"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -1442,7 +1442,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload", func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). - RequiredTopologyRequest(testing.DefaultRackTopologyLevel). + RequiredTopologyRequest(utiltesting.DefaultRackTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -1512,8 +1512,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("x2"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x2"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("5"), @@ -1524,8 +1524,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("x1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r2"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r2"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("5"), @@ -1580,7 +1580,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). Priority(1). PodSets(*utiltestingapi.MakePodSet("worker", 1). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "5").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -1595,7 +1595,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl2 = utiltestingapi.MakeWorkload("wl2", ns.Name). Priority(2). PodSets(*utiltestingapi.MakePodSet("worker", 1). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "5").Obj() util.MustCreate(ctx, k8sClient, wl2) @@ -1610,7 +1610,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl3 = utiltestingapi.MakeWorkload("wl3", ns.Name). Priority(3). PodSets(*utiltestingapi.MakePodSet("worker", 2). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "5").Obj() util.MustCreate(ctx, k8sClient, wl3) @@ -1640,8 +1640,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("x2"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x2"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("5"), @@ -1652,8 +1652,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("x1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r2"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r2"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("5"), @@ -1726,7 +1726,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { ginkgo.By("creating a workload which can fit only when borrowing quota", func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). PodSets(*utiltestingapi.MakePodSet("worker", 2). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "4").Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -1745,7 +1745,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl2 = utiltestingapi.MakeWorkload("wl2", ns.Name). Priority(2). PodSets(*utiltestingapi.MakePodSet("worker", 1). - PreferredTopologyRequest(testing.DefaultBlockTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultBlockTopologyLevel). Obj()). Queue(kueue.LocalQueueName(localQueueB.Name)).Request(corev1.ResourceCPU, "2").Obj() util.MustCreate(ctx, k8sClient, wl2) @@ -1806,7 +1806,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl1.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), } util.MustCreate(ctx, k8sClient, wl1) }) @@ -1819,8 +1819,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("b1-r1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), @@ -1886,8 +1886,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("b1-r1-x1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "b1-r1-x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -1909,7 +1909,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { wl1 = utiltestingapi.MakeWorkload("wl1", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl1.Spec.PodSets[0].TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), } util.MustCreate(ctx, k8sClient, wl1) }) @@ -1944,8 +1944,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("node-missing-label"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "node-missing-label"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -2076,7 +2076,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl1.Spec.PodSets[0] = *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "1"). - PreferredTopologyRequest(testing.DefaultRackTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultRackTopologyLevel). Image("image"). Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -2109,8 +2109,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("x1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -2206,7 +2206,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl1.Spec.PodSets[0] = *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "1"). - PreferredTopologyRequest(testing.DefaultRackTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultRackTopologyLevel). Image("image"). Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -2245,8 +2245,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { *testingnode.MakeNode("x2"). Label("node-group", "tas"). Label("dedicated-selector-key", "dedicated-selector-value-abc"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x2"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -2258,8 +2258,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { *testingnode.MakeNode("x1"). Label("node-group", "tas"). Label("dedicated-selector-key", "dedicated-selector-value-xyz"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r2"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r2"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("2"), @@ -2326,7 +2326,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl1.Spec.PodSets[0] = *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "1"). - PreferredTopologyRequest(testing.DefaultRackTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultRackTopologyLevel). Image("image"). Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -2383,8 +2383,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("x1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -2474,7 +2474,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1").Obj() wl1.Spec.PodSets[0] = *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "1"). - PreferredTopologyRequest(testing.DefaultRackTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultRackTopologyLevel). Image("image"). Obj() util.MustCreate(ctx, k8sClient, wl1) @@ -2502,8 +2502,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("x1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -2582,7 +2582,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Queue(kueue.LocalQueueName(localQueue.Name)).Request(corev1.ResourceCPU, "1"). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1). Request(corev1.ResourceCPU, "1"). - PreferredTopologyRequest(testing.DefaultRackTopologyLevel). + PreferredTopologyRequest(utiltesting.DefaultRackTopologyLevel). Image("image"). Obj(), ).Obj() @@ -2616,8 +2616,8 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("x1"). Label("node-group", "tas"). - Label(testing.DefaultBlockTopologyLevel, "b1"). - Label(testing.DefaultRackTopologyLevel, "r1"). + Label(utiltesting.DefaultBlockTopologyLevel, "b1"). + Label(utiltesting.DefaultRackTopologyLevel, "r1"). Label(corev1.LabelHostname, "x1"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), @@ -2731,7 +2731,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { nodes = []corev1.Node{ *testingnode.MakeNode("cpu-node"). Label(corev1.LabelInstanceTypeStable, "cpu-node"). - Label(testing.DefaultRackTopologyLevel, "cpu-rack"). + Label(utiltesting.DefaultRackTopologyLevel, "cpu-rack"). StatusAllocatable(corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("5"), corev1.ResourcePods: resource.MustParse("10"), @@ -2740,7 +2740,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { Obj(), *testingnode.MakeNode("gpu-node"). Label(corev1.LabelInstanceTypeStable, "gpu-node"). - Label(testing.DefaultRackTopologyLevel, "gpu-rack"). + Label(utiltesting.DefaultRackTopologyLevel, "gpu-rack"). StatusAllocatable(corev1.ResourceList{ gpuResName: resource.MustParse("4"), corev1.ResourcePods: resource.MustParse("10"), @@ -2751,7 +2751,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { util.CreateNodesWithStatus(ctx, k8sClient, nodes) topology = utiltestingapi.MakeTopology("default").Levels( - testing.DefaultRackTopologyLevel, + utiltesting.DefaultRackTopologyLevel, ).Obj() util.MustCreate(ctx, k8sClient, topology) @@ -2800,13 +2800,13 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { map[string]string{corev1.LabelInstanceTypeStable: "cpu-node"}, ).Request(corev1.ResourceCPU, "5").Obj() ps1.TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), } ps2 := *utiltestingapi.MakePodSet("worker", 2).NodeSelector( map[string]string{corev1.LabelInstanceTypeStable: "gpu-node"}, ).Request(gpuResName, "2").Obj() ps2.TopologyRequest = &kueue.PodSetTopologyRequest{ - Required: ptr.To(testing.DefaultRackTopologyLevel), + Required: ptr.To(utiltesting.DefaultRackTopologyLevel), } wl1.Spec.PodSets = []kueue.PodSet{ps1, ps2} util.MustCreate(ctx, k8sClient, wl1) @@ -2822,7 +2822,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { gomega.Expect(wl1.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ Levels: []string{ - testing.DefaultRackTopologyLevel, + utiltesting.DefaultRackTopologyLevel, }, Domains: []kueue.TopologyDomainAssignment{ { @@ -2837,7 +2837,7 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { gomega.Expect(wl1.Status.Admission.PodSetAssignments[1].TopologyAssignment).Should(gomega.BeComparableTo( &kueue.TopologyAssignment{ Levels: []string{ - testing.DefaultRackTopologyLevel, + utiltesting.DefaultRackTopologyLevel, }, Domains: []kueue.TopologyDomainAssignment{ { @@ -3350,21 +3350,21 @@ var _ = ginkgo.Describe("Topology validations", func() { gomega.Succeed()), ginkgo.Entry("no levels", utiltestingapi.MakeTopology("no-levels").Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("invalid levels", utiltestingapi.MakeTopology("invalid-level").Levels("@invalid").Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("non-unique levels", - utiltestingapi.MakeTopology("default").Levels(testing.DefaultBlockTopologyLevel, testing.DefaultBlockTopologyLevel).Obj(), - testing.BeInvalidError()), + utiltestingapi.MakeTopology("default").Levels(utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultBlockTopologyLevel).Obj(), + utiltesting.BeInvalidError()), ginkgo.Entry("kubernetes.io/hostname first", - utiltestingapi.MakeTopology("default").Levels(corev1.LabelHostname, testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel).Obj(), - testing.BeInvalidError()), + utiltestingapi.MakeTopology("default").Levels(corev1.LabelHostname, utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel).Obj(), + utiltesting.BeInvalidError()), ginkgo.Entry("kubernetes.io/hostname middle", - utiltestingapi.MakeTopology("default").Levels(testing.DefaultBlockTopologyLevel, corev1.LabelHostname, testing.DefaultRackTopologyLevel).Obj(), - testing.BeInvalidError()), + utiltestingapi.MakeTopology("default").Levels(utiltesting.DefaultBlockTopologyLevel, corev1.LabelHostname, utiltesting.DefaultRackTopologyLevel).Obj(), + utiltesting.BeInvalidError()), ginkgo.Entry("kubernetes.io/hostname last", - utiltestingapi.MakeTopology("default").Levels(testing.DefaultBlockTopologyLevel, testing.DefaultRackTopologyLevel, corev1.LabelHostname).Obj(), + utiltestingapi.MakeTopology("default").Levels(utiltesting.DefaultBlockTopologyLevel, utiltesting.DefaultRackTopologyLevel, corev1.LabelHostname).Obj(), gomega.Succeed()), ) }) @@ -3394,13 +3394,13 @@ var _ = ginkgo.Describe("Topology validations", func() { NodeLabel: "added", }) }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("updating levels order is prohibited", utiltestingapi.MakeDefaultThreeLevelTopology("default"), func(topology *kueue.Topology) { topology.Spec.Levels[0], topology.Spec.Levels[1] = topology.Spec.Levels[1], topology.Spec.Levels[0] }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ) }) }) diff --git a/test/integration/singlecluster/webhook/core/admissioncheck_test.go b/test/integration/singlecluster/webhook/core/admissioncheck_test.go index 93a44473400..0255dbd5b1e 100644 --- a/test/integration/singlecluster/webhook/core/admissioncheck_test.go +++ b/test/integration/singlecluster/webhook/core/admissioncheck_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -91,7 +91,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("Should fail to create AdmissionCheck with bad ref api group", kueue.AdmissionCheck{ @@ -104,7 +104,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("Should fail to create AdmissionCheck with no ref api group", kueue.AdmissionCheck{ @@ -116,7 +116,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("Should fail to create AdmissionCheck with bad ref kind", kueue.AdmissionCheck{ @@ -129,7 +129,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("Should fail to create AdmissionCheck with no ref kind", kueue.AdmissionCheck{ @@ -141,7 +141,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("Should fail to create AdmissionCheck with bad ref name", kueue.AdmissionCheck{ @@ -154,7 +154,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("Should fail to create AdmissionCheck with no ref name", kueue.AdmissionCheck{ @@ -166,7 +166,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("Should allow to create AdmissionCheck with no parameters", kueue.AdmissionCheck{ @@ -256,7 +256,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { var updateAC kueue.AdmissionCheck g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(ac), &updateAC)).Should(gomega.Succeed()) updateAC.Spec.Parameters.Name = "" - g.Expect(k8sClient.Update(ctx, &updateAC)).Should(testing.BeInvalidError()) + g.Expect(k8sClient.Update(ctx, &updateAC)).Should(utiltesting.BeInvalidError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -276,7 +276,7 @@ var _ = ginkgo.Describe("AdmissionCheck Webhook", ginkgo.Ordered, func() { var updateAC kueue.AdmissionCheck g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(ac), &updateAC)).Should(gomega.Succeed()) updateAC.Spec.ControllerName = "controller-name2" - g.Expect(k8sClient.Update(ctx, &updateAC)).Should(testing.BeInvalidError()) + g.Expect(k8sClient.Update(ctx, &updateAC)).Should(utiltesting.BeInvalidError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/integration/singlecluster/webhook/core/clusterqueue_test.go b/test/integration/singlecluster/webhook/core/clusterqueue_test.go index 956d7132fab..28947371d62 100644 --- a/test/integration/singlecluster/webhook/core/clusterqueue_test.go +++ b/test/integration/singlecluster/webhook/core/clusterqueue_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -185,7 +185,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { var updateCQ kueue.ClusterQueue g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cq), &updateCQ)).Should(gomega.Succeed()) updateCQ.Spec.ResourceGroups[0].Flavors[0].Name = "@x86" - g.Expect(k8sClient.Update(ctx, &updateCQ)).Should(testing.BeInvalidError()) + g.Expect(k8sClient.Update(ctx, &updateCQ)).Should(utiltesting.BeInvalidError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -232,30 +232,30 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { ResourceGroup(*utiltestingapi.MakeFlavorQuotas("x86").Resource(corev1.ResourceCPU, "2", "-1").Obj()). Cohort("cohort"). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should have non-negative quota value", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("x86").Resource(corev1.ResourceCPU, "-1").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should have at least one flavor", utiltestingapi.MakeClusterQueue("cluster-queue").ResourceGroup().Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should have at least one resource", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("foo").Obj()). Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should have qualified flavor name", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("invalid_name").Resource(corev1.ResourceCPU, "5").Obj()). Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should have qualified resource name", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("x86").Resource("@cpu", "5").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should have valid resources quantity", func() *kueue.ClusterQueue { flvQuotas := utiltestingapi.MakeFlavorQuotas("flavor") @@ -264,7 +264,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { } return utiltestingapi.MakeClusterQueue("cluster-queue").ResourceGroup(*flvQuotas.Obj()).Obj() }(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should have valid flavors quantity", func() *kueue.ClusterQueue { flavors := make([]kueue.FlavorQuotas, flavorsMaxItems+1) @@ -275,15 +275,15 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { } return utiltestingapi.MakeClusterQueue("cluster-queue").ResourceGroup(flavors...).Obj() }(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should forbid clusterQueue creation with unqualified labelSelector", utiltestingapi.MakeClusterQueue("cluster-queue").NamespaceSelector(&metav1.LabelSelector{ MatchLabels: map[string]string{"nospecialchars^=@": "bar"}, }).Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should forbid to create clusterQueue with unknown clusterQueueingStrategy", utiltestingapi.MakeClusterQueue("cluster-queue").QueueingStrategy("unknown").Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should allow to create clusterQueue with empty clusterQueueingStrategy", utiltestingapi.MakeClusterQueue("cluster-queue").QueueingStrategy("").Obj(), gomega.Succeed()), @@ -298,10 +298,10 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { gomega.Succeed()), ginkgo.Entry("Should forbid to create clusterQueue with unknown preemption.withinCohort", utiltestingapi.MakeClusterQueue("cluster-queue").Preemption(kueue.ClusterQueuePreemption{ReclaimWithinCohort: "unknown"}).Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should forbid to create clusterQueue with unknown preemption.withinClusterQueue", utiltestingapi.MakeClusterQueue("cluster-queue").Preemption(kueue.ClusterQueuePreemption{WithinClusterQueue: "unknown"}).Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should allow to create clusterQueue with built-in resources with qualified names", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU).Obj()). @@ -311,13 +311,13 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource("@cpu").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should allow to create clusterQueue with valid cohort", utiltestingapi.MakeClusterQueue("cluster-queue").Cohort("prod").Obj(), gomega.Succeed()), ginkgo.Entry("Should forbid to create clusterQueue with invalid cohort", utiltestingapi.MakeClusterQueue("cluster-queue").Cohort("@prod").Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should allow to create clusterQueue with extended resources with qualified names", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource("example.com/gpu").Obj()). @@ -332,13 +332,13 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("invalid_name").Obj()). Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should forbid to create clusterQueue with flavor quota with negative value", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("x86").Resource(corev1.ResourceCPU, "-1").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should allow to create clusterQueue with flavor quota with zero values", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup( @@ -358,13 +358,13 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { *utiltestingapi.MakeFlavorQuotas("x86").Resource(corev1.ResourceCPU, "1", "-1").Obj()). Cohort("cohort"). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should forbid to create clusterQueue with flavor quota with borrowingLimit and empty cohort", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("x86").Resource(corev1.ResourceCPU, "1", "1").Obj()). Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should allow to create clusterQueue with empty queueing strategy", utiltestingapi.MakeClusterQueue("cluster-queue"). QueueingStrategy(""). @@ -374,7 +374,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { utiltestingapi.MakeClusterQueue("cluster-queue").NamespaceSelector(&metav1.LabelSelector{ MatchLabels: map[string]string{"nospecialchars^=@": "bar"}, }).Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should forbid to create clusterQueue with namespaceSelector with invalid expressions", utiltestingapi.MakeClusterQueue("cluster-queue").NamespaceSelector(&metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -384,7 +384,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }, }, }).Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should allow to create clusterQueue with multiple resource groups", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup( @@ -430,7 +430,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should forbid to create clusterQueue missing resources in a flavor", &kueue.ClusterQueue{ ObjectMeta: metav1.ObjectMeta{ @@ -449,7 +449,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should forbid to create clusterQueue missing resources in a flavor", &kueue.ClusterQueue{ ObjectMeta: metav1.ObjectMeta{ @@ -469,7 +469,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should forbid to create clusterQueue missing resources in a flavor and mismatch", &kueue.ClusterQueue{ ObjectMeta: metav1.ObjectMeta{ @@ -489,7 +489,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should forbid to create clusterQueue with resource in more than one resource group", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup( @@ -504,7 +504,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { Obj(), ). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should forbid to create clusterQueue with flavor in more than one resource group", utiltestingapi.MakeClusterQueue("cluster-queue"). ResourceGroup( @@ -515,7 +515,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { *utiltestingapi.MakeFlavorQuotas("beta").Resource(corev1.ResourceMemory).Obj(), ). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should forbid to create clusterQueue missing with invalid preemption due to reclaimWithinCohort=Never, while borrowWithinCohort!=nil", &kueue.ClusterQueue{ ObjectMeta: metav1.ObjectMeta{ @@ -530,7 +530,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should allow to create clusterQueue with valid preemption with borrowWithinCohort", &kueue.ClusterQueue{ ObjectMeta: metav1.ObjectMeta{ @@ -634,7 +634,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should forbid collapsed FairSharing weight", &kueue.ClusterQueue{ ObjectMeta: metav1.ObjectMeta{ @@ -647,7 +647,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should forbid negative FairSharing weight", &kueue.ClusterQueue{ ObjectMeta: metav1.ObjectMeta{ @@ -659,7 +659,7 @@ var _ = ginkgo.Describe("ClusterQueue Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ) }) }) diff --git a/test/integration/singlecluster/webhook/core/cohort_test.go b/test/integration/singlecluster/webhook/core/cohort_test.go index cf2e37e9fd8..7f27ea30cd2 100644 --- a/test/integration/singlecluster/webhook/core/cohort_test.go +++ b/test/integration/singlecluster/webhook/core/cohort_test.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -56,7 +56,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { }, ginkgo.Entry("Should disallow empty name", utiltestingapi.MakeCohort("").Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should allow default Cohort", utiltestingapi.MakeCohort("cohort").Obj(), gomega.Succeed()), @@ -65,25 +65,25 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { gomega.Succeed()), ginkgo.Entry("Should reject invalid parent name", utiltestingapi.MakeCohort("cohort").Parent("@prod").Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("ResourceGroup should have at least one flavor", utiltestingapi.MakeCohort("cohort").ResourceGroup().Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("FlavorQuota should have at least one resource", utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("foo").Obj()). Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should reject invalid flavor name", utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("@x86").Resource(corev1.ResourceCPU, "5").Obj()). Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should allow valid resource name", utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("x86").Resource("@cpu", "5").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should reject too many flavors in resource group", func() *kueue.Cohort { var flavors []kueue.FlavorQuotas @@ -98,7 +98,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { ResourceGroup(flavors...). Obj() }(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should reject too many resources in resource group", func() *kueue.Cohort { fq := utiltestingapi.MakeFlavorQuotas("flavor") @@ -109,7 +109,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { ResourceGroup(*fq.Obj()). Obj() }(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should allow resource with valid name", utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource(corev1.ResourceCPU).Obj()). @@ -119,7 +119,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource("@cpu").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should allow extended resources with valid name", utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("default").Resource("example.com/gpu").Obj()). @@ -134,28 +134,28 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("x_86").Resource(corev1.ResourceCPU).Obj()). Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should reject negative nominal quota", utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("x86").Resource(corev1.ResourceCPU, "-1").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should reject negative borrowing limit", utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("x86").Resource(corev1.ResourceCPU, "1", "-1").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should reject negative lending limit", utiltestingapi.MakeCohort("cohort"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas("x86").Resource(corev1.ResourceCPU, "1", "", "-1").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should reject borrowingLimit when no parent", utiltestingapi.MakeCohort("cohort"). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("x86").Resource(corev1.ResourceCPU, "1", "1").Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should allow borrowingLimit 0 when parent exists", utiltestingapi.MakeCohort("cohort"). ResourceGroup( @@ -178,7 +178,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { Obj(), ). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should allow lendingLimit when parent exists", utiltestingapi.MakeCohort("cohort"). ResourceGroup( @@ -254,7 +254,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should reject missing resources in a flavor", &kueue.Cohort{ ObjectMeta: metav1.ObjectMeta{ @@ -273,7 +273,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should reject resource not defined in resource group", &kueue.Cohort{ ObjectMeta: metav1.ObjectMeta{ @@ -293,7 +293,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { }, }, }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should reject resource in more than one resource group", utiltestingapi.MakeCohort("cohort"). ResourceGroup( @@ -308,7 +308,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { Obj(), ). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should reject flavor in more than one resource group", utiltestingapi.MakeCohort("cohort"). ResourceGroup( @@ -319,7 +319,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { *utiltestingapi.MakeFlavorQuotas("beta").Resource(corev1.ResourceMemory).Obj(), ). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should allow FairSharing weight", utiltestingapi.MakeCohort("cohort").FairWeight(resource.MustParse("1")).Obj(), gomega.Succeed()), @@ -328,7 +328,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { gomega.Succeed()), ginkgo.Entry("Should forbid negative FairSharing weight", utiltestingapi.MakeCohort("cohort").FairWeight(resource.MustParse("-1")).Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should allow fractional FairSharing weight", utiltestingapi.MakeCohort("cohort").FairWeight(resource.MustParse("0.5")).Obj(), gomega.Succeed()), @@ -347,11 +347,11 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() { ginkgo.Entry("Should forbid threshold FairSharing weight", // 10^-9 utiltestingapi.MakeCohort("cohort").FairWeight(resource.MustParse("1n")).Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should forbid collapsed FairSharing weight", // 10^-10 utiltestingapi.MakeCohort("cohort").FairWeight(resource.MustParse("0.0000000001")).Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ) }) diff --git a/test/integration/singlecluster/webhook/core/localqueue_test.go b/test/integration/singlecluster/webhook/core/localqueue_test.go index 534caa64f9a..6e7e7795eb6 100644 --- a/test/integration/singlecluster/webhook/core/localqueue_test.go +++ b/test/integration/singlecluster/webhook/core/localqueue_test.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -51,7 +51,7 @@ var _ = ginkgo.Describe("Queue validating webhook", ginkgo.Ordered, func() { ginkgo.It("Should reject bad value for spec.clusterQueue", func() { ginkgo.By("Creating a new Queue") obj := utiltestingapi.MakeLocalQueue(queueName, ns.Name).ClusterQueue("invalid_name").Obj() - gomega.Expect(k8sClient.Create(ctx, obj)).Should(testing.BeInvalidError()) + gomega.Expect(k8sClient.Create(ctx, obj)).Should(utiltesting.BeInvalidError()) }) ginkgo.It("Should reject the change of spec.clusterQueue", func() { ginkgo.By("Creating a new Queue") @@ -63,7 +63,7 @@ var _ = ginkgo.Describe("Queue validating webhook", ginkgo.Ordered, func() { var updatedQ kueue.LocalQueue g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(obj), &updatedQ)).Should(gomega.Succeed()) updatedQ.Spec.ClusterQueue = "bar" - g.Expect(k8sClient.Update(ctx, &updatedQ)).Should(testing.BeInvalidError()) + g.Expect(k8sClient.Update(ctx, &updatedQ)).Should(utiltesting.BeInvalidError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/integration/singlecluster/webhook/core/resourceflavor_test.go b/test/integration/singlecluster/webhook/core/resourceflavor_test.go index e957d7910ed..1238f2cef40 100644 --- a/test/integration/singlecluster/webhook/core/resourceflavor_test.go +++ b/test/integration/singlecluster/webhook/core/resourceflavor_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/test/util" ) @@ -108,7 +108,7 @@ var _ = ginkgo.Describe("ResourceFlavor Webhook", ginkgo.Ordered, func() { err := k8sClient.Create(ctx, resourceFlavor) if isInvalid { gomega.Expect(err).To(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeInvalidError()) + gomega.Expect(err).Should(utiltesting.BeInvalidError()) } else { gomega.Expect(err).To(gomega.Succeed()) defer func() { @@ -149,7 +149,7 @@ var _ = ginkgo.Describe("ResourceFlavor Webhook", ginkgo.Ordered, func() { ginkgo.By("Updating the resourceFlavor with invalid labels") err := k8sClient.Update(ctx, &created) gomega.Expect(err).To(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeInvalidError()) + gomega.Expect(err).Should(utiltesting.BeInvalidError()) }) }) @@ -172,10 +172,10 @@ var _ = ginkgo.Describe("ResourceFlavor Webhook", ginkgo.Ordered, func() { Value: "bar", Effect: corev1.TaintEffectNoSchedule, }).Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("Should fail to create with invalid label name", utiltestingapi.MakeResourceFlavor("resource-flavor").NodeLabel("@abc", "foo").Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("Should fail to create with invalid tolerations", utiltestingapi.MakeResourceFlavor("resource-flavor"). Toleration(corev1.Toleration{ @@ -203,6 +203,6 @@ var _ = ginkgo.Describe("ResourceFlavor Webhook", ginkgo.Ordered, func() { Effect: corev1.TaintEffectNoSchedule, }). Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ) }) diff --git a/test/integration/singlecluster/webhook/core/workload_test.go b/test/integration/singlecluster/webhook/core/workload_test.go index 632c6be0e85..56904268e76 100644 --- a/test/integration/singlecluster/webhook/core/workload_test.go +++ b/test/integration/singlecluster/webhook/core/workload_test.go @@ -36,7 +36,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/pkg/workloadslicing" @@ -109,7 +109,7 @@ var _ = ginkgo.Describe("Workload defaulting webhook", ginkgo.Ordered, func() { }, }, } - gomega.Expect(k8sClient.Create(ctx, &workload)).Should(testing.BeInvalidError()) + gomega.Expect(k8sClient.Create(ctx, &workload)).Should(utiltesting.BeInvalidError()) }) }) }) @@ -142,7 +142,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { workload := utiltestingapi.MakeWorkload(workloadName, ns.Name).PodSets(podSets...).Obj() err := k8sClient.Create(ctx, workload) if isInvalid { - gomega.Expect(err).Should(testing.BeInvalidError()) + gomega.Expect(err).Should(utiltesting.BeInvalidError()) } else { gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) } @@ -170,7 +170,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { *utiltestingapi.MakePodSet("@driver", 1).Obj(), ).Obj() }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("invalid priorityClassName", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -178,7 +178,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { Priority(0). Obj() }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("empty priorityClassName is valid", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name).Priority(0).Obj() @@ -190,26 +190,26 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { PriorityClass("priority"). Obj() }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("invalid queueName", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). Queue("@invalid"). Obj() }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("should not request num-pods resource", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). PodSets( *utiltestingapi.MakePodSet("bad", 1). InitContainers( - testing.SingleContainerForRequest(map[corev1.ResourceName]string{ + utiltesting.SingleContainerForRequest(map[corev1.ResourceName]string{ corev1.ResourcePods: "1", })..., ). Containers( - testing.SingleContainerForRequest(map[corev1.ResourceName]string{ + utiltesting.SingleContainerForRequest(map[corev1.ResourceName]string{ corev1.ResourcePods: "1", })..., ). @@ -217,7 +217,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ). Obj() }, - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("empty podSetUpdates should be valid since it is optional", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -279,7 +279,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ). Obj() }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("invalid podSet minCount (too big)", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -288,7 +288,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ). Obj() }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("too many variable count podSets", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -298,14 +298,14 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ). Obj() }, - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ginkgo.Entry("invalid maximumExexcutionTimeSeconds", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). MaximumExecutionTimeSeconds(0). Obj() }, - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("valid maximumExexcutionTimeSeconds", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -333,14 +333,14 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { Obj() }, utiltestingapi.MakeAdmission("@invalid").Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("invalid podSet name in status assignment", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). Obj() }, utiltestingapi.MakeAdmission("cluster-queue", "@invalid").Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("mismatched names in admission with names in podSets", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -351,7 +351,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { Obj() }, utiltestingapi.MakeAdmission("cluster-queue", "main1", "main2", "main3").Obj(), - testing.BeInvalidError()), + utiltesting.BeInvalidError()), ginkgo.Entry("assignment usage should be divisible by count", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -368,7 +368,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { Count(3). Obj()). Obj(), - testing.BeForbiddenError()), + utiltesting.BeForbiddenError()), ) ginkgo.DescribeTable("Should have valid values when setting AdmissionCheckState", func(w func() *kueue.Workload, acs kueue.AdmissionCheckState) { @@ -378,7 +378,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), wl)).To(gomega.Succeed()) workload.SetAdmissionCheckState(&wl.Status.AdmissionChecks, acs, realClock) - g.Expect(k8sClient.Status().Update(ctx, wl)).Should(testing.BeForbiddenError()) + g.Expect(k8sClient.Status().Update(ctx, wl)).Should(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }, ginkgo.Entry("mismatched names in podSetUpdates with names in podSets", @@ -440,7 +440,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { {Name: "ps1", Count: 4}, {Name: "ps2", Count: 1}, }) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) }) @@ -453,7 +453,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ) ginkgo.BeforeEach(func() { workloadPriorityClass = utiltestingapi.MakeWorkloadPriorityClass("workload-priority-class").PriorityValue(200).Obj() - priorityClass = testing.MakePriorityClass("priority-class").PriorityValue(100).Obj() + priorityClass = utiltesting.MakePriorityClass("priority-class").PriorityValue(100).Obj() util.MustCreate(ctx, k8sClient, workloadPriorityClass) util.MustCreate(ctx, k8sClient, priorityClass) }) @@ -487,7 +487,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { func(newWL *kueue.Workload) { newWL.Spec.PodSets = []kueue.PodSet{*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 2).Obj()} }, - testing.BeForbiddenError(), + utiltesting.BeForbiddenError(), ), ginkgo.Entry("podSets should not be updated: podSpec", func() *kueue.Workload { @@ -512,7 +512,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }, }} }, - testing.BeForbiddenError(), + utiltesting.BeForbiddenError(), ), ginkgo.Entry("queueName can be updated when not admitted", func() *kueue.Workload { @@ -542,7 +542,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { func(newWL *kueue.Workload) { newWL.Spec.QueueName = "q2" }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("queueName can be updated when admission is reset", func() *kueue.Workload { @@ -599,7 +599,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { func(newWL *kueue.Workload) { newWL.Spec.PriorityClassSource = constants.WorkloadPriorityClassSource }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("priorityClassName should not be updated", func() *kueue.Workload { @@ -613,7 +613,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { func(newWL *kueue.Workload) { newWL.Spec.PriorityClassName = "test-class-2" }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("should change other fields of admissionchecks when podSetUpdates is immutable", func() *kueue.Workload { @@ -714,7 +714,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { func(newWL *kueue.Workload) { newWL.Spec.PodSets[0].Count = 10 }, - testing.BeForbiddenError(), + utiltesting.BeForbiddenError(), ), ginkgo.Entry("reclaimable pod count can go to 0 if the job is suspended", func() *kueue.Workload { @@ -779,7 +779,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { func(newWL *kueue.Workload) { newWL.Spec.MaximumExecutionTimeSeconds = ptr.To[int32](1) }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("cannot update maximum execution time when admitted", func() *kueue.Workload { @@ -791,7 +791,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { func(newWL *kueue.Workload) { *newWL.Spec.MaximumExecutionTimeSeconds = 2 }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("can set workload priority class when QuotaReserved=false", func() *kueue.Workload { @@ -815,7 +815,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { newWL.Spec.PriorityClassName = "low" newWL.Spec.Priority = ptr.To[int32](100) }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("can update workload priority class when QuotaReserved=false", func() *kueue.Workload { @@ -877,7 +877,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { newWL.Spec.PriorityClassName = "" newWL.Spec.Priority = nil }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("can set pod priority class QuotaReserved=false", func() *kueue.Workload { @@ -901,7 +901,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { newWL.Spec.PriorityClassName = "low" newWL.Spec.Priority = ptr.To[int32](100) }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("can update pod priority class when QuotaReserved=false", func() *kueue.Workload { @@ -931,7 +931,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { newWL.Spec.PriorityClassName = "low" newWL.Spec.Priority = ptr.To[int32](100) }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("can delete pod priority class when QuotaReserved=false", func() *kueue.Workload { @@ -963,7 +963,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { newWL.Spec.PriorityClassName = "" newWL.Spec.Priority = nil }, - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ) @@ -978,7 +978,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { var newWL kueue.Workload g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(workload), &newWL)).To(gomega.Succeed()) newWL.Spec.QueueName = "queue2" - g.Expect(k8sClient.Update(ctx, &newWL)).Should(testing.BeInvalidError()) + g.Expect(k8sClient.Update(ctx, &newWL)).Should(utiltesting.BeInvalidError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -998,7 +998,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(workload), &newWL)).To(gomega.Succeed()) g.Expect(newWL.Status.Admission).NotTo(gomega.BeNil()) newWL.Status.Admission.ClusterQueue = "foo-cluster-queue" - g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(testing.BeForbiddenError()) + g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1006,7 +1006,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.By("Creating a new Workload") workload := utiltestingapi.MakeWorkload(workloadName, ns.Name).PriorityClass("priority").Obj() err := k8sClient.Create(ctx, workload) - gomega.Expect(err).Should(testing.BeInvalidError()) + gomega.Expect(err).Should(utiltesting.BeInvalidError()) }) ginkgo.It("workload's priority should be mutable when referencing WorkloadPriorityClass", func() { @@ -1064,7 +1064,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { var newWL kueue.Workload g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(workload), &newWL)).To(gomega.Succeed()) newWL.Status.Admission = utiltestingapi.MakeAdmission("cluster-queue").PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName).Assignment("on-demand", "5", "1").Obj()).Obj() - g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(testing.BeForbiddenError()) + g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1121,7 +1121,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { err := workload.UpdateReclaimablePods(ctx, k8sClient, wl, []kueue.ReclaimablePod{ {Name: "ps1", Count: 1}, }) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) ginkgo.It("podSetUpdates should be immutable when state is ready", func() { @@ -1157,7 +1157,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { PodSetUpdates: []kueue.PodSetUpdate{{Name: "first", Labels: map[string]string{"foo": "baz"}}, {Name: "second"}}, State: kueue.CheckStateReady, }, realClock) - g.Expect(k8sClient.Status().Update(ctx, wl)).Should(testing.BeForbiddenError()) + g.Expect(k8sClient.Status().Update(ctx, wl)).Should(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1190,7 +1190,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), wl)).To(gomega.Succeed()) wl.Spec.PodSets[0].Count++ // Increase from 1 -> 2. - g.Expect(k8sClient.Update(ctx, wl)).Should(testing.BeForbiddenError()) + g.Expect(k8sClient.Update(ctx, wl)).Should(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.It("Should allow workload podSets count decrease when ElasticJobsViaWorkloadSlices feature gate is enabled", func() { @@ -1252,7 +1252,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al ) ginkgo.BeforeEach(func() { workloadPriorityClass = utiltestingapi.MakeWorkloadPriorityClass("workload-priority-class").PriorityValue(200).Obj() - priorityClass = testing.MakePriorityClass("priority-class").PriorityValue(100).Obj() + priorityClass = utiltesting.MakePriorityClass("priority-class").PriorityValue(100).Obj() util.MustCreate(ctx, k8sClient, workloadPriorityClass) util.MustCreate(ctx, k8sClient, priorityClass) }) @@ -1304,7 +1304,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al wl.Status.ClusterName = ptr.To("worker3") }, gomega.Succeed(), - testing.BeForbiddenError(), + utiltesting.BeForbiddenError(), ), ginkgo.Entry("Invalid: ClusterName is set when NominatedClusters is empty", func(wl *kueue.Workload) { @@ -1316,7 +1316,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al wl.Status.NominatedClusterNames = nil }, gomega.Succeed(), - testing.BeForbiddenError(), + utiltesting.BeForbiddenError(), ), ginkgo.Entry("Invalid: ClusterName and NominatedClusters are mutually exclusive", func(wl *kueue.Workload) {}, @@ -1325,7 +1325,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher Al wl.Status.NominatedClusterNames = []string{"worker1", "worker2"} }, gomega.Succeed(), - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("Valid: neither ClusterName nor NominatedClusters is set", func(wl *kueue.Workload) {}, @@ -1362,7 +1362,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher In ) ginkgo.BeforeEach(func() { workloadPriorityClass = utiltestingapi.MakeWorkloadPriorityClass("workload-priority-class").PriorityValue(200).Obj() - priorityClass = testing.MakePriorityClass("priority-class").PriorityValue(100).Obj() + priorityClass = utiltesting.MakePriorityClass("priority-class").PriorityValue(100).Obj() util.MustCreate(ctx, k8sClient, workloadPriorityClass) util.MustCreate(ctx, k8sClient, priorityClass) }) @@ -1414,7 +1414,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher In wl.Status.ClusterName = ptr.To("worker3") }, gomega.Succeed(), - testing.BeForbiddenError(), + utiltesting.BeForbiddenError(), ), ginkgo.Entry("Invalid: ClusterName is changed", func(wl *kueue.Workload) { @@ -1425,8 +1425,8 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher In wl.Status.NominatedClusterNames = nil wl.Status.ClusterName = ptr.To("worker2") }, - testing.BeForbiddenError(), - testing.BeForbiddenError(), + utiltesting.BeForbiddenError(), + utiltesting.BeForbiddenError(), ), ginkgo.Entry("Invalid: ClusterName is set when NominatedClusters is empty", func(wl *kueue.Workload) { @@ -1438,7 +1438,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher In wl.Status.NominatedClusterNames = nil }, gomega.Succeed(), - testing.BeForbiddenError(), + utiltesting.BeForbiddenError(), ), ginkgo.Entry("Invalid: ClusterName and NominatedClusters are mutually exclusive", func(wl *kueue.Workload) {}, @@ -1447,7 +1447,7 @@ var _ = ginkgo.Describe("Workload validating webhook ClusterName - Dispatcher In wl.Status.NominatedClusterNames = []string{"worker1", "worker2"} }, gomega.Succeed(), - testing.BeInvalidError(), + utiltesting.BeInvalidError(), ), ginkgo.Entry("Valid: neither ClusterName nor NominatedClusters is set", func(wl *kueue.Workload) {}, diff --git a/test/integration/singlecluster/webhook/jobs/appwrapper_webhook_test.go b/test/integration/singlecluster/webhook/jobs/appwrapper_webhook_test.go index 754c5693557..69f84dcae30 100644 --- a/test/integration/singlecluster/webhook/jobs/appwrapper_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/appwrapper_webhook_test.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingaw "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" "sigs.k8s.io/kueue/test/util" ) @@ -61,6 +61,6 @@ var _ = ginkgo.Describe("AppWrapper Webhook", ginkgo.Ordered, func() { appwrapper := testingaw.MakeAppWrapper("aw-with-invalid-queue", ns.Name).Queue("indexed_job").Obj() err := k8sClient.Create(ctx, appwrapper) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) }) diff --git a/test/integration/singlecluster/webhook/jobs/deployment_webhook_test.go b/test/integration/singlecluster/webhook/jobs/deployment_webhook_test.go index c3ba159cf22..00d6d4ff74d 100644 --- a/test/integration/singlecluster/webhook/jobs/deployment_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/deployment_webhook_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" deploymentcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/deployment" "sigs.k8s.io/kueue/pkg/util/kubeversion" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingdeployment "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" "sigs.k8s.io/kueue/test/util" ) @@ -114,7 +114,7 @@ var _ = ginkgo.Describe("Deployment Webhook", ginkgo.Ordered, ginkgo.ContinueOnF ginkgo.By("Try to remove queue label", func() { gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment), createdDeployment)).Should(gomega.Succeed()) delete(createdDeployment.Labels, constants.QueueLabel) - gomega.Expect(k8sClient.Update(ctx, createdDeployment)).To(testing.BeForbiddenError()) + gomega.Expect(k8sClient.Update(ctx, createdDeployment)).To(utiltesting.BeForbiddenError()) }) ginkgo.By("Check that queue label not deleted from pod template spec", func() { @@ -142,7 +142,7 @@ var _ = ginkgo.Describe("Deployment Webhook", ginkgo.Ordered, ginkgo.ContinueOnF g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(deployment), createdDeployment)).Should(gomega.Succeed()) deploymentWrapper := &testingdeployment.DeploymentWrapper{Deployment: *createdDeployment} updatedDeployment := deploymentWrapper.Queue("another-queue").Obj() - g.Expect(k8sClient.Update(ctx, updatedDeployment)).To(testing.BeForbiddenError()) + g.Expect(k8sClient.Update(ctx, updatedDeployment)).To(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/webhook/jobs/job_webhook_test.go b/test/integration/singlecluster/webhook/jobs/job_webhook_test.go index 9f8cc76f92d..511784beba9 100644 --- a/test/integration/singlecluster/webhook/jobs/job_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/job_webhook_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/kubeversion" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" ) @@ -49,7 +49,7 @@ var _ = ginkgo.Describe("Job Webhook With manageJobsWithoutQueueName enabled", g jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")), jobframework.WithKubeServerVersion(serverVersionFetcher), )) - unmanagedNamespace := testing.MakeNamespace("unmanaged-ns") + unmanagedNamespace := utiltesting.MakeNamespace("unmanaged-ns") util.MustCreate(ctx, k8sClient, unmanagedNamespace) }) ginkgo.BeforeEach(func() { @@ -67,7 +67,7 @@ var _ = ginkgo.Describe("Job Webhook With manageJobsWithoutQueueName enabled", g job := testingjob.MakeJob("job-with-queue-name", ns.Name).Queue("foo").SetAnnotation(job.JobMinParallelismAnnotation, "a").Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) ginkgo.It("Should suspend a Job even no queue name specified", func() { diff --git a/test/integration/singlecluster/webhook/jobs/jobset_webhook_test.go b/test/integration/singlecluster/webhook/jobs/jobset_webhook_test.go index 788e9d20de3..e103bd2bd0b 100644 --- a/test/integration/singlecluster/webhook/jobs/jobset_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/jobset_webhook_test.go @@ -23,7 +23,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" "sigs.k8s.io/kueue/test/util" ) @@ -54,7 +54,7 @@ var _ = ginkgo.Describe("JobSet Webhook", ginkgo.Ordered, ginkgo.ContinueOnFailu Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) }) @@ -74,7 +74,7 @@ var _ = ginkgo.Describe("JobSet Webhook", ginkgo.Ordered, ginkgo.ContinueOnFailu Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) ginkgo.It("should reject a JobSet whose pod‑slice size exceeds total pod count", func() { @@ -92,8 +92,8 @@ var _ = ginkgo.Describe("JobSet Webhook", ginkgo.Ordered, ginkgo.ContinueOnFailu Parallelism: parallelism, Completions: parallelism, PodAnnotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, - kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetSliceSizeAnnotation: invalidSliceSize, }, }). diff --git a/test/integration/singlecluster/webhook/jobs/mpijob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/mpijob_webhook_test.go index af4758b55dc..1c2162f761d 100644 --- a/test/integration/singlecluster/webhook/jobs/mpijob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/mpijob_webhook_test.go @@ -22,7 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/kueue/pkg/controller/jobs/mpijob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" "sigs.k8s.io/kueue/test/util" ) @@ -49,7 +49,7 @@ var _ = ginkgo.Describe("MPIJob Webhook", ginkgo.Ordered, func() { job := testingjob.MakeMPIJob("job", ns.Name).Queue("indexed_job").Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) }) }) diff --git a/test/integration/singlecluster/webhook/jobs/paddlejob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/paddlejob_webhook_test.go index 1c01c4b81a5..90bf659c32c 100644 --- a/test/integration/singlecluster/webhook/jobs/paddlejob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/paddlejob_webhook_test.go @@ -24,7 +24,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/paddlejob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingjobspaddlejob "sigs.k8s.io/kueue/pkg/util/testingjobs/paddlejob" "sigs.k8s.io/kueue/test/util" ) @@ -63,7 +63,7 @@ var _ = ginkgo.Describe("PaddleJob Webhook", ginkgo.Ordered, func() { Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError(), "error: %v", err) + gomega.Expect(err).Should(utiltesting.BeForbiddenError(), "error: %v", err) }) }) }) diff --git a/test/integration/singlecluster/webhook/jobs/pytorchjob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/pytorchjob_webhook_test.go index ef756f82b61..2badeeac94c 100644 --- a/test/integration/singlecluster/webhook/jobs/pytorchjob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/pytorchjob_webhook_test.go @@ -24,7 +24,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/pytorchjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingjobspytorchjob "sigs.k8s.io/kueue/pkg/util/testingjobs/pytorchjob" "sigs.k8s.io/kueue/test/util" ) @@ -63,7 +63,7 @@ var _ = ginkgo.Describe("PyTorchJob Webhook", ginkgo.Ordered, func() { Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError(), "error: %v", err) + gomega.Expect(err).Should(utiltesting.BeForbiddenError(), "error: %v", err) }) }) }) diff --git a/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go b/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go index 4109aab05fb..8b0a29788a0 100644 --- a/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/raycluster_webhook_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/jobs/raycluster" workloadrayjob "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingraycluster "sigs.k8s.io/kueue/pkg/util/testingjobs/raycluster" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" @@ -62,7 +62,7 @@ var _ = ginkgo.Describe("RayCluster Webhook", func() { job := testingraycluster.MakeCluster("raycluster", ns.Name).Queue("indexed_job").Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) }) diff --git a/test/integration/singlecluster/webhook/jobs/rayjob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/rayjob_webhook_test.go index 51e91e6c0ec..dde79bc7e18 100644 --- a/test/integration/singlecluster/webhook/jobs/rayjob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/rayjob_webhook_test.go @@ -22,7 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" "sigs.k8s.io/kueue/pkg/controller/jobs/rayjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" "sigs.k8s.io/kueue/test/util" ) @@ -49,7 +49,7 @@ var _ = ginkgo.Describe("RayJob Webhook", func() { job := testingjob.MakeJob("rayjob", ns.Name).Queue("indexed_job").Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) ginkgo.It("invalid configuration shutdown after job finishes", func() { @@ -59,7 +59,7 @@ var _ = ginkgo.Describe("RayJob Webhook", func() { Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) ginkgo.It("should reject RayJob with clusterSelector and RayClusterSpec", func() { @@ -70,7 +70,7 @@ var _ = ginkgo.Describe("RayJob Webhook", func() { // clusterSelector + RayClusterSpec -> validation error err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) ginkgo.It("should allow RayJob with clusterSelector but no RayClusterSpec", func() { @@ -91,7 +91,7 @@ var _ = ginkgo.Describe("RayJob Webhook", func() { // no clusterSelector + nil RayClusterSpec -> validation error err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) ginkgo.It("should allow RayJob with clusterSelector and no queue label", func() { diff --git a/test/integration/singlecluster/webhook/jobs/tfjob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/tfjob_webhook_test.go index 34eaad0a794..71a29511f70 100644 --- a/test/integration/singlecluster/webhook/jobs/tfjob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/tfjob_webhook_test.go @@ -24,7 +24,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/tfjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingjobstfjob "sigs.k8s.io/kueue/pkg/util/testingjobs/tfjob" "sigs.k8s.io/kueue/test/util" ) @@ -63,7 +63,7 @@ var _ = ginkgo.Describe("TFJob Webhook", ginkgo.Ordered, func() { Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError(), "error: %v", err) + gomega.Expect(err).Should(utiltesting.BeForbiddenError(), "error: %v", err) }) }) }) diff --git a/test/integration/singlecluster/webhook/jobs/xgboostjob_webhook_test.go b/test/integration/singlecluster/webhook/jobs/xgboostjob_webhook_test.go index 001360f90e9..f4674ae07be 100644 --- a/test/integration/singlecluster/webhook/jobs/xgboostjob_webhook_test.go +++ b/test/integration/singlecluster/webhook/jobs/xgboostjob_webhook_test.go @@ -24,7 +24,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/kubeflow/jobs/xgboostjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" testingjobsxgboostjob "sigs.k8s.io/kueue/pkg/util/testingjobs/xgboostjob" "sigs.k8s.io/kueue/test/util" ) @@ -63,7 +63,7 @@ var _ = ginkgo.Describe("XGBoostJob Webhook", ginkgo.Ordered, func() { Obj() err := k8sClient.Create(ctx, job) gomega.Expect(err).Should(gomega.HaveOccurred()) - gomega.Expect(err).Should(testing.BeForbiddenError(), "error: %v", err) + gomega.Expect(err).Should(utiltesting.BeForbiddenError(), "error: %v", err) }) }) }) From 22e249f8adceb4ad339ecc7dce57a3071b1d5987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Wo=C5=BAniak?= Date: Thu, 6 Nov 2025 15:44:54 +0100 Subject: [PATCH 107/119] Update main with the latest v0.14.4 (#7559) --- CHANGELOG/CHANGELOG-0.14.md | 17 +++++++++++++++++ Makefile | 2 +- README.md | 2 +- SECURITY-INSIGHTS.yaml | 12 ++++++------ charts/kueue/Chart.yaml | 4 ++-- charts/kueue/README.md | 8 ++++---- charts/kueue/README.md.gotmpl | 6 +++--- cmd/kueueviz/INSTALL.md | 6 +++--- cmd/kueueviz/frontend/package-lock.json | 4 ++-- cmd/kueueviz/frontend/package.json | 2 +- site/hugo.toml | 4 ++-- test/e2e/kueueviz/package-lock.json | 4 ++-- test/e2e/kueueviz/package.json | 2 +- 13 files changed, 45 insertions(+), 28 deletions(-) diff --git a/CHANGELOG/CHANGELOG-0.14.md b/CHANGELOG/CHANGELOG-0.14.md index f6e672d5214..a92b1d2b43e 100644 --- a/CHANGELOG/CHANGELOG-0.14.md +++ b/CHANGELOG/CHANGELOG-0.14.md @@ -1,3 +1,20 @@ +## v0.14.4 + +Changes since `v0.14.3`: + +## Changes by Kind + +### Feature + +- `ReclaimablePods` feature gate is introduced to enable users switching on and off the reclaimable Pods feature (#7537, @PBundyra) + +### Bug or Regression + +- Fix eviction of jobs with memory requests in decimal format (#7556, @brejman) +- Fix the bug for the StatefulSet integration that the scale up could get stuck if + triggered immediately after scale down to zero. (#7500, @IrvingMg) +- MultiKueue: Remove remoteClient from clusterReconciler when kubeconfig is detected as invalid or insecure, preventing workloads from being admitted to misconfigured clusters. (#7517, @mszadkow) + ## v0.14.3 Changes since `v0.14.2`: diff --git a/Makefile b/Makefile index c55aa88fe85..57e8e53e339 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ LD_FLAGS += -X '$(version_pkg).BuildDate=$(shell date -u +%Y-%m-%dT%H:%M:%SZ)' # Update these variables when preparing a new release or a release branch. # Then run `make prepare-release-branch` -RELEASE_VERSION=v0.14.3 +RELEASE_VERSION=v0.14.4 RELEASE_BRANCH=main # Application version for Helm and npm (strips leading 'v' from RELEASE_VERSION) APP_VERSION := $(shell echo $(RELEASE_VERSION) | cut -c2-) diff --git a/README.md b/README.md index 3938f3a6277..1e8aa80d999 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Read the [overview](https://kueue.sigs.k8s.io/docs/overview/) and watch the Kueu To install the latest release of Kueue in your cluster, run the following command: ```shell -kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.3/manifests.yaml +kubectl apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.4/manifests.yaml ``` The controller runs in the `kueue-system` namespace. diff --git a/SECURITY-INSIGHTS.yaml b/SECURITY-INSIGHTS.yaml index 57ef963334c..4232b0635c9 100644 --- a/SECURITY-INSIGHTS.yaml +++ b/SECURITY-INSIGHTS.yaml @@ -1,11 +1,11 @@ header: schema-version: 1.0.0 expiration-date: '2024-09-28T01:00:00.000Z' - last-updated: '' - last-reviewed: '' - commit-hash: "" + last-updated: '2025-11-06' + last-reviewed: '2025-11-06' + commit-hash: "28700e89ee89a0e33eb4c6a0ca8859fbe8b9b8b1" project-url: 'https://github.com/kubernetes-sigs/kueue' - project-release: "0.14.3" + project-release: "0.14.4" changelog: 'https://github.com/kubernetes-sigs/kueue/tree/main/CHANGELOG' license: 'https://github.com/kubernetes-sigs/kueue/blob/main/LICENSE' project-lifecycle: @@ -28,7 +28,7 @@ documentation: - 'https://kueue.sigs.k8s.io/docs/' distribution-points: - >- - https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.3/manifests.yaml + https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.4/manifests.yaml security-artifacts: threat-model: threat-model-created: false @@ -62,5 +62,5 @@ dependencies: dependencies-lists: - 'https://github.com/kubernetes-sigs/kueue/blob/main/go.mod' sbom: - - sbom-file: https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.3/kueue-v0.14.3.spdx.json + - sbom-file: https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.4/kueue-v0.14.4.spdx.json sbom-format: SPDX diff --git a/charts/kueue/Chart.yaml b/charts/kueue/Chart.yaml index 6c5a9b64983..ef89a600287 100644 --- a/charts/kueue/Chart.yaml +++ b/charts/kueue/Chart.yaml @@ -16,9 +16,9 @@ type: application # NOTE: Do not modify manually. In Kueue, the version and appVersion are # overridden to GIT_TAG when building the artifacts, including the helm charts, # via Makefile. -version: 0.14.3 +version: 0.14.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v0.14.3" +appVersion: "v0.14.4" diff --git a/charts/kueue/README.md b/charts/kueue/README.md index 6ae7ae56572..236097cfa5e 100644 --- a/charts/kueue/README.md +++ b/charts/kueue/README.md @@ -1,6 +1,6 @@ # kueue -![Version: 0.14.3](https://img.shields.io/badge/Version-0.14.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.14.3](https://img.shields.io/badge/AppVersion-v0.14.3-informational?style=flat-square) +![Version: 0.14.4](https://img.shields.io/badge/Version-0.14.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.14.4](https://img.shields.io/badge/AppVersion-v0.14.4-informational?style=flat-square) Kueue is a set of APIs and controllers for job queueing. It is a job-level manager that decides when a job should be admitted to start (as in pods can be created) and when it should stop (as in active pods should be deleted). @@ -28,7 +28,7 @@ $ helm install kueue kueue/ --create-namespace --namespace kueue-system Or use the charts pushed to `oci://registry.k8s.io/kueue/charts/kueue`: ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" --create-namespace --namespace=kueue-system +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.4" --create-namespace --namespace=kueue-system ``` For more advanced parametrization of Kueue, we recommend using a local overrides file, passed via the `--values` flag. For example: @@ -50,7 +50,7 @@ controllerManager: ``` ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.4" \ --create-namespace --namespace=kueue-system \ --values overrides.yaml ``` @@ -58,7 +58,7 @@ helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ You can also use the `--set` flag. For example, to enable a feature gate (e.g., `TopologyAwareScheduling`): ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.4" \ --create-namespace --namespace=kueue-system \ --set "controllerManager.featureGates[0].name=TopologyAwareScheduling" \ --set "controllerManager.featureGates[0].enabled=true" diff --git a/charts/kueue/README.md.gotmpl b/charts/kueue/README.md.gotmpl index 404ad302df0..63bce8d8704 100644 --- a/charts/kueue/README.md.gotmpl +++ b/charts/kueue/README.md.gotmpl @@ -30,7 +30,7 @@ $ helm install kueue kueue/ --create-namespace --namespace kueue-system Or use the charts pushed to `oci://registry.k8s.io/kueue/charts/kueue`: ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" --create-namespace --namespace=kueue-system +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.4" --create-namespace --namespace=kueue-system ``` For more advanced parametrization of Kueue, we recommend using a local overrides file, passed via the `--values` flag. For example: @@ -52,7 +52,7 @@ controllerManager: ``` ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.4" \ --create-namespace --namespace=kueue-system \ --values overrides.yaml ``` @@ -60,7 +60,7 @@ helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ You can also use the `--set` flag. For example, to enable a feature gate (e.g., `TopologyAwareScheduling`): ```bash -helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.3" \ +helm install kueue oci://registry.k8s.io/kueue/charts/kueue --version="0.14.4" \ --create-namespace --namespace=kueue-system \ --set "controllerManager.featureGates[0].name=TopologyAwareScheduling" \ --set "controllerManager.featureGates[0].enabled=true" diff --git a/cmd/kueueviz/INSTALL.md b/cmd/kueueviz/INSTALL.md index 36ac6f55d82..2453845e359 100644 --- a/cmd/kueueviz/INSTALL.md +++ b/cmd/kueueviz/INSTALL.md @@ -3,7 +3,7 @@ KueueViz can be installed using `kubectl` with the following command: ``` -kubectl create -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.3/kueueviz.yaml +kubectl create -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.14.4/kueueviz.yaml ``` If you are using `kind` and that you don't have an `ingress` controller, you can use `port-forward` to configure and run `KueueViz`: @@ -23,7 +23,7 @@ by ensuring that `enableKueueViz` is set to `true`: ``` helm upgrade --install kueue oci://registry.k8s.io/kueue/charts/kueue \ - --version="0.14.3" + --version="0.14.4" --namespace kueue-system \ --set enableKueueViz=true \ --create-namespace @@ -44,7 +44,7 @@ kind create cluster kind get kubeconfig > kubeconfig export KUBECONFIG=$PWD/kubeconfig helm install kueue oci://us-central1-docker.pkg.dev/k8s-staging-images/charts/kueue \ - --version="0.14.3" --create-namespace --namespace=kueue-system + --version="0.14.4" --create-namespace --namespace=kueue-system ``` ## Build diff --git a/cmd/kueueviz/frontend/package-lock.json b/cmd/kueueviz/frontend/package-lock.json index 6d594e9f4de..5e0232b995d 100644 --- a/cmd/kueueviz/frontend/package-lock.json +++ b/cmd/kueueviz/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "kueueviz-frontend", - "version": "0.14.3", + "version": "0.14.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "kueueviz-frontend", - "version": "0.14.3", + "version": "0.14.4", "license": "Apache 2.0", "dependencies": { "@emotion/react": "^11.14.0", diff --git a/cmd/kueueviz/frontend/package.json b/cmd/kueueviz/frontend/package.json index 63ed18fc410..c575317fa64 100644 --- a/cmd/kueueviz/frontend/package.json +++ b/cmd/kueueviz/frontend/package.json @@ -1,6 +1,6 @@ { "name": "kueueviz-frontend", - "version": "0.14.3", + "version": "0.14.4", "private": true, "description": "Frontend dashboard for visualizing Kueue status", "main": "src/index.jsx", diff --git a/site/hugo.toml b/site/hugo.toml index 9a7f6754706..8c4850f00d3 100644 --- a/site/hugo.toml +++ b/site/hugo.toml @@ -99,10 +99,10 @@ ignoreFiles = [] # The major.minor version tag for the version of the docs represented in this # branch of the repository. Used in the "version-banner" partial to display a # version number for this doc set. - version = "v0.14.3" + version = "v0.14.4" # Version of Kueue without the leading "v", as used for Helm charts. - chart_version = "0.14.3" + chart_version = "0.14.4" # Flag used in the "version-banner" partial to decide whether to display a # banner on every page indicating that this is an archived version of the docs. diff --git a/test/e2e/kueueviz/package-lock.json b/test/e2e/kueueviz/package-lock.json index 6d2b85090bf..83e752042bb 100644 --- a/test/e2e/kueueviz/package-lock.json +++ b/test/e2e/kueueviz/package-lock.json @@ -1,12 +1,12 @@ { "name": "kueueviz-frontend-tests", - "version": "0.14.3", + "version": "0.14.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "kueueviz-frontend-tests", - "version": "0.14.3", + "version": "0.14.4", "devDependencies": { "@testing-library/cypress": "^10.1.0", "cypress": "^15.5.0", diff --git a/test/e2e/kueueviz/package.json b/test/e2e/kueueviz/package.json index a3dcded44b3..22106b64938 100644 --- a/test/e2e/kueueviz/package.json +++ b/test/e2e/kueueviz/package.json @@ -1,6 +1,6 @@ { "name": "kueueviz-frontend-tests", - "version": "0.14.3", + "version": "0.14.4", "scripts": { "cypress:run": "cypress run --headless", "check-unused": "depcheck" From 25f46f80dddbd1594b76c4bb6625c19640fd32c7 Mon Sep 17 00:00:00 2001 From: Sohan Kunkerkar Date: Thu, 6 Nov 2025 10:38:55 -0500 Subject: [PATCH 108/119] api/kueue/v1beta1: add unit tests for workload conversion (#7546) Related to #7394 --- .../kueue/v1beta1/workload_conversion_test.go | 285 ++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 apis/kueue/v1beta1/workload_conversion_test.go diff --git a/apis/kueue/v1beta1/workload_conversion_test.go b/apis/kueue/v1beta1/workload_conversion_test.go new file mode 100644 index 00000000000..6c90084e964 --- /dev/null +++ b/apis/kueue/v1beta1/workload_conversion_test.go @@ -0,0 +1,285 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +func TestWorkloadConvertTo(t *testing.T) { + defaultObjectMeta := metav1.ObjectMeta{ + Name: "test-workload", + Namespace: "default", + } + + testCases := map[string]struct { + input *Workload + expected *v1beta2.Workload + }{ + "nil AccumulatedPastExexcutionTimeSeconds": { + input: &Workload{ + ObjectMeta: defaultObjectMeta, + Status: WorkloadStatus{ + AccumulatedPastExexcutionTimeSeconds: nil, + }, + }, + expected: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.WorkloadStatus{ + AccumulatedPastExecutionTimeSeconds: nil, + }, + }, + }, + "zero AccumulatedPastExexcutionTimeSeconds": { + input: &Workload{ + ObjectMeta: defaultObjectMeta, + Status: WorkloadStatus{ + AccumulatedPastExexcutionTimeSeconds: ptr.To[int32](0), + }, + }, + expected: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.WorkloadStatus{ + AccumulatedPastExecutionTimeSeconds: ptr.To[int32](0), + }, + }, + }, + "non-zero AccumulatedPastExexcutionTimeSeconds": { + input: &Workload{ + ObjectMeta: defaultObjectMeta, + Status: WorkloadStatus{ + AccumulatedPastExexcutionTimeSeconds: ptr.To[int32](3600), + }, + }, + expected: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.WorkloadStatus{ + AccumulatedPastExecutionTimeSeconds: ptr.To[int32](3600), + }, + }, + }, + "with conditions and other fields": { + input: &Workload{ + ObjectMeta: defaultObjectMeta, + Status: WorkloadStatus{ + Conditions: []metav1.Condition{ + { + Type: "Admitted", + Status: metav1.ConditionTrue, + Reason: "AdmittedByClusterQueue", + }, + }, + AccumulatedPastExexcutionTimeSeconds: ptr.To[int32](7200), + }, + }, + expected: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.WorkloadStatus{ + Conditions: []metav1.Condition{ + { + Type: "Admitted", + Status: metav1.ConditionTrue, + Reason: "AdmittedByClusterQueue", + }, + }, + AccumulatedPastExecutionTimeSeconds: ptr.To[int32](7200), + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + result := &v1beta2.Workload{} + if err := tc.input.ConvertTo(result); err != nil { + t.Fatalf("ConvertTo failed: %v", err) + } + if diff := cmp.Diff(tc.expected, result); diff != "" { + t.Errorf("unexpected conversion result (-want +got):\n%s", diff) + } + }) + } +} + +func TestWorkloadConvertFrom(t *testing.T) { + defaultObjectMeta := metav1.ObjectMeta{ + Name: "test-workload", + Namespace: "default", + } + + testCases := map[string]struct { + input *v1beta2.Workload + expected *Workload + }{ + "nil AccumulatedPastExecutionTimeSeconds": { + input: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.WorkloadStatus{ + AccumulatedPastExecutionTimeSeconds: nil, + }, + }, + expected: &Workload{ + ObjectMeta: defaultObjectMeta, + Status: WorkloadStatus{ + AccumulatedPastExexcutionTimeSeconds: nil, + }, + }, + }, + "zero AccumulatedPastExecutionTimeSeconds": { + input: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.WorkloadStatus{ + AccumulatedPastExecutionTimeSeconds: ptr.To[int32](0), + }, + }, + expected: &Workload{ + ObjectMeta: defaultObjectMeta, + Status: WorkloadStatus{ + AccumulatedPastExexcutionTimeSeconds: ptr.To[int32](0), + }, + }, + }, + "non-zero AccumulatedPastExecutionTimeSeconds": { + input: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.WorkloadStatus{ + AccumulatedPastExecutionTimeSeconds: ptr.To[int32](3600), + }, + }, + expected: &Workload{ + ObjectMeta: defaultObjectMeta, + Status: WorkloadStatus{ + AccumulatedPastExexcutionTimeSeconds: ptr.To[int32](3600), + }, + }, + }, + "with conditions and other fields": { + input: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.WorkloadStatus{ + Conditions: []metav1.Condition{ + { + Type: "Finished", + Status: metav1.ConditionTrue, + Reason: "JobFinished", + }, + }, + AccumulatedPastExecutionTimeSeconds: ptr.To[int32](1800), + }, + }, + expected: &Workload{ + ObjectMeta: defaultObjectMeta, + Status: WorkloadStatus{ + Conditions: []metav1.Condition{ + { + Type: "Finished", + Status: metav1.ConditionTrue, + Reason: "JobFinished", + }, + }, + AccumulatedPastExexcutionTimeSeconds: ptr.To[int32](1800), + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + result := &Workload{} + if err := result.ConvertFrom(tc.input); err != nil { + t.Fatalf("ConvertFrom failed: %v", err) + } + if diff := cmp.Diff(tc.expected, result); diff != "" { + t.Errorf("unexpected conversion result (-want +got):\n%s", diff) + } + }) + } +} + +func TestWorkloadConversion_RoundTrip(t *testing.T) { + testCases := map[string]struct { + v1beta1Obj *Workload + }{ + "complete Workload with AccumulatedPastExexcutionTimeSeconds": { + v1beta1Obj: &Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-workload", + Namespace: "default", + }, + Status: WorkloadStatus{ + Conditions: []metav1.Condition{ + { + Type: "Admitted", + Status: metav1.ConditionTrue, + Reason: "AdmittedByClusterQueue", + }, + }, + AccumulatedPastExexcutionTimeSeconds: ptr.To[int32](5400), + }, + }, + }, + "Workload with nil AccumulatedPastExexcutionTimeSeconds": { + v1beta1Obj: &Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: "simple-workload", + Namespace: "test-ns", + }, + Status: WorkloadStatus{ + AccumulatedPastExexcutionTimeSeconds: nil, + }, + }, + }, + "Workload with zero AccumulatedPastExexcutionTimeSeconds": { + v1beta1Obj: &Workload{ + ObjectMeta: metav1.ObjectMeta{ + Name: "zero-workload", + Namespace: "test-ns", + }, + Status: WorkloadStatus{ + AccumulatedPastExexcutionTimeSeconds: ptr.To[int32](0), + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + // Convert v1beta1 -> v1beta2 + v1beta2Obj := &v1beta2.Workload{} + if err := tc.v1beta1Obj.ConvertTo(v1beta2Obj); err != nil { + t.Fatalf("ConvertTo failed: %v", err) + } + + // Convert v1beta2 -> v1beta1 (round-trip) + roundTripped := &Workload{} + if err := roundTripped.ConvertFrom(v1beta2Obj); err != nil { + t.Fatalf("ConvertFrom failed: %v", err) + } + + // Verify round-trip + if diff := cmp.Diff(tc.v1beta1Obj, roundTripped); diff != "" { + t.Errorf("round-trip conversion produced diff (-original +roundtripped):\n%s", diff) + } + }) + } +} From 1e9c2cdd96471c026e6ba26a9ae3af294024bf1d Mon Sep 17 00:00:00 2001 From: Sohan Kunkerkar Date: Thu, 6 Nov 2025 12:29:01 -0500 Subject: [PATCH 109/119] test: Add conversion unit tests for LocalQueue and ClusterQueue (#7567) * fix: ClusterQueue conversion for FlavorFungibility and AdmissionChecks Run autoConvert before modifying fields to ensure structs are populated. Fixes Preempt/Borrow->MayStopSearch and AdmissionChecks conversions. Signed-off-by: Sohan Kunkerkar * test: Add conversion tests for LocalQueue and ClusterQueue Signed-off-by: Sohan Kunkerkar --------- Signed-off-by: Sohan Kunkerkar --- apis/kueue/v1beta1/clusterqueue_conversion.go | 15 +- .../v1beta1/clusterqueue_conversion_test.go | 455 +++++++++++++++ .../v1beta1/localqueue_conversion_test.go | 530 ++++++++++++++++++ 3 files changed, 998 insertions(+), 2 deletions(-) create mode 100644 apis/kueue/v1beta1/clusterqueue_conversion_test.go create mode 100644 apis/kueue/v1beta1/localqueue_conversion_test.go diff --git a/apis/kueue/v1beta1/clusterqueue_conversion.go b/apis/kueue/v1beta1/clusterqueue_conversion.go index 99935586974..6af66ab35c6 100644 --- a/apis/kueue/v1beta1/clusterqueue_conversion.go +++ b/apis/kueue/v1beta1/clusterqueue_conversion.go @@ -39,7 +39,14 @@ func (dst *ClusterQueue) ConvertFrom(srcRaw conversion.Hub) error { func Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in *ClusterQueueSpec, out *v1beta2.ClusterQueueSpec, s conversionapi.Scope) error { out.CohortName = v1beta2.CohortReference(in.Cohort) - // Convert AdmissionChecks to AdmissionChecksStrategy before autoConvert + // Run autoConvert first to populate out.FlavorFungibility. + // We need out.FlavorFungibility to be non-nil before we can update its fields. + if err := autoConvert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in, out, s); err != nil { + return err + } + + // Convert AdmissionChecks to AdmissionChecksStrategy after autoConvert. + // This must happen after autoConvert to avoid being overwritten. if len(in.AdmissionChecks) > 0 { out.AdmissionChecksStrategy = &v1beta2.AdmissionChecksStrategy{ AdmissionChecks: make([]v1beta2.AdmissionCheckStrategyRule, len(in.AdmissionChecks)), @@ -51,6 +58,10 @@ func Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in *ClusterQue } } + // Convert deprecated FlavorFungibility values to their v1beta2 equivalents. + // This must happen after autoConvert, which creates out.FlavorFungibility. + // Preempt -> MayStopSearch + // Borrow -> MayStopSearch if in.FlavorFungibility != nil && out.FlavorFungibility != nil { if in.FlavorFungibility.WhenCanPreempt == Preempt { out.FlavorFungibility.WhenCanPreempt = v1beta2.MayStopSearch @@ -60,7 +71,7 @@ func Convert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in *ClusterQue } } - return autoConvert_v1beta1_ClusterQueueSpec_To_v1beta2_ClusterQueueSpec(in, out, s) + return nil } func Convert_v1beta2_ClusterQueueSpec_To_v1beta1_ClusterQueueSpec(in *v1beta2.ClusterQueueSpec, out *ClusterQueueSpec, s conversionapi.Scope) error { diff --git a/apis/kueue/v1beta1/clusterqueue_conversion_test.go b/apis/kueue/v1beta1/clusterqueue_conversion_test.go new file mode 100644 index 00000000000..e5c15565b85 --- /dev/null +++ b/apis/kueue/v1beta1/clusterqueue_conversion_test.go @@ -0,0 +1,455 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +func TestClusterQueueConvertTo(t *testing.T) { + defaultObjectMeta := metav1.ObjectMeta{ + Name: "test-clusterqueue", + } + + testCases := map[string]struct { + input *ClusterQueue + expected *v1beta2.ClusterQueue + }{ + "nil cohort": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + Cohort: "", + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + CohortName: "", + }, + }, + }, + "cohort to cohortName conversion": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + Cohort: "test-cohort", + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + CohortName: "test-cohort", + }, + }, + }, + "AdmissionChecks to AdmissionChecksStrategy conversion": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + AdmissionChecks: []AdmissionCheckReference{"check1", "check2"}, + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + AdmissionChecksStrategy: &v1beta2.AdmissionChecksStrategy{ + AdmissionChecks: []v1beta2.AdmissionCheckStrategyRule{ + {Name: "check1"}, + {Name: "check2"}, + }, + }, + }, + }, + }, + "AdmissionChecksStrategy preserved": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + AdmissionChecksStrategy: &AdmissionChecksStrategy{ + AdmissionChecks: []AdmissionCheckStrategyRule{ + {Name: "strategy-check1"}, + {Name: "strategy-check2"}, + }, + }, + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + AdmissionChecksStrategy: &v1beta2.AdmissionChecksStrategy{ + AdmissionChecks: []v1beta2.AdmissionCheckStrategyRule{ + {Name: "strategy-check1"}, + {Name: "strategy-check2"}, + }, + }, + }, + }, + }, + "empty AdmissionChecks": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + AdmissionChecks: []AdmissionCheckReference{}, + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{}, + }, + }, + "FlavorFungibility Preempt to MayStopSearch": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + FlavorFungibility: &FlavorFungibility{ + WhenCanPreempt: Preempt, + }, + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + FlavorFungibility: &v1beta2.FlavorFungibility{ + WhenCanPreempt: v1beta2.MayStopSearch, + }, + }, + }, + }, + "FlavorFungibility Borrow to MayStopSearch": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + FlavorFungibility: &FlavorFungibility{ + WhenCanBorrow: Borrow, + }, + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + FlavorFungibility: &v1beta2.FlavorFungibility{ + WhenCanBorrow: v1beta2.MayStopSearch, + }, + }, + }, + }, + "FlavorFungibility MayStopSearch preserved": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + FlavorFungibility: &FlavorFungibility{ + WhenCanPreempt: MayStopSearch, + WhenCanBorrow: MayStopSearch, + }, + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + FlavorFungibility: &v1beta2.FlavorFungibility{ + WhenCanPreempt: v1beta2.MayStopSearch, + WhenCanBorrow: v1beta2.MayStopSearch, + }, + }, + }, + }, + "FlavorFungibility TryNextFlavor preserved": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + FlavorFungibility: &FlavorFungibility{ + WhenCanPreempt: TryNextFlavor, + WhenCanBorrow: TryNextFlavor, + }, + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + FlavorFungibility: &v1beta2.FlavorFungibility{ + WhenCanPreempt: v1beta2.TryNextFlavor, + WhenCanBorrow: v1beta2.TryNextFlavor, + }, + }, + }, + }, + "complete ClusterQueue with all fields": { + input: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + Cohort: "prod-cohort", + AdmissionChecks: []AdmissionCheckReference{"multikueue-check"}, + FlavorFungibility: &FlavorFungibility{ + WhenCanPreempt: TryNextFlavor, + WhenCanBorrow: MayStopSearch, + }, + QueueingStrategy: StrictFIFO, + }, + Status: ClusterQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + PendingWorkloads: 5, + }, + }, + expected: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + CohortName: "prod-cohort", + AdmissionChecksStrategy: &v1beta2.AdmissionChecksStrategy{ + AdmissionChecks: []v1beta2.AdmissionCheckStrategyRule{ + {Name: "multikueue-check"}, + }, + }, + FlavorFungibility: &v1beta2.FlavorFungibility{ + WhenCanPreempt: v1beta2.TryNextFlavor, + WhenCanBorrow: v1beta2.MayStopSearch, + }, + QueueingStrategy: v1beta2.StrictFIFO, + }, + Status: v1beta2.ClusterQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + PendingWorkloads: 5, + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + result := &v1beta2.ClusterQueue{} + if err := tc.input.ConvertTo(result); err != nil { + t.Fatalf("ConvertTo failed: %v", err) + } + if diff := cmp.Diff(tc.expected, result); diff != "" { + t.Errorf("unexpected conversion result (-want +got):\n%s", diff) + } + }) + } +} + +func TestClusterQueueConvertFrom(t *testing.T) { + defaultObjectMeta := metav1.ObjectMeta{ + Name: "test-clusterqueue", + } + + testCases := map[string]struct { + input *v1beta2.ClusterQueue + expected *ClusterQueue + }{ + "nil cohortName": { + input: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + CohortName: "", + }, + }, + expected: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + Cohort: "", + }, + }, + }, + "cohortName to cohort conversion": { + input: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + CohortName: "test-cohort", + }, + }, + expected: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + Cohort: "test-cohort", + }, + }, + }, + "FlavorFungibility MayStopSearch preserved": { + input: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + FlavorFungibility: &v1beta2.FlavorFungibility{ + WhenCanPreempt: v1beta2.MayStopSearch, + WhenCanBorrow: v1beta2.MayStopSearch, + }, + }, + }, + expected: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + FlavorFungibility: &FlavorFungibility{ + WhenCanPreempt: MayStopSearch, + WhenCanBorrow: MayStopSearch, + }, + }, + }, + }, + "FlavorFungibility TryNextFlavor preserved": { + input: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + FlavorFungibility: &v1beta2.FlavorFungibility{ + WhenCanPreempt: v1beta2.TryNextFlavor, + WhenCanBorrow: v1beta2.TryNextFlavor, + }, + }, + }, + expected: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + FlavorFungibility: &FlavorFungibility{ + WhenCanPreempt: TryNextFlavor, + WhenCanBorrow: TryNextFlavor, + }, + }, + }, + }, + "complete ClusterQueue with all fields": { + input: &v1beta2.ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.ClusterQueueSpec{ + CohortName: "prod-cohort", + FlavorFungibility: &v1beta2.FlavorFungibility{ + WhenCanPreempt: v1beta2.MayStopSearch, + WhenCanBorrow: v1beta2.TryNextFlavor, + }, + QueueingStrategy: v1beta2.StrictFIFO, + }, + Status: v1beta2.ClusterQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + PendingWorkloads: 5, + }, + }, + expected: &ClusterQueue{ + ObjectMeta: defaultObjectMeta, + Spec: ClusterQueueSpec{ + Cohort: "prod-cohort", + FlavorFungibility: &FlavorFungibility{ + WhenCanPreempt: MayStopSearch, + WhenCanBorrow: TryNextFlavor, + }, + QueueingStrategy: StrictFIFO, + }, + Status: ClusterQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + PendingWorkloads: 5, + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + result := &ClusterQueue{} + if err := result.ConvertFrom(tc.input); err != nil { + t.Fatalf("ConvertFrom failed: %v", err) + } + if diff := cmp.Diff(tc.expected, result); diff != "" { + t.Errorf("unexpected conversion result (-want +got):\n%s", diff) + } + }) + } +} + +func TestClusterQueueConversion_RoundTrip(t *testing.T) { + testCases := map[string]struct { + v1beta1Obj *ClusterQueue + }{ + "complete ClusterQueue with cohort and FlavorFungibility": { + v1beta1Obj: &ClusterQueue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-clusterqueue", + }, + Spec: ClusterQueueSpec{ + Cohort: "prod-cohort", + FlavorFungibility: &FlavorFungibility{ + WhenCanPreempt: TryNextFlavor, + WhenCanBorrow: MayStopSearch, + }, + QueueingStrategy: StrictFIFO, + }, + Status: ClusterQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + PendingWorkloads: 10, + }, + }, + }, + "minimal ClusterQueue": { + v1beta1Obj: &ClusterQueue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "minimal-cq", + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + // Convert v1beta1 -> v1beta2 + v1beta2Obj := &v1beta2.ClusterQueue{} + if err := tc.v1beta1Obj.ConvertTo(v1beta2Obj); err != nil { + t.Fatalf("ConvertTo failed: %v", err) + } + + // Convert v1beta2 -> v1beta1 (round-trip) + roundTripped := &ClusterQueue{} + if err := roundTripped.ConvertFrom(v1beta2Obj); err != nil { + t.Fatalf("ConvertFrom failed: %v", err) + } + + // Verify round-trip + if diff := cmp.Diff(tc.v1beta1Obj, roundTripped); diff != "" { + t.Errorf("round-trip conversion produced diff (-original +roundtripped):\n%s", diff) + } + }) + } +} diff --git a/apis/kueue/v1beta1/localqueue_conversion_test.go b/apis/kueue/v1beta1/localqueue_conversion_test.go new file mode 100644 index 00000000000..f7c5821ef24 --- /dev/null +++ b/apis/kueue/v1beta1/localqueue_conversion_test.go @@ -0,0 +1,530 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +func TestLocalQueueConvertTo(t *testing.T) { + defaultObjectMeta := metav1.ObjectMeta{ + Name: "test-localqueue", + Namespace: "default", + } + + testCases := map[string]struct { + input *LocalQueue + expected *v1beta2.LocalQueue + }{ + "nil FlavorUsage": { + input: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: LocalQueueStatus{ + FlavorUsage: nil, + }, + }, + expected: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.LocalQueueStatus{ + FlavorsUsage: nil, + }, + }, + }, + "empty FlavorUsage": { + input: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: LocalQueueStatus{ + FlavorUsage: []LocalQueueFlavorUsage{}, + }, + }, + expected: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.LocalQueueStatus{ + FlavorsUsage: []v1beta2.LocalQueueFlavorUsage{}, + }, + }, + }, + "single FlavorUsage": { + input: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: LocalQueueStatus{ + FlavorUsage: []LocalQueueFlavorUsage{ + { + Name: "default-flavor", + Resources: []LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expected: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.LocalQueueStatus{ + FlavorsUsage: []v1beta2.LocalQueueFlavorUsage{ + { + Name: "default-flavor", + Resources: []v1beta2.LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + }, + "multiple FlavorUsage with various resources": { + input: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: LocalQueueStatus{ + FlavorUsage: []LocalQueueFlavorUsage{ + { + Name: "flavor-1", + Resources: []LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + { + Name: corev1.ResourceMemory, + Total: resource.MustParse("20Gi"), + }, + }, + }, + { + Name: "flavor-2", + Resources: []LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("5"), + }, + }, + }, + }, + PendingWorkloads: 3, + ReservingWorkloads: 2, + AdmittedWorkloads: 1, + }, + }, + expected: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.LocalQueueStatus{ + FlavorsUsage: []v1beta2.LocalQueueFlavorUsage{ + { + Name: "flavor-1", + Resources: []v1beta2.LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + { + Name: corev1.ResourceMemory, + Total: resource.MustParse("20Gi"), + }, + }, + }, + { + Name: "flavor-2", + Resources: []v1beta2.LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("5"), + }, + }, + }, + }, + PendingWorkloads: 3, + ReservingWorkloads: 2, + AdmittedWorkloads: 1, + }, + }, + }, + "with conditions and status fields": { + input: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Spec: LocalQueueSpec{ + ClusterQueue: "cluster-queue-1", + }, + Status: LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + FlavorUsage: []LocalQueueFlavorUsage{ + { + Name: "default-flavor", + Resources: []LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + }, + }, + }, + PendingWorkloads: 5, + }, + }, + expected: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.LocalQueueSpec{ + ClusterQueue: "cluster-queue-1", + }, + Status: v1beta2.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + FlavorsUsage: []v1beta2.LocalQueueFlavorUsage{ + { + Name: "default-flavor", + Resources: []v1beta2.LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + }, + }, + }, + PendingWorkloads: 5, + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + result := &v1beta2.LocalQueue{} + if err := tc.input.ConvertTo(result); err != nil { + t.Fatalf("ConvertTo failed: %v", err) + } + if diff := cmp.Diff(tc.expected, result); diff != "" { + t.Errorf("unexpected conversion result (-want +got):\n%s", diff) + } + }) + } +} + +func TestLocalQueueConvertFrom(t *testing.T) { + defaultObjectMeta := metav1.ObjectMeta{ + Name: "test-localqueue", + Namespace: "default", + } + + testCases := map[string]struct { + input *v1beta2.LocalQueue + expected *LocalQueue + }{ + "nil FlavorsUsage": { + input: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.LocalQueueStatus{ + FlavorsUsage: nil, + }, + }, + expected: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: LocalQueueStatus{ + FlavorUsage: nil, + }, + }, + }, + "empty FlavorsUsage": { + input: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.LocalQueueStatus{ + FlavorsUsage: []v1beta2.LocalQueueFlavorUsage{}, + }, + }, + expected: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: LocalQueueStatus{ + FlavorUsage: []LocalQueueFlavorUsage{}, + }, + }, + }, + "single FlavorsUsage": { + input: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.LocalQueueStatus{ + FlavorsUsage: []v1beta2.LocalQueueFlavorUsage{ + { + Name: "default-flavor", + Resources: []v1beta2.LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expected: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: LocalQueueStatus{ + FlavorUsage: []LocalQueueFlavorUsage{ + { + Name: "default-flavor", + Resources: []LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + }, + "multiple FlavorsUsage with various resources": { + input: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: v1beta2.LocalQueueStatus{ + FlavorsUsage: []v1beta2.LocalQueueFlavorUsage{ + { + Name: "flavor-1", + Resources: []v1beta2.LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + { + Name: corev1.ResourceMemory, + Total: resource.MustParse("20Gi"), + }, + }, + }, + { + Name: "flavor-2", + Resources: []v1beta2.LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("5"), + }, + }, + }, + }, + PendingWorkloads: 3, + ReservingWorkloads: 2, + AdmittedWorkloads: 1, + }, + }, + expected: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Status: LocalQueueStatus{ + FlavorUsage: []LocalQueueFlavorUsage{ + { + Name: "flavor-1", + Resources: []LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + { + Name: corev1.ResourceMemory, + Total: resource.MustParse("20Gi"), + }, + }, + }, + { + Name: "flavor-2", + Resources: []LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("5"), + }, + }, + }, + }, + PendingWorkloads: 3, + ReservingWorkloads: 2, + AdmittedWorkloads: 1, + }, + }, + }, + "with conditions and status fields": { + input: &v1beta2.LocalQueue{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.LocalQueueSpec{ + ClusterQueue: "cluster-queue-1", + }, + Status: v1beta2.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + FlavorsUsage: []v1beta2.LocalQueueFlavorUsage{ + { + Name: "default-flavor", + Resources: []v1beta2.LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + }, + }, + }, + PendingWorkloads: 5, + }, + }, + expected: &LocalQueue{ + ObjectMeta: defaultObjectMeta, + Spec: LocalQueueSpec{ + ClusterQueue: "cluster-queue-1", + }, + Status: LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + FlavorUsage: []LocalQueueFlavorUsage{ + { + Name: "default-flavor", + Resources: []LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + }, + }, + }, + PendingWorkloads: 5, + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + result := &LocalQueue{} + if err := result.ConvertFrom(tc.input); err != nil { + t.Fatalf("ConvertFrom failed: %v", err) + } + if diff := cmp.Diff(tc.expected, result); diff != "" { + t.Errorf("unexpected conversion result (-want +got):\n%s", diff) + } + }) + } +} + +func TestLocalQueueConversion_RoundTrip(t *testing.T) { + testCases := map[string]struct { + v1beta1Obj *LocalQueue + }{ + "complete LocalQueue with multiple flavors": { + v1beta1Obj: &LocalQueue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-localqueue", + Namespace: "default", + }, + Spec: LocalQueueSpec{ + ClusterQueue: "main-cluster-queue", + }, + Status: LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: "Active", + Status: metav1.ConditionTrue, + Reason: "Ready", + }, + }, + FlavorUsage: []LocalQueueFlavorUsage{ + { + Name: "flavor-1", + Resources: []LocalQueueResourceUsage{ + { + Name: corev1.ResourceCPU, + Total: resource.MustParse("10"), + }, + }, + }, + }, + PendingWorkloads: 5, + ReservingWorkloads: 3, + AdmittedWorkloads: 2, + }, + }, + }, + "LocalQueue with nil FlavorUsage": { + v1beta1Obj: &LocalQueue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "simple-localqueue", + Namespace: "test-ns", + }, + Status: LocalQueueStatus{ + FlavorUsage: nil, + }, + }, + }, + "LocalQueue with empty FlavorUsage": { + v1beta1Obj: &LocalQueue{ + ObjectMeta: metav1.ObjectMeta{ + Name: "empty-flavor-localqueue", + Namespace: "test-ns", + }, + Status: LocalQueueStatus{ + FlavorUsage: []LocalQueueFlavorUsage{}, + }, + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + // Convert v1beta1 -> v1beta2 + v1beta2Obj := &v1beta2.LocalQueue{} + if err := tc.v1beta1Obj.ConvertTo(v1beta2Obj); err != nil { + t.Fatalf("ConvertTo failed: %v", err) + } + + // Convert v1beta2 -> v1beta1 (round-trip) + roundTripped := &LocalQueue{} + if err := roundTripped.ConvertFrom(v1beta2Obj); err != nil { + t.Fatalf("ConvertFrom failed: %v", err) + } + + // Verify round-trip + if diff := cmp.Diff(tc.v1beta1Obj, roundTripped); diff != "" { + t.Errorf("round-trip conversion produced diff (-original +roundtripped):\n%s", diff) + } + }) + } +} From 89f6f2713d0e4dc7b57dc0b9a09f92c013f58094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Thu, 6 Nov 2025 23:52:52 +0100 Subject: [PATCH 110/119] Set default image in wrappers to agnhost (#7551) --- test/e2e/certmanager/certmanager_test.go | 1 + test/e2e/customconfigs/objectretentionpolicies_test.go | 2 ++ test/e2e/customconfigs/waitforpodsready_test.go | 1 + test/e2e/singlecluster/e2e_test.go | 5 +++++ test/e2e/singlecluster/e2e_v1beta1_test.go | 1 + test/e2e/singlecluster/metrics_test.go | 6 ++++++ test/e2e/singlecluster/tas_test.go | 4 +--- test/e2e/tas/job_test.go | 8 -------- test/e2e/tas/pod_group_test.go | 2 ++ 9 files changed, 19 insertions(+), 11 deletions(-) diff --git a/test/e2e/certmanager/certmanager_test.go b/test/e2e/certmanager/certmanager_test.go index 2aa7a582f13..0ec8c6b70f8 100644 --- a/test/e2e/certmanager/certmanager_test.go +++ b/test/e2e/certmanager/certmanager_test.go @@ -68,6 +68,7 @@ var _ = ginkgo.Describe("CertManager", ginkgo.Ordered, func() { ginkgo.It("should admit a Job", func() { testJob := testingjob.MakeJob("test-job", ns.Name). Queue("main"). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). Suspend(false). Obj() util.MustCreate(ctx, k8sClient, testJob) diff --git a/test/e2e/customconfigs/objectretentionpolicies_test.go b/test/e2e/customconfigs/objectretentionpolicies_test.go index 05a150ba198..1117786eae8 100644 --- a/test/e2e/customconfigs/objectretentionpolicies_test.go +++ b/test/e2e/customconfigs/objectretentionpolicies_test.go @@ -86,6 +86,7 @@ var _ = ginkgo.Describe("ObjectRetentionPolicies", ginkgo.Ordered, ginkgo.Contin job := testingjob.MakeJob("job", ns.Name). Queue(kueue.LocalQueueName(lq.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). RequestAndLimit(corev1.ResourceCPU, "1"). Obj() ginkgo.By("Creating a Job", func() { @@ -318,6 +319,7 @@ var _ = ginkgo.Describe("ObjectRetentionPolicies with TinyTimeout and RequeuingL ginkgo.It("should delete Job", func() { job := testingjob.MakeJob("job", ns.Name). Queue(kueue.LocalQueueName(lq.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). RequestAndLimit(corev1.ResourceCPU, "1"). Obj() ginkgo.By("Creating a Job", func() { diff --git a/test/e2e/customconfigs/waitforpodsready_test.go b/test/e2e/customconfigs/waitforpodsready_test.go index 863d1796c88..5dff224c9cd 100644 --- a/test/e2e/customconfigs/waitforpodsready_test.go +++ b/test/e2e/customconfigs/waitforpodsready_test.go @@ -135,6 +135,7 @@ var _ = ginkgo.Describe("WaitForPodsReady with tiny Timeout and no RecoveryTimeo ginkgo.By("creating a suspended job so its pods never report Ready", func() { job = testingjob.MakeJob("job-timeout", ns.Name). Queue(kueue.LocalQueueName(lq.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). Request(corev1.ResourceCPU, "2"). Parallelism(1). Obj() diff --git a/test/e2e/singlecluster/e2e_test.go b/test/e2e/singlecluster/e2e_test.go index c673cb2e544..6171cb3140d 100644 --- a/test/e2e/singlecluster/e2e_test.go +++ b/test/e2e/singlecluster/e2e_test.go @@ -50,6 +50,7 @@ var _ = ginkgo.Describe("Kueue", func() { ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "e2e-") sampleJob = testingjob.MakeJob("test-job", ns.Name). Queue("main"). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). RequestAndLimit(corev1.ResourceCPU, "1"). RequestAndLimit(corev1.ResourceMemory, "20Mi"). Obj() @@ -300,6 +301,7 @@ var _ = ginkgo.Describe("Kueue", func() { ginkgo.By("Job is preempted by higher priority job", func() { job := testingjob.MakeJob("high", ns.Name). Queue("main"). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). PriorityClass("high"). RequestAndLimit(corev1.ResourceCPU, "1"). NodeSelector("instance-type", "on-demand"). // target the same flavor to cause preemption @@ -336,6 +338,7 @@ var _ = ginkgo.Describe("Kueue", func() { ginkgo.By("Job is preempted by higher priority job", func() { job := testingjob.MakeJob("high-with-wpc", ns.Name). Queue("main"). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). WorkloadPriorityClass("high-workload"). RequestAndLimit(corev1.ResourceCPU, "1"). NodeSelector("instance-type", "on-demand"). // target the same flavor to cause preemption @@ -505,6 +508,7 @@ var _ = ginkgo.Describe("Kueue", func() { lowJob := testingjob.MakeJob("low", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). Parallelism(1). NodeSelector("instance-type", "on-demand"). Containers( @@ -539,6 +543,7 @@ var _ = ginkgo.Describe("Kueue", func() { highJob := testingjob.MakeJob("high", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). Parallelism(1). PriorityClass(highPriorityClass.Name). Request(corev1.ResourceCPU, "1"). diff --git a/test/e2e/singlecluster/e2e_v1beta1_test.go b/test/e2e/singlecluster/e2e_v1beta1_test.go index 25437afae16..25ed6833b8f 100644 --- a/test/e2e/singlecluster/e2e_v1beta1_test.go +++ b/test/e2e/singlecluster/e2e_v1beta1_test.go @@ -42,6 +42,7 @@ var _ = ginkgo.Describe("Kueue v1beta2", func() { ns = util.CreateNamespaceFromPrefixWithLog(ctx, k8sClient, "e2e-") sampleJob = testingjob.MakeJob("test-job", ns.Name). Queue("main"). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). RequestAndLimit(corev1.ResourceCPU, "1"). RequestAndLimit(corev1.ResourceMemory, "20Mi"). Obj() diff --git a/test/e2e/singlecluster/metrics_test.go b/test/e2e/singlecluster/metrics_test.go index 6dbb0a0ee12..d0db766af4e 100644 --- a/test/e2e/singlecluster/metrics_test.go +++ b/test/e2e/singlecluster/metrics_test.go @@ -261,6 +261,7 @@ var _ = ginkgo.Describe("Metrics", func() { createdJob = testingjob.MakeJob("admission-checked-job", ns.Name). Queue(kueue.LocalQueueName(localQueue.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). RequestAndLimit(corev1.ResourceCPU, "1"). Obj() util.MustCreate(ctx, k8sClient, createdJob) @@ -404,6 +405,7 @@ var _ = ginkgo.Describe("Metrics", func() { lowerJob1 = testingjob.MakeJob("lower-job-1", ns.Name). Queue(kueue.LocalQueueName(localQueue1.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). RequestAndLimit(corev1.ResourceCPU, "1"). Obj() util.MustCreate(ctx, k8sClient, lowerJob1) @@ -423,6 +425,7 @@ var _ = ginkgo.Describe("Metrics", func() { lowerJob2 = testingjob.MakeJob("lower-job-2", ns.Name). Queue(kueue.LocalQueueName(localQueue2.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). RequestAndLimit(corev1.ResourceCPU, "1"). Obj() util.MustCreate(ctx, k8sClient, lowerJob2) @@ -442,6 +445,7 @@ var _ = ginkgo.Describe("Metrics", func() { blockerJob = testingjob.MakeJob("blocker", ns.Name). Queue(kueue.LocalQueueName(localQueue2.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). PriorityClass(highPriorityClass.Name). RequestAndLimit(corev1.ResourceCPU, "3"). Obj() @@ -462,6 +466,7 @@ var _ = ginkgo.Describe("Metrics", func() { higherJob1 = testingjob.MakeJob("high-large-1", ns.Name). Queue(kueue.LocalQueueName(localQueue1.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). PriorityClass(highPriorityClass.Name). RequestAndLimit(corev1.ResourceCPU, "4"). Obj() @@ -469,6 +474,7 @@ var _ = ginkgo.Describe("Metrics", func() { higherJob2 = testingjob.MakeJob("high-large-2", ns.Name). Queue(kueue.LocalQueueName(localQueue2.Name)). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). PriorityClass(highPriorityClass.Name). RequestAndLimit(corev1.ResourceCPU, "4"). Obj() diff --git a/test/e2e/singlecluster/tas_test.go b/test/e2e/singlecluster/tas_test.go index 43f568aca10..66b59d84a78 100644 --- a/test/e2e/singlecluster/tas_test.go +++ b/test/e2e/singlecluster/tas_test.go @@ -98,9 +98,6 @@ var _ = ginkgo.Describe("TopologyAwareScheduling", func() { Queue(kueue.LocalQueueName(localQueue.Name)). RequestAndLimit(corev1.ResourceCPU, "700m"). RequestAndLimit(corev1.ResourceMemory, "20Mi"). - Obj() - jobKey := client.ObjectKeyFromObject(sampleJob) - sampleJob = (&testingjob.JobWrapper{Job: *sampleJob}). PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, corev1.LabelHostname). Image(util.GetAgnHostImage(), util.BehaviorExitFast). Obj() @@ -109,6 +106,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling", func() { createdWorkload := &kueue.Workload{} // The job might have finished at this point. That shouldn't be a problem for the purpose of this test + jobKey := client.ObjectKeyFromObject(sampleJob) util.ExpectJobUnsuspendedWithNodeSelectors(ctx, k8sClient, jobKey, map[string]string{ "instance-type": "on-demand", }) diff --git a/test/e2e/tas/job_test.go b/test/e2e/tas/job_test.go index 2bae3bed4c9..cc1f0bdfa52 100644 --- a/test/e2e/tas/job_test.go +++ b/test/e2e/tas/job_test.go @@ -120,8 +120,6 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { Parallelism(3). Completions(3). RequestAndLimit(extraResource, "1"). - Obj() - sampleJob = (&testingjob.JobWrapper{Job: *sampleJob}). PodAnnotation(kueue.PodSetPreferredTopologyAnnotation, testing.DefaultRackTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorExitFast). Obj() @@ -172,8 +170,6 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { Parallelism(3). Completions(3). RequestAndLimit(extraResource, "1"). - Obj() - sampleJob = (&testingjob.JobWrapper{Job: *sampleJob}). PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorExitFast). Obj() @@ -225,8 +221,6 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { Parallelism(2). Completions(3). RequestAndLimit(extraResource, "1"). - Obj() - sampleJob = (&testingjob.JobWrapper{Job: *sampleJob}). PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorExitFast). Obj() @@ -253,8 +247,6 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { Completions(int32(numPods)). Indexed(true). RequestAndLimit(extraResource, "1"). - Obj() - sampleJob = (&testingjob.JobWrapper{Job: *sampleJob}). PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). Obj() diff --git a/test/e2e/tas/pod_group_test.go b/test/e2e/tas/pod_group_test.go index 7e82a6c88f3..d0075dacd8e 100644 --- a/test/e2e/tas/pod_group_test.go +++ b/test/e2e/tas/pod_group_test.go @@ -134,6 +134,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Pod group", func() { numPods := 4 basePod := testingpod.MakePod("test-pod", ns.Name). Queue("test-queue"). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). Request(extraResource, "1"). Limit(extraResource, "1") podGroup := basePod.MakeIndexedGroup(numPods) @@ -216,6 +217,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Pod group", func() { numPods := 4 basePod := testingpod.MakePod("test-pod", ns.Name). Queue("test-queue"). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). Request(extraResource, "1"). Limit(extraResource, "1") podGroup := basePod.MakeIndexedGroup(numPods) From c00e7281886c6bea006c2f8eb559dc92da6e6b6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 14:52:58 -0800 Subject: [PATCH 111/119] Bump github.com/containerd/containerd in /hack/internal/tools (#7568) Bumps [github.com/containerd/containerd](https://github.com/containerd/containerd) from 1.7.28 to 1.7.29. - [Release notes](https://github.com/containerd/containerd/releases) - [Changelog](https://github.com/containerd/containerd/blob/main/RELEASES.md) - [Commits](https://github.com/containerd/containerd/compare/v1.7.28...v1.7.29) --- updated-dependencies: - dependency-name: github.com/containerd/containerd dependency-version: 1.7.29 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- hack/internal/tools/go.mod | 2 +- hack/internal/tools/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/internal/tools/go.mod b/hack/internal/tools/go.mod index c9289d1dcae..80852c90d27 100644 --- a/hack/internal/tools/go.mod +++ b/hack/internal/tools/go.mod @@ -164,7 +164,7 @@ require ( github.com/clbanning/mxj/v2 v2.7.0 // indirect github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect - github.com/containerd/containerd v1.7.28 // indirect + github.com/containerd/containerd v1.7.29 // indirect github.com/containerd/errdefs v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect diff --git a/hack/internal/tools/go.sum b/hack/internal/tools/go.sum index 445ff09bdcb..25c83e90817 100644 --- a/hack/internal/tools/go.sum +++ b/hack/internal/tools/go.sum @@ -324,8 +324,8 @@ github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= -github.com/containerd/containerd v1.7.28 h1:Nsgm1AtcmEh4AHAJ4gGlNSaKgXiNccU270Dnf81FQ3c= -github.com/containerd/containerd v1.7.28/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs= +github.com/containerd/containerd v1.7.29 h1:90fWABQsaN9mJhGkoVnuzEY+o1XDPbg9BTC9QTAHnuE= +github.com/containerd/containerd v1.7.29/go.mod h1:azUkWcOvHrWvaiUjSQH0fjzuHIwSPg1WL5PshGP4Szs= github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4= github.com/containerd/errdefs v0.3.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= From ae767302109530f9249e98c8251a447cd7aa5378 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 7 Nov 2025 12:08:52 +0530 Subject: [PATCH 112/119] Use util.RealClock in tests. (#7574) --- test/e2e/singlecluster/e2e_test.go | 6 +++--- test/e2e/singlecluster/metrics_test.go | 2 +- test/e2e/singlecluster/suite_test.go | 2 -- test/integration/multikueue/dispatcher_test.go | 18 +++++++----------- test/integration/multikueue/jobs_test.go | 6 ++---- test/integration/multikueue/no_gc_test.go | 4 +--- .../core/workload_controller_test.go | 18 ++++++++---------- .../appwrapper/appwrapper_controller_test.go | 7 ++----- .../jobs/jaxjob/jaxjob_controller_test.go | 5 +---- .../controller/jobs/job/job_controller_test.go | 13 +++++-------- .../jobs/jobset/jobset_controller_test.go | 7 ++----- .../jobs/mpijob/mpijob_controller_test.go | 5 +---- .../controller/jobs/pod/pod_controller_test.go | 15 ++++++--------- .../pytorchjob/pytorchjob_controller_test.go | 5 +---- .../jobs/trainjob/trainjob_controller_test.go | 3 +-- .../scheduler/podsready/scheduler_test.go | 5 +---- .../scheduler/workload_controller_test.go | 6 ++---- .../webhook/core/workload_test.go | 17 +++++++---------- .../scheduler/runner/controller/controller.go | 7 ++----- test/util/constants.go | 3 +++ test/util/util.go | 9 ++++----- test/util/util_scheduling.go | 4 +--- 22 files changed, 61 insertions(+), 106 deletions(-) diff --git a/test/e2e/singlecluster/e2e_test.go b/test/e2e/singlecluster/e2e_test.go index 6171cb3140d..3bb0cc0ab13 100644 --- a/test/e2e/singlecluster/e2e_test.go +++ b/test/e2e/singlecluster/e2e_test.go @@ -731,7 +731,7 @@ var _ = ginkgo.Describe("Kueue", func() { workload.SetAdmissionCheckState(&patch.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: "check1", State: kueue.CheckStateReady, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Patch(ctx, patch, client.Apply, client.FieldOwner("test-admission-check-controller"), client.ForceOwnership)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -769,7 +769,7 @@ var _ = ginkgo.Describe("Kueue", func() { workload.SetAdmissionCheckState(&patch.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: "check1", State: kueue.CheckStateReady, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Patch(ctx, patch, client.Apply, client.FieldOwner("test-admission-check-controller"), client.ForceOwnership)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -785,7 +785,7 @@ var _ = ginkgo.Describe("Kueue", func() { workload.SetAdmissionCheckState(&patch.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: "check1", State: kueue.CheckStateRejected, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Patch(ctx, patch, client.Apply, client.FieldOwner("test-admission-check-controller"), client.ForceOwnership)).Should(gomega.Succeed()) diff --git a/test/e2e/singlecluster/metrics_test.go b/test/e2e/singlecluster/metrics_test.go index d0db766af4e..0593fcc0133 100644 --- a/test/e2e/singlecluster/metrics_test.go +++ b/test/e2e/singlecluster/metrics_test.go @@ -296,7 +296,7 @@ var _ = ginkgo.Describe("Metrics", func() { workload.SetAdmissionCheckState(&patch.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: kueue.AdmissionCheckReference(admissionCheck.Name), State: kueue.CheckStateReady, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status(). Patch(ctx, patch, client.Apply, client.FieldOwner("test-admission-check-controller"), client.ForceOwnership)). Should(gomega.Succeed()) diff --git a/test/e2e/singlecluster/suite_test.go b/test/e2e/singlecluster/suite_test.go index afe32b742f8..5c47eb93946 100644 --- a/test/e2e/singlecluster/suite_test.go +++ b/test/e2e/singlecluster/suite_test.go @@ -27,7 +27,6 @@ import ( "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "k8s.io/client-go/rest" - "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" kueueclientset "sigs.k8s.io/kueue/client-go/clientset/versioned" @@ -37,7 +36,6 @@ import ( var ( kueuectlPath = filepath.Join("..", "..", "..", "bin", "kubectl-kueue") - realClock = clock.RealClock{} k8sClient client.WithWatch cfg *rest.Config restClient *rest.RESTClient diff --git a/test/integration/multikueue/dispatcher_test.go b/test/integration/multikueue/dispatcher_test.go index 6e1348dedb1..8d51c379cbc 100644 --- a/test/integration/multikueue/dispatcher_test.go +++ b/test/integration/multikueue/dispatcher_test.go @@ -27,7 +27,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -63,7 +62,6 @@ var _ = ginkgo.Describe("MultiKueueDispatcherIncremental", ginkgo.Ordered, ginkg worker2Cq *kueue.ClusterQueue worker2Lq *kueue.LocalQueue - realClock = clock.RealClock{} ) ginkgo.BeforeAll(func() { @@ -211,13 +209,13 @@ var _ = ginkgo.Describe("MultiKueueDispatcherIncremental", ginkgo.Ordered, ginkg ginkgo.By("setting the check conditions for eviction", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, createdWorkload, realClock, func() (*kueue.Workload, bool, error) { + g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, createdWorkload, util.RealClock, func() (*kueue.Workload, bool, error) { acs := kueue.AdmissionCheckState{ Name: kueue.AdmissionCheckReference(multiKueueAC.Name), State: kueue.CheckStateRejected, Message: "check rejected", } - return createdWorkload, workload.SetAdmissionCheckState(&createdWorkload.Status.AdmissionChecks, acs, realClock), nil + return createdWorkload, workload.SetAdmissionCheckState(&createdWorkload.Status.AdmissionChecks, acs, util.RealClock), nil })).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -246,7 +244,6 @@ var _ = ginkgo.Describe("MultiKueueDispatcherExternal", ginkgo.Ordered, ginkgo.C multiKueueAC *kueue.AdmissionCheck managerCq *kueue.ClusterQueue managerLq *kueue.LocalQueue - realClock = clock.RealClock{} worker1Cq *kueue.ClusterQueue worker1Lq *kueue.LocalQueue @@ -379,7 +376,7 @@ var _ = ginkgo.Describe("MultiKueueDispatcherExternal", ginkgo.Ordered, ginkgo.C gomega.Eventually(func(g gomega.Gomega) { managerWl := &kueue.Workload{} g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) - g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, managerWl, realClock, func() (*kueue.Workload, bool, error) { + g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, managerWl, util.RealClock, func() (*kueue.Workload, bool, error) { managerWl.Status.NominatedClusterNames = []string{workerCluster2.Name} return managerWl, true, nil })).To(gomega.Succeed()) @@ -409,7 +406,7 @@ var _ = ginkgo.Describe("MultiKueueDispatcherExternal", ginkgo.Ordered, ginkgo.C managerWl := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) - g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, managerWl, realClock, func() (*kueue.Workload, bool, error) { + g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, managerWl, util.RealClock, func() (*kueue.Workload, bool, error) { managerWl.Status.NominatedClusterNames = []string{workerCluster1.Name, workerCluster2.Name} return managerWl, true, nil })).To(gomega.Succeed()) @@ -451,13 +448,13 @@ var _ = ginkgo.Describe("MultiKueueDispatcherExternal", ginkgo.Ordered, ginkgo.C ginkgo.By("setting the check conditions for eviction", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, createdWorkload, realClock, func() (*kueue.Workload, bool, error) { + g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, createdWorkload, util.RealClock, func() (*kueue.Workload, bool, error) { acs := kueue.AdmissionCheckState{ Name: kueue.AdmissionCheckReference(multiKueueAC.Name), State: kueue.CheckStateRejected, Message: "check rejected", } - return createdWorkload, workload.SetAdmissionCheckState(&createdWorkload.Status.AdmissionChecks, acs, realClock), nil + return createdWorkload, workload.SetAdmissionCheckState(&createdWorkload.Status.AdmissionChecks, acs, util.RealClock), nil })).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -492,7 +489,6 @@ var _ = ginkgo.Describe("MultiKueueDispatcherAllAtOnce", ginkgo.Ordered, ginkgo. worker2Cq *kueue.ClusterQueue worker2Lq *kueue.LocalQueue - realClock = clock.RealClock{} ) ginkgo.BeforeAll(func() { @@ -663,7 +659,7 @@ var _ = ginkgo.Describe("MultiKueueDispatcherAllAtOnce", ginkgo.Ordered, ginkgo. ginkgo.By("Resetting ClusterName field in the manager", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, createdWorkload, realClock, func() (*kueue.Workload, bool, error) { + g.Expect(workload.PatchAdmissionStatus(managerTestCluster.ctx, managerTestCluster.client, createdWorkload, util.RealClock, func() (*kueue.Workload, bool, error) { createdWorkload.Status.ClusterName = nil return createdWorkload, true, nil })).To(gomega.Succeed()) diff --git a/test/integration/multikueue/jobs_test.go b/test/integration/multikueue/jobs_test.go index 1505dd1f436..3a6b8fcb39d 100644 --- a/test/integration/multikueue/jobs_test.go +++ b/test/integration/multikueue/jobs_test.go @@ -35,7 +35,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -1632,7 +1631,6 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, manager := managerTestCluster worker1 := worker1TestCluster worker2 := worker2TestCluster - realClock := clock.RealClock{} features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.ElasticJobsViaWorkloadSlices, true) @@ -1765,7 +1763,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, ginkgo.By("admit the new workload and finish the old workload in the manager cluster", func() { gomega.Eventually(func(g gomega.Gomega) { oldWorkload := getWorkload(g, manager.ctx, manager.client, workloadKey) - g.Expect(workloadslicing.Finish(manager.ctx, manager.client, realClock, oldWorkload, kueue.WorkloadSliceReplaced, "Replaced to accommodate a new slice")).To(gomega.Succeed()) + g.Expect(workloadslicing.Finish(manager.ctx, manager.client, util.RealClock, oldWorkload, kueue.WorkloadSliceReplaced, "Replaced to accommodate a new slice")).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.SetQuotaReservation(manager.ctx, manager.client, newWorkloadKey, utiltestingapi.MakeAdmission(managerCq.Name).Obj()) }) @@ -1802,7 +1800,7 @@ var _ = ginkgo.Describe("MultiKueue", ginkgo.Ordered, ginkgo.ContinueOnFailure, util.SetQuotaReservation(worker1.ctx, worker1.client, newWorkloadKey, utiltestingapi.MakeAdmission(managerCq.Name).Obj()) gomega.Eventually(func(g gomega.Gomega) { workload := getWorkload(g, worker1.ctx, worker1.client, workloadKey) - g.Expect(workloadslicing.Finish(worker1.ctx, worker1.client, realClock, workload, kueue.WorkloadSliceReplaced, "Replaced to accommodate a new slice")).To(gomega.Succeed()) + g.Expect(workloadslicing.Finish(worker1.ctx, worker1.client, util.RealClock, workload, kueue.WorkloadSliceReplaced, "Replaced to accommodate a new slice")).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/multikueue/no_gc_test.go b/test/integration/multikueue/no_gc_test.go index 960e77683a3..2c7c82de62a 100644 --- a/test/integration/multikueue/no_gc_test.go +++ b/test/integration/multikueue/no_gc_test.go @@ -25,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -62,7 +61,6 @@ var _ = ginkgo.Describe("MultiKueue no GC", ginkgo.Ordered, ginkgo.ContinueOnFai worker2Cq *kueue.ClusterQueue worker2Lq *kueue.LocalQueue - realClock clock.RealClock ) ginkgo.BeforeAll(func() { @@ -258,7 +256,7 @@ var _ = ginkgo.Describe("MultiKueue no GC", ginkgo.Ordered, ginkgo.ContinueOnFai workload.SetAdmissionCheckState(&createdWorkload.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: kueue.AdmissionCheckReference(multiKueueAC.Name), State: kueue.CheckStateRejected, - }, realClock) + }, util.RealClock) g.Expect(managerTestCluster.client.Status().Update(managerTestCluster.ctx, createdWorkload)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go index c58ca2a2424..77ccc3b7e96 100644 --- a/test/integration/singlecluster/controller/core/workload_controller_test.go +++ b/test/integration/singlecluster/controller/core/workload_controller_test.go @@ -25,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -52,7 +51,6 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn clusterQueue *kueue.ClusterQueue workloadPriorityClass *kueue.WorkloadPriorityClass updatedWorkloadPriorityClass *kueue.WorkloadPriorityClass - realClock = clock.RealClock{} ) ginkgo.BeforeAll(func() { @@ -227,12 +225,12 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn Name: "check1", State: kueue.CheckStateReady, Message: "check successfully passed", - }, realClock) + }, util.RealClock) workload.SetAdmissionCheckState(&createdWl.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: "check2", State: kueue.CheckStateRetry, Message: "check rejected", - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &createdWl)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -293,7 +291,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn Name: "check1", State: kueue.CheckStateRejected, Message: "check rejected", - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &createdWl)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -345,12 +343,12 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn Name: "check1", State: kueue.CheckStateReady, Message: "check ready", - }, realClock) + }, util.RealClock) workload.SetAdmissionCheckState(&createdWl.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: "check2", State: kueue.CheckStateReady, Message: "check ready", - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &createdWl)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -374,7 +372,7 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn Name: "check1", State: kueue.CheckStateRejected, Message: "check rejected", - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &createdWl)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -506,8 +504,8 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn ginkgo.By("evicting the workload, the accumulated admission time is updated", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, key, wl)).To(gomega.Succeed()) - g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, wl, realClock, func() (*kueue.Workload, bool, error) { - return wl, workload.SetEvictedCondition(wl, realClock.Now(), "ByTest", "by test"), nil + g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, wl, util.RealClock, func() (*kueue.Workload, bool, error) { + return wl, workload.SetEvictedCondition(wl, util.RealClock.Now(), "ByTest", "by test"), nil })).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) util.FinishEvictionForWorkloads(ctx, k8sClient, wl) diff --git a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go index 8ed59b7f9c2..2011b69adde 100644 --- a/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/appwrapper/appwrapper_controller_test.go @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/clock" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -57,8 +56,6 @@ const ( ) var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.ContinueOnFailure, ginkgo.ContinueOnFailure, func() { - var realClock = clock.RealClock{} - ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) @@ -285,7 +282,7 @@ var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.Continue ginkgo.By("preempt the workload", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(workload.UpdateStatus(ctx, k8sClient, createdWorkload, kueue.WorkloadEvicted, metav1.ConditionTrue, kueue.WorkloadEvictedByPreemption, "By test", "evict", clock.RealClock{})).To(gomega.Succeed()) + g.Expect(workload.UpdateStatus(ctx, k8sClient, createdWorkload, kueue.WorkloadEvicted, metav1.ConditionTrue, kueue.WorkloadEvictedByPreemption, "By test", "evict", util.RealClock)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -430,7 +427,7 @@ var _ = ginkgo.Describe("AppWrapper controller", ginkgo.Ordered, ginkgo.Continue }, }, }, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go b/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go index 3de5e2c5235..ca5866111b6 100644 --- a/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/jaxjob/jaxjob_controller_test.go @@ -25,7 +25,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -92,8 +91,6 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu }) var _ = ginkgo.Describe("Job controller for workloads when only jobs with queue are managed", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { - var realClock = clock.RealClock{} - ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup()) }) @@ -229,7 +226,7 @@ var _ = ginkgo.Describe("Job controller for workloads when only jobs with queue }, }, }, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go index 55e6ec82736..a04eef776da 100644 --- a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go @@ -35,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/utils/clock" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -68,8 +67,6 @@ const ( ) var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { - var realClock = clock.RealClock{} - ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup( jobframework.WithManageJobsWithoutQueueName(true), @@ -639,7 +636,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu kueue.WorkloadEvicted, metav1.ConditionTrue, kueue.WorkloadEvictedByPreemption, "By test", "evict", - clock.RealClock{}, + util.RealClock, )).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -773,7 +770,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu }, }, }, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -891,7 +888,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu }, }, }, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -2405,7 +2402,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con workload.SetAdmissionCheckState(&createdLowWorkload.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: kueue.AdmissionCheckReference(admissionCheck.Name), State: kueue.CheckStateReady, - }, clock.RealClock{}) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, createdLowWorkload)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -2451,7 +2448,7 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con workload.SetAdmissionCheckState(&createdHighWorkload.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: kueue.AdmissionCheckReference(admissionCheck.Name), State: kueue.CheckStateReady, - }, clock.RealClock{}) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, createdHighWorkload)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go index 251b06b0fcd..43b7d055096 100644 --- a/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/jobset/jobset_controller_test.go @@ -28,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/clock" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -55,8 +54,6 @@ const ( ) var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFailure, ginkgo.ContinueOnFailure, func() { - var realClock = clock.RealClock{} - ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) @@ -354,7 +351,7 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa ginkgo.By("preempt the workload", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(workload.UpdateStatus(ctx, k8sClient, createdWorkload, kueue.WorkloadEvicted, metav1.ConditionTrue, kueue.WorkloadEvictedByPreemption, "By test", "evict", clock.RealClock{})).To(gomega.Succeed()) + g.Expect(workload.UpdateStatus(ctx, k8sClient, createdWorkload, kueue.WorkloadEvicted, metav1.ConditionTrue, kueue.WorkloadEvictedByPreemption, "By test", "evict", util.RealClock)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -591,7 +588,7 @@ var _ = ginkgo.Describe("JobSet controller", ginkgo.Ordered, ginkgo.ContinueOnFa }, }, }, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go index 22af84fd500..7fd268284cb 100644 --- a/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/mpijob/mpijob_controller_test.go @@ -28,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/clock" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -55,8 +54,6 @@ const ( ) var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailure, ginkgo.ContinueOnFailure, func() { - var realClock = clock.RealClock{} - ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup(false, jobframework.WithManageJobsWithoutQueueName(true), jobframework.WithManagedJobsNamespaceSelector(util.NewNamespaceSelectorExcluding("unmanaged-ns")))) @@ -369,7 +366,7 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu }, }, }, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go index d617e41fe93..36f0ed51b1c 100644 --- a/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pod/pod_controller_test.go @@ -28,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -61,8 +60,6 @@ var ( ) var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { - var realClock = clock.RealClock{} - ginkgo.When("manageJobsWithoutQueueName is disabled", func() { var defaultFlavor = utiltestingapi.MakeResourceFlavor("default").NodeLabel(corev1.LabelArchStable, "arm64").Obj() var clusterQueue = utiltestingapi.MakeClusterQueue("cluster-queue"). @@ -331,7 +328,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu gomega.Expect( workload.UpdateStatus(ctx, k8sClient, createdWorkload, kueue.WorkloadEvicted, metav1.ConditionTrue, - kueue.WorkloadEvictedByPreemption, "By test", "evict", clock.RealClock{}), + kueue.WorkloadEvictedByPreemption, "By test", "evict", util.RealClock), ).Should(gomega.Succeed()) util.FinishEvictionForWorkloads(ctx, k8sClient, createdWorkload) @@ -509,7 +506,7 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu }, }, }, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -834,8 +831,8 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu createdWorkload := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, createdWorkload, realClock, func() (*kueue.Workload, bool, error) { - return createdWorkload, workload.SetEvictedCondition(createdWorkload, realClock.Now(), "ByTest", "by test"), nil + g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, createdWorkload, util.RealClock, func() (*kueue.Workload, bool, error) { + return createdWorkload, workload.SetEvictedCondition(createdWorkload, util.RealClock.Now(), "ByTest", "by test"), nil })).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -1427,8 +1424,8 @@ var _ = ginkgo.Describe("Pod controller", ginkgo.Ordered, ginkgo.ContinueOnFailu ginkgo.By("setting evicted condition to true", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, wl)).Should(gomega.Succeed()) - g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, wl, realClock, func() (*kueue.Workload, bool, error) { - return wl, workload.SetEvictedCondition(wl, realClock.Now(), kueue.WorkloadEvictedByPreemption, "By test"), nil + g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, wl, util.RealClock, func() (*kueue.Workload, bool, error) { + return wl, workload.SetEvictedCondition(wl, util.RealClock.Now(), kueue.WorkloadEvictedByPreemption, "By test"), nil })).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go b/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go index d00a656eacf..2cb35dedcc3 100644 --- a/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/pytorchjob/pytorchjob_controller_test.go @@ -25,7 +25,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -96,8 +95,6 @@ var _ = ginkgo.Describe("Job controller", ginkgo.Ordered, ginkgo.ContinueOnFailu }) var _ = ginkgo.Describe("Job controller for workloads when only jobs with queue are managed", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { - var realClock = clock.RealClock{} - ginkgo.BeforeAll(func() { fwk.StartManager(ctx, cfg, managerSetup()) }) @@ -233,7 +230,7 @@ var _ = ginkgo.Describe("Job controller for workloads when only jobs with queue }, }, }, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &newWL)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go b/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go index db477799ab1..4f2c9a8275f 100644 --- a/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/trainjob/trainjob_controller_test.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" jobsetapi "sigs.k8s.io/jobset/api/jobset/v1alpha2" @@ -259,7 +258,7 @@ var _ = ginkgo.Describe("Trainjob controller", ginkgo.Ordered, ginkgo.ContinueOn ginkgo.By("preempt the workload", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(workload.UpdateStatus(ctx, k8sClient, createdWorkload, kueue.WorkloadEvicted, metav1.ConditionTrue, kueue.WorkloadEvictedByPreemption, "By test", "evict", clock.RealClock{})).To(gomega.Succeed()) + g.Expect(workload.UpdateStatus(ctx, k8sClient, createdWorkload, kueue.WorkloadEvicted, metav1.ConditionTrue, kueue.WorkloadEvictedByPreemption, "By test", "evict", util.RealClock)).To(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go index 225ae08530e..9ce02c2c70b 100644 --- a/test/integration/singlecluster/scheduler/podsready/scheduler_test.go +++ b/test/integration/singlecluster/scheduler/podsready/scheduler_test.go @@ -26,7 +26,6 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -244,8 +243,6 @@ var _ = ginkgo.Describe("SchedulerWithWaitForPodsReady", func() { }) var _ = ginkgo.Context("Short PodsReady timeout", func() { - var realClock = clock.RealClock{} - ginkgo.BeforeEach(func() { podsReadyTimeout = util.ShortTimeout requeueingBackoffLimitCount = ptr.To[int32](2) @@ -332,7 +329,7 @@ var _ = ginkgo.Describe("SchedulerWithWaitForPodsReady", func() { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(prodWl), prodWl)).Should(gomega.Succeed()) g.Expect(workload.IsActive(prodWl)).Should(gomega.BeFalse()) g.Expect(prodWl.Status.RequeueState).Should(gomega.BeNil()) - g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, prodWl, realClock, func() (*kueue.Workload, bool, error) { + g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, prodWl, util.RealClock, func() (*kueue.Workload, bool, error) { return prodWl, workload.SetRequeuedCondition(prodWl, kueue.WorkloadDeactivated, "by test", false), nil })).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) diff --git a/test/integration/singlecluster/scheduler/workload_controller_test.go b/test/integration/singlecluster/scheduler/workload_controller_test.go index dd3d2e888f0..e486d5f9483 100644 --- a/test/integration/singlecluster/scheduler/workload_controller_test.go +++ b/test/integration/singlecluster/scheduler/workload_controller_test.go @@ -24,7 +24,6 @@ import ( nodev1 "k8s.io/api/node/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" @@ -126,7 +125,6 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { }) ginkgo.It("the workload should have appropriate AdditionalChecks added", framework.SlowSpec, func() { - var realClock = clock.RealClock{} wl := utiltestingapi.MakeWorkload("wl", ns.Name). Queue("queue"). Request(resourceGPU, "3"). @@ -168,12 +166,12 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { Name: "check1", State: kueue.CheckStateReady, Message: "check successfully passed", - }, realClock) + }, util.RealClock) workload.SetAdmissionCheckState(&updatedWl.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: "check3", State: kueue.CheckStateReady, Message: "check successfully passed", - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, &updatedWl)).Should(gomega.Succeed()) g.Expect(k8sClient.Get(ctx, wlKey, &updatedWl)).Should(gomega.Succeed()) g.Expect(workload.IsAdmitted(&updatedWl)).Should(gomega.BeTrue(), "should have been admitted") diff --git a/test/integration/singlecluster/webhook/core/workload_test.go b/test/integration/singlecluster/webhook/core/workload_test.go index 56904268e76..ef3f8ced1cf 100644 --- a/test/integration/singlecluster/webhook/core/workload_test.go +++ b/test/integration/singlecluster/webhook/core/workload_test.go @@ -28,7 +28,6 @@ import ( schedulingv1 "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -131,8 +130,6 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { fwk.StopManager(ctx) }) - var realClock = clock.RealClock{} - ginkgo.Context("When creating a Workload", func() { ginkgo.DescribeTable("Should have valid PodSet when creating", func(podSetsCapacity int, podSetCount int, isInvalid bool) { podSets := make([]kueue.PodSet, podSetsCapacity) @@ -321,8 +318,8 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), wl)).To(gomega.Succeed()) - err := workload.PatchAdmissionStatus(ctx, k8sClient, wl, clock.RealClock{}, func() (*kueue.Workload, bool, error) { - return wl, workload.SetQuotaReservation(wl, a, clock.RealClock{}), nil + err := workload.PatchAdmissionStatus(ctx, k8sClient, wl, util.RealClock, func() (*kueue.Workload, bool, error) { + return wl, workload.SetQuotaReservation(wl, a, util.RealClock), nil }) g.Expect(err).Should(matcher) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -377,7 +374,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), wl)).To(gomega.Succeed()) - workload.SetAdmissionCheckState(&wl.Status.AdmissionChecks, acs, realClock) + workload.SetAdmissionCheckState(&wl.Status.AdmissionChecks, acs, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, wl)).Should(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }, @@ -1143,7 +1140,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { LastTransitionTime: metav1.NewTime(time.Now()), PodSetUpdates: []kueue.PodSetUpdate{{Name: "first", Labels: map[string]string{"foo": "bar"}}, {Name: "second"}}, State: kueue.CheckStateReady, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, wl)).Should(gomega.Succeed()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) @@ -1156,7 +1153,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { LastTransitionTime: metav1.NewTime(time.Now()), PodSetUpdates: []kueue.PodSetUpdate{{Name: "first", Labels: map[string]string{"foo": "baz"}}, {Name: "second"}}, State: kueue.CheckStateReady, - }, realClock) + }, util.RealClock) g.Expect(k8sClient.Status().Update(ctx, wl)).Should(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1180,7 +1177,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { Assignment(corev1.ResourceCPU, "default", "1"). Obj()). Obj() - workload.SetQuotaReservation(wl, admission, realClock) + workload.SetQuotaReservation(wl, admission, util.RealClock) wl.Status.Admission = admission g.Expect(k8sClient.Status().Update(ctx, wl)).Should(gomega.Succeed()) @@ -1213,7 +1210,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { Count(10). Obj()). Obj() - workload.SetQuotaReservation(wl, admission, realClock) + workload.SetQuotaReservation(wl, admission, util.RealClock) wl.Status.Admission = admission g.Expect(k8sClient.Status().Update(ctx, wl)).Should(gomega.Succeed()) diff --git a/test/performance/scheduler/runner/controller/controller.go b/test/performance/scheduler/runner/controller/controller.go index 4fefd877deb..e8a4827a419 100644 --- a/test/performance/scheduler/runner/controller/controller.go +++ b/test/performance/scheduler/runner/controller/controller.go @@ -42,6 +42,7 @@ import ( "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/performance/scheduler/runner/generator" "sigs.k8s.io/kueue/test/performance/scheduler/runner/recorder" + "sigs.k8s.io/kueue/test/util" ) type reconciler struct { @@ -52,10 +53,6 @@ type reconciler struct { clock clock.Clock } -var ( - realClock = clock.RealClock{} -) - func (r *reconciler) getAdmittedTime(uid types.UID) (time.Time, bool) { r.atLock.RLock() defer r.atLock.RUnlock() @@ -178,7 +175,7 @@ type options struct { type Option func(*options) var defaultOptions = options{ - clock: realClock, + clock: util.RealClock, } func WithClock(_ testing.TB, c clock.Clock) Option { diff --git a/test/util/constants.go b/test/util/constants.go index f7519d9cf5e..88993760431 100644 --- a/test/util/constants.go +++ b/test/util/constants.go @@ -24,6 +24,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/clock" ) const ( @@ -82,3 +83,5 @@ var ( // The agnhost container will print args passed and `exit 0` BehaviorExitFast = []string{"entrypoint-tester"} ) + +var RealClock = clock.RealClock{} diff --git a/test/util/util.go b/test/util/util.go index 68ab01fb0d2..981eeb10df0 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -56,7 +56,6 @@ import ( "k8s.io/client-go/tools/remotecommand" "k8s.io/component-base/metrics/testutil" "k8s.io/klog/v2" - "k8s.io/utils/clock" testingclock "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -428,7 +427,7 @@ func SetRequeuedConditionWithPodsReadyTimeout(ctx context.Context, k8sClient cli gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { var wl kueue.Workload g.Expect(k8sClient.Get(ctx, wlKey, &wl)).Should(gomega.Succeed()) - g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, &wl, clock.RealClock{}, func() (*kueue.Workload, bool, error) { + g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, &wl, RealClock, func() (*kueue.Workload, bool, error) { return &wl, workload.SetRequeuedCondition(&wl, kueue.WorkloadEvictedByPodsReadyTimeout, fmt.Sprintf("Exceeded the PodsReady timeout %s", klog.KObj(&wl).String()), false), nil })).Should(gomega.Succeed()) }, Timeout, Interval).Should(gomega.Succeed()) @@ -831,7 +830,7 @@ func SyncAdmittedConditionForWorkloads(ctx context.Context, k8sClient client.Cli for _, wl := range wls { gomega.EventuallyWithOffset(1, func(g gomega.Gomega) { g.ExpectWithOffset(1, k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedWorkload)).To(gomega.Succeed()) - g.ExpectWithOffset(1, workload.PatchAdmissionStatus(ctx, k8sClient, &updatedWorkload, clock.RealClock{}, func() (*kueue.Workload, bool, error) { + g.ExpectWithOffset(1, workload.PatchAdmissionStatus(ctx, k8sClient, &updatedWorkload, RealClock, func() (*kueue.Workload, bool, error) { return &updatedWorkload, workload.SyncAdmittedCondition(&updatedWorkload, time.Now()), nil })).To(gomega.Succeed()) }, Timeout, Interval).Should(gomega.Succeed()) @@ -857,7 +856,7 @@ func FinishEvictionForWorkloads(ctx context.Context, k8sClient client.Client, wl var updatedWorkload kueue.Workload g.Expect(k8sClient.Get(ctx, key, &updatedWorkload)).Should(gomega.Succeed()) if apimeta.IsStatusConditionTrue(updatedWorkload.Status.Conditions, kueue.WorkloadQuotaReserved) { - g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, &updatedWorkload, clock.RealClock{}, func() (*kueue.Workload, bool, error) { + g.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, &updatedWorkload, RealClock, func() (*kueue.Workload, bool, error) { return &updatedWorkload, workload.UnsetQuotaReservationWithCondition(&updatedWorkload, "Pending", "By test", time.Now()), nil }), ).Should(gomega.Succeed(), fmt.Sprintf("Unable to unset quota reservation for %q", key)) @@ -892,7 +891,7 @@ func SetWorkloadsAdmissionCheck(ctx context.Context, k8sClient client.Client, wl workload.SetAdmissionCheckState(&updatedWorkload.Status.AdmissionChecks, kueue.AdmissionCheckState{ Name: check, State: state, - }, clock.RealClock{}) + }, RealClock) } g.Expect(k8sClient.Status().Update(ctx, &updatedWorkload)).To(gomega.Succeed()) }, Timeout, Interval).Should(gomega.Succeed()) diff --git a/test/util/util_scheduling.go b/test/util/util_scheduling.go index 3c6bc7f6158..2ab6ea68bc8 100644 --- a/test/util/util_scheduling.go +++ b/test/util/util_scheduling.go @@ -24,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/utils/clock" "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" @@ -46,7 +45,6 @@ func FinishRunningWorkloadsInCQ(ctx context.Context, k8sClient client.Client, cq } func finishEvictionsOfAnyWorkloadsInCq(ctx context.Context, k8sClient client.Client, cq *kueue.ClusterQueue) sets.Set[types.UID] { - var realClock = clock.RealClock{} finished := sets.New[types.UID]() var wList kueue.WorkloadList gomega.Expect(k8sClient.List(ctx, &wList)).To(gomega.Succeed()) @@ -57,7 +55,7 @@ func finishEvictionsOfAnyWorkloadsInCq(ctx context.Context, k8sClient client.Cli evicted := meta.IsStatusConditionTrue(wl.Status.Conditions, kueue.WorkloadEvicted) quotaReserved := meta.IsStatusConditionTrue(wl.Status.Conditions, kueue.WorkloadQuotaReserved) if evicted && quotaReserved { - gomega.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, &wl, realClock, func() (*kueue.Workload, bool, error) { + gomega.Expect(workload.PatchAdmissionStatus(ctx, k8sClient, &wl, RealClock, func() (*kueue.Workload, bool, error) { return &wl, workload.UnsetQuotaReservationWithCondition(&wl, "Pending", "Eviction finished by test", time.Now()), nil }), ).To(gomega.Succeed()) From ec1b73d8fa74add877bbd8a1c845a75cb48e7194 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 7 Nov 2025 02:10:51 -0500 Subject: [PATCH 113/119] enable linter via regular expressions (#7571) --- .golangci-kal.yml | 37 ++++++++++++++ apis/kueue/v1beta2/admissioncheck_types.go | 6 +-- apis/kueue/v1beta2/clusterqueue_types.go | 32 ++++++------- apis/kueue/v1beta2/cohort_types.go | 6 +-- apis/kueue/v1beta2/fairsharing_types.go | 6 +-- apis/kueue/v1beta2/localqueue_types.go | 18 +++---- apis/kueue/v1beta2/multikueue_types.go | 8 ++-- .../provisioningrequestconfig_types.go | 2 +- apis/kueue/v1beta2/resourceflavor_types.go | 4 +- apis/kueue/v1beta2/topology_types.go | 2 +- apis/kueue/v1beta2/workload_types.go | 48 +++++++++---------- .../v1beta2/workloadpriorityclass_types.go | 4 +- 12 files changed, 105 insertions(+), 68 deletions(-) diff --git a/.golangci-kal.yml b/.golangci-kal.yml index 344621545e8..5f356aaff1e 100644 --- a/.golangci-kal.yml +++ b/.golangci-kal.yml @@ -68,6 +68,43 @@ linters: linters: - kubeapilinter + ## Exclusions for intentional API design decisions + # PodSets field intentionally excludes omitempty + - text: "optionalfields: field PodSets should have the omitempty tag." + path: "apis/kueue/v1beta2" + + # Spec and Status fields - Kubernetes convention (not pointers, specific omitempty rules) + - text: "optionalfields: field (Spec|Status) should (be a pointer|have the omitempty tag)." + path: "apis/kueue/v1beta2" + + # Various optional fields intentionally not pointers for API stability + - text: "optionalfields: field (QueueName|PriorityClassName|PriorityClassSource|ClusterQueue|ParentName|CohortName|QueueingStrategy|LocationType|Description|Policy|WhenCanBorrow|WhenCanPreempt|ReclaimWithinCohort|WithinClusterQueue|Borrowed|Total|PendingWorkloads|ReservingWorkloads|AdmittedWorkloads) should (be a pointer|have the omitempty tag)." + path: "apis/kueue/v1beta2" + + # Name and Count fields - intentional design decisions + - text: "(optionalfields|requiredfields): field (Name|Count) should (be a pointer|have the omitempty tag)." + path: "apis/kueue/v1beta2" + + # PodSetAssignments intentionally excludes omitempty + - text: "optionalfields: field PodSetAssignments should have the omitempty tag." + path: "apis/kueue/v1beta2" + + # Required fields with valid zero values - validation is intentionally incomplete + - text: "requiredfields: field .* has a valid zero value.*should be a pointer" + path: "apis/kueue/v1beta2" + + # Required fields that intentionally exclude omitempty + - text: "requiredfields: field (ControllerName|LastUpdate|Reason|UnderlyingCause|Message|WeightedShare|Value|ConsumedResources) should have the omitempty tag." + path: "apis/kueue/v1beta2" + + # ResourceList fields (map types required for Kubernetes compatibility) + - text: "(nomaps|optionalfields): (field )?(ResourceUsage|Resources|ConsumedResources) should (not use a map type|be a pointer)" + path: "apis/kueue/v1beta2" + + # Bool fields for API compatibility (Active mirrors Job.Suspend, Unconstrained is existing API) + - text: "(nobools|optionalorrequired): field (Active|Unconstrained) .*(should not use a bool|must be marked as optional or required)" + path: "apis/kueue/v1beta2" + issues: max-same-issues: 0 max-issues-per-linter: 0 diff --git a/apis/kueue/v1beta2/admissioncheck_types.go b/apis/kueue/v1beta2/admissioncheck_types.go index 9ddb90f0679..afd2211859e 100644 --- a/apis/kueue/v1beta2/admissioncheck_types.go +++ b/apis/kueue/v1beta2/admissioncheck_types.go @@ -52,7 +52,7 @@ type AdmissionCheckSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable" // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:MinLength=1 - ControllerName string `json:"controllerName"` //nolint:kubeapilinter // ignore omitempty + ControllerName string `json:"controllerName"` // parameters identifies a configuration with additional parameters for the // check. @@ -116,11 +116,11 @@ type AdmissionCheck struct { // spec is the specification of the AdmissionCheck. // +optional - Spec AdmissionCheckSpec `json:"spec"` //nolint:kubeapilinter // spec should not be a pointer + Spec AdmissionCheckSpec `json:"spec"` // status is the status of the AdmissionCheck. // +optional - Status AdmissionCheckStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer + Status AdmissionCheckStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index 195497331e4..60605f4339e 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -78,7 +78,7 @@ type ClusterQueueSpec struct { // A cohort is a name that links CQs together, but it doesn't reference any // object. // +optional - CohortName CohortReference `json:"cohortName,omitempty"` //nolint:kubeapilinter // should not be a pointer + CohortName CohortReference `json:"cohortName,omitempty"` // queueingStrategy indicates the queueing strategy of the workloads // across the queues in this ClusterQueue. @@ -94,7 +94,7 @@ type ClusterQueueSpec struct { // +optional // +kubebuilder:default=BestEffortFIFO // +kubebuilder:validation:Enum=StrictFIFO;BestEffortFIFO - QueueingStrategy QueueingStrategy `json:"queueingStrategy,omitempty"` //nolint:kubeapilinter // should not be a pointer + QueueingStrategy QueueingStrategy `json:"queueingStrategy,omitempty"` // namespaceSelector defines which namespaces are allowed to submit workloads to // this clusterQueue. Beyond this basic support for policy, a policy agent like @@ -217,7 +217,7 @@ type FlavorQuotas struct { // ResourceFlavor. If a matching ResourceFlavor does not exist, the // ClusterQueue will have an Active condition set to False. // +required - Name ResourceFlavorReference `json:"name"` //nolint:kubeapilinter // should not be a pointer + Name ResourceFlavorReference `json:"name"` // resources is the list of quotas for this flavor per resource. // There could be up to 64 resources. @@ -311,17 +311,17 @@ type ClusterQueueStatus struct { // pendingWorkloads is the number of workloads currently waiting to be // admitted to this clusterQueue. // +optional - PendingWorkloads int32 `json:"pendingWorkloads"` //nolint:kubeapilinter // should not be a pointer + PendingWorkloads int32 `json:"pendingWorkloads"` // reservingWorkloads is the number of workloads currently reserving quota in this // clusterQueue. // +optional - ReservingWorkloads int32 `json:"reservingWorkloads"` //nolint:kubeapilinter // should not be a pointer + ReservingWorkloads int32 `json:"reservingWorkloads"` // admittedWorkloads is the number of workloads currently admitted to this // clusterQueue and haven't finished yet. // +optional - AdmittedWorkloads int32 `json:"admittedWorkloads"` //nolint:kubeapilinter // should not be a pointer + AdmittedWorkloads int32 `json:"admittedWorkloads"` // fairSharing contains the current state for this ClusterQueue // when participating in Fair Sharing. @@ -333,7 +333,7 @@ type ClusterQueueStatus struct { type FlavorUsage struct { // name of the flavor. // +required - Name ResourceFlavorReference `json:"name,omitempty"` //nolint:kubeapilinter // should not be a pointer + Name ResourceFlavorReference `json:"name,omitempty"` // resources lists the quota usage for the resources in this flavor. // +listType=map @@ -351,12 +351,12 @@ type ResourceUsage struct { // total is the total quantity of used quota, including the amount borrowed // from the cohort. // +optional - Total resource.Quantity `json:"total,omitempty"` //nolint:kubeapilinter // should not be a pointer + Total resource.Quantity `json:"total,omitempty"` // borrowed is quantity of quota that is borrowed from the cohort. In other // words, it's the used quota that is over the nominalQuota. // +optional - Borrowed resource.Quantity `json:"borrowed,omitempty"` //nolint:kubeapilinter // should not be a pointer + Borrowed resource.Quantity `json:"borrowed,omitempty"` } const ( @@ -394,7 +394,7 @@ type FlavorFungibility struct { // +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor} // +kubebuilder:default="MayStopSearch" // +optional - WhenCanBorrow FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"` //nolint:kubeapilinter // should not be a pointer + WhenCanBorrow FlavorFungibilityPolicy `json:"whenCanBorrow,omitempty"` // whenCanPreempt determines whether a workload should try the next flavor // before borrowing in current flavor. The possible values are: // @@ -406,7 +406,7 @@ type FlavorFungibility struct { // +kubebuilder:validation:Enum={MayStopSearch,TryNextFlavor} // +kubebuilder:default="TryNextFlavor" // +optional - WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` //nolint:kubeapilinter // should not be a pointer + WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt,omitempty"` } // ClusterQueuePreemption contains policies to preempt Workloads from this @@ -450,7 +450,7 @@ type ClusterQueuePreemption struct { // +optional // +kubebuilder:default=Never // +kubebuilder:validation:Enum=Never;LowerPriority;Any - ReclaimWithinCohort PreemptionPolicy `json:"reclaimWithinCohort,omitempty"` //nolint:kubeapilinter // should not be a pointer + ReclaimWithinCohort PreemptionPolicy `json:"reclaimWithinCohort,omitempty"` // borrowWithinCohort determines whether a pending Workload can preempt // Workloads from other ClusterQueues in the cohort if the workload requires borrowing. @@ -473,7 +473,7 @@ type ClusterQueuePreemption struct { // +kubebuilder:default=Never // +kubebuilder:validation:Enum=Never;LowerPriority;LowerOrNewerEqualPriority // +optional - WithinClusterQueue PreemptionPolicy `json:"withinClusterQueue,omitempty"` //nolint:kubeapilinter // should not be a pointer + WithinClusterQueue PreemptionPolicy `json:"withinClusterQueue,omitempty"` } type BorrowWithinCohortPolicy string @@ -498,7 +498,7 @@ type BorrowWithinCohort struct { // +kubebuilder:default=Never // +kubebuilder:validation:Enum=Never;LowerPriority // +optional - Policy BorrowWithinCohortPolicy `json:"policy,omitempty"` //nolint:kubeapilinter // should not be a pointer + Policy BorrowWithinCohortPolicy `json:"policy,omitempty"` // maxPriorityThreshold allows to restrict the set of workloads which // might be preempted by a borrowing workload, to only workloads with // priority less than or equal to the specified threshold priority. @@ -528,10 +528,10 @@ type ClusterQueue struct { // spec is the specification of the ClusterQueue. // +optional - Spec ClusterQueueSpec `json:"spec,omitempty"` //nolint:kubeapilinter // spec should not be a pointer + Spec ClusterQueueSpec `json:"spec,omitempty"` // status is the status of the ClusterQueue. // +optional - Status ClusterQueueStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer + Status ClusterQueueStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/cohort_types.go b/apis/kueue/v1beta2/cohort_types.go index e53c824f396..8542b109718 100644 --- a/apis/kueue/v1beta2/cohort_types.go +++ b/apis/kueue/v1beta2/cohort_types.go @@ -33,7 +33,7 @@ type CohortSpec struct { // removed. We prevent further admission while the cycle // exists. // +optional - ParentName CohortReference `json:"parentName,omitempty"` //nolint:kubeapilinter // should not be a pointer + ParentName CohortReference `json:"parentName,omitempty"` // resourceGroups describes groupings of Resources and // Flavors. Each ResourceGroup defines a list of Resources @@ -98,10 +98,10 @@ type Cohort struct { // spec is the specification of the Cohort. // +optional - Spec CohortSpec `json:"spec"` //nolint:kubeapilinter // spec should not be a pointer + Spec CohortSpec `json:"spec"` // status is the status of the Cohort. // +optional - Status CohortStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer + Status CohortStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/fairsharing_types.go b/apis/kueue/v1beta2/fairsharing_types.go index fbcb0bf97bc..2bab0cc89dd 100644 --- a/apis/kueue/v1beta2/fairsharing_types.go +++ b/apis/kueue/v1beta2/fairsharing_types.go @@ -55,7 +55,7 @@ type FairSharingStatus struct { // weight of zero and is borrowing, this will return // 9223372036854775807, the maximum possible share value. // +required - WeightedShare int64 `json:"weightedShare"` //nolint:kubeapilinter // do not add omitempty + WeightedShare int64 `json:"weightedShare"` // admissionFairSharingStatus represents information relevant to the Admission Fair Sharing // +optional @@ -67,11 +67,11 @@ type AdmissionFairSharingStatus struct { // with decaying function applied. // The value is populated if usage consumption functionality is enabled in Kueue config. // +required - ConsumedResources corev1.ResourceList `json:"consumedResources"` //nolint:kubeapilinter // map type is required for standard Kubernetes ResourceList + ConsumedResources corev1.ResourceList `json:"consumedResources"` // lastUpdate is the time when share and consumed resources were updated. // +required - LastUpdate metav1.Time `json:"lastUpdate"` //nolint:kubeapilinter // exclude omitempty + LastUpdate metav1.Time `json:"lastUpdate"` } type AdmissionScope struct { diff --git a/apis/kueue/v1beta2/localqueue_types.go b/apis/kueue/v1beta2/localqueue_types.go index b6386d08bb0..e047c7367e2 100644 --- a/apis/kueue/v1beta2/localqueue_types.go +++ b/apis/kueue/v1beta2/localqueue_types.go @@ -34,7 +34,7 @@ type LocalQueueSpec struct { // clusterQueue is a reference to a clusterQueue that backs this localQueue. // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable" // +optional - ClusterQueue ClusterQueueReference `json:"clusterQueue,omitempty"` //nolint:kubeapilinter // should not be a pointer + ClusterQueue ClusterQueueReference `json:"clusterQueue,omitempty"` // stopPolicy - if set to a value different from None, the LocalQueue is considered Inactive, // no new reservation being made. @@ -61,7 +61,7 @@ type TopologyInfo struct { // name is the name of the topology. // // +required - Name TopologyReference `json:"name"` //nolint:kubeapilinter // should not be a pointer + Name TopologyReference `json:"name"` // levels define the levels of topology. // @@ -88,17 +88,17 @@ type LocalQueueStatus struct { // pendingWorkloads is the number of Workloads in the LocalQueue not yet admitted to a ClusterQueue // +optional - PendingWorkloads int32 `json:"pendingWorkloads"` //nolint:kubeapilinter // should not be a pointer + PendingWorkloads int32 `json:"pendingWorkloads"` // reservingWorkloads is the number of workloads in this LocalQueue // reserving quota in a ClusterQueue and that haven't finished yet. // +optional - ReservingWorkloads int32 `json:"reservingWorkloads"` //nolint:kubeapilinter // should not be a pointer + ReservingWorkloads int32 `json:"reservingWorkloads"` // admittedWorkloads is the number of workloads in this LocalQueue // admitted to a ClusterQueue and that haven't finished yet. // +optional - AdmittedWorkloads int32 `json:"admittedWorkloads"` //nolint:kubeapilinter // should not be a pointer + AdmittedWorkloads int32 `json:"admittedWorkloads"` // flavorsReservation are the reserved quotas, by flavor currently in use by the // workloads assigned to this LocalQueue. @@ -130,7 +130,7 @@ const ( type LocalQueueFlavorUsage struct { // name of the flavor. // +required - Name ResourceFlavorReference `json:"name"` //nolint:kubeapilinter // should not be a pointer + Name ResourceFlavorReference `json:"name"` // resources lists the quota usage for the resources in this flavor. // +listType=map @@ -147,7 +147,7 @@ type LocalQueueResourceUsage struct { // total is the total quantity of used quota. // +optional - Total resource.Quantity `json:"total,omitempty"` //nolint:kubeapilinter // should not be a pointer + Total resource.Quantity `json:"total,omitempty"` } // +genclient @@ -167,10 +167,10 @@ type LocalQueue struct { // spec is the specification of the LocalQueue. // +optional - Spec LocalQueueSpec `json:"spec"` //nolint:kubeapilinter // should not be a pointer + Spec LocalQueueSpec `json:"spec"` // status is the status of the LocalQueue. // +optional - Status LocalQueueStatus `json:"status,omitempty"` //nolint:kubeapilinter // should not be a pointer + Status LocalQueueStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/multikueue_types.go b/apis/kueue/v1beta2/multikueue_types.go index 629558cb385..e27f2c1199c 100644 --- a/apis/kueue/v1beta2/multikueue_types.go +++ b/apis/kueue/v1beta2/multikueue_types.go @@ -59,7 +59,7 @@ type KubeConfig struct { // +kubebuilder:default=Secret // +optional // +kubebuilder:validation:Enum=Secret;Path - LocationType LocationType `json:"locationType"` //nolint:kubeapilinter // should not be a pointer + LocationType LocationType `json:"locationType"` } type MultiKueueClusterSpec struct { @@ -98,11 +98,11 @@ type MultiKueueCluster struct { // spec is the specification of the MultiKueueCluster. // +optional - Spec MultiKueueClusterSpec `json:"spec,omitempty,omitzero"` //nolint:kubeapilinter // spec should not be a pointer + Spec MultiKueueClusterSpec `json:"spec,omitempty,omitzero"` // status is the status of the MultiKueueCluster. // +optional - Status MultiKueueClusterStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer + Status MultiKueueClusterStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true @@ -140,7 +140,7 @@ type MultiKueueConfig struct { // spec is the specification of the MultiKueueConfig. // +optional - Spec MultiKueueConfigSpec `json:"spec,omitempty"` //nolint:kubeapilinter // spec should not be a pointer + Spec MultiKueueConfigSpec `json:"spec,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/provisioningrequestconfig_types.go b/apis/kueue/v1beta2/provisioningrequestconfig_types.go index 2e3f75c1db5..61c1619133d 100644 --- a/apis/kueue/v1beta2/provisioningrequestconfig_types.go +++ b/apis/kueue/v1beta2/provisioningrequestconfig_types.go @@ -171,7 +171,7 @@ type ProvisioningRequestConfig struct { // spec is the specification of the ProvisioningRequestConfig. // +optional - Spec ProvisioningRequestConfigSpec `json:"spec"` //nolint:kubeapilinter // should not be a pointer + Spec ProvisioningRequestConfigSpec `json:"spec"` } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/resourceflavor_types.go b/apis/kueue/v1beta2/resourceflavor_types.go index e1dc741963a..c5ea8eda778 100644 --- a/apis/kueue/v1beta2/resourceflavor_types.go +++ b/apis/kueue/v1beta2/resourceflavor_types.go @@ -35,7 +35,7 @@ type ResourceFlavor struct { // spec is the specification of the ResourceFlavor. // +optional - Spec ResourceFlavorSpec `json:"spec,omitempty"` //nolint:kubeapilinter // spec should not be a pointer + Spec ResourceFlavorSpec `json:"spec,omitempty"` } // TopologyReference is the name of the Topology. @@ -105,7 +105,7 @@ type ResourceFlavorSpec struct { // nodes matching to the Resource Flavor node labels. // // +optional - TopologyName *TopologyReference `json:"topologyName,omitempty"` //nolint:kubeapilinter // should be a pointer + TopologyName *TopologyReference `json:"topologyName,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/topology_types.go b/apis/kueue/v1beta2/topology_types.go index e29b7350a09..ab1a0df560a 100644 --- a/apis/kueue/v1beta2/topology_types.go +++ b/apis/kueue/v1beta2/topology_types.go @@ -136,7 +136,7 @@ type Topology struct { // spec is the specification of the Topology. // +optional - Spec TopologySpec `json:"spec"` //nolint:kubeapilinter // spec should not be a pointer + Spec TopologySpec `json:"spec"` } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index c2e9077cbe3..b956e9cafe0 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -36,12 +36,12 @@ type WorkloadSpec struct { // +kubebuilder:validation:MaxItems=8 // +kubebuilder:validation:MinItems=1 // +optional - PodSets []PodSet `json:"podSets"` //nolint:kubeapilinter // do not add omitempty tag + PodSets []PodSet `json:"podSets"` // queueName is the name of the LocalQueue the Workload is associated with. // queueName cannot be changed while .status.admission is not null. // +optional - QueueName LocalQueueName `json:"queueName,omitempty"` //nolint:kubeapilinter // should not be a pointer + QueueName LocalQueueName `json:"queueName,omitempty"` // priorityClassName is the name of the PriorityClass the Workload is associated with. // If specified, indicates the workload's priority. @@ -54,7 +54,7 @@ type WorkloadSpec struct { // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" // +optional - PriorityClassName string `json:"priorityClassName,omitempty"` //nolint:kubeapilinter // should not be a pointer + PriorityClassName string `json:"priorityClassName,omitempty"` // priority determines the order of access to the resources managed by the // ClusterQueue where the workload is queued. @@ -70,7 +70,7 @@ type WorkloadSpec struct { // +kubebuilder:default="" // +kubebuilder:validation:Enum=kueue.x-k8s.io/workloadpriorityclass;scheduling.k8s.io/priorityclass;"" // +optional - PriorityClassSource string `json:"priorityClassSource,omitempty"` //nolint:kubeapilinter // should not be a pointer + PriorityClassSource string `json:"priorityClassSource,omitempty"` // active determines if a workload can be admitted into a queue. // Changing active from true to false will evict any running workloads. @@ -81,7 +81,7 @@ type WorkloadSpec struct { // // Defaults to true // +kubebuilder:default=true - Active *bool `json:"active,omitempty"` //nolint:kubeapilinter // bool for the ease of use, inspired by k8s Job's Suspend field. + Active *bool `json:"active,omitempty"` // maximumExecutionTimeSeconds if provided, determines the maximum time, in seconds, // the workload can be admitted before it's automatically deactivated. @@ -119,7 +119,7 @@ type PodSetTopologyRequest struct { // // +optional // +kubebuilder:validation:Type=boolean - Unconstrained *bool `json:"unconstrained,omitempty"` //nolint:kubeapilinter // disabling to avoid changing existing API + Unconstrained *bool `json:"unconstrained,omitempty"` // podIndexLabel indicates the name of the label indexing the pods. // For example, in the context of @@ -169,14 +169,14 @@ type PodSetTopologyRequest struct { type Admission struct { // clusterQueue is the name of the ClusterQueue that admitted this workload. // +optional - ClusterQueue ClusterQueueReference `json:"clusterQueue"` //nolint:kubeapilinter // should not be a pointer + ClusterQueue ClusterQueueReference `json:"clusterQueue"` // podSetAssignments hold the admission results for each of the .spec.podSets entries. // +listType=map // +listMapKey=name // +kubebuilder:validation:MaxItems=8 // +optional - PodSetAssignments []PodSetAssignment `json:"podSetAssignments"` //nolint:kubeapilinter // do not add omitempty + PodSetAssignments []PodSetAssignment `json:"podSetAssignments"` } // PodSetReference is the name of a PodSet. @@ -192,7 +192,7 @@ type PodSetAssignment struct { // name is the name of the podSet. It should match one of the names in .spec.podSets. // +kubebuilder:default=main // +optional - Name PodSetReference `json:"name"` //nolint:kubeapilinter // should not be a pointer + Name PodSetReference `json:"name"` // flavors are the flavors assigned to the workload for each resource. // +optional @@ -204,7 +204,7 @@ type PodSetAssignment struct { // the LimitRange defaults and RuntimeClass overheads at the moment of admission. // This field will not change in case of quota reclaim. // +optional - ResourceUsage corev1.ResourceList `json:"resourceUsage,omitempty"` //nolint:kubeapilinter // map type is required for standard Kubernetes ResourceList + ResourceUsage corev1.ResourceList `json:"resourceUsage,omitempty"` // count is the number of pods taken into account at admission time. // This field will not change in case of quota reclaim. @@ -334,7 +334,7 @@ type PodSet struct { // name is the PodSet name. // +kubebuilder:default=main // +optional - Name PodSetReference `json:"name,omitempty"` //nolint:kubeapilinter // should not be a pointer + Name PodSetReference `json:"name,omitempty"` // template is the Pod template. // @@ -356,7 +356,7 @@ type PodSet struct { // +kubebuilder:default=1 // +kubebuilder:validation:Minimum=0 // +optional - Count int32 `json:"count"` //nolint:kubeapilinter // should not be a pointer + Count int32 `json:"count"` // minCount is the minimum number of pods for the spec acceptable // if the workload supports partial admission. @@ -509,19 +509,19 @@ type WorkloadSchedulingStatsEviction struct { // // +required // +kubebuilder:validation:MaxLength=316 - Reason string `json:"reason"` //nolint:kubeapilinter // should not be a pointer + Reason string `json:"reason"` // underlyingCause specifies a finer-grained explanation that complements the eviction reason. // This may be an empty string. // // +required - UnderlyingCause EvictionUnderlyingCause `json:"underlyingCause"` //nolint:kubeapilinter // should not be a pointer + UnderlyingCause EvictionUnderlyingCause `json:"underlyingCause"` // count tracks the number of evictions for this reason and detailed reason. // // +required // +kubebuilder:validation:Minimum=0 - Count int32 `json:"count"` //nolint:kubeapilinter // should not be a pointer + Count int32 `json:"count"` } type UnhealthyNode struct { @@ -529,7 +529,7 @@ type UnhealthyNode struct { // // +required // +kubebuilder:validation:MaxLength=63 - Name string `json:"name"` //nolint:kubeapilinter // should not be a pointer + Name string `json:"name"` } type RequeueState struct { @@ -572,7 +572,7 @@ type AdmissionCheckState struct { // This may be an empty string. // +required // +kubebuilder:validation:MaxLength=32768 - Message string `json:"message" protobuf:"bytes,6,opt,name=message"` //nolint:kubeapilinter // disable should be a pointer rule + Message string `json:"message" protobuf:"bytes,6,opt,name=message"` // podSetUpdates contains a list of pod set modifications suggested by AdmissionChecks. // +optional @@ -588,7 +588,7 @@ type AdmissionCheckState struct { type PodSetUpdate struct { // name of the PodSet to modify. Should match to one of the Workload's PodSets. // +required - Name PodSetReference `json:"name"` //nolint:kubeapilinter // duplicatemarkers seems to say these are duplicates + Name PodSetReference `json:"name"` // labels of the PodSet to modify. // +optional @@ -617,18 +617,18 @@ type PodSetUpdate struct { type ReclaimablePod struct { // name is the PodSet name. // +required - Name PodSetReference `json:"name"` //nolint:kubeapilinter // duplicatemarkers seems to say these are duplicates + Name PodSetReference `json:"name"` // count is the number of pods for which the requested resources are no longer needed. // +kubebuilder:validation:Minimum=0 // +required - Count int32 `json:"count"` //nolint:kubeapilinter // should not be a pointer + Count int32 `json:"count"` } type PodSetRequest struct { // name is the name of the podSet. It should match one of the names in .spec.podSets. // +required - Name PodSetReference `json:"name"` //nolint:kubeapilinter // duplicatemarkers seems to say these are duplicates + Name PodSetReference `json:"name"` // resources is the total resources all the pods in the podset need to run. // @@ -636,7 +636,7 @@ type PodSetRequest struct { // the LimitRange defaults and RuntimeClass overheads at the moment of consideration // and the application of resource.excludeResourcePrefixes and resource.transformations. // +optional - Resources corev1.ResourceList `json:"resources,omitempty"` //nolint:kubeapilinter // map type is required for standard Kubernetes ResourceList + Resources corev1.ResourceList `json:"resources,omitempty"` } const ( @@ -812,10 +812,10 @@ type Workload struct { // spec is the specification of the Workload. // +optional - Spec WorkloadSpec `json:"spec"` //nolint:kubeapilinter // spec should not be a pointer + Spec WorkloadSpec `json:"spec"` // status is the status of the Workload. // +optional - Status WorkloadStatus `json:"status,omitempty"` //nolint:kubeapilinter // status should not be a pointer + Status WorkloadStatus `json:"status,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/kueue/v1beta2/workloadpriorityclass_types.go b/apis/kueue/v1beta2/workloadpriorityclass_types.go index c6727a659ec..49c0e3e57dd 100644 --- a/apis/kueue/v1beta2/workloadpriorityclass_types.go +++ b/apis/kueue/v1beta2/workloadpriorityclass_types.go @@ -37,14 +37,14 @@ type WorkloadPriorityClass struct { // receive when jobs have the name of this class in their workloadPriorityClass label. // Changing the value of workloadPriorityClass doesn't affect the priority of workloads that were already created. // +required - Value int32 `json:"value"` //nolint:kubeapilinter // disabling should be a pointer + Value int32 `json:"value"` // description is an arbitrary string that usually provides guidelines on // when this workloadPriorityClass should be used. // The description is limited to a maximum of 2048 characters. // +optional // +kubebuilder:validation:MaxLength=2048 - Description string `json:"description,omitempty"` //nolint:kubeapilinter // disabling should not be a pointer + Description string `json:"description,omitempty"` } // +kubebuilder:object:root=true From 15a6257df77b12162da42207aa87ed4526ad5103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Szadkowski?= Date: Fri, 7 Nov 2025 09:32:54 +0100 Subject: [PATCH 114/119] [Cleanup] Update e2e tests to use utiltesting alias (#7564) * Update e2e tests to use utiltesting alias * Update remaining utiltesting alias in intergation tests --- .../managejobswithoutqueuename_test.go | 22 ++++---- .../objectretentionpolicies_test.go | 10 ++-- .../podintegrationautoenablement_test.go | 4 +- .../customconfigs/waitforpodsready_test.go | 18 +++--- test/e2e/singlecluster/appwrapper_test.go | 4 +- test/e2e/singlecluster/deployment_test.go | 8 +-- test/e2e/singlecluster/e2e_test.go | 24 ++++---- test/e2e/singlecluster/e2e_v1beta1_test.go | 4 +- test/e2e/singlecluster/jobset_test.go | 4 +- .../e2e/singlecluster/leaderworkerset_test.go | 56 +++++++++---------- test/e2e/singlecluster/pod_test.go | 16 +++--- test/e2e/singlecluster/statefulset_test.go | 6 +- test/e2e/singlecluster/tas_test.go | 12 ++-- test/e2e/singlecluster/trainjob_test.go | 8 +-- test/e2e/singlecluster/visibility_test.go | 26 ++++----- test/e2e/tas/appwrapper_test.go | 6 +- test/e2e/tas/hotswap_test.go | 10 ++-- test/e2e/tas/job_test.go | 18 +++--- test/e2e/tas/jobset_test.go | 10 ++-- test/e2e/tas/leaderworkerset_test.go | 18 +++--- test/e2e/tas/mpijob_test.go | 6 +- test/e2e/tas/pod_group_test.go | 4 +- test/e2e/tas/pytorch_test.go | 6 +- test/e2e/tas/rayjob_test.go | 8 +-- test/e2e/tas/statefulset_test.go | 4 +- test/e2e/tas/trainjob_test.go | 10 ++-- .../scheduler/workload_controller_test.go | 12 ++-- 27 files changed, 167 insertions(+), 167 deletions(-) diff --git a/test/e2e/customconfigs/managejobswithoutqueuename_test.go b/test/e2e/customconfigs/managejobswithoutqueuename_test.go index a3b925011de..2250a9efe06 100644 --- a/test/e2e/customconfigs/managejobswithoutqueuename_test.go +++ b/test/e2e/customconfigs/managejobswithoutqueuename_test.go @@ -42,7 +42,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobs/statefulset" "sigs.k8s.io/kueue/pkg/features" utilpod "sigs.k8s.io/kueue/pkg/util/pod" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" awtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" testingdeploy "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" @@ -523,7 +523,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { ginkgo.By("deleting the pod", func() { gomega.Expect(k8sClient.Delete(ctx, testPod)).Should(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, podLookupKey, createdPod)).Should(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, podLookupKey, createdPod)).Should(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -608,7 +608,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { ginkgo.By("check that workload is created and not admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusFalse(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusFalse(kueue.WorkloadQuotaReserved)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -647,7 +647,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { ginkgo.By("check that workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -742,7 +742,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { ginkgo.By("verifying that the Deployment doesn't create", func() { createdDeployment := &appsv1.Deployment{} gomega.Consistently(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, deploymentKey, createdDeployment)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, deploymentKey, createdDeployment)).To(utiltesting.BeNotFoundError()) }, util.ConsistentDuration, util.ShortInterval).Should(gomega.Succeed()) }) @@ -769,7 +769,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(aw.Namespace))).To(gomega.Succeed()) g.Expect(createdWorkloads.Items).To(gomega.HaveLen(1)) g.Expect(createdWorkloads.Items[0].Name).To(gomega.Equal(appwrapper.GetWorkloadNameForAppWrapper(aw.Name, aw.UID))) - g.Expect(createdWorkloads.Items[0].Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkloads.Items[0].Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -801,7 +801,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(lws.Namespace))).To(gomega.Succeed()) g.Expect(createdWorkloads.Items).To(gomega.HaveLen(1)) g.Expect(createdWorkloads.Items[0].Name).To(gomega.Equal(leaderworkerset.GetWorkloadName(lws.UID, lws.Name, "0"))) - g.Expect(createdWorkloads.Items[0].Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkloads.Items[0].Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) util.MustHaveOwnerReference(g, createdWorkloads.Items[0].OwnerReferences, lws, k8sClient.Scheme()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -811,7 +811,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrue("Available")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrue("Available")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -838,7 +838,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { } createdWorkload := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -862,7 +862,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(lws.Namespace))).To(gomega.Succeed()) g.Expect(createdWorkloads.Items).To(gomega.HaveLen(1)) g.Expect(createdWorkloads.Items[0].Name).To(gomega.Equal(leaderworkerset.GetWorkloadName(lws.UID, lws.Name, "0"))) - g.Expect(createdWorkloads.Items[0].Status.Conditions).To(testing.HaveConditionStatusFalse(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkloads.Items[0].Status.Conditions).To(utiltesting.HaveConditionStatusFalse(kueue.WorkloadQuotaReserved)) util.MustHaveOwnerReference(g, createdWorkloads.Items[0].OwnerReferences, lws, k8sClient.Scheme()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -900,7 +900,7 @@ var _ = ginkgo.Describe("ManageJobsWithoutQueueName", ginkgo.Ordered, func() { } createdWorkload := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/e2e/customconfigs/objectretentionpolicies_test.go b/test/e2e/customconfigs/objectretentionpolicies_test.go index 1117786eae8..8002aea3e76 100644 --- a/test/e2e/customconfigs/objectretentionpolicies_test.go +++ b/test/e2e/customconfigs/objectretentionpolicies_test.go @@ -30,7 +30,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" @@ -121,13 +121,13 @@ var _ = ginkgo.Describe("ObjectRetentionPolicies", ginkgo.Ordered, ginkgo.Contin ginkgo.By("Checking that the Job is deleted", func() { createdJob := &batchv1.Job{} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(job), createdJob)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(job), createdJob)).To(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.By("Checking that the Workload is deleted", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlKey, wl)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlKey, wl)).To(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -203,7 +203,7 @@ var _ = ginkgo.Describe("ObjectRetentionPolicies with TinyTimeout", ginkgo.Order ginkgo.By("Checking that the Workload is deleted after it is finished", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlKey, wl)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlKey, wl)).To(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -337,7 +337,7 @@ var _ = ginkgo.Describe("ObjectRetentionPolicies with TinyTimeout and RequeuingL } wl := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlKey, wl)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlKey, wl)).To(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/e2e/customconfigs/podintegrationautoenablement_test.go b/test/e2e/customconfigs/podintegrationautoenablement_test.go index a4b15b88faa..fbae082e2b3 100644 --- a/test/e2e/customconfigs/podintegrationautoenablement_test.go +++ b/test/e2e/customconfigs/podintegrationautoenablement_test.go @@ -28,7 +28,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/statefulset" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" testingsts "sigs.k8s.io/kueue/pkg/util/testingjobs/statefulset" @@ -93,7 +93,7 @@ var _ = ginkgo.Describe("Auto-Enablement of Pod Integration for Pod-Dependent Fr createdWorkload := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/e2e/customconfigs/waitforpodsready_test.go b/test/e2e/customconfigs/waitforpodsready_test.go index 5dff224c9cd..3646286b02d 100644 --- a/test/e2e/customconfigs/waitforpodsready_test.go +++ b/test/e2e/customconfigs/waitforpodsready_test.go @@ -30,7 +30,7 @@ import ( configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingjobspod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" @@ -157,8 +157,8 @@ var _ = ginkgo.Describe("WaitForPodsReady with tiny Timeout and no RecoveryTimeo ginkgo.By("waiting for the workload to be evicted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, &wl)).Should(gomega.Succeed()) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusFalseAndReason(kueue.WorkloadPodsReady, kueue.WorkloadWaitForStart)) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason(kueue.WorkloadEvicted, kueue.WorkloadEvictedByPodsReadyTimeout)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusFalseAndReason(kueue.WorkloadPodsReady, kueue.WorkloadWaitForStart)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason(kueue.WorkloadEvicted, kueue.WorkloadEvictedByPodsReadyTimeout)) g.Expect(wl.Status.SchedulingStats.Evictions).To( gomega.BeComparableTo([]kueue.WorkloadSchedulingStatsEviction{{ Reason: kueue.WorkloadEvictedByPodsReadyTimeout, @@ -309,7 +309,7 @@ var _ = ginkgo.Describe("WaitForPodsReady with default Timeout and a tiny Recove ginkgo.By("checking workload availability", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, &wl)).Should(gomega.Succeed()) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadPodsReady)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadPodsReady)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -322,9 +322,9 @@ var _ = ginkgo.Describe("WaitForPodsReady with default Timeout and a tiny Recove g.Expect(k8sClient.Get(ctx, wlKey, &wl)).Should(gomega.Succeed()) g.Expect(wl.Status.RequeueState).ShouldNot(gomega.BeNil()) g.Expect(*wl.Status.RequeueState.Count).To(gomega.Equal(int32(1))) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason(kueue.WorkloadPodsReady, kueue.WorkloadStarted)) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusFalse(kueue.WorkloadEvicted)) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadRequeued)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason(kueue.WorkloadPodsReady, kueue.WorkloadStarted)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusFalse(kueue.WorkloadEvicted)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadRequeued)) g.Expect(wl.Status.SchedulingStats.Evictions).To( gomega.BeComparableTo([]kueue.WorkloadSchedulingStatsEviction{{ @@ -454,7 +454,7 @@ var _ = ginkgo.Describe("WaitForPodsReady with default Timeout and a long Recove ginkgo.By("checking workload availability", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, &wl)).Should(gomega.Succeed()) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadPodsReady)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadPodsReady)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -473,7 +473,7 @@ var _ = ginkgo.Describe("WaitForPodsReady with default Timeout and a long Recove ginkgo.By("verifying the pod is recovered before recoveryTimeout", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlKey, &wl)).Should(gomega.Succeed()) - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason(kueue.WorkloadPodsReady, kueue.WorkloadRecovered)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason(kueue.WorkloadPodsReady, kueue.WorkloadRecovered)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/e2e/singlecluster/appwrapper_test.go b/test/e2e/singlecluster/appwrapper_test.go index 90d56da15cb..528355fb219 100644 --- a/test/e2e/singlecluster/appwrapper_test.go +++ b/test/e2e/singlecluster/appwrapper_test.go @@ -27,7 +27,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/jobs/appwrapper" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" awtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" testingdeploy "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" @@ -158,7 +158,7 @@ var _ = ginkgo.Describe("AppWrapper", func() { g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(aw.Namespace))).To(gomega.Succeed()) g.Expect(createdWorkloads.Items).To(gomega.HaveLen(1)) g.Expect(createdWorkloads.Items[0].Name).To(gomega.Equal(appwrapper.GetWorkloadNameForAppWrapper(aw.Name, aw.UID))) - g.Expect(createdWorkloads.Items[0].Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkloads.Items[0].Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/e2e/singlecluster/deployment_test.go b/test/e2e/singlecluster/deployment_test.go index f39a408e343..10073b93730 100644 --- a/test/e2e/singlecluster/deployment_test.go +++ b/test/e2e/singlecluster/deployment_test.go @@ -27,7 +27,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/pod" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" deploymenttesting "sigs.k8s.io/kueue/pkg/util/testingjobs/deployment" "sigs.k8s.io/kueue/test/util" @@ -111,7 +111,7 @@ var _ = ginkgo.Describe("Deployment", func() { Namespace: p.Namespace, } gomega.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - gomega.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + gomega.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) createdWorkloads = append(createdWorkloads, createdWorkload) } }) @@ -161,7 +161,7 @@ var _ = ginkgo.Describe("Deployment", func() { g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(ns.Name))).To(gomega.Succeed()) g.Expect(createdWorkloads.Items).To(gomega.HaveLen(3)) for _, wl := range createdWorkloads.Items { - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusFalse(kueue.WorkloadQuotaReserved)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusFalse(kueue.WorkloadQuotaReserved)) } }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -202,7 +202,7 @@ var _ = ginkgo.Describe("Deployment", func() { g.Expect(k8sClient.List(ctx, createdWorkloads, client.InNamespace(ns.Name))).To(gomega.Succeed()) g.Expect(createdWorkloads.Items).To(gomega.HaveLen(3)) for _, wl := range createdWorkloads.Items { - g.Expect(wl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(wl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) } }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/e2e/singlecluster/e2e_test.go b/test/e2e/singlecluster/e2e_test.go index 3bb0cc0ab13..961753d8b10 100644 --- a/test/e2e/singlecluster/e2e_test.go +++ b/test/e2e/singlecluster/e2e_test.go @@ -34,7 +34,7 @@ import ( "sigs.k8s.io/kueue/pkg/controller/jobframework" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/util/slices" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workload" @@ -216,7 +216,7 @@ var _ = ginkgo.Describe("Kueue", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -278,7 +278,7 @@ var _ = ginkgo.Describe("Kueue", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), createdWorkload)).To(gomega.Succeed()) g.Expect(createdWorkload.Finalizers).NotTo(gomega.ContainElement(kueue.ResourceInUseFinalizerName)) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason(kueue.WorkloadFinished, kueue.WorkloadFinishedReasonFailed)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason(kueue.WorkloadFinished, kueue.WorkloadFinishedReasonFailed)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -286,7 +286,7 @@ var _ = ginkgo.Describe("Kueue", func() { ginkgo.It("Should readmit preempted job with priorityClass into a separate flavor", func() { util.MustCreate(ctx, k8sClient, sampleJob) - highPriorityClass := testing.MakePriorityClass("high").PriorityValue(100).Obj() + highPriorityClass := utiltesting.MakePriorityClass("high").PriorityValue(100).Obj() util.MustCreate(ctx, k8sClient, highPriorityClass) ginkgo.DeferCleanup(func() { gomega.Expect(k8sClient.Delete(ctx, highPriorityClass)).To(gomega.Succeed()) @@ -389,7 +389,7 @@ var _ = ginkgo.Describe("Kueue", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -437,7 +437,7 @@ var _ = ginkgo.Describe("Kueue", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, jobKey, createdJob)).Should(gomega.Succeed()) createdJob.Labels[constants.WorkloadPriorityClassLabel] = "" - g.Expect(k8sClient.Update(ctx, createdJob)).Should(testing.BeForbiddenError()) + g.Expect(k8sClient.Update(ctx, createdJob)).Should(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -500,7 +500,7 @@ var _ = ginkgo.Describe("Kueue", func() { }) ginkgo.It("Should deduplicate env variables", func() { - highPriorityClass := testing.MakePriorityClass("high").PriorityValue(100).Obj() + highPriorityClass := utiltesting.MakePriorityClass("high").PriorityValue(100).Obj() util.MustCreate(ctx, k8sClient, highPriorityClass) ginkgo.DeferCleanup(func() { gomega.Expect(k8sClient.Delete(ctx, highPriorityClass)).To(gomega.Succeed()) @@ -512,7 +512,7 @@ var _ = ginkgo.Describe("Kueue", func() { Parallelism(1). NodeSelector("instance-type", "on-demand"). Containers( - *testing.MakeContainer(). + *utiltesting.MakeContainer(). Name("c"). Image("sleep"). WithResourceReq(corev1.ResourceCPU, "1"). @@ -568,7 +568,7 @@ var _ = ginkgo.Describe("Kueue", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowWlLookupKey, lowCreatedWorkload)).Should(gomega.Succeed()) g.Expect(workload.IsEvicted(lowCreatedWorkload)).Should(gomega.BeTrue()) - g.Expect(lowCreatedWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadPreempted)) + g.Expect(lowCreatedWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadPreempted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -647,7 +647,7 @@ var _ = ginkgo.Describe("Kueue", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, jobKey, createdJob)).Should(gomega.Succeed()) createdJob.Labels[constants.WorkloadPriorityClassLabel] = "" - g.Expect(k8sClient.Update(ctx, createdJob)).Should(testing.BeForbiddenError()) + g.Expect(k8sClient.Update(ctx, createdJob)).Should(utiltesting.BeForbiddenError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -711,7 +711,7 @@ var _ = ginkgo.Describe("Kueue", func() { ginkgo.By("waiting for the workload to be assigned", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadQuotaReserved)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -743,7 +743,7 @@ var _ = ginkgo.Describe("Kueue", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/e2e/singlecluster/e2e_v1beta1_test.go b/test/e2e/singlecluster/e2e_v1beta1_test.go index 25ed6833b8f..20d8dbfa80c 100644 --- a/test/e2e/singlecluster/e2e_v1beta1_test.go +++ b/test/e2e/singlecluster/e2e_v1beta1_test.go @@ -27,7 +27,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta1" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" @@ -132,7 +132,7 @@ var _ = ginkgo.Describe("Kueue v1beta2", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(hasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/e2e/singlecluster/jobset_test.go b/test/e2e/singlecluster/jobset_test.go index 058839d4350..e515a98ae43 100644 --- a/test/e2e/singlecluster/jobset_test.go +++ b/test/e2e/singlecluster/jobset_test.go @@ -26,7 +26,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjobset "sigs.k8s.io/kueue/pkg/controller/jobs/jobset" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" "sigs.k8s.io/kueue/test/util" @@ -97,7 +97,7 @@ var _ = ginkgo.Describe("JobSet", func() { ginkgo.By("Waiting for the jobSet to finish", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdLeaderWorkload)).To(gomega.Succeed()) - g.Expect(createdLeaderWorkload.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason(kueue.WorkloadFinished, kueue.WorkloadFinishedReasonSucceeded)) + g.Expect(createdLeaderWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason(kueue.WorkloadFinished, kueue.WorkloadFinishedReasonSucceeded)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/e2e/singlecluster/leaderworkerset_test.go b/test/e2e/singlecluster/leaderworkerset_test.go index d740860d14c..58fd06a396e 100644 --- a/test/e2e/singlecluster/leaderworkerset_test.go +++ b/test/e2e/singlecluster/leaderworkerset_test.go @@ -31,7 +31,7 @@ import ( ctrlconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/leaderworkerset" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" leaderworkersettesting "sigs.k8s.io/kueue/pkg/util/testingjobs/leaderworkerset" "sigs.k8s.io/kueue/test/util" @@ -101,7 +101,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -152,7 +152,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -201,7 +201,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(2))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -259,7 +259,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -281,7 +281,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(2))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -340,7 +340,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(2))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -368,7 +368,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -424,7 +424,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -454,7 +454,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -466,7 +466,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { wlLookupKey2 := types.NamespacedName{Name: leaderworkerset.GetWorkloadName(lws.UID, lws.Name, "1"), Namespace: ns.Name} ginkgo.By("Check workload for group 2 is deleted", func() { gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).To(utiltesting.BeNotFoundError()) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -533,7 +533,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(2))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -609,7 +609,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(2))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -642,7 +642,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { createdLeaderWorkerSet := &leaderworkersetv1.LeaderWorkerSet{} g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(2))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -727,7 +727,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lowPriorityLWS), createdLowPriorityLWS)).To(gomega.Succeed()) g.Expect(createdLowPriorityLWS.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLowPriorityLWS.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLowPriorityLWS.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -744,7 +744,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { ginkgo.By("Check the low priority Workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowPriorityWlLookupKey, createdLowPriorityWl)).To(gomega.Succeed()) - g.Expect(createdLowPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdLowPriorityWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -766,7 +766,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(highPriorityLWS), createdHighPriorityLWS)).To(gomega.Succeed()) g.Expect(createdHighPriorityLWS.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdHighPriorityLWS.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdHighPriorityLWS.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -775,8 +775,8 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lowPriorityLWS), createdLowPriorityLWS)).To(gomega.Succeed()) g.Expect(createdLowPriorityLWS.Status.ReadyReplicas).To(gomega.Equal(int32(0))) - g.Expect(createdLowPriorityLWS.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Progressing", "GroupsProgressing")) - g.Expect(createdLowPriorityLWS.Status.Conditions).To(testing.HaveConditionStatusFalseAndReason("Available", "AllGroupsReady")) + g.Expect(createdLowPriorityLWS.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Progressing", "GroupsProgressing")) + g.Expect(createdLowPriorityLWS.Status.Conditions).To(utiltesting.HaveConditionStatusFalseAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -793,15 +793,15 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { ginkgo.By("Await for the high priority Workload to be admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, highPriorityWlLookupKey, createdHighPriorityWl)).To(gomega.Succeed()) - g.Expect(createdHighPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdHighPriorityWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) ginkgo.By("Check the low priority Workload is preempted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowPriorityWlLookupKey, createdLowPriorityWl)).To(gomega.Succeed()) - g.Expect(createdLowPriorityWl.Status.Conditions).To(testing.HaveConditionStatusFalse(kueue.WorkloadAdmitted)) - g.Expect(createdLowPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadPreempted)) + g.Expect(createdLowPriorityWl.Status.Conditions).To(utiltesting.HaveConditionStatusFalse(kueue.WorkloadAdmitted)) + g.Expect(createdLowPriorityWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadPreempted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -848,7 +848,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { ginkgo.By("Check the low priority Workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowPriorityWlLookupKey, createdLowPriorityWl)).To(gomega.Succeed()) - g.Expect(createdLowPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdLowPriorityWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -911,7 +911,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { ginkgo.By("Await for the high priority Workload to be admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, updatablePriorityWlLookupKey, createdUpdatablePriorityWl)).To(gomega.Succeed()) - g.Expect(createdUpdatablePriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdUpdatablePriorityWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -938,7 +938,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -967,7 +967,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(0))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusFalseAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusFalseAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -1008,7 +1008,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(int32(1))) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -1016,7 +1016,7 @@ var _ = ginkgo.Describe("LeaderWorkerSet integration", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) g.Expect(createdWorkload.UID).Should(gomega.Equal(createdWorkloadUID)) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/e2e/singlecluster/pod_test.go b/test/e2e/singlecluster/pod_test.go index b78052bcd42..c7c7d2145dc 100644 --- a/test/e2e/singlecluster/pod_test.go +++ b/test/e2e/singlecluster/pod_test.go @@ -33,7 +33,7 @@ import ( controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/pod" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" podtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/test/util" @@ -117,7 +117,7 @@ var _ = ginkgo.Describe("Pod groups", func() { gomega.Eventually(func(g gomega.Gomega) { for _, p := range group { var pCopy corev1.Pod - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(p), &pCopy)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(p), &pCopy)).To(utiltesting.BeNotFoundError()) } }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) util.ExpectWorkloadsFinalizedOrGone(ctx, k8sClient, gKey) @@ -151,7 +151,7 @@ var _ = ginkgo.Describe("Pod groups", func() { gomega.Eventually(func(g gomega.Gomega) { for _, origPod := range group[:2] { var p corev1.Pod - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(origPod), &p)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(origPod), &p)).To(utiltesting.BeNotFoundError()) } }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -356,7 +356,7 @@ var _ = ginkgo.Describe("Pod groups", func() { var p corev1.Pod g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(rep), &p)).To(gomega.Succeed()) g.Expect(p.Spec.SchedulingGates).To(gomega.BeEmpty()) - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(group[0]), &p)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(group[0]), &p)).To(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -395,7 +395,7 @@ var _ = ginkgo.Describe("Pod groups", func() { gomega.Eventually(func(g gomega.Gomega) { for _, p := range group { var pCopy corev1.Pod - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(p), &pCopy)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(p), &pCopy)).To(utiltesting.BeNotFoundError()) } }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) util.ExpectWorkloadsFinalizedOrGone(ctx, k8sClient, gKey) @@ -413,7 +413,7 @@ var _ = ginkgo.Describe("Pod groups", func() { eventWatcher.Stop() }) - highPriorityClass := testing.MakePriorityClass("high").PriorityValue(100).Obj() + highPriorityClass := utiltesting.MakePriorityClass("high").PriorityValue(100).Obj() util.MustCreate(ctx, k8sClient, highPriorityClass) ginkgo.DeferCleanup(func() { gomega.Expect(k8sClient.Delete(ctx, highPriorityClass)).To(gomega.Succeed()) @@ -505,7 +505,7 @@ var _ = ginkgo.Describe("Pod groups", func() { var p corev1.Pod for _, origPod := range defaultPriorityGroup { origKey := client.ObjectKeyFromObject(origPod) - g.Expect(k8sClient.Get(ctx, origKey, &p)).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, origKey, &p)).To(utiltesting.BeNotFoundError()) } }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -609,7 +609,7 @@ var _ = ginkgo.Describe("Pod groups", func() { gomega.Expect(k8sClient.Delete(ctx, &p)).To(gomega.Succeed()) gomega.Eventually(func(g gomega.Gomega) { - g.Expect(k8sClient.Get(ctx, pKey, &corev1.Pod{})).To(testing.BeNotFoundError()) + g.Expect(k8sClient.Get(ctx, pKey, &corev1.Pod{})).To(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) gKey := client.ObjectKey{Namespace: ns.Name, Name: "test-group"} diff --git a/test/e2e/singlecluster/statefulset_test.go b/test/e2e/singlecluster/statefulset_test.go index d2d24c3c81b..1d8f4da4a00 100644 --- a/test/e2e/singlecluster/statefulset_test.go +++ b/test/e2e/singlecluster/statefulset_test.go @@ -30,7 +30,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" controllerconstants "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobs/statefulset" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" statefulsettesting "sigs.k8s.io/kueue/pkg/util/testingjobs/statefulset" "sigs.k8s.io/kueue/test/util" @@ -489,7 +489,7 @@ var _ = ginkgo.Describe("StatefulSet integration", func() { ginkgo.By("Check the low-priority Workload is created and admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, lowPriorityWlKey, createdLowPriorityWl)).To(gomega.Succeed()) - g.Expect(createdLowPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdLowPriorityWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -534,7 +534,7 @@ var _ = ginkgo.Describe("StatefulSet integration", func() { ginkgo.By("Await for the high-priority Workload to be admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, highPriorityWlKey, createdHighPriorityWl)).To(gomega.Succeed()) - g.Expect(createdHighPriorityWl.Status.Conditions).To(testing.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) + g.Expect(createdHighPriorityWl.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/e2e/singlecluster/tas_test.go b/test/e2e/singlecluster/tas_test.go index 66b59d84a78..e04cc768e63 100644 --- a/test/e2e/singlecluster/tas_test.go +++ b/test/e2e/singlecluster/tas_test.go @@ -34,7 +34,7 @@ import ( podcontroller "sigs.k8s.io/kueue/pkg/controller/jobs/pod" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" workloadtrainjob "sigs.k8s.io/kueue/pkg/controller/jobs/trainjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" @@ -139,7 +139,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -274,7 +274,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -375,7 +375,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -439,7 +439,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -553,7 +553,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) diff --git a/test/e2e/singlecluster/trainjob_test.go b/test/e2e/singlecluster/trainjob_test.go index 9efa0daa733..256289d155f 100644 --- a/test/e2e/singlecluster/trainjob_test.go +++ b/test/e2e/singlecluster/trainjob_test.go @@ -27,7 +27,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadtrainjob "sigs.k8s.io/kueue/pkg/controller/jobs/trainjob" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" "sigs.k8s.io/kueue/test/util" @@ -90,7 +90,7 @@ var _ = ginkgo.Describe("TrainJob", func() { ginkgo.By("Ensuring that a workload is created and the job admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatus(kueue.WorkloadAdmitted, metav1.ConditionTrue)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatus(kueue.WorkloadAdmitted, metav1.ConditionTrue)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -104,7 +104,7 @@ var _ = ginkgo.Describe("TrainJob", func() { ginkgo.By("Waiting for the trainjob to finish", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason(kueue.WorkloadFinished, kueue.WorkloadFinishedReasonSucceeded)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason(kueue.WorkloadFinished, kueue.WorkloadFinishedReasonSucceeded)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -173,7 +173,7 @@ var _ = ginkgo.Describe("TrainJob", func() { ginkgo.By("Ensuring that a workload is created and the job admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) - g.Expect(createdWorkload.Status.Conditions).To(testing.HaveConditionStatus(kueue.WorkloadAdmitted, metav1.ConditionTrue)) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatus(kueue.WorkloadAdmitted, metav1.ConditionTrue)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/e2e/singlecluster/visibility_test.go b/test/e2e/singlecluster/visibility_test.go index 60912474e50..9bcff13e860 100644 --- a/test/e2e/singlecluster/visibility_test.go +++ b/test/e2e/singlecluster/visibility_test.go @@ -31,7 +31,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" visibility "sigs.k8s.io/kueue/apis/visibility/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/test/util" @@ -92,13 +92,13 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { localQueueB = utiltestingapi.MakeLocalQueue("b", nsA.Name).ClusterQueue(clusterQueue.Name).Obj() util.MustCreate(ctx, k8sClient, localQueueB) - highPriorityClass = testing.MakePriorityClass("high").PriorityValue(100).Obj() + highPriorityClass = utiltesting.MakePriorityClass("high").PriorityValue(100).Obj() util.MustCreate(ctx, k8sClient, highPriorityClass) - midPriorityClass = testing.MakePriorityClass("mid").PriorityValue(75).Obj() + midPriorityClass = utiltesting.MakePriorityClass("mid").PriorityValue(75).Obj() util.MustCreate(ctx, k8sClient, midPriorityClass) - lowPriorityClass = testing.MakePriorityClass("low").PriorityValue(50).Obj() + lowPriorityClass = utiltesting.MakePriorityClass("low").PriorityValue(50).Obj() util.MustCreate(ctx, k8sClient, lowPriorityClass) ginkgo.By("Schedule a job that when admitted workload blocks the queue", func() { @@ -541,7 +541,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.By("Wait for ResourceNotFound error instead of Forbidden to make sure the role bindings work", func() { gomega.Eventually(func(g gomega.Gomega) { _, err := impersonatedVisibilityClient.ClusterQueues().GetPendingWorkloadsSummary(ctx, "non-existent", metav1.GetOptions{}) - g.Expect(err).Should(testing.BeNotFoundError()) + g.Expect(err).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -553,11 +553,11 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.It("Should return an appropriate error", func() { ginkgo.By("Returning a ResourceNotFound error for a nonexistent ClusterQueue", func() { _, err := impersonatedVisibilityClient.ClusterQueues().GetPendingWorkloadsSummary(ctx, "non-existent", metav1.GetOptions{}) - gomega.Expect(err).Should(testing.BeNotFoundError()) + gomega.Expect(err).Should(utiltesting.BeNotFoundError()) }) ginkgo.By("Returning a ResourceNotFound error for a nonexistent LocalQueue", func() { _, err := impersonatedVisibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, "non-existent", metav1.GetOptions{}) - gomega.Expect(err).Should(testing.BeNotFoundError()) + gomega.Expect(err).Should(utiltesting.BeNotFoundError()) }) }) }) @@ -577,7 +577,7 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.By("Wait for ResourceNotFound error instead of Forbidden to make sure the role bindings work", func() { gomega.Eventually(func(g gomega.Gomega) { _, err := impersonatedVisibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, "non-existent", metav1.GetOptions{}) - g.Expect(err).Should(testing.BeNotFoundError()) + g.Expect(err).Should(utiltesting.BeNotFoundError()) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -589,15 +589,15 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.It("Should return an appropriate error", func() { ginkgo.By("Returning a Forbidden error due to insufficient permissions for the ClusterQueue request", func() { _, err := impersonatedVisibilityClient.ClusterQueues().GetPendingWorkloadsSummary(ctx, "non-existent", metav1.GetOptions{}) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) ginkgo.By("Returning a ResourceNotFound error for a nonexistent LocalQueue", func() { _, err := impersonatedVisibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, "non-existent", metav1.GetOptions{}) - gomega.Expect(err).Should(testing.BeNotFoundError()) + gomega.Expect(err).Should(utiltesting.BeNotFoundError()) }) ginkgo.By("Returning a Forbidden error due to insufficient permissions for the LocalQueue request in different namespace", func() { _, err := impersonatedVisibilityClient.LocalQueues("default").GetPendingWorkloadsSummary(ctx, "non-existent", metav1.GetOptions{}) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) }) }) @@ -606,11 +606,11 @@ var _ = ginkgo.Describe("Kueue visibility server", func() { ginkgo.It("Should return an appropriate error", func() { ginkgo.By("Returning a Forbidden error due to insufficient permissions for the ClusterQueue request", func() { _, err := impersonatedVisibilityClient.ClusterQueues().GetPendingWorkloadsSummary(ctx, "non-existent", metav1.GetOptions{}) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) ginkgo.By("Returning a Forbidden error due to insufficient permissions for the LocalQueue request", func() { _, err := impersonatedVisibilityClient.LocalQueues(nsA.Name).GetPendingWorkloadsSummary(ctx, "non-existent", metav1.GetOptions{}) - gomega.Expect(err).Should(testing.BeForbiddenError()) + gomega.Expect(err).Should(utiltesting.BeForbiddenError()) }) }) }) diff --git a/test/e2e/tas/appwrapper_test.go b/test/e2e/tas/appwrapper_test.go index f002731420e..32159e1f915 100644 --- a/test/e2e/tas/appwrapper_test.go +++ b/test/e2e/tas/appwrapper_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" awtesting "sigs.k8s.io/kueue/pkg/util/testingjobs/appwrapper" utiltestingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" @@ -91,7 +91,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for AppWrapper", func() { RequestAndLimit(extraResource, "1"). Suspend(false). Image(util.GetAgnHostImage(), util.BehaviorExitFast). - PodAnnotation(kueue.PodSetPreferredTopologyAnnotation, testing.DefaultRackTopologyLevel). + PodAnnotation(kueue.PodSetPreferredTopologyAnnotation, utiltesting.DefaultRackTopologyLevel). SetTypeMeta().Obj(), }). Queue(localQueue.Name). @@ -133,7 +133,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for AppWrapper", func() { Indexed(true). Suspend(false). RequestAndLimit(extraResource, "1"). - PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel). + PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, utiltesting.DefaultBlockTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorExitFast). SetTypeMeta(). Obj() diff --git a/test/e2e/tas/hotswap_test.go b/test/e2e/tas/hotswap_test.go index 6ce198734f0..a8d56c72859 100644 --- a/test/e2e/tas/hotswap_test.go +++ b/test/e2e/tas/hotswap_test.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" @@ -129,8 +129,8 @@ var _ = ginkgo.Describe("Hotswap for Topology Aware Scheduling", ginkgo.Ordered, Parallelism: int32(parallelism), Completions: int32(parallelism), PodAnnotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, - kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetSliceSizeAnnotation: "3", }, }, @@ -207,8 +207,8 @@ var _ = ginkgo.Describe("Hotswap for Topology Aware Scheduling", ginkgo.Ordered, Parallelism: int32(parallelism), Completions: int32(parallelism), PodAnnotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, - kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, kueue.PodSetSliceSizeAnnotation: "2", }, }, diff --git a/test/e2e/tas/job_test.go b/test/e2e/tas/job_test.go index cc1f0bdfa52..27d01b2108e 100644 --- a/test/e2e/tas/job_test.go +++ b/test/e2e/tas/job_test.go @@ -30,7 +30,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" "sigs.k8s.io/kueue/pkg/workload" @@ -99,7 +99,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { RequestAndLimit(extraResource, "1"). Obj() sampleJob = (&testingjob.JobWrapper{Job: *sampleJob}). - PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultRackTopologyLevel). + PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, utiltesting.DefaultRackTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorExitFast). Obj() util.MustCreate(ctx, k8sClient, sampleJob) @@ -120,7 +120,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { Parallelism(3). Completions(3). RequestAndLimit(extraResource, "1"). - PodAnnotation(kueue.PodSetPreferredTopologyAnnotation, testing.DefaultRackTopologyLevel). + PodAnnotation(kueue.PodSetPreferredTopologyAnnotation, utiltesting.DefaultRackTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorExitFast). Obj() util.MustCreate(ctx, k8sClient, sampleJob) @@ -159,7 +159,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -170,7 +170,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { Parallelism(3). Completions(3). RequestAndLimit(extraResource, "1"). - PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel). + PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, utiltesting.DefaultBlockTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorExitFast). Obj() util.MustCreate(ctx, k8sClient, sampleJob) @@ -210,7 +210,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -221,7 +221,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { Parallelism(2). Completions(3). RequestAndLimit(extraResource, "1"). - PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel). + PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, utiltesting.DefaultBlockTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorExitFast). Obj() util.MustCreate(ctx, k8sClient, sampleJob) @@ -234,7 +234,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { g.Expect(k8sClient.Get(ctx, wlLookupKey, createdWorkload)).Should(gomega.Succeed()) g.Expect(createdWorkload.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(1)) g.Expect(createdWorkload.Status.Admission.PodSetAssignments[0].TopologyAssignment).ShouldNot(gomega.BeNil()) - g.Expect(createdWorkload.Status.Conditions).Should(testing.HaveConditionStatusTrue(kueue.WorkloadFinished)) + g.Expect(createdWorkload.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadFinished)) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) }) @@ -247,7 +247,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Job", func() { Completions(int32(numPods)). Indexed(true). RequestAndLimit(extraResource, "1"). - PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel). + PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, utiltesting.DefaultBlockTopologyLevel). Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). Obj() util.MustCreate(ctx, k8sClient, sampleJob) diff --git a/test/e2e/tas/jobset_test.go b/test/e2e/tas/jobset_test.go index df3f6e34e09..514155e2826 100644 --- a/test/e2e/tas/jobset_test.go +++ b/test/e2e/tas/jobset_test.go @@ -29,7 +29,7 @@ import ( jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" "sigs.k8s.io/kueue/test/util" @@ -98,7 +98,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for JobSet", func() { Parallelism: int32(parallelism), Completions: int32(parallelism), PodAnnotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, ). @@ -161,8 +161,8 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for JobSet", func() { Parallelism: int32(parallelism), Completions: int32(parallelism), PodAnnotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, - kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetSliceSizeAnnotation: "3", }, }, @@ -226,7 +226,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for JobSet", func() { Parallelism: int32(parallelism), Completions: int32(parallelism), PodAnnotations: map[string]string{ - kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetSliceSizeAnnotation: "3", }, }, diff --git a/test/e2e/tas/leaderworkerset_test.go b/test/e2e/tas/leaderworkerset_test.go index 18fe68f40f0..5be87e3b048 100644 --- a/test/e2e/tas/leaderworkerset_test.go +++ b/test/e2e/tas/leaderworkerset_test.go @@ -29,7 +29,7 @@ import ( leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" leaderworkersettesting "sigs.k8s.io/kueue/pkg/util/testingjobs/leaderworkerset" "sigs.k8s.io/kueue/test/util" @@ -92,7 +92,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for LeaderWorkerSet", func() { WorkerTemplate(corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, Spec: corev1.PodSpec{ @@ -124,7 +124,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for LeaderWorkerSet", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(replicas)) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -185,7 +185,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for LeaderWorkerSet", func() { WorkerTemplate(corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetGroupName: "same-group", }, }, @@ -210,7 +210,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for LeaderWorkerSet", func() { LeaderTemplate(corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetGroupName: "same-group", }, }, @@ -243,7 +243,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for LeaderWorkerSet", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(replicas)) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) @@ -294,7 +294,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for LeaderWorkerSet", func() { WorkerTemplate(corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetGroupName: "same-group", }, }, @@ -321,7 +321,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for LeaderWorkerSet", func() { LeaderTemplate(corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetGroupName: "same-group", }, }, @@ -354,7 +354,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for LeaderWorkerSet", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(lws), createdLeaderWorkerSet)).To(gomega.Succeed()) g.Expect(createdLeaderWorkerSet.Status.ReadyReplicas).To(gomega.Equal(replicas)) - g.Expect(createdLeaderWorkerSet.Status.Conditions).To(testing.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) + g.Expect(createdLeaderWorkerSet.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason("Available", "AllGroupsReady")) }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/e2e/tas/mpijob_test.go b/test/e2e/tas/mpijob_test.go index ccd8585afb9..2f902f9bbf5 100644 --- a/test/e2e/tas/mpijob_test.go +++ b/test/e2e/tas/mpijob_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingmpijob "sigs.k8s.io/kueue/pkg/util/testingjobs/mpijob" "sigs.k8s.io/kueue/test/util" @@ -97,7 +97,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for MPIJob", func() { ReplicaCount: launcherReplicas, RestartPolicy: corev1.RestartPolicyOnFailure, Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, }, testingmpijob.MPIJobReplicaSpecRequirement{ @@ -105,7 +105,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for MPIJob", func() { ReplicaCount: workerReplicas, RestartPolicy: corev1.RestartPolicyOnFailure, Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, ). diff --git a/test/e2e/tas/pod_group_test.go b/test/e2e/tas/pod_group_test.go index d0075dacd8e..9e254fba53d 100644 --- a/test/e2e/tas/pod_group_test.go +++ b/test/e2e/tas/pod_group_test.go @@ -24,7 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpod "sigs.k8s.io/kueue/pkg/util/testingjobs/pod" "sigs.k8s.io/kueue/test/util" @@ -86,7 +86,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for Pod group", func() { RequestAndLimit(extraResource, "1"). Limit(extraResource, "1"). Image(util.GetAgnHostImage(), util.BehaviorExitFast). - Annotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel) + Annotation(kueue.PodSetRequiredTopologyAnnotation, utiltesting.DefaultBlockTopologyLevel) podGroup := basePod.MakeIndexedGroup(numPods) for _, pod := range podGroup { diff --git a/test/e2e/tas/pytorch_test.go b/test/e2e/tas/pytorch_test.go index 9da357209f9..40574c84369 100644 --- a/test/e2e/tas/pytorch_test.go +++ b/test/e2e/tas/pytorch_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingpytorchjob "sigs.k8s.io/kueue/pkg/util/testingjobs/pytorchjob" "sigs.k8s.io/kueue/test/util" @@ -96,7 +96,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for PyTorchJob", func() { ReplicaCount: masterReplicas, RestartPolicy: kftraining.RestartPolicyOnFailure, Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, }, testingpytorchjob.PyTorchReplicaSpecRequirement{ @@ -104,7 +104,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for PyTorchJob", func() { ReplicaCount: workerReplicas, RestartPolicy: kftraining.RestartPolicyOnFailure, Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, ). diff --git a/test/e2e/tas/rayjob_test.go b/test/e2e/tas/rayjob_test.go index eb7d9cfbede..364c0416ad3 100644 --- a/test/e2e/tas/rayjob_test.go +++ b/test/e2e/tas/rayjob_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingrayjob "sigs.k8s.io/kueue/pkg/util/testingjobs/rayjob" "sigs.k8s.io/kueue/test/util" @@ -116,7 +116,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for RayJob", ginkgo.Ordered, fu }, ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultRackTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultRackTopologyLevel, }, }, }, @@ -148,7 +148,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for RayJob", ginkgo.Ordered, fu }, ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, }, @@ -174,7 +174,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for RayJob", ginkgo.Ordered, fu }, ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - kueue.PodSetRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }, }). diff --git a/test/e2e/tas/statefulset_test.go b/test/e2e/tas/statefulset_test.go index 0cf805c7e94..2033360a7d0 100644 --- a/test/e2e/tas/statefulset_test.go +++ b/test/e2e/tas/statefulset_test.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/util/testingjobs/statefulset" "sigs.k8s.io/kueue/test/util" @@ -81,7 +81,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for StatefulSet", func() { RequestAndLimit(extraResource, "1"). Replicas(replicas). Queue(localQueue.Name). - PodTemplateSpecAnnotation(kueue.PodSetRequiredTopologyAnnotation, testing.DefaultBlockTopologyLevel). + PodTemplateSpecAnnotation(kueue.PodSetRequiredTopologyAnnotation, utiltesting.DefaultBlockTopologyLevel). TerminationGracePeriod(1). Obj() util.MustCreate(ctx, k8sClient, sts) diff --git a/test/e2e/tas/trainjob_test.go b/test/e2e/tas/trainjob_test.go index 6255454e21c..5501014bbaf 100644 --- a/test/e2e/tas/trainjob_test.go +++ b/test/e2e/tas/trainjob_test.go @@ -30,7 +30,7 @@ import ( jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" testingjobset "sigs.k8s.io/kueue/pkg/util/testingjobs/jobset" testingtrainjob "sigs.k8s.io/kueue/pkg/util/testingjobs/trainjob" @@ -101,7 +101,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for TrainJob", func() { Parallelism: int32(parallelism), Completions: int32(parallelism), PodAnnotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, }, }). RequestAndLimit("node", extraResource, "1").Obj().Spec) @@ -171,8 +171,8 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for TrainJob", func() { Parallelism: int32(parallelism), Completions: int32(parallelism), PodAnnotations: map[string]string{ - kueue.PodSetPreferredTopologyAnnotation: testing.DefaultBlockTopologyLevel, - kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetPreferredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetSliceSizeAnnotation: "3", }, }). @@ -243,7 +243,7 @@ var _ = ginkgo.Describe("TopologyAwareScheduling for TrainJob", func() { Parallelism: int32(parallelism), Completions: int32(parallelism), PodAnnotations: map[string]string{ - kueue.PodSetSliceRequiredTopologyAnnotation: testing.DefaultBlockTopologyLevel, + kueue.PodSetSliceRequiredTopologyAnnotation: utiltesting.DefaultBlockTopologyLevel, kueue.PodSetSliceSizeAnnotation: "3", }, }). diff --git a/test/integration/singlecluster/scheduler/workload_controller_test.go b/test/integration/singlecluster/scheduler/workload_controller_test.go index e486d5f9483..a9c33fd8eb4 100644 --- a/test/integration/singlecluster/scheduler/workload_controller_test.go +++ b/test/integration/singlecluster/scheduler/workload_controller_test.go @@ -28,7 +28,7 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" "sigs.k8s.io/kueue/pkg/util/slices" - "sigs.k8s.io/kueue/pkg/util/testing" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" "sigs.k8s.io/kueue/pkg/workload" "sigs.k8s.io/kueue/test/integration/framework" @@ -184,7 +184,7 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.BeforeEach(func() { util.MustCreate(ctx, k8sClient, onDemandFlavor) - runtimeClass = testing.MakeRuntimeClass("kata", "bar-handler").PodOverhead(resources).Obj() + runtimeClass = utiltesting.MakeRuntimeClass("kata", "bar-handler").PodOverhead(resources).Obj() util.MustCreate(ctx, k8sClient, runtimeClass) clusterQueue = utiltestingapi.MakeClusterQueue("clusterqueue"). ResourceGroup(*utiltestingapi.MakeFlavorQuotas(onDemandFlavor.Name). @@ -293,7 +293,7 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.When("LimitRanges are defined", func() { ginkgo.BeforeEach(func() { - limitRange := testing.MakeLimitRange("limits", ns.Name).WithValue("DefaultRequest", corev1.ResourceCPU, "3").Obj() + limitRange := utiltesting.MakeLimitRange("limits", ns.Name).WithValue("DefaultRequest", corev1.ResourceCPU, "3").Obj() util.MustCreate(ctx, k8sClient, limitRange) util.MustCreate(ctx, k8sClient, onDemandFlavor) clusterQueue = utiltestingapi.MakeClusterQueue("clusterqueue"). @@ -569,7 +569,7 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.When("RuntimeClass is defined and change", func() { ginkgo.BeforeEach(func() { - runtimeClass = testing.MakeRuntimeClass("kata", "bar-handler"). + runtimeClass = utiltesting.MakeRuntimeClass("kata", "bar-handler"). PodOverhead(corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2")}). Obj() util.MustCreate(ctx, k8sClient, runtimeClass) @@ -661,7 +661,7 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.When("LimitRanges are defined and change", func() { var limitRange *corev1.LimitRange ginkgo.BeforeEach(func() { - limitRange = testing.MakeLimitRange("limits", ns.Name).WithValue("DefaultRequest", corev1.ResourceCPU, "3").Obj() + limitRange = utiltesting.MakeLimitRange("limits", ns.Name).WithValue("DefaultRequest", corev1.ResourceCPU, "3").Obj() util.MustCreate(ctx, k8sClient, limitRange) util.MustCreate(ctx, k8sClient, onDemandFlavor) clusterQueue = utiltestingapi.MakeClusterQueue("clusterqueue"). @@ -747,7 +747,7 @@ var _ = ginkgo.Describe("Workload controller with scheduler", func() { ginkgo.When("a LimitRange event occurs near workload deletion time", func() { var limitRange *corev1.LimitRange ginkgo.BeforeEach(func() { - limitRange = testing.MakeLimitRange("limits", ns.Name).WithValue("DefaultRequest", corev1.ResourceCPU, "3").Obj() + limitRange = utiltesting.MakeLimitRange("limits", ns.Name).WithValue("DefaultRequest", corev1.ResourceCPU, "3").Obj() util.MustCreate(ctx, k8sClient, limitRange) util.MustCreate(ctx, k8sClient, onDemandFlavor) clusterQueue = utiltestingapi.MakeClusterQueue("clusterqueue"). From 3b811648242ab8dc2a00ed4d2d11e5c0ee292ec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Irving=20Mondrag=C3=B3n?= Date: Fri, 7 Nov 2025 10:12:55 +0100 Subject: [PATCH 115/119] Fix - Workloads requesting TAS cannot run via MultiKueue (#5361) * Enable MultiKueue + TAS + ProvisioningRequest integration * Avoid leaking topology assignment details to manager cluster * Remove runtime node labeling for TAS in MultiKueue e2e tests * Refactoring --- apis/kueue/v1beta1/clusterqueue_types.go | 1 - apis/kueue/v1beta2/clusterqueue_types.go | 1 - hack/multikueue/manager-cluster.kind.yaml | 1 + hack/multikueue/worker-cluster.kind.yaml | 1 + pkg/cache/scheduler/clusterqueue.go | 14 +- pkg/cache/scheduler/clusterqueue_snapshot.go | 5 + pkg/cache/scheduler/clusterqueue_test.go | 6 +- pkg/cache/scheduler/snapshot.go | 1 + .../admissionchecks/multikueue/workload.go | 128 +++- .../flavorassigner/tas_flavorassigner.go | 6 +- test/e2e/multikueue/tas_test.go | 384 +++++++++++ test/integration/multikueue/tas/suite_test.go | 230 +++++++ test/integration/multikueue/tas/tas_test.go | 606 ++++++++++++++++++ .../integration/singlecluster/tas/tas_test.go | 51 -- 14 files changed, 1349 insertions(+), 86 deletions(-) create mode 100644 test/e2e/multikueue/tas_test.go create mode 100644 test/integration/multikueue/tas/suite_test.go create mode 100644 test/integration/multikueue/tas/tas_test.go diff --git a/apis/kueue/v1beta1/clusterqueue_types.go b/apis/kueue/v1beta1/clusterqueue_types.go index 8a5e9975b43..71e48be8609 100644 --- a/apis/kueue/v1beta1/clusterqueue_types.go +++ b/apis/kueue/v1beta1/clusterqueue_types.go @@ -31,7 +31,6 @@ const ( ClusterQueueActiveReasonAdmissionCheckInactive = "AdmissionCheckInactive" ClusterQueueActiveReasonMultipleMultiKueueAdmissionChecks = "MultipleMultiKueueAdmissionChecks" ClusterQueueActiveReasonMultiKueueAdmissionCheckAppliedPerFlavor = "MultiKueueAdmissionCheckAppliedPerFlavor" - ClusterQueueActiveReasonNotSupportedWithTopologyAwareScheduling = "NotSupportedWithTopologyAwareScheduling" ClusterQueueActiveReasonTopologyNotFound = "TopologyNotFound" ClusterQueueActiveReasonUnknown = "Unknown" ClusterQueueActiveReasonReady = "Ready" diff --git a/apis/kueue/v1beta2/clusterqueue_types.go b/apis/kueue/v1beta2/clusterqueue_types.go index 60605f4339e..63549156ad6 100644 --- a/apis/kueue/v1beta2/clusterqueue_types.go +++ b/apis/kueue/v1beta2/clusterqueue_types.go @@ -31,7 +31,6 @@ const ( ClusterQueueActiveReasonAdmissionCheckInactive = "AdmissionCheckInactive" ClusterQueueActiveReasonMultipleMultiKueueAdmissionChecks = "MultipleMultiKueueAdmissionChecks" ClusterQueueActiveReasonMultiKueueAdmissionCheckAppliedPerFlavor = "MultiKueueAdmissionCheckAppliedPerFlavor" - ClusterQueueActiveReasonNotSupportedWithTopologyAwareScheduling = "NotSupportedWithTopologyAwareScheduling" ClusterQueueActiveReasonTopologyNotFound = "TopologyNotFound" ClusterQueueActiveReasonUnknown = "Unknown" ClusterQueueActiveReasonReady = "Ready" diff --git a/hack/multikueue/manager-cluster.kind.yaml b/hack/multikueue/manager-cluster.kind.yaml index 8962e93640f..7c4f0cedbda 100644 --- a/hack/multikueue/manager-cluster.kind.yaml +++ b/hack/multikueue/manager-cluster.kind.yaml @@ -24,6 +24,7 @@ nodes: - role: worker labels: instance-type: on-demand + cloud.provider.com/node-group: tas-node kubeadmConfigPatches: - | diff --git a/hack/multikueue/worker-cluster.kind.yaml b/hack/multikueue/worker-cluster.kind.yaml index 8962e93640f..7c4f0cedbda 100644 --- a/hack/multikueue/worker-cluster.kind.yaml +++ b/hack/multikueue/worker-cluster.kind.yaml @@ -24,6 +24,7 @@ nodes: - role: worker labels: instance-type: on-demand + cloud.provider.com/node-group: tas-node kubeadmConfigPatches: - | diff --git a/pkg/cache/scheduler/clusterqueue.go b/pkg/cache/scheduler/clusterqueue.go index 53bb2ed0edf..5b098d46f9b 100644 --- a/pkg/cache/scheduler/clusterqueue.go +++ b/pkg/cache/scheduler/clusterqueue.go @@ -276,10 +276,6 @@ func (c *clusterQueue) inactiveReason() (string, string) { } if features.Enabled(features.TopologyAwareScheduling) && len(c.tasFlavors) > 0 { - if len(c.multiKueueAdmissionChecks) > 0 { - reasons = append(reasons, kueue.ClusterQueueActiveReasonNotSupportedWithTopologyAwareScheduling) - messages = append(messages, "TAS is not supported with MultiKueue admission check") - } for tasFlavor, topology := range c.tasFlavors { if c.tasCache.Get(tasFlavor) == nil { reasons = append(reasons, kueue.ClusterQueueActiveReasonTopologyNotFound) @@ -301,10 +297,14 @@ func (c *clusterQueue) isTASViolated() bool { if !features.Enabled(features.TopologyAwareScheduling) || len(c.tasFlavors) == 0 { return false } + // Skip TAS cache validation when MultiKueue is enabled; topology runs on worker clusters. + if c.hasMultiKueueAdmissionCheck() { + return false + } if !c.isTASSynced() { return true } - return len(c.multiKueueAdmissionChecks) > 0 + return false } // UpdateWithFlavors updates a ClusterQueue based on the passed ResourceFlavors set. @@ -641,3 +641,7 @@ func (c *clusterQueue) flavorsWithProvReqAdmissionCheck() sets.Set[kueue.Resourc } return flvs } + +func (c *clusterQueue) hasMultiKueueAdmissionCheck() bool { + return len(c.multiKueueAdmissionChecks) > 0 +} diff --git a/pkg/cache/scheduler/clusterqueue_snapshot.go b/pkg/cache/scheduler/clusterqueue_snapshot.go index 6ba1320980a..1dfa4bdf3a8 100644 --- a/pkg/cache/scheduler/clusterqueue_snapshot.go +++ b/pkg/cache/scheduler/clusterqueue_snapshot.go @@ -59,6 +59,7 @@ type ClusterQueueSnapshot struct { tasOnly bool flavorsForProvReqACs sets.Set[kueue.ResourceFlavorReference] + hasMultiKueueAC bool } // RGByResource returns the ResourceGroup which contains capacity @@ -216,6 +217,10 @@ func (c *ClusterQueueSnapshot) HasProvRequestAdmissionCheck(rf kueue.ResourceFla return c.flavorsForProvReqACs.Has(rf) } +func (c *ClusterQueueSnapshot) HasMultiKueueAdmissionCheck() bool { + return c.hasMultiKueueAC +} + // Returns all ancestors starting with parent and ending with root func (c *ClusterQueueSnapshot) PathParentToRoot() iter.Seq[*CohortSnapshot] { return func(yield func(*CohortSnapshot) bool) { diff --git a/pkg/cache/scheduler/clusterqueue_test.go b/pkg/cache/scheduler/clusterqueue_test.go index 31cdf2c89fa..6289f5a8507 100644 --- a/pkg/cache/scheduler/clusterqueue_test.go +++ b/pkg/cache/scheduler/clusterqueue_test.go @@ -548,7 +548,7 @@ func TestClusterQueueReadinessWithTAS(t *testing.T) { wantMessage: "Can admit new workloads", }, { - name: "TAS do not support MultiKueue AdmissionCheck", + name: "TAS supports MultiKueue AdmissionCheck", cq: utiltestingapi.MakeClusterQueue("cq"). ResourceGroup( *utiltestingapi.MakeFlavorQuotas("tas-flavor"). @@ -561,8 +561,8 @@ func TestClusterQueueReadinessWithTAS(t *testing.T) { ResourceQuotaWrapper("example.com/gpu").NominalQuota("5").Append(). Obj(), ).AdmissionChecks("mk-check").Obj(), - wantReason: kueue.ClusterQueueActiveReasonNotSupportedWithTopologyAwareScheduling, - wantMessage: "Can't admit new workloads: TAS is not supported with MultiKueue admission check.", + wantReason: kueue.ClusterQueueActiveReasonReady, + wantMessage: "Can admit new workloads", }, { name: "Referenced TAS flavor without topology", diff --git a/pkg/cache/scheduler/snapshot.go b/pkg/cache/scheduler/snapshot.go index da410c69fa1..b3c640502d3 100644 --- a/pkg/cache/scheduler/snapshot.go +++ b/pkg/cache/scheduler/snapshot.go @@ -222,6 +222,7 @@ func (c *Cache) snapshotClusterQueue(ctx context.Context, cq *clusterQueue, afsE TASFlavors: make(map[kueue.ResourceFlavorReference]*TASFlavorSnapshot), tasOnly: cq.isTASOnly(), flavorsForProvReqACs: cq.flavorsWithProvReqAdmissionCheck(), + hasMultiKueueAC: cq.hasMultiKueueAdmissionCheck(), } for i, rg := range cq.ResourceGroups { cc.ResourceGroups[i] = rg.Clone() diff --git a/pkg/controller/admissionchecks/multikueue/workload.go b/pkg/controller/admissionchecks/multikueue/workload.go index 7c3f692d96d..29fc411c91a 100644 --- a/pkg/controller/admissionchecks/multikueue/workload.go +++ b/pkg/controller/admissionchecks/multikueue/workload.go @@ -34,6 +34,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" "k8s.io/utils/clock" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -420,28 +421,8 @@ func (w *wlReconciler) reconcileGroup(ctx context.Context, group *wlGroup) (reco return reconcile.Result{}, err } - if acs.State != kueue.CheckStateRetry && acs.State != kueue.CheckStateRejected { - if err := workload.PatchAdmissionStatus(ctx, w.client, group.local, w.clock, func() (*kueue.Workload, bool, error) { - if group.jobAdapter.KeepAdmissionCheckPending() { - acs.State = kueue.CheckStatePending - } else { - acs.State = kueue.CheckStateReady - } - // update the message - acs.Message = fmt.Sprintf("The workload got reservation on %q", reservingRemote) - // update the transition time since is used to detect the lost worker state. - acs.LastTransitionTime = metav1.NewTime(w.clock.Now()) - - workload.SetAdmissionCheckState(&group.local.Status.AdmissionChecks, *acs, w.clock) - // Set the cluster name to the reserving remote and clear the nominated clusters. - group.local.Status.ClusterName = &reservingRemote - group.local.Status.NominatedClusterNames = nil - return group.local, true, nil - }); err != nil { - log.V(2).Error(err, "Failed to patch workload", "workload", klog.KObj(group.local)) - return reconcile.Result{}, err - } - w.recorder.Eventf(group.local, corev1.EventTypeNormal, "MultiKueue", acs.Message) + if err := w.syncReservingRemoteState(ctx, group, reservingRemote, acs); err != nil { + return reconcile.Result{}, err } return reconcile.Result{RequeueAfter: w.workerLostTimeout}, nil } else if acs.State == kueue.CheckStateReady { @@ -637,6 +618,109 @@ func (w *wlReconciler) setupWithManager(mgr ctrl.Manager) error { Complete(w) } +func findPodSetAssignment(assignments []kueue.PodSetAssignment, name kueue.PodSetReference) *kueue.PodSetAssignment { + for i := range assignments { + if assignments[i].Name == name { + return &assignments[i] + } + } + return nil +} + +func needsDelayedTopologyUpdate(local, remote *kueue.Workload) bool { + if remote == nil || remote.Status.Admission == nil || local == nil || local.Status.Admission == nil { + return false + } + + for _, remotePSA := range remote.Status.Admission.PodSetAssignments { + if remotePSA.TopologyAssignment == nil { + continue + } + + localPSA := findPodSetAssignment(local.Status.Admission.PodSetAssignments, remotePSA.Name) + if localPSA == nil { + continue + } + + if localPSA.TopologyAssignment == nil { + return true + } + + if localPSA.DelayedTopologyRequest != nil && + *localPSA.DelayedTopologyRequest == kueue.DelayedTopologyRequestStatePending { + return true + } + } + return false +} + +func (w *wlReconciler) syncReservingRemoteState(ctx context.Context, group *wlGroup, reservingRemote string, acs *kueue.AdmissionCheckState) error { + log := ctrl.LoggerFrom(ctx) + + needsTopologyUpdate := needsDelayedTopologyUpdate(group.local, group.remotes[reservingRemote]) + needsACUpdate := acs.State != kueue.CheckStateRetry && acs.State != kueue.CheckStateRejected + + if !needsTopologyUpdate && !needsACUpdate { + return nil + } + + if err := workload.PatchAdmissionStatus(ctx, w.client, group.local, w.clock, func() (*kueue.Workload, bool, error) { + if needsTopologyUpdate { + updateDelayedTopologyRequest(group.local, group.remotes[reservingRemote]) + } + + if needsACUpdate { + if group.jobAdapter.KeepAdmissionCheckPending() { + acs.State = kueue.CheckStatePending + } else { + acs.State = kueue.CheckStateReady + } + // update the message + acs.Message = fmt.Sprintf("The workload got reservation on %q", reservingRemote) + // update the transition time since is used to detect the lost worker state. + acs.LastTransitionTime = metav1.NewTime(w.clock.Now()) + + workload.SetAdmissionCheckState(&group.local.Status.AdmissionChecks, *acs, w.clock) + // Set the cluster name to the reserving remote and clear the nominated clusters. + group.local.Status.ClusterName = &reservingRemote + group.local.Status.NominatedClusterNames = nil + } + + return group.local, true, nil + }); err != nil { + log.V(2).Error(err, "Failed to patch workload", "workload", klog.KObj(group.local)) + return err + } + + if needsACUpdate { + w.recorder.Eventf(group.local, corev1.EventTypeNormal, "MultiKueue", acs.Message) + } + + return nil +} + +func updateDelayedTopologyRequest(local, remote *kueue.Workload) { + if remote == nil || remote.Status.Admission == nil || local == nil || local.Status.Admission == nil { + return + } + + for _, remotePSA := range remote.Status.Admission.PodSetAssignments { + if remotePSA.TopologyAssignment == nil { + continue + } + + localPSA := findPodSetAssignment(local.Status.Admission.PodSetAssignments, remotePSA.Name) + if localPSA == nil { + continue + } + + if localPSA.DelayedTopologyRequest != nil && + *localPSA.DelayedTopologyRequest == kueue.DelayedTopologyRequestStatePending { + localPSA.DelayedTopologyRequest = ptr.To(kueue.DelayedTopologyRequestStateReady) + } + } +} + func cloneForCreate(orig *kueue.Workload, origin string) *kueue.Workload { remoteWl := &kueue.Workload{} remoteWl.ObjectMeta = api.CloneObjectMetaForCreation(&orig.ObjectMeta) diff --git a/pkg/scheduler/flavorassigner/tas_flavorassigner.go b/pkg/scheduler/flavorassigner/tas_flavorassigner.go index dd96bb58882..957207476ed 100644 --- a/pkg/scheduler/flavorassigner/tas_flavorassigner.go +++ b/pkg/scheduler/flavorassigner/tas_flavorassigner.go @@ -77,9 +77,9 @@ func podSetTopologyRequest(psAssignment *PodSetAssignment, if err != nil { return nil, err } - if !workload.HasQuotaReservation(wl.Obj) && cq.HasProvRequestAdmissionCheck(*tasFlvr) { - // We delay TAS as this is the first scheduling pass, and there is a - // ProvisioningRequest admission check used for the flavor. + if cq.HasMultiKueueAdmissionCheck() || (!workload.HasQuotaReservation(wl.Obj) && cq.HasProvRequestAdmissionCheck(*tasFlvr)) { + // Delay TAS when MultiKueue is used (topology always assigned on worker cluster). + // For ProvisioningRequest, delay TAS on first scheduling pass only (topology assigned after provisioning). psAssignment.DelayedTopologyRequest = ptr.To(kueue.DelayedTopologyRequestStatePending) return nil, nil } diff --git a/test/e2e/multikueue/tas_test.go b/test/e2e/multikueue/tas_test.go new file mode 100644 index 00000000000..938e528d7b3 --- /dev/null +++ b/test/e2e/multikueue/tas_test.go @@ -0,0 +1,384 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mke2e + +import ( + "context" + "fmt" + + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" + "sigs.k8s.io/kueue/pkg/util/admissioncheck" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" + "sigs.k8s.io/kueue/pkg/workload" + "sigs.k8s.io/kueue/test/util" +) + +const ( + tasNodeGroupLabel = "cloud.provider.com/node-group" + instanceType = "tas-node" +) + +var _ = ginkgo.Describe("MultiKueue with TopologyAwareScheduling", func() { + var ( + managerNs *corev1.Namespace + worker1Ns *corev1.Namespace + worker2Ns *corev1.Namespace + + workerCluster1 *kueue.MultiKueueCluster + workerCluster2 *kueue.MultiKueueCluster + multiKueueConfig *kueue.MultiKueueConfig + multiKueueAc *kueue.AdmissionCheck + + managerTopology *kueue.Topology + managerFlavor *kueue.ResourceFlavor + managerCq *kueue.ClusterQueue + managerLq *kueue.LocalQueue + + worker1Topology *kueue.Topology + worker1Flavor *kueue.ResourceFlavor + worker1Cq *kueue.ClusterQueue + worker1Lq *kueue.LocalQueue + + worker2Topology *kueue.Topology + worker2Flavor *kueue.ResourceFlavor + worker2Cq *kueue.ClusterQueue + worker2Lq *kueue.LocalQueue + ) + + ginkgo.BeforeEach(func() { + managerNs = util.CreateNamespaceFromPrefixWithLog(ctx, k8sManagerClient, "multikueue-tas-") + worker1Ns = util.CreateNamespaceWithLog(ctx, k8sWorker1Client, managerNs.Name) + worker2Ns = util.CreateNamespaceWithLog(ctx, k8sWorker2Client, managerNs.Name) + + workerCluster1 = utiltestingapi.MakeMultiKueueCluster("worker1").KubeConfig(kueue.SecretLocationType, "multikueue1").Obj() + util.MustCreate(ctx, k8sManagerClient, workerCluster1) + + workerCluster2 = utiltestingapi.MakeMultiKueueCluster("worker2").KubeConfig(kueue.SecretLocationType, "multikueue2").Obj() + util.MustCreate(ctx, k8sManagerClient, workerCluster2) + + multiKueueConfig = utiltestingapi.MakeMultiKueueConfig("multikueueconfig").Clusters("worker1", "worker2").Obj() + util.MustCreate(ctx, k8sManagerClient, multiKueueConfig) + + multiKueueAc = utiltestingapi.MakeAdmissionCheck("ac1"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", multiKueueConfig.Name). + Obj() + util.MustCreate(ctx, k8sManagerClient, multiKueueAc) + + ginkgo.By("wait for check active", func() { + updatedAc := kueue.AdmissionCheck{} + acKey := client.ObjectKeyFromObject(multiKueueAc) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sManagerClient.Get(ctx, acKey, &updatedAc)).To(gomega.Succeed()) + g.Expect(updatedAc.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.AdmissionCheckActive)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + managerTopology = utiltestingapi.MakeTopology("default").Levels(corev1.LabelHostname).Obj() + util.MustCreate(ctx, k8sManagerClient, managerTopology) + + managerFlavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). + NodeLabel(tasNodeGroupLabel, instanceType). + TopologyName(managerTopology.Name). + Obj() + util.MustCreate(ctx, k8sManagerClient, managerFlavor) + + managerCq = utiltestingapi.MakeClusterQueue("q1"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(managerFlavor.Name). + Resource(corev1.ResourceCPU, "8"). + Resource(corev1.ResourceMemory, "8Gi"). + Obj(), + ). + AdmissionChecks(kueue.AdmissionCheckReference(multiKueueAc.Name)). + Obj() + util.MustCreate(ctx, k8sManagerClient, managerCq) + + managerLq = utiltestingapi.MakeLocalQueue(managerCq.Name, managerNs.Name).ClusterQueue(managerCq.Name).Obj() + util.MustCreate(ctx, k8sManagerClient, managerLq) + + worker1Topology = utiltestingapi.MakeTopology("default").Levels(corev1.LabelHostname).Obj() + util.MustCreate(ctx, k8sWorker1Client, worker1Topology) + + worker1Flavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). + NodeLabel(tasNodeGroupLabel, instanceType). + TopologyName(worker1Topology.Name). + Obj() + util.MustCreate(ctx, k8sWorker1Client, worker1Flavor) + + worker1Cq = utiltestingapi.MakeClusterQueue("q1"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(worker1Flavor.Name). + Resource(corev1.ResourceCPU, "8"). + Resource(corev1.ResourceMemory, "4Gi"). + Obj(), + ). + Obj() + util.MustCreate(ctx, k8sWorker1Client, worker1Cq) + + worker1Lq = utiltestingapi.MakeLocalQueue(worker1Cq.Name, worker1Ns.Name).ClusterQueue(worker1Cq.Name).Obj() + util.MustCreate(ctx, k8sWorker1Client, worker1Lq) + + worker2Topology = utiltestingapi.MakeTopology("default").Levels(corev1.LabelHostname).Obj() + util.MustCreate(ctx, k8sWorker2Client, worker2Topology) + + worker2Flavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). + NodeLabel(tasNodeGroupLabel, instanceType). + TopologyName(worker2Topology.Name). + Obj() + util.MustCreate(ctx, k8sWorker2Client, worker2Flavor) + + worker2Cq = utiltestingapi.MakeClusterQueue("q1"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(worker2Flavor.Name). + Resource(corev1.ResourceCPU, "4"). + Resource(corev1.ResourceMemory, "8Gi"). + Obj(), + ). + Obj() + util.MustCreate(ctx, k8sWorker2Client, worker2Cq) + + worker2Lq = utiltestingapi.MakeLocalQueue(worker2Cq.Name, worker2Ns.Name).ClusterQueue(worker2Cq.Name).Obj() + util.MustCreate(ctx, k8sWorker2Client, worker2Lq) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sManagerClient, managerNs)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(ctx, k8sWorker1Client, worker1Ns)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(ctx, k8sWorker2Client, worker2Ns)).To(gomega.Succeed()) + + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker1Client, worker1Cq, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker1Client, worker1Flavor, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker1Client, worker1Topology, true, util.LongTimeout) + + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker2Client, worker2Cq, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker2Client, worker2Flavor, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sWorker2Client, worker2Topology, true, util.LongTimeout) + + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sManagerClient, managerCq, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sManagerClient, managerFlavor, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sManagerClient, managerTopology, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sManagerClient, multiKueueAc, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sManagerClient, multiKueueConfig, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sManagerClient, workerCluster1, true, util.LongTimeout) + util.ExpectObjectToBeDeletedWithTimeout(ctx, k8sManagerClient, workerCluster2, true, util.LongTimeout) + + util.ExpectAllPodsInNamespaceDeleted(ctx, k8sManagerClient, managerNs) + util.ExpectAllPodsInNamespaceDeleted(ctx, k8sWorker1Client, worker1Ns) + util.ExpectAllPodsInNamespaceDeleted(ctx, k8sWorker2Client, worker2Ns) + }) + + ginkgo.When("Creating a Job with TAS requirements", func() { + ginkgo.It("Should admit a Job and assign topology in the worker cluster", func() { + job := testingjob.MakeJob("tas-job", managerNs.Name). + Queue(kueue.LocalQueueName(managerLq.Name)). + Parallelism(2). + Completions(2). + RequestAndLimit(corev1.ResourceCPU, "500m"). + RequestAndLimit(corev1.ResourceMemory, "200Mi"). + TerminationGracePeriod(1). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). + Obj() + job = (&testingjob.JobWrapper{Job: *job}). + PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, corev1.LabelHostname). + Obj() + + ginkgo.By("Creating the job", func() { + util.MustCreate(ctx, k8sManagerClient, job) + gomega.Eventually(func(g gomega.Gomega) { + createdJob := &batchv1.Job{} + g.Expect(k8sManagerClient.Get(ctx, client.ObjectKeyFromObject(job), createdJob)).To(gomega.Succeed()) + g.Expect(ptr.Deref(createdJob.Spec.ManagedBy, "")).To(gomega.BeEquivalentTo(kueue.MultiKueueControllerName)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + createdWorkload := &kueue.Workload{} + wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job.Name, job.UID), Namespace: managerNs.Name} + + var assignedWorkerCluster client.Client + var assignedWorkerCtx context.Context + var assignedClusterName string + ginkgo.By("checking which worker cluster was assigned", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(k8sManagerClient.Get(ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.ClusterName).NotTo(gomega.BeNil()) + + assignedClusterName = *managerWl.Status.ClusterName + g.Expect(assignedClusterName).To(gomega.Or(gomega.Equal(workerCluster1.Name), gomega.Equal(workerCluster2.Name))) + if assignedClusterName == workerCluster1.Name { + assignedWorkerCluster = k8sWorker1Client + assignedWorkerCtx = ctx + } else { + assignedWorkerCluster = k8sWorker2Client + assignedWorkerCtx = ctx + } + + g.Expect(admissioncheck.FindAdmissionCheck(managerWl.Status.AdmissionChecks, kueue.AdmissionCheckReference(multiKueueAc.Name))).To(gomega.BeComparableTo(&kueue.AdmissionCheckState{ + Name: kueue.AdmissionCheckReference(multiKueueAc.Name), + State: kueue.CheckStateReady, + Message: fmt.Sprintf(`The workload got reservation on "%s"`, assignedClusterName), + }, cmpopts.IgnoreFields(kueue.AdmissionCheckState{}, "LastTransitionTime"))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By(fmt.Sprintf("Waiting for TopologyAssignment to be computed in %s", assignedClusterName), func() { + workerWlLookupKey := types.NamespacedName{Name: wlLookupKey.Name, Namespace: managerNs.Name} + workerWl := &kueue.Workload{} + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(assignedWorkerCluster.Get(assignedWorkerCtx, workerWlLookupKey, workerWl)).To(gomega.Succeed()) + g.Expect(workerWl.Status.Admission).NotTo(gomega.BeNil()) + g.Expect(workerWl.Status.Admission.PodSetAssignments).To(gomega.HaveLen(1)) + g.Expect(workerWl.Status.Admission.PodSetAssignments[0].TopologyAssignment).NotTo(gomega.BeNil()) + g.Expect(workerWl.Status.Admission.PodSetAssignments[0].TopologyAssignment.Levels).To(gomega.Equal([]string{corev1.LabelHostname})) + g.Expect(workerWl.Status.Admission.PodSetAssignments[0].TopologyAssignment.Domains).NotTo(gomega.BeEmpty()) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Waiting for DelayedTopologyRequest to be marked Ready on manager", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(k8sManagerClient.Get(ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.Admission).NotTo(gomega.BeNil()) + g.Expect(managerWl.Status.Admission.PodSetAssignments).To(gomega.HaveLen(1)) + g.Expect(managerWl.Status.Admission.PodSetAssignments[0].DelayedTopologyRequest).To(gomega.Equal(ptr.To(kueue.DelayedTopologyRequestStateReady))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Waiting for the job to get status updates", func() { + gomega.Eventually(func(g gomega.Gomega) { + createdJob := batchv1.Job{} + g.Expect(k8sManagerClient.Get(ctx, client.ObjectKeyFromObject(job), &createdJob)).To(gomega.Succeed()) + g.Expect(createdJob.Status.StartTime).NotTo(gomega.BeNil()) + g.Expect(createdJob.Status.Active).To(gomega.Equal(int32(2))) + g.Expect(createdJob.Status.CompletionTime).To(gomega.BeNil()) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Finishing the job's pods", func() { + listOpts := util.GetListOptsFromLabel(fmt.Sprintf("batch.kubernetes.io/job-name=%s", job.Name)) + if assignedClusterName == workerCluster1.Name { + util.WaitForActivePodsAndTerminate(ctx, k8sWorker1Client, worker1RestClient, worker1Cfg, job.Namespace, 2, 0, listOpts) + } else { + util.WaitForActivePodsAndTerminate(ctx, k8sWorker2Client, worker2RestClient, worker2Cfg, job.Namespace, 2, 0, listOpts) + } + }) + + ginkgo.By("Waiting for the job to finish", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sManagerClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason(kueue.WorkloadFinished, kueue.WorkloadFinishedReasonSucceeded)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Checking no objects are left in the worker clusters and the job is completed", func() { + expectObjectToBeDeletedOnWorkerClusters(ctx, createdWorkload) + expectObjectToBeDeletedOnWorkerClusters(ctx, job) + + createdJob := &batchv1.Job{} + gomega.Expect(k8sManagerClient.Get(ctx, client.ObjectKeyFromObject(job), createdJob)).To(gomega.Succeed()) + gomega.Expect(createdJob.Status.Conditions).To(gomega.ContainElement(gomega.BeComparableTo( + batchv1.JobCondition{ + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }, + cmpopts.IgnoreFields(batchv1.JobCondition{}, "LastTransitionTime", "LastProbeTime", "Reason", "Message")))) + }) + }) + + ginkgo.It("Should handle implicit TAS", func() { + job := testingjob.MakeJob("implicit-tas-job", managerNs.Name). + Queue(kueue.LocalQueueName(managerLq.Name)). + Parallelism(2). + Completions(2). + RequestAndLimit(corev1.ResourceCPU, "500m"). + RequestAndLimit(corev1.ResourceMemory, "200Mi"). + TerminationGracePeriod(1). + Image(util.GetAgnHostImage(), util.BehaviorWaitForDeletion). + Obj() + + ginkgo.By("Creating the job without TAS annotation", func() { + util.MustCreate(ctx, k8sManagerClient, job) + gomega.Eventually(func(g gomega.Gomega) { + createdJob := &batchv1.Job{} + g.Expect(k8sManagerClient.Get(ctx, client.ObjectKeyFromObject(job), createdJob)).To(gomega.Succeed()) + g.Expect(ptr.Deref(createdJob.Spec.ManagedBy, "")).To(gomega.BeEquivalentTo(kueue.MultiKueueControllerName)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + createdWorkload := &kueue.Workload{} + wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job.Name, job.UID), Namespace: managerNs.Name} + + var assignedClusterName string + ginkgo.By("checking which worker cluster was assigned", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(k8sManagerClient.Get(ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.ClusterName).NotTo(gomega.BeNil()) + assignedClusterName = *managerWl.Status.ClusterName + g.Expect(apimeta.FindStatusCondition(managerWl.Status.Conditions, kueue.WorkloadAdmitted)).To(gomega.BeComparableTo(&metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + }, util.IgnoreConditionTimestampsAndObservedGeneration)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Waiting for DelayedTopologyRequest to be marked Ready on manager", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(k8sManagerClient.Get(ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.Admission).NotTo(gomega.BeNil()) + g.Expect(managerWl.Status.Admission.PodSetAssignments).To(gomega.HaveLen(1)) + g.Expect(managerWl.Status.Admission.PodSetAssignments[0].DelayedTopologyRequest).To(gomega.Equal(ptr.To(kueue.DelayedTopologyRequestStateReady))) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("Finishing the job", func() { + listOpts := util.GetListOptsFromLabel(fmt.Sprintf("batch.kubernetes.io/job-name=%s", job.Name)) + if assignedClusterName == workerCluster1.Name { + util.WaitForActivePodsAndTerminate(ctx, k8sWorker1Client, worker1RestClient, worker1Cfg, job.Namespace, 2, 0, listOpts) + } else { + util.WaitForActivePodsAndTerminate(ctx, k8sWorker2Client, worker2RestClient, worker2Cfg, job.Namespace, 2, 0, listOpts) + } + }) + + ginkgo.By("Waiting for the job to finish", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sManagerClient.Get(ctx, wlLookupKey, createdWorkload)).To(gomega.Succeed()) + g.Expect(workload.HasQuotaReservation(createdWorkload)).Should(gomega.BeTrue()) + g.Expect(createdWorkload.Status.Conditions).To(utiltesting.HaveConditionStatusTrueAndReason(kueue.WorkloadFinished, kueue.WorkloadFinishedReasonSucceeded)) + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) +}) diff --git a/test/integration/multikueue/tas/suite_test.go b/test/integration/multikueue/tas/suite_test.go new file mode 100644 index 00000000000..fec5e754ab1 --- /dev/null +++ b/test/integration/multikueue/tas/suite_test.go @@ -0,0 +1,230 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multikueue + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + versionutil "k8s.io/apimachinery/pkg/util/version" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + config "sigs.k8s.io/kueue/apis/config/v1beta2" + qcache "sigs.k8s.io/kueue/pkg/cache/queue" + schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" + "sigs.k8s.io/kueue/pkg/constants" + "sigs.k8s.io/kueue/pkg/controller/admissionchecks/multikueue" + "sigs.k8s.io/kueue/pkg/controller/admissionchecks/provisioning" + "sigs.k8s.io/kueue/pkg/controller/core" + "sigs.k8s.io/kueue/pkg/controller/core/indexer" + "sigs.k8s.io/kueue/pkg/controller/jobframework" + workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" + "sigs.k8s.io/kueue/pkg/controller/tas" + tasindexer "sigs.k8s.io/kueue/pkg/controller/tas/indexer" + "sigs.k8s.io/kueue/pkg/scheduler" + "sigs.k8s.io/kueue/pkg/util/kubeversion" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" + "sigs.k8s.io/kueue/pkg/webhooks" + "sigs.k8s.io/kueue/test/integration/framework" + "sigs.k8s.io/kueue/test/util" +) + +const ( + testingWorkerLostTimeout = 3 * time.Second +) + +type cluster struct { + cfg *rest.Config + client client.Client + ctx context.Context + fwk *framework.Framework +} + +func (c *cluster) kubeConfigBytes() ([]byte, error) { + return utiltesting.RestConfigToKubeConfig(c.cfg) +} + +var ( + managerK8sVersion *versionutil.Version + managerTestCluster cluster + worker1TestCluster cluster + worker2TestCluster cluster + managersConfigNamespace *corev1.Namespace + + // makes sure there is only one fwk.Init and setupClient at the same time + // since these functions are not thread safe due to adding to the common + // schema. + mu sync.Mutex +) + +func TestMultiKueue(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, + "MultiKueue TAS Suite", + ) +} + +func createCluster(setupFnc framework.ManagerSetup, apiFeatureGates ...string) cluster { + c := cluster{} + c.fwk = &framework.Framework{ + WebhookPath: util.WebhookPath, + DepCRDPaths: []string{ + util.AutoscalerCrds, + }, + APIServerFeatureGates: apiFeatureGates, + } + mu.Lock() + c.cfg = c.fwk.Init() + c.ctx, c.client = c.fwk.SetupClient(c.cfg) + mu.Unlock() + + // skip the manager setup if setup func is not provided + if setupFnc != nil { + c.fwk.StartManager(c.ctx, c.cfg, setupFnc) + } + return c +} + +func managerSetup(ctx context.Context, mgr manager.Manager) { + err := indexer.Setup(ctx, mgr.GetFieldIndexer()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + cCache := schdcache.New(mgr.GetClient()) + queues := qcache.NewManager(mgr.GetClient(), cCache) + + configuration := &config.Configuration{} + mgr.GetScheme().Default(configuration) + + failedCtrl, err := core.SetupControllers(mgr, queues, cCache, configuration) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "controller", failedCtrl) + + failedCtrl, err = tas.SetupControllers(mgr, queues, cCache, configuration) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "TAS controller", failedCtrl) + + err = tasindexer.SetupIndexes(ctx, mgr.GetFieldIndexer()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + failedWebhook, err := webhooks.Setup(mgr) + gomega.Expect(err).ToNot(gomega.HaveOccurred(), "webhook", failedWebhook) + + err = workloadjob.SetupIndexes(ctx, mgr.GetFieldIndexer()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + jobReconciler, err := workloadjob.NewReconciler( + ctx, + mgr.GetClient(), + mgr.GetFieldIndexer(), + mgr.GetEventRecorderFor(constants.JobControllerName)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + err = jobReconciler.SetupWithManager(mgr) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = workloadjob.SetupWebhook(mgr, jobframework.WithCache(cCache), jobframework.WithQueues(queues)) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = provisioning.SetupIndexer(ctx, mgr.GetFieldIndexer()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + reconciler, err := provisioning.NewController( + mgr.GetClient(), + mgr.GetEventRecorderFor("kueue-provisioning-request-controller")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = reconciler.SetupWithManager(mgr) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + sched := scheduler.New(queues, cCache, mgr.GetClient(), mgr.GetEventRecorderFor(constants.AdmissionName)) + err = sched.Start(ctx) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func managerAndMultiKueueSetup( + ctx context.Context, + mgr manager.Manager, + gcInterval time.Duration, + enabledIntegrations sets.Set[string], + dispatcherName string, +) { + managerSetup(ctx, mgr) + + err := multikueue.SetupIndexer(ctx, mgr.GetFieldIndexer(), managersConfigNamespace.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + adapters, err := jobframework.GetMultiKueueAdapters(enabledIntegrations) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = multikueue.SetupControllers(mgr, managersConfigNamespace.Name, + multikueue.WithGCInterval(gcInterval), + multikueue.WithWorkerLostTimeout(testingWorkerLostTimeout), + multikueue.WithEventsBatchPeriod(100*time.Millisecond), + multikueue.WithAdapters(adapters), + multikueue.WithDispatcherName(dispatcherName), + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +var _ = ginkgo.BeforeSuite(func() { + var managerFeatureGates []string + ginkgo.By("creating the clusters", func() { + mu = sync.Mutex{} + wg := sync.WaitGroup{} + wg.Add(3) + go func() { + defer ginkgo.GinkgoRecover() + defer wg.Done() + // pass nil setup since the manager for the manage cluster is different in some specs. + c := createCluster(nil, managerFeatureGates...) + managerTestCluster = c + }() + go func() { + defer ginkgo.GinkgoRecover() + defer wg.Done() + c := createCluster(managerSetup) + worker1TestCluster = c + }() + go func() { + defer ginkgo.GinkgoRecover() + defer wg.Done() + c := createCluster(managerSetup) + worker2TestCluster = c + }() + wg.Wait() + }) + + discoveryClient, err := discovery.NewDiscoveryClientForConfig(managerTestCluster.cfg) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + managerK8sVersion, err = kubeversion.FetchServerVersion(discoveryClient) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + managersConfigNamespace = utiltesting.MakeNamespace("kueue-system") + util.MustCreate(managerTestCluster.ctx, managerTestCluster.client, managersConfigNamespace) +}) + +var _ = ginkgo.AfterSuite(func() { + managerTestCluster.fwk.Teardown() + worker1TestCluster.fwk.Teardown() + worker2TestCluster.fwk.Teardown() +}) diff --git a/test/integration/multikueue/tas/tas_test.go b/test/integration/multikueue/tas/tas_test.go new file mode 100644 index 00000000000..e3a3baa47ba --- /dev/null +++ b/test/integration/multikueue/tas/tas_test.go @@ -0,0 +1,606 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multikueue + +import ( + "context" + "time" + + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + autoscaling "k8s.io/autoscaler/cluster-autoscaler/apis/provisioningrequest/autoscaling.x-k8s.io/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + config "sigs.k8s.io/kueue/apis/config/v1beta2" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" + "sigs.k8s.io/kueue/pkg/controller/admissionchecks/provisioning" + workloadjob "sigs.k8s.io/kueue/pkg/controller/jobs/job" + "sigs.k8s.io/kueue/pkg/features" + utiltesting "sigs.k8s.io/kueue/pkg/util/testing" + utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" + testingjob "sigs.k8s.io/kueue/pkg/util/testingjobs/job" + testingnode "sigs.k8s.io/kueue/pkg/util/testingjobs/node" + "sigs.k8s.io/kueue/test/util" +) + +var defaultEnabledIntegrations sets.Set[string] = sets.New( + "batch/job", "kubeflow.org/mpijob", "ray.io/rayjob", "ray.io/raycluster", + "jobset.x-k8s.io/jobset", "kubeflow.org/paddlejob", + "kubeflow.org/pytorchjob", "kubeflow.org/tfjob", "kubeflow.org/xgboostjob", "kubeflow.org/jaxjob", + "pod", "workload.codeflare.dev/appwrapper") + +var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { + var ( + managerNs *corev1.Namespace + worker1Ns *corev1.Namespace + worker2Ns *corev1.Namespace + + managerMultiKueueSecret1 *corev1.Secret + managerMultiKueueSecret2 *corev1.Secret + workerCluster1 *kueue.MultiKueueCluster + workerCluster2 *kueue.MultiKueueCluster + managerMultiKueueConfig *kueue.MultiKueueConfig + multiKueueAC *kueue.AdmissionCheck + + managerTopology *kueue.Topology + worker1Topology *kueue.Topology + worker2Topology *kueue.Topology + + managerTasFlavor *kueue.ResourceFlavor + worker1TasFlavor *kueue.ResourceFlavor + worker2TasFlavor *kueue.ResourceFlavor + + managerCq *kueue.ClusterQueue + worker1Cq *kueue.ClusterQueue + worker2Cq *kueue.ClusterQueue + + managerLq *kueue.LocalQueue + worker1Lq *kueue.LocalQueue + worker2Lq *kueue.LocalQueue + + worker1Ac *kueue.AdmissionCheck + worker2Ac *kueue.AdmissionCheck + + worker1Nodes []corev1.Node + worker2Nodes []corev1.Node + ) + + ginkgo.BeforeAll(func() { + managerTestCluster.fwk.StartManager(managerTestCluster.ctx, managerTestCluster.cfg, func(ctx context.Context, mgr manager.Manager) { + managerAndMultiKueueSetup(ctx, mgr, 2*time.Second, defaultEnabledIntegrations, config.MultiKueueDispatcherModeAllAtOnce) + }) + }) + + ginkgo.AfterAll(func() { + managerTestCluster.fwk.StopManager(managerTestCluster.ctx) + }) + + ginkgo.BeforeEach(func() { + managerNs = util.CreateNamespaceFromPrefixWithLog(managerTestCluster.ctx, managerTestCluster.client, "multikueue-") + worker1Ns = util.CreateNamespaceWithLog(worker1TestCluster.ctx, worker1TestCluster.client, managerNs.Name) + worker2Ns = util.CreateNamespaceWithLog(worker2TestCluster.ctx, worker2TestCluster.client, managerNs.Name) + + w1Kubeconfig, err := worker1TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + w2Kubeconfig, err := worker2TestCluster.kubeConfigBytes() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + managerMultiKueueSecret1 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multikueue1", + Namespace: managersConfigNamespace.Name, + }, + Data: map[string][]byte{ + kueue.MultiKueueConfigSecretKey: w1Kubeconfig, + }, + } + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultiKueueSecret1)).To(gomega.Succeed()) + + managerMultiKueueSecret2 = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multikueue2", + Namespace: managersConfigNamespace.Name, + }, + Data: map[string][]byte{ + kueue.MultiKueueConfigSecretKey: w2Kubeconfig, + }, + } + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultiKueueSecret2)).To(gomega.Succeed()) + + workerCluster1 = utiltestingapi.MakeMultiKueueCluster("worker1").KubeConfig(kueue.SecretLocationType, managerMultiKueueSecret1.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, workerCluster1)).To(gomega.Succeed()) + + workerCluster2 = utiltestingapi.MakeMultiKueueCluster("worker2").KubeConfig(kueue.SecretLocationType, managerMultiKueueSecret2.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, workerCluster2)).To(gomega.Succeed()) + + managerMultiKueueConfig = utiltestingapi.MakeMultiKueueConfig("multikueueconfig").Clusters(workerCluster1.Name, workerCluster2.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerMultiKueueConfig)).Should(gomega.Succeed()) + + multiKueueAC = utiltestingapi.MakeAdmissionCheck("ac1"). + ControllerName(kueue.MultiKueueControllerName). + Parameters(kueue.GroupVersion.Group, "MultiKueueConfig", managerMultiKueueConfig.Name). + Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, multiKueueAC)).Should(gomega.Succeed()) + + ginkgo.By("wait for check active", func() { + updatedAc := kueue.AdmissionCheck{} + acKey := client.ObjectKeyFromObject(multiKueueAC) + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, acKey, &updatedAc)).To(gomega.Succeed()) + g.Expect(updatedAc.Status.Conditions).To(utiltesting.HaveConditionStatusTrue(kueue.AdmissionCheckActive)) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + managerTopology = utiltestingapi.MakeDefaultOneLevelTopology("default") + util.MustCreate(managerTestCluster.ctx, managerTestCluster.client, managerTopology) + + managerTasFlavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). + NodeLabel("node-group", "tas"). + TopologyName(managerTopology.Name).Obj() + util.MustCreate(managerTestCluster.ctx, managerTestCluster.client, managerTasFlavor) + + worker1Topology = utiltestingapi.MakeDefaultOneLevelTopology("default") + util.MustCreate(worker1TestCluster.ctx, worker1TestCluster.client, worker1Topology) + + worker1TasFlavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). + NodeLabel("node-group", "tas"). + TopologyName(worker1Topology.Name).Obj() + util.MustCreate(worker1TestCluster.ctx, worker1TestCluster.client, worker1TasFlavor) + + worker2Topology = utiltestingapi.MakeDefaultOneLevelTopology("default") + util.MustCreate(worker2TestCluster.ctx, worker2TestCluster.client, worker2Topology) + + worker2TasFlavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). + NodeLabel("node-group", "tas"). + TopologyName(worker2Topology.Name).Obj() + util.MustCreate(worker2TestCluster.ctx, worker2TestCluster.client, worker2TasFlavor) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(managerTestCluster.ctx, managerTestCluster.client, managerNs)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(worker1TestCluster.ctx, worker1TestCluster.client, worker1Ns)).To(gomega.Succeed()) + gomega.Expect(util.DeleteNamespace(worker2TestCluster.ctx, worker2TestCluster.client, worker2Ns)).To(gomega.Succeed()) + + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerLq, true) + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1Lq, true) + util.ExpectObjectToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, worker2Lq, true) + + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerCq, true) + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1Cq, true) + util.ExpectObjectToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, worker2Cq, true) + + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1Ac, true) + util.ExpectObjectToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, worker2Ac, true) + + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerTasFlavor, true) + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1TasFlavor, true) + util.ExpectObjectToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, worker2TasFlavor, true) + + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerTopology, true) + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1Topology, true) + util.ExpectObjectToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, worker2Topology, true) + + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, multiKueueAC, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerMultiKueueConfig, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, workerCluster1, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, workerCluster2, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerMultiKueueSecret1, true) + util.ExpectObjectToBeDeleted(managerTestCluster.ctx, managerTestCluster.client, managerMultiKueueSecret2, true) + }) + + ginkgo.When("Topology has a single level", func() { + ginkgo.BeforeEach(func() { + managerCq = utiltestingapi.MakeClusterQueue("mgr-cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(managerTasFlavor.Name).Resource(corev1.ResourceCPU, "5").Obj(), + ). + AdmissionChecks(kueue.AdmissionCheckReference(multiKueueAC.Name)). + Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerCq)).Should(gomega.Succeed()) + util.ExpectClusterQueuesToBeActive(managerTestCluster.ctx, managerTestCluster.client, managerCq) + + managerLq = utiltestingapi.MakeLocalQueue("local-queue", managerNs.Name).ClusterQueue(managerCq.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerLq)).Should(gomega.Succeed()) + + worker1Cq = utiltestingapi.MakeClusterQueue("wr-cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(worker1TasFlavor.Name).Resource(corev1.ResourceCPU, "5").Obj(), + ). + Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Cq)).Should(gomega.Succeed()) + util.ExpectClusterQueuesToBeActive(worker1TestCluster.ctx, worker1TestCluster.client, worker1Cq) + + worker1Lq = utiltestingapi.MakeLocalQueue("local-queue", worker1Ns.Name).ClusterQueue(worker1Cq.Name).Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Lq)).Should(gomega.Succeed()) + + worker2Cq = utiltestingapi.MakeClusterQueue("cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(worker2TasFlavor.Name).Resource(corev1.ResourceCPU, "5").Obj(), + ). + Obj() + gomega.Expect(worker2TestCluster.client.Create(worker2TestCluster.ctx, worker2Cq)).Should(gomega.Succeed()) + util.ExpectClusterQueuesToBeActive(worker2TestCluster.ctx, worker2TestCluster.client, worker2Cq) + + worker2Lq = utiltestingapi.MakeLocalQueue("local-queue", worker2Ns.Name).ClusterQueue(worker2Cq.Name).Obj() + gomega.Expect(worker2TestCluster.client.Create(worker2TestCluster.ctx, worker2Lq)).Should(gomega.Succeed()) + + worker1Nodes = []corev1.Node{ + *testingnode.MakeNode("single-node"). + Label(corev1.LabelHostname, "host-1"). + Label("node-group", "tas"). + StatusAllocatable(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5"), + corev1.ResourcePods: resource.MustParse("10"), + }). + Ready(). + Obj(), + } + worker2Nodes = []corev1.Node{ + *testingnode.MakeNode("single-node"). + Label(corev1.LabelHostname, "host-1"). + Label("node-group", "tas"). + StatusAllocatable(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("5"), + corev1.ResourcePods: resource.MustParse("10"), + }). + Ready(). + Obj(), + } + util.CreateNodesWithStatus(worker1TestCluster.ctx, worker1TestCluster.client, worker1Nodes) + util.CreateNodesWithStatus(worker2TestCluster.ctx, worker2TestCluster.client, worker2Nodes) + }) + + ginkgo.AfterEach(func() { + for _, node := range worker1Nodes { + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, &node, true) + } + for _, node := range worker2Nodes { + util.ExpectObjectToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, &node, true) + } + }) + + ginkgo.It("should admit workload which fits in a required topology domain", func() { + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.MultiKueueBatchJobWithManagedBy, true) + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.TopologyAwareScheduling, true) + + job := testingjob.MakeJob("job", managerNs.Name). + ManagedBy(kueue.MultiKueueControllerName). + Queue(kueue.LocalQueueName(managerLq.Name)). + PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj() + ginkgo.By("creating a job which requires block", func() { + util.MustCreate(managerTestCluster.ctx, managerTestCluster.client, job) + }) + + wl := &kueue.Workload{} + wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job.Name, job.UID), Namespace: managerNs.Name} + + ginkgo.By("verify the workload is created in manager cluster and has QuotaReserved", func() { + managerWl := &kueue.Workload{} + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).Should(gomega.Succeed()) + g.Expect(apimeta.IsStatusConditionTrue(managerWl.Status.Conditions, kueue.WorkloadQuotaReserved)).Should(gomega.BeTrue()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + var assignedWorkerCluster client.Client + var assignedWorkerCtx context.Context + var assignedClusterName string + ginkgo.By("checking which worker cluster was assigned", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.ClusterName).NotTo(gomega.BeNil()) + + assignedClusterName = *managerWl.Status.ClusterName + g.Expect(assignedClusterName).To(gomega.Or(gomega.Equal(workerCluster1.Name), gomega.Equal(workerCluster2.Name))) + if assignedClusterName == workerCluster1.Name { + assignedWorkerCluster = worker1TestCluster.client + assignedWorkerCtx = worker1TestCluster.ctx + } else { + assignedWorkerCluster = worker2TestCluster.client + assignedWorkerCtx = worker2TestCluster.ctx + } + + g.Expect(assignedWorkerCluster.Get(assignedWorkerCtx, wlLookupKey, wl)).To(gomega.Succeed()) + g.Expect(wl.Spec.PodSets).Should(gomega.BeComparableTo([]kueue.PodSet{{ + Name: kueue.DefaultPodSetName, + Count: 1, + TopologyRequest: &kueue.PodSetTopologyRequest{ + Required: ptr.To(string(corev1.LabelHostname)), + PodIndexLabel: ptr.To(batchv1.JobCompletionIndexAnnotation), + }, + }}, cmpopts.IgnoreFields(kueue.PodSet{}, "Template"))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verify the workload is admitted in the assigned worker cluster", func() { + util.ExpectWorkloadsToBeAdmitted(assignedWorkerCtx, assignedWorkerCluster, wl) + }) + + ginkgo.By("verify TopologyAssignment for the workload in the assigned worker cluster", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(assignedWorkerCluster.Get(assignedWorkerCtx, wlLookupKey, wl)).Should(gomega.Succeed()) + g.Expect(wl.Status.Admission).ShouldNot(gomega.BeNil()) + g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(1)) + g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( + &kueue.TopologyAssignment{ + Levels: []string{corev1.LabelHostname}, + Domains: []kueue.TopologyDomainAssignment{{Count: 1, Values: []string{"host-1"}}}, + }, + )) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verify DelayedTopologyRequest is marked Ready on manager", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.Admission).NotTo(gomega.BeNil()) + g.Expect(managerWl.Status.Admission.PodSetAssignments).To(gomega.HaveLen(1)) + g.Expect(managerWl.Status.Admission.PodSetAssignments[0].DelayedTopologyRequest).To(gomega.Equal(ptr.To(kueue.DelayedTopologyRequestStateReady))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) + + ginkgo.When("ProvisioningRequest is used", func() { + var ( + worker1Prc *kueue.ProvisioningRequestConfig + worker2Prc *kueue.ProvisioningRequestConfig + + createdRequest autoscaling.ProvisioningRequest + ) + + ginkgo.BeforeEach(func() { + worker1Prc = utiltestingapi.MakeProvisioningRequestConfig("prov-config"). + ProvisioningClass("provisioning-class"). + RetryLimit(1). + BaseBackoff(1). + PodSetUpdate(kueue.ProvisioningRequestPodSetUpdates{ + NodeSelector: []kueue.ProvisioningRequestPodSetUpdatesNodeSelector{{ + Key: "dedicated-selector-key", + ValueFromProvisioningClassDetail: "dedicated-selector-detail", + }}, + }). + Obj() + util.MustCreate(worker1TestCluster.ctx, worker1TestCluster.client, worker1Prc) + + worker2Prc = utiltestingapi.MakeProvisioningRequestConfig("prov-config"). + ProvisioningClass("provisioning-class"). + RetryLimit(1). + BaseBackoff(1). + PodSetUpdate(kueue.ProvisioningRequestPodSetUpdates{ + NodeSelector: []kueue.ProvisioningRequestPodSetUpdatesNodeSelector{{ + Key: "dedicated-selector-key", + ValueFromProvisioningClassDetail: "dedicated-selector-detail", + }}, + }). + Obj() + util.MustCreate(worker2TestCluster.ctx, worker2TestCluster.client, worker2Prc) + + worker1Ac = utiltestingapi.MakeAdmissionCheck("provisioning"). + ControllerName(kueue.ProvisioningRequestControllerName). + Parameters(kueue.GroupVersion.Group, "ProvisioningRequestConfig", worker1Prc.Name). + Obj() + util.MustCreate(worker1TestCluster.ctx, worker1TestCluster.client, worker1Ac) + + worker2Ac = utiltestingapi.MakeAdmissionCheck("provisioning"). + ControllerName(kueue.ProvisioningRequestControllerName). + Parameters(kueue.GroupVersion.Group, "ProvisioningRequestConfig", worker2Prc.Name). + Obj() + util.MustCreate(worker2TestCluster.ctx, worker2TestCluster.client, worker2Ac) + + util.SetAdmissionCheckActive(worker1TestCluster.ctx, worker1TestCluster.client, worker1Ac, metav1.ConditionTrue) + util.SetAdmissionCheckActive(worker2TestCluster.ctx, worker2TestCluster.client, worker2Ac, metav1.ConditionTrue) + + managerCq = utiltestingapi.MakeClusterQueue("mgr-cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(managerTasFlavor.Name).Resource(corev1.ResourceCPU, "5").Obj(), + ). + AdmissionChecks(kueue.AdmissionCheckReference(multiKueueAC.Name)). + Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerCq)).Should(gomega.Succeed()) + util.ExpectClusterQueuesToBeActive(managerTestCluster.ctx, managerTestCluster.client, managerCq) + + managerLq = utiltestingapi.MakeLocalQueue("local-queue", managerNs.Name).ClusterQueue(managerCq.Name).Obj() + gomega.Expect(managerTestCluster.client.Create(managerTestCluster.ctx, managerLq)).Should(gomega.Succeed()) + + worker1Cq = utiltestingapi.MakeClusterQueue("wr-cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(worker1TasFlavor.Name).Resource(corev1.ResourceCPU, "5").Obj(), + ). + AdmissionChecks(kueue.AdmissionCheckReference(worker1Ac.Name)). + Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Cq)).Should(gomega.Succeed()) + util.ExpectClusterQueuesToBeActive(worker1TestCluster.ctx, worker1TestCluster.client, worker1Cq) + + worker1Lq = utiltestingapi.MakeLocalQueue("local-queue", worker1Ns.Name).ClusterQueue(worker1Cq.Name).Obj() + gomega.Expect(worker1TestCluster.client.Create(worker1TestCluster.ctx, worker1Lq)).Should(gomega.Succeed()) + + worker2Cq = utiltestingapi.MakeClusterQueue("cluster-queue"). + ResourceGroup( + *utiltestingapi.MakeFlavorQuotas(worker2TasFlavor.Name).Resource(corev1.ResourceCPU, "5").Obj(), + ). + AdmissionChecks(kueue.AdmissionCheckReference(worker2Ac.Name)). + Obj() + gomega.Expect(worker2TestCluster.client.Create(worker2TestCluster.ctx, worker2Cq)).Should(gomega.Succeed()) + util.ExpectClusterQueuesToBeActive(worker2TestCluster.ctx, worker2TestCluster.client, worker2Cq) + + worker2Lq = utiltestingapi.MakeLocalQueue("local-queue", worker2Ns.Name).ClusterQueue(worker2Cq.Name).Obj() + gomega.Expect(worker2TestCluster.client.Create(worker2TestCluster.ctx, worker2Lq)).Should(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, &createdRequest, true) + + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, worker1Prc, true) + util.ExpectObjectToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, worker2Prc, true) + + for _, node := range worker1Nodes { + util.ExpectObjectToBeDeleted(worker1TestCluster.ctx, worker1TestCluster.client, &node, true) + } + for _, node := range worker2Nodes { + util.ExpectObjectToBeDeleted(worker2TestCluster.ctx, worker2TestCluster.client, &node, true) + } + }) + + ginkgo.It("should admit workload when nodes are provisioned", func() { + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.MultiKueueBatchJobWithManagedBy, true) + features.SetFeatureGateDuringTest(ginkgo.GinkgoTB(), features.TopologyAwareScheduling, true) + + job := testingjob.MakeJob("job", managerNs.Name). + ManagedBy(kueue.MultiKueueControllerName). + Queue(kueue.LocalQueueName(managerLq.Name)). + PodAnnotation(kueue.PodSetRequiredTopologyAnnotation, corev1.LabelHostname). + Request(corev1.ResourceCPU, "1"). + Obj() + ginkgo.By("creating a job which requires block", func() { + util.MustCreate(managerTestCluster.ctx, managerTestCluster.client, job) + }) + + wl := &kueue.Workload{} + wlLookupKey := types.NamespacedName{Name: workloadjob.GetWorkloadNameForJob(job.Name, job.UID), Namespace: managerNs.Name} + + ginkgo.By("verify the workload reserves the quota", func() { + util.ExpectQuotaReservedWorkloadsTotalMetric(managerCq, "", 1) + util.ExpectReservingActiveWorkloadsMetric(managerCq, 1) + }) + + ginkgo.By("provision the nodes on both workers", func() { + worker1Nodes = []corev1.Node{ + *testingnode.MakeNode("x1"). + Label("node-group", "tas"). + Label(corev1.LabelHostname, "x1"). + StatusAllocatable(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("10"), + }). + Ready(). + Obj(), + } + worker2Nodes = []corev1.Node{ + *testingnode.MakeNode("x1"). + Label("node-group", "tas"). + Label(corev1.LabelHostname, "x1"). + StatusAllocatable(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourcePods: resource.MustParse("10"), + }). + Ready(). + Obj(), + } + util.CreateNodesWithStatus(worker1TestCluster.ctx, worker1TestCluster.client, worker1Nodes) + util.CreateNodesWithStatus(worker2TestCluster.ctx, worker2TestCluster.client, worker2Nodes) + }) + + var assignedWorkerCluster client.Client + var assignedWorkerCtx context.Context + var assignedClusterName string + var assignedAcName string + ginkgo.By("checking which worker cluster was assigned", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.ClusterName).NotTo(gomega.BeNil()) + + assignedClusterName = *managerWl.Status.ClusterName + g.Expect(assignedClusterName).To(gomega.Or(gomega.Equal(workerCluster1.Name), gomega.Equal(workerCluster2.Name))) + if assignedClusterName == workerCluster1.Name { + assignedWorkerCluster = worker1TestCluster.client + assignedWorkerCtx = worker1TestCluster.ctx + assignedAcName = worker1Ac.Name + } else { + assignedWorkerCluster = worker2TestCluster.client + assignedWorkerCtx = worker2TestCluster.ctx + assignedAcName = worker2Ac.Name + } + + g.Expect(assignedWorkerCluster.Get(assignedWorkerCtx, wlLookupKey, wl)).To(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + provReqKey := types.NamespacedName{ + Namespace: wlLookupKey.Namespace, + Name: provisioning.ProvisioningRequestName(wlLookupKey.Name, kueue.AdmissionCheckReference(assignedAcName), 1), + } + + ginkgo.By("set the ProvisioningRequest as Provisioned in the assigned worker cluster", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(assignedWorkerCluster.Get(assignedWorkerCtx, provReqKey, &createdRequest)).Should(gomega.Succeed()) + apimeta.SetStatusCondition(&createdRequest.Status.Conditions, metav1.Condition{ + Type: autoscaling.Provisioned, + Status: metav1.ConditionTrue, + Reason: autoscaling.Provisioned, + }) + g.Expect(assignedWorkerCluster.Status().Update(assignedWorkerCtx, &createdRequest)).Should(gomega.Succeed()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verify the workload is admitted in the assigned worker cluster", func() { + util.ExpectWorkloadsToBeAdmitted(assignedWorkerCtx, assignedWorkerCluster, wl) + }) + + ginkgo.By("verify TopologyAssignment for the workload in the assigned worker cluster", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(assignedWorkerCluster.Get(assignedWorkerCtx, wlLookupKey, wl)).To(gomega.Succeed()) + g.Expect(wl.Status.Admission).ShouldNot(gomega.BeNil()) + g.Expect(wl.Status.Admission.PodSetAssignments).Should(gomega.HaveLen(1)) + g.Expect(wl.Status.Admission.PodSetAssignments[0].TopologyAssignment).Should(gomega.BeComparableTo( + &kueue.TopologyAssignment{ + Levels: []string{ + corev1.LabelHostname, + }, + Domains: []kueue.TopologyDomainAssignment{ + { + Count: 1, + Values: []string{ + "x1", + }, + }, + }, + }, + )) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + + ginkgo.By("verify DelayedTopologyRequest is marked Ready on manager", func() { + gomega.Eventually(func(g gomega.Gomega) { + managerWl := &kueue.Workload{} + g.Expect(managerTestCluster.client.Get(managerTestCluster.ctx, wlLookupKey, managerWl)).To(gomega.Succeed()) + g.Expect(managerWl.Status.Admission).NotTo(gomega.BeNil()) + g.Expect(managerWl.Status.Admission.PodSetAssignments).To(gomega.HaveLen(1)) + g.Expect(managerWl.Status.Admission.PodSetAssignments[0].DelayedTopologyRequest).To(gomega.Equal(ptr.To(kueue.DelayedTopologyRequestStateReady))) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) +}) diff --git a/test/integration/singlecluster/tas/tas_test.go b/test/integration/singlecluster/tas/tas_test.go index a3d8fd9d975..a6e90e4e2a0 100644 --- a/test/integration/singlecluster/tas/tas_test.go +++ b/test/integration/singlecluster/tas/tas_test.go @@ -212,57 +212,6 @@ var _ = ginkgo.Describe("Topology Aware Scheduling", ginkgo.Ordered, func() { }) }) - ginkgo.When("Negative scenarios for ClusterQueue configuration", func() { - var ( - topology *kueue.Topology - tasFlavor *kueue.ResourceFlavor - clusterQueue *kueue.ClusterQueue - admissionCheck *kueue.AdmissionCheck - ) - - ginkgo.BeforeEach(func() { - topology = utiltestingapi.MakeDefaultTwoLevelTopology("default") - util.MustCreate(ctx, k8sClient, topology) - - tasFlavor = utiltestingapi.MakeResourceFlavor("tas-flavor"). - NodeLabel("node-group", "tas"). - TopologyName("default").Obj() - util.MustCreate(ctx, k8sClient, tasFlavor) - }) - - ginkgo.AfterEach(func() { - util.ExpectObjectToBeDeleted(ctx, k8sClient, clusterQueue, true) - util.ExpectObjectToBeDeleted(ctx, k8sClient, tasFlavor, true) - util.ExpectObjectToBeDeleted(ctx, k8sClient, topology, true) - util.ExpectObjectToBeDeleted(ctx, k8sClient, admissionCheck, true) - }) - - ginkgo.It("should mark TAS ClusterQueue as inactive if used with MultiKueue", func() { - admissionCheck = utiltestingapi.MakeAdmissionCheck("multikueue").ControllerName(kueue.MultiKueueControllerName).Obj() - util.MustCreate(ctx, k8sClient, admissionCheck) - util.SetAdmissionCheckActive(ctx, k8sClient, admissionCheck, metav1.ConditionTrue) - - clusterQueue = utiltestingapi.MakeClusterQueue("cq"). - ResourceGroup( - *utiltestingapi.MakeFlavorQuotas(tasFlavor.Name).Resource(corev1.ResourceCPU, "5").Obj(), - ).AdmissionChecks(kueue.AdmissionCheckReference(admissionCheck.Name)).Obj() - util.MustCreate(ctx, k8sClient, clusterQueue) - - gomega.Eventually(func(g gomega.Gomega) { - var updatedCq kueue.ClusterQueue - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedCq)).To(gomega.Succeed()) - g.Expect(updatedCq.Status.Conditions).Should(gomega.BeComparableTo([]metav1.Condition{ - { - Type: kueue.ClusterQueueActive, - Status: metav1.ConditionFalse, - Reason: "NotSupportedWithTopologyAwareScheduling", - Message: `Can't admit new workloads: TAS is not supported with MultiKueue admission check.`, - }, - }, util.IgnoreConditionTimestampsAndObservedGeneration)) - }, util.Timeout, util.Interval).Should(gomega.Succeed()) - }) - }) - ginkgo.When("Single TAS Resource Flavor", func() { var ( tasFlavor *kueue.ResourceFlavor From e2ecd6a893a0faf8c125ff9cb20acbf6a8198936 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 7 Nov 2025 14:43:02 +0530 Subject: [PATCH 116/119] Fixed DelayedTopologyRequestState enum validation. (#7573) --- apis/kueue/v1beta2/workload_types.go | 6 +-- .../crd/kueue.x-k8s.io_workloads.yaml | 3 ++ .../crd/bases/kueue.x-k8s.io_workloads.yaml | 3 ++ .../webhook/core/workload_test.go | 54 +++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index b956e9cafe0..556e9151b17 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -276,15 +276,15 @@ type PodSetAssignment struct { // DelayedTopologyRequestState indicates the state of the delayed TopologyRequest. // +enum -// +kubebuilder:validation:enum=Pending;Ready +// +kubebuilder:validation:Enum=Pending;Ready // +kubebuilder:validation:MaxLength=7 type DelayedTopologyRequestState string const ( - // This state indicates the delayed TopologyRequest is waiting for determining. + // DelayedTopologyRequestStatePending indicates the delayed TopologyRequest is waiting for determining. DelayedTopologyRequestStatePending DelayedTopologyRequestState = "Pending" - // This state indicates the delayed TopologyRequest is was requested and completed. + // DelayedTopologyRequestStateReady indicates the delayed TopologyRequest is was requested and completed. DelayedTopologyRequestStateReady DelayedTopologyRequestState = "Ready" ) diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index 4659b61c8ca..437d20e290e 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -17200,6 +17200,9 @@ spec: Kueue schedules the second pass of scheduling for each workload with at least one PodSet which has delayedTopologyRequest=true and without topologyAssignment. + enum: + - Pending + - Ready maxLength: 7 type: string flavors: diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index cd69125907f..60039d16bb9 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -18191,6 +18191,9 @@ spec: Kueue schedules the second pass of scheduling for each workload with at least one PodSet which has delayedTopologyRequest=true and without topologyAssignment. + enum: + - Pending + - Ready maxLength: 7 type: string flavors: diff --git a/test/integration/singlecluster/webhook/core/workload_test.go b/test/integration/singlecluster/webhook/core/workload_test.go index ef3f8ced1cf..9ad6638c702 100644 --- a/test/integration/singlecluster/webhook/core/workload_test.go +++ b/test/integration/singlecluster/webhook/core/workload_test.go @@ -366,6 +366,60 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { Obj()). Obj(), utiltesting.BeForbiddenError()), + ginkgo.Entry("Pending delayedTopologyRequest", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PodSets( + *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 3). + Request(corev1.ResourceCPU, "1"). + Obj(), + ). + Obj() + }, + utiltestingapi.MakeAdmission("cluster-queue"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "flv", "3"). + Count(3). + DelayedTopologyRequest(kueue.DelayedTopologyRequestStatePending). + Obj()). + Obj(), + gomega.Succeed()), + ginkgo.Entry("Ready delayedTopologyRequest", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PodSets( + *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 3). + Request(corev1.ResourceCPU, "1"). + Obj(), + ). + Obj() + }, + utiltestingapi.MakeAdmission("cluster-queue"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "flv", "3"). + Count(3). + DelayedTopologyRequest(kueue.DelayedTopologyRequestStateReady). + Obj()). + Obj(), + gomega.Succeed()), + ginkgo.Entry("invalid delayedTopologyRequest", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PodSets( + *utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 3). + Request(corev1.ResourceCPU, "1"). + Obj(), + ). + Obj() + }, + utiltestingapi.MakeAdmission("cluster-queue"). + PodSets(utiltestingapi.MakePodSetAssignment(kueue.DefaultPodSetName). + Assignment(corev1.ResourceCPU, "flv", "3"). + Count(3). + DelayedTopologyRequest("invalid"). + Obj()). + Obj(), + utiltesting.BeInvalidError()), ) ginkgo.DescribeTable("Should have valid values when setting AdmissionCheckState", func(w func() *kueue.Workload, acs kueue.AdmissionCheckState) { From 84b835cee73d59537217ac4adffd1496293a21c9 Mon Sep 17 00:00:00 2001 From: Sabari Narayana Date: Fri, 7 Nov 2025 15:00:56 +0530 Subject: [PATCH 117/119] Add documentation for Kubeflow Trainer v2 TrainJob integration with Kueue (#7533) * Docs for kf trainer v2 Signed-off-by: narayanasabari * updated on trainjobs Signed-off-by: narayanasabari * Updated docs on andreyvelich feedback * update render for overview * Restructure docs: Move TrainJob to dedicated section Signed-off-by: narayanasabari * update docker image for trainjob Signed-off-by: narayanasabari * File Structure and batch user updated for trainjobs * Update docker image for ClusterTrainingRuntime Signed-off-by: narayanasabari * Added docs for multi-cluster trainjobs Signed-off-by: narayanasabari --------- Signed-off-by: narayanasabari --- site/content/en/docs/tasks/_index.md | 4 +- .../en/docs/tasks/run/kubeflow/_index.md | 32 ++- .../en/docs/tasks/run/multikueue/kubeflow.md | 14 +- .../en/docs/tasks/run/multikueue/trainjob.md | 103 +++++++ site/content/en/docs/tasks/run/trainjobs.md | 254 ++++++++++++++++++ 5 files changed, 385 insertions(+), 22 deletions(-) create mode 100644 site/content/en/docs/tasks/run/multikueue/trainjob.md create mode 100644 site/content/en/docs/tasks/run/trainjobs.md diff --git a/site/content/en/docs/tasks/_index.md b/site/content/en/docs/tasks/_index.md index f2d62f98e0f..52c2d8b0d90 100755 --- a/site/content/en/docs/tasks/_index.md +++ b/site/content/en/docs/tasks/_index.md @@ -36,7 +36,9 @@ batch user is a researcher, AI/ML engineer, data scientist, among others. As a batch user, you can learn how to: - [Run a Kueue managed batch/Job](run/jobs). -- [Run a Kueue managed Kubeflow Job](run/kubeflow). +- [Run a Kueue managed Kubeflow TrainJob (v2)](run/trainjobs). + Kueue supports Kubeflow Trainer v2's unified TrainJob API. +- [Run a Kueue managed Kubeflow Job (v1)](run/kubeflow). Kueue supports MPIJob v2beta1, PyTorchJob, TFJob, XGBoostJob and PaddleJob. - [Run a Kueue managed KubeRay RayJob](run/rayjobs). - [Run a Kueue managed KubeRay RayCluster](run/rayclusters). diff --git a/site/content/en/docs/tasks/run/kubeflow/_index.md b/site/content/en/docs/tasks/run/kubeflow/_index.md index bf6cea017db..23916cd6546 100755 --- a/site/content/en/docs/tasks/run/kubeflow/_index.md +++ b/site/content/en/docs/tasks/run/kubeflow/_index.md @@ -1,27 +1,25 @@ --- - -title: "Run with Kubeflow" -linkTitle: "Kubeflow Jobs" -weight: 6 +title: "Kubeflow Jobs (v1)" +linkTitle: "Kubeflow Jobs (v1)" +weight: 7 date: 2023-08-23 description: > - How to run Kueue with Kubeflow + Run Kueue managed Kubeflow Trainer v1 Jobs no_list: true --- -The tasks below show you how to run Kueue managed Kubeflow Jobs. - -### [MPI Operator](https://github.com/kubeflow/mpi-operator) Integration -- [Run a Kueue managed Kubeflow MPIJob](/docs/tasks/run_kubeflow_jobs/run_mpijobs). - -### [Trainer](https://github.com/kubeflow/trainer) Integration +The tasks below show you how to run Kueue managed Kubeflow Trainer v1 Jobs. {{% alert title="Note" color="primary" %}} -Kueue supports only traditional Jobs served until Trainer v1.9.x and does not support new TrainJob. +**Kubeflow Trainer v2 is now available:** Consider using [TrainJob](/docs/tasks/run/trainjobs/) which provides a unified API for all training frameworks. Trainer v1 APIs (PyTorchJob, TFJob, etc.) are stable and production-ready. {{% /alert %}} -- [Run a Kueue managed Kubeflow PyTorchJob](/docs/tasks/run_kubeflow_jobs/run_pytorchjobs). -- [Run a Kueue managed Kubeflow TFJob](/docs/tasks/run_kubeflow_jobs/run_tfjobs). -- [Run a Kueue managed Kubeflow XGBoostJob](/docs/tasks/run_kubeflow_jobs/run_xgboostjobs). -- [Run a Kueue managed kubeflow PaddleJob](/docs/tasks/run_kubeflow_jobs/run_paddlejobs). -- [Run a Kueue managed kubeflow JAXJob](/docs/tasks/run_kubeflow_jobs/run_jaxjobs). +### [Trainer v1](https://github.com/kubeflow/trainer) Integration +- [Run a Kueue managed Kubeflow PyTorchJob](/docs/tasks/run/kubeflow/pytorchjobs/). +- [Run a Kueue managed Kubeflow TFJob](/docs/tasks/run/kubeflow/tfjobs/). +- [Run a Kueue managed Kubeflow XGBoostJob](/docs/tasks/run/kubeflow/xgboostjobs/). +- [Run a Kueue managed Kubeflow PaddleJob](/docs/tasks/run/kubeflow/paddlejobs/). +- [Run a Kueue managed Kubeflow JAXJob](/docs/tasks/run/kubeflow/jaxjobs/). + +### [MPI Operator](https://github.com/kubeflow/mpi-operator) Integration +- [Run a Kueue managed Kubeflow MPIJob](/docs/tasks/run/kubeflow/mpijobs/). diff --git a/site/content/en/docs/tasks/run/multikueue/kubeflow.md b/site/content/en/docs/tasks/run/multikueue/kubeflow.md index 9f8884cca55..60e3c8a9f4d 100644 --- a/site/content/en/docs/tasks/run/multikueue/kubeflow.md +++ b/site/content/en/docs/tasks/run/multikueue/kubeflow.md @@ -1,12 +1,18 @@ ---- -title: "Run Kubeflow Jobs in Multi-Cluster" -linkTitle: "Kubeflow" + --- +title: "Run Kubeflow Jobs (v1) in Multi-Cluster" +linkTitle: "Kubeflow (v1)" weight: 4 date: 2024-09-25 description: > - Run a MultiKueue scheduled Kubeflow Jobs. + Run MultiKueue scheduled Kubeflow Trainer v1 Jobs (PyTorchJob, TFJob, etc.). --- +{{% alert title="Note" color="primary" %}} +This page covers legacy Kubeflow Trainer v1 jobs (PyTorchJob, TFJob, XGBoostJob, etc.). + +For Kubeflow Trainer v2 TrainJob, see [Run TrainJobs in Multi-Cluster](/docs/tasks/run/multikueue/trainjob/). +{{% /alert %}} + ## Before you begin Check the [MultiKueue installation guide](/docs/tasks/manage/setup_multikueue) on how to properly setup MultiKueue clusters. diff --git a/site/content/en/docs/tasks/run/multikueue/trainjob.md b/site/content/en/docs/tasks/run/multikueue/trainjob.md new file mode 100644 index 00000000000..bf4a452e481 --- /dev/null +++ b/site/content/en/docs/tasks/run/multikueue/trainjob.md @@ -0,0 +1,103 @@ +--- +title: "Run TrainJobs in Multi-Cluster" +linkTitle: "TrainJob" +weight: 3 +date: 2025-11-06 +description: > + Run a MultiKueue scheduled TrainJob from Kubeflow Trainer v2. +--- + +## Before you begin + +Check the [MultiKueue installation guide](/docs/tasks/manage/setup_multikueue) on how to properly setup MultiKueue clusters. + +For the ease of setup and use we recommend using at least Kueue v0.11.0 and for Kubeflow Trainer at least v2.0.0. + +See [Trainer Installation](https://www.kubeflow.org/docs/components/trainer/operator-guides/installation/) for installation and configuration details of Kubeflow Trainer v2. + +{{% alert title="Note" color="primary" %}} +TrainJob is part of Kubeflow Trainer v2, which introduces a unified API for all training frameworks. It replaces the framework-specific job types (PyTorchJob, TFJob, etc.) from Trainer v1. + +For legacy Kubeflow Jobs (v1), see [Run Kubeflow Jobs in Multi-Cluster](/docs/tasks/run/multikueue/kubeflow/). +{{% /alert %}} + +## MultiKueue integration + +Once the setup is complete you can test it by running a TrainJob [`sample-trainjob`](/docs/tasks/run/trainjobs/#using-clustertrainingruntime). + +{{% alert title="Note" color="primary" %}} +Kueue defaults the `spec.managedBy` field to `kueue.x-k8s.io/multikueue` on the management cluster for TrainJob. + +This allows the Trainer to ignore the Jobs managed by MultiKueue on the management cluster, and in particular skip Pod creation. + +The pods are created and the actual computation will happen on the mirror copy of the TrainJob on the selected worker cluster. +The mirror copy of the TrainJob does not have the field set. +{{% /alert %}} + +## ClusterTrainingRuntime and TrainingRuntime + +TrainJobs in MultiKueue environments work with both ClusterTrainingRuntime and TrainingRuntime: + +- **ClusterTrainingRuntime**: Must be installed on all worker clusters. The runtime definition should be consistent across clusters. +- **TrainingRuntime**: Must be installed in the corresponding namespaces on the worker clusters where TrainJobs will run. + +Ensure that the runtime configurations referenced by your TrainJobs are available on the target worker clusters before dispatching the jobs. + +## Example + +Here's a complete example of running a TrainJob with MultiKueue: + +1. **Create a ClusterTrainingRuntime** on all clusters (management and workers): + +```yaml +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: ClusterTrainingRuntime +metadata: + name: torch-distributed + labels: + trainer.kubeflow.org/framework: torch +spec: + mlPolicy: + numNodes: 1 + torch: + numProcPerNode: auto + template: + spec: + replicatedJobs: + - name: node + template: + metadata: + labels: + trainer.kubeflow.org/trainjob-ancestor-step: trainer + spec: + template: + spec: + containers: + - name: node + image: pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime +``` + +2. **Create a MultiKueue-enabled TrainJob** on the management cluster: + +```yaml +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: TrainJob +metadata: + name: pytorch-multikueue + namespace: default + labels: + kueue.x-k8s.io/queue-name: multikueue-queue +spec: + runtimeRef: + name: torch-distributed + kind: ClusterTrainingRuntime + trainer: + numNodes: 2 + resourcesPerNode: + requests: + cpu: "4" + memory: "8Gi" + nvidia.com/gpu: "1" +``` + +The TrainJob will be dispatched to a worker cluster based on the MultiKueue configuration and available resources. The training pods will be created and executed on the selected worker cluster. diff --git a/site/content/en/docs/tasks/run/trainjobs.md b/site/content/en/docs/tasks/run/trainjobs.md new file mode 100644 index 00000000000..5484b15c650 --- /dev/null +++ b/site/content/en/docs/tasks/run/trainjobs.md @@ -0,0 +1,254 @@ +--- +title: "Run A TrainJob" +linkTitle: "TrainJobs" +date: 2024-11-05 +weight: 6 +description: > + Run a Kueue scheduled TrainJob from Kubeflow Trainer v2 +--- + +This page shows how to leverage Kueue's scheduling and resource management capabilities when running [Kubeflow Trainer](https://www.kubeflow.org/docs/components/trainer/) TrainJobs. + +Kubeflow Trainer v2 introduces the `TrainJob` API that works seamlessly with Kueue for batch scheduling and resource management. TrainJobs can be configured to use either: + +- **ClusterTrainingRuntime**: Cluster-scoped training runtimes that can be used across all namespaces +- **TrainingRuntime**: Namespace-scoped training runtimes that are only available within a specific namespace + +Kueue manages TrainJobs by scheduling their underlying jobs according to available quota and priority. + +This guide is for [batch users](/docs/tasks#batch-user) that have a basic understanding of Kueue. For more information, see [Kueue's overview](/docs/overview). + +## Before you begin + +1. Check [administer cluster quotas](/docs/tasks/manage/administer_cluster_quotas) for details on the initial cluster setup. + +2. Install Kubeflow Trainer v2. Check [the Trainer installation guide](https://www.kubeflow.org/docs/components/trainer/operator-guides/installation/). + + **Note**: The minimum required Trainer version is v2.0.0. + +3. Enable TrainJob integration in Kueue. You can [modify kueue configurations from installed releases](/docs/installation#install-a-custom-configured-released-version) to include TrainJobs as an allowed workload. + +## TrainJob definition + +### a. Queue selection + +The target [local queue](/docs/concepts/local_queue) should be specified in the `metadata.labels` section of the TrainJob configuration: + +```yaml +metadata: + labels: + kueue.x-k8s.io/queue-name: user-queue +``` + +### b. Suspend field + +By default, Kueue will set `suspend` to true via webhook and unsuspend it when the TrainJob is admitted: + +```yaml +spec: + suspend: true +``` + +## Using ClusterTrainingRuntime + +ClusterTrainingRuntimes are cluster-scoped resources that define training configurations accessible across all namespaces. They are typically created by platform administrators. For more details on ClusterTrainingRuntime configuration, see the [Kubeflow Runtime Guide](https://www.kubeflow.org/docs/components/trainer/operator-guides/runtime/). + +### Example: PyTorch Distributed Training with ClusterTrainingRuntime + +First, create a ClusterTrainingRuntime for PyTorch distributed training: + +```yaml +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: ClusterTrainingRuntime +metadata: + name: torch-distributed + labels: + trainer.kubeflow.org/framework: torch +spec: + mlPolicy: + numNodes: 1 + torch: + numProcPerNode: auto + template: + spec: + replicatedJobs: + - name: node + template: + metadata: + labels: + trainer.kubeflow.org/trainjob-ancestor-step: trainer + spec: + template: + spec: + containers: + - name: node + image: pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime +``` + +Now, create a TrainJob that references this ClusterTrainingRuntime and will be scheduled by Kueue: + +```yaml +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: TrainJob +metadata: + name: pytorch-distributed + namespace: default + labels: + kueue.x-k8s.io/queue-name: user-queue +spec: + runtimeRef: + name: torch-distributed + kind: ClusterTrainingRuntime + trainer: + numNodes: 2 + resourcesPerNode: + requests: + cpu: "4" + memory: "8Gi" + nvidia.com/gpu: "1" +``` + +**Key Points:** +- The `kueue.x-k8s.io/queue-name` label is the only Kueue-specific addition needed +- The `runtimeRef` points to the `ClusterTrainingRuntime` named `torch-distributed` +- This example uses the default executable image from the runtime definition +- Kueue will manage the lifecycle and admission of this TrainJob based on available quota +- For custom training code, see the [Kubeflow Trainer examples](https://github.com/kubeflow/trainer/tree/master/examples) + +## Using TrainingRuntime (Namespace-scoped) + +TrainingRuntimes are namespace-scoped resources that provide more granular control per namespace. They are useful when different teams need customized training configurations. + +### Example: Custom PyTorch Training with TrainingRuntime + +Create a namespace-scoped TrainingRuntime: + +```yaml +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: TrainingRuntime +metadata: + name: torch-custom + namespace: team-a +spec: + mlPolicy: + numNodes: 1 + torch: + numProcPerNode: auto + template: + spec: + replicatedJobs: + - name: node + template: + metadata: + labels: + trainer.kubeflow.org/trainjob-ancestor-step: trainer + spec: + template: + spec: + containers: + - name: trainer + image: pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime + env: + - name: CUSTOM_ENV + value: "team-a-value" +``` + +Create a TrainJob that uses this namespace-scoped runtime: + +```yaml +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: TrainJob +metadata: + name: pytorch-custom + namespace: team-a + labels: + kueue.x-k8s.io/queue-name: team-a-queue +spec: + runtimeRef: + name: torch-custom + kind: TrainingRuntime + apiGroup: trainer.kubeflow.org + trainer: + image: docker.io/team-a/custom-training:latest + numNodes: 1 + resourcesPerNode: + requests: + cpu: "2" + memory: "4Gi" +``` + +**Key Points:** +- The TrainingRuntime is created in the same namespace as the TrainJob (`team-a`) +- The `runtimeRef` specifies `kind: TrainingRuntime` to use the namespace-scoped runtime +- Each namespace can have its own customized runtimes with different configurations + +## Using Workload Priority + +To prioritize TrainJobs, use Kueue's workload priority classes. See [Run job with WorkloadPriority](/docs/tasks/manage/run_job_with_workload_priority) for details on configuring and using workload priority classes. + +TrainJobs use the same priority mechanism as other Kueue workloads via the `kueue.x-k8s.io/priority-class` label. + +## LLM Fine-Tuning with Kueue + +Kubeflow Trainer v2 supports LLM fine-tuning with TorchTune and DeepSpeed. For comprehensive examples, see: + +- [Fine-tune Llama-3.2-1B with Alpaca Dataset](https://github.com/kubeflow/trainer/blob/master/examples/torchtune/llama3_2/alpaca-trainjob-yaml.ipynb) +- [Fine-tune Qwen2.5-1.5B with Alpaca Dataset](https://github.com/kubeflow/trainer/blob/master/examples/torchtune/qwen2_5/qwen2.5-1.5B-with-alpaca.ipynb) +- [T5 Fine-Tuning with DeepSpeed](https://github.com/kubeflow/trainer/blob/master/examples/deepspeed/text-summarization/T5-Fine-Tuning.ipynb) + +To use Kueue scheduling with these examples, add the queue label to your TrainJob: + +```yaml +metadata: + labels: + kueue.x-k8s.io/queue-name: gpu-queue # Add this label for Kueue scheduling +spec: + runtimeRef: + name: torchtune-llama3.2-1b + kind: ClusterTrainingRuntime + # ... rest of the TrainJob spec as shown in Kubeflow examples +``` + +## Differences from Kubeflow Training Operator V1 + +{{% alert title="Important" color="warning" %}} +Kubeflow Trainer v2 introduces a new API that is not compatible with Training Operator v1 APIs (PyTorchJob, TFJob, etc.). The key differences are: + +- **Unified API**: TrainJob replaces framework-specific CRDs like PyTorchJob, TFJob +- **Runtime-based**: Training configurations are defined in reusable Runtimes +- **Built on JobSet**: Uses Kubernetes JobSet as the underlying infrastructure +- **Better integration**: Native support for Kueue scheduling from the start + +For migration guidance, refer to the [Kubeflow Trainer documentation](https://www.kubeflow.org/docs/components/trainer/operator-guides/migration/). +{{% /alert %}} + +## Best Practices + +1. **Use ClusterTrainingRuntimes for common patterns**: Create cluster-scoped runtimes for frequently used training configurations +2. **Use TrainingRuntimes for team-specific needs**: Leverage namespace-scoped runtimes for customizations per team +3. **Set appropriate resource requests**: Ensure your TrainJob resource requests match the ResourceFlavor in your ClusterQueue +4. **Monitor quota usage**: Use `kubectl get clusterqueue` to track resource utilization +5. **Use priority classes**: Assign priorities to TrainJobs to ensure critical workloads are scheduled first +6. **Test with small configurations**: Before scaling up, test your TrainJob configuration with minimal resources + +## Additional Resources + +- [Kubeflow Trainer Documentation](https://www.kubeflow.org/docs/components/trainer/) +- [Kubeflow Runtime Guide](https://www.kubeflow.org/docs/components/trainer/operator-guides/runtime/) +- [Fine-tune LLMs with TorchTune](https://www.kubeflow.org/docs/components/trainer/user-guides/builtin-trainer/torchtune/) +- [Kueue Concepts](/docs/concepts/) +- [Run job with WorkloadPriority](/docs/tasks/manage/run_job_with_workload_priority) +- [Monitor Pending Workloads](/docs/tasks/manage/monitor_pending_workloads) +- [Troubleshooting](/docs/tasks/troubleshooting) +- [Kubeflow Python SDK](https://github.com/kubeflow/sdk/) + +## Troubleshooting + +For general troubleshooting guidance, see the [Kueue troubleshooting guide](/docs/tasks/troubleshooting). + +For TrainJob-specific issues, verify that the referenced ClusterTrainingRuntime or TrainingRuntime exists: + +```bash +kubectl get clustertrainingruntime +kubectl get trainingruntime -n +``` From 71cd4e1ac60c36776b7089e96cf65934e017119b Mon Sep 17 00:00:00 2001 From: walker <51307483+olderTaoist@users.noreply.github.com> Date: Fri, 7 Nov 2025 19:18:56 +0800 Subject: [PATCH 118/119] JobReconciler don't update PodsReady condition timely (#7364) --- pkg/controller/jobframework/reconciler.go | 1 + .../jobframework/reconciler_test.go | 139 ++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index ef7a5a6c8ab..c1830f7bc9a 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -522,6 +522,7 @@ func (r *JobReconciler) ReconcileGenericJob(ctx context.Context, req ctrl.Reques err := workload.UpdateStatus(ctx, r.client, wl, condition.Type, condition.Status, condition.Reason, condition.Message, constants.JobControllerName, r.clock) if err != nil { log.Error(err, "Updating workload status") + return ctrl.Result{}, client.IgnoreNotFound(err) } // update the metrics only when PodsReady condition status is true if condition.Status == metav1.ConditionTrue { diff --git a/pkg/controller/jobframework/reconciler_test.go b/pkg/controller/jobframework/reconciler_test.go index 26592bd7d72..315196b5210 100644 --- a/pkg/controller/jobframework/reconciler_test.go +++ b/pkg/controller/jobframework/reconciler_test.go @@ -17,6 +17,8 @@ limitations under the License. package jobframework_test import ( + "context" + "errors" "testing" "time" @@ -28,6 +30,7 @@ import ( appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/clock" @@ -35,6 +38,7 @@ import ( "k8s.io/utils/ptr" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/jobset/api/jobset/v1alpha2" configapi "sigs.k8s.io/kueue/apis/config/v1beta2" @@ -42,6 +46,7 @@ import ( mocks "sigs.k8s.io/kueue/internal/mocks/controller/jobframework" "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/core/indexer" + "sigs.k8s.io/kueue/pkg/controller/jobs/job" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/kubeversion" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" @@ -709,3 +714,137 @@ func TestProcessOptions(t *testing.T) { }) } } + +func TestReconcileGenericJobWithWaitForPodsReady(t *testing.T) { + var ( + testLocalQueueName = kueue.LocalQueueName("default") + testGVK = batchv1.SchemeGroupVersion.WithKind("Job") + ) + testCases := map[string]struct { + workload *kueue.Workload + job GenericJob + wantError error + }{ + "update podready condition failed": { + workload: utiltestingapi.MakeWorkload("job-test-job-podready-fail", metav1.NamespaceDefault). + Finalizers(kueue.ResourceInUseFinalizerName). + Label(constants.JobUIDLabel, "test-job-podready-fail"). + ControllerReference(testGVK, "test-job-podready-fail", "test-job-podready-fail"). + Queue(testLocalQueueName). + PodSets(*utiltestingapi.MakePodSet("main", 1).Obj()). + Conditions(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(time.Now()), + }, metav1.Condition{ + Type: kueue.WorkloadPodsReady, + Status: metav1.ConditionFalse, + Reason: kueue.WorkloadWaitForStart, + Message: "Not all pods are ready or succeeded", + LastTransitionTime: metav1.NewTime(time.Now()), + }). + Admission(&kueue.Admission{ + ClusterQueue: "default-cq", + }). + Obj(), + job: (*job.Job)(testingjob.MakeJob("test-job-podready-fail", metav1.NamespaceDefault). + UID("test-job-podready-fail"). + Label(constants.QueueLabel, string(testLocalQueueName)). + Parallelism(1). + Suspend(false). + Containers(corev1.Container{ + Name: "c", + Resources: corev1.ResourceRequirements{ + Requests: make(corev1.ResourceList), + }, + }). + Ready(1). + Obj()), + wantError: apierrors.NewInternalError(errors.New("failed calling webhook")), + }, + "update podready condition success": { + workload: utiltestingapi.MakeWorkload("job-test-job-podready-success", metav1.NamespaceDefault). + Finalizers(kueue.ResourceInUseFinalizerName). + Label(constants.JobUIDLabel, "job-test-job-podready-success"). + ControllerReference(testGVK, "test-job-podready-success", "test-job-podready-success"). + Queue(testLocalQueueName). + PodSets(*utiltestingapi.MakePodSet("main", 1).Obj()). + Conditions(metav1.Condition{ + Type: kueue.WorkloadAdmitted, + Status: metav1.ConditionTrue, + Reason: "Admitted", + Message: "The workload is admitted", + LastTransitionTime: metav1.NewTime(time.Now()), + }, metav1.Condition{ + Type: kueue.WorkloadPodsReady, + Status: metav1.ConditionFalse, + Reason: kueue.WorkloadWaitForStart, + Message: "Not all pods are ready or succeeded", + LastTransitionTime: metav1.NewTime(time.Now()), + }). + Admission(&kueue.Admission{ + ClusterQueue: "default-cq", + }). + Obj(), + job: (*job.Job)(testingjob.MakeJob("test-job-podready-success", metav1.NamespaceDefault). + UID("test-job-podready-success"). + Label(constants.QueueLabel, string(testLocalQueueName)). + Parallelism(1). + Suspend(false). + Containers(corev1.Container{ + Name: "c", + Resources: corev1.ResourceRequirements{ + Requests: make(corev1.ResourceList), + }, + }). + Ready(1). + Obj()), + wantError: nil, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, _ := utiltesting.ContextWithLog(t) + managedNamespace := utiltesting.MakeNamespaceWrapper(metav1.NamespaceDefault). + Label("managed-by-kueue", "true"). + Obj() + builder := utiltesting.NewClientBuilder(batchv1.AddToScheme, kueue.AddToScheme). + WithObjects(tc.workload, tc.job.Object(), managedNamespace). + WithStatusSubresource(tc.workload, tc.job.Object()). + WithIndex(&kueue.Workload{}, indexer.OwnerReferenceIndexKey(testGVK), indexer.WorkloadOwnerIndexFunc(testGVK)). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourcePatch: func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + if _, ok := obj.(*kueue.Workload); ok && subResourceName == "status" && tc.wantError != nil { + return tc.wantError + } + return utiltesting.TreatSSAAsStrategicMerge(ctx, client, subResourceName, obj, patch, opts...) + }, + }) + + cl := builder.Build() + + testStartTime := time.Now().Truncate(time.Second) + + fakeClock := testingclock.NewFakeClock(testStartTime) + options := []Option{ + WithClock(nil, fakeClock), + WithWaitForPodsReady(&configapi.WaitForPodsReady{ + Enable: true, + }), + } + recorder := &utiltesting.EventRecorder{} + r := NewReconciler(cl, recorder, options...) + _, err := r.ReconcileGenericJob(ctx, controllerruntime.Request{ + NamespacedName: types.NamespacedName{ + Name: tc.job.Object().GetName(), + Namespace: tc.job.Object().GetNamespace(), + }}, tc.job) + if !errors.Is(err, tc.wantError) { + t.Errorf("unexpected reconcile error want %s got %s)", tc.wantError, err) + } + }) + } +} From 7d2f0ffb10b7de6d61673f68ddb705a99b7d5569 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Fri, 7 Nov 2025 17:52:52 +0530 Subject: [PATCH 119/119] v1beta2: change the API for Workload's spec.priorityClassSource (#7540) * v1beta2: change the API for Workload's spec.priorityClassSource * Use defaultObjectMeta. --- apis/kueue/v1beta1/workload_conversion.go | 24 ++ .../kueue/v1beta1/workload_conversion_test.go | 124 ++++++++- apis/kueue/v1beta1/workload_types.go | 5 + apis/kueue/v1beta1/zz_generated.conversion.go | 37 +-- apis/kueue/v1beta2/workload_types.go | 109 ++++++-- apis/kueue/v1beta2/zz_generated.deepcopy.go | 20 ++ .../crd/kueue.x-k8s.io_workloads.yaml | 85 ++++--- .../kueue/v1beta2/priorityclassref.go | 60 +++++ .../kueue/v1beta2/workloadspec.go | 29 +-- client-go/applyconfiguration/utils.go | 2 + cmd/importer/pod/import.go | 3 +- .../crd/bases/kueue.x-k8s.io_workloads.yaml | 113 +++++---- pkg/cache/queue/cluster_queue_test.go | 8 +- pkg/constants/constants.go | 7 +- pkg/controller/core/workload_controller.go | 17 +- pkg/controller/jobframework/reconciler.go | 20 +- .../jobs/job/job_controller_test.go | 40 +-- .../jobs/jobset/jobset_controller_test.go | 10 +- .../leaderworkerset_reconciler_test.go | 4 +- .../jobs/mpijob/mpijob_controller_test.go | 16 +- pkg/scheduler/scheduler.go | 18 +- pkg/util/priority/priority.go | 28 +-- pkg/util/priority/priority_test.go | 71 +++--- pkg/util/testing/v1beta2/wrappers.go | 21 +- pkg/workload/workload.go | 20 +- .../en/docs/reference/kueue.v1beta2.md | 101 ++++++-- .../core/workload_controller_test.go | 5 +- .../jobs/job/job_controller_test.go | 4 - .../webhook/core/workload_test.go | 237 ++++++++++-------- test/util/util.go | 13 +- 30 files changed, 803 insertions(+), 448 deletions(-) create mode 100644 client-go/applyconfiguration/kueue/v1beta2/priorityclassref.go diff --git a/apis/kueue/v1beta1/workload_conversion.go b/apis/kueue/v1beta1/workload_conversion.go index 55d407ec5bb..729b7f8637e 100644 --- a/apis/kueue/v1beta1/workload_conversion.go +++ b/apis/kueue/v1beta1/workload_conversion.go @@ -45,3 +45,27 @@ func Convert_v1beta1_WorkloadStatus_To_v1beta2_WorkloadStatus(in *WorkloadStatus out.AccumulatedPastExecutionTimeSeconds = in.AccumulatedPastExexcutionTimeSeconds return autoConvert_v1beta1_WorkloadStatus_To_v1beta2_WorkloadStatus(in, out, s) } + +func Convert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(in *WorkloadSpec, out *v1beta2.WorkloadSpec, s conversionapi.Scope) error { + switch in.PriorityClassSource { + case WorkloadPriorityClassSource: + out.PriorityClassRef = v1beta2.NewWorkloadPriorityClassRef(in.PriorityClassName) + case PodPriorityClassSource: + out.PriorityClassRef = v1beta2.NewPodPriorityClassRef(in.PriorityClassName) + } + return autoConvert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(in, out, s) +} + +func Convert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(in *v1beta2.WorkloadSpec, out *WorkloadSpec, s conversionapi.Scope) error { + if in.PriorityClassRef != nil { + switch { + case in.PriorityClassRef.Group == v1beta2.WorkloadPriorityClassGroup && in.PriorityClassRef.Kind == v1beta2.WorkloadPriorityClassKind: + out.PriorityClassSource = WorkloadPriorityClassSource + out.PriorityClassName = in.PriorityClassRef.Name + case in.PriorityClassRef.Group == v1beta2.PodPriorityClassGroup && in.PriorityClassRef.Kind == v1beta2.PodPriorityClassKind: + out.PriorityClassSource = PodPriorityClassSource + out.PriorityClassName = in.PriorityClassRef.Name + } + } + return autoConvert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(in, out, s) +} diff --git a/apis/kueue/v1beta1/workload_conversion_test.go b/apis/kueue/v1beta1/workload_conversion_test.go index 6c90084e964..e9b9bdd0a10 100644 --- a/apis/kueue/v1beta1/workload_conversion_test.go +++ b/apis/kueue/v1beta1/workload_conversion_test.go @@ -106,6 +106,48 @@ func TestWorkloadConvertTo(t *testing.T) { }, }, }, + "with PodPriorityClassSource": { + input: &Workload{ + ObjectMeta: defaultObjectMeta, + Spec: WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassSource: PodPriorityClassSource, + PriorityClassName: "low", + }, + }, + expected: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassRef: &v1beta2.PriorityClassRef{ + Group: v1beta2.PodPriorityClassGroup, + Kind: v1beta2.PodPriorityClassKind, + Name: "low", + }, + }, + }, + }, + "with WorkloadPriorityClassSource": { + input: &Workload{ + ObjectMeta: defaultObjectMeta, + Spec: WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassSource: WorkloadPriorityClassSource, + PriorityClassName: "low", + }, + }, + expected: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassRef: &v1beta2.PriorityClassRef{ + Group: v1beta2.WorkloadPriorityClassGroup, + Kind: v1beta2.WorkloadPriorityClassKind, + Name: "low", + }, + }, + }, + }, } for name, tc := range testCases { @@ -201,6 +243,48 @@ func TestWorkloadConvertFrom(t *testing.T) { }, }, }, + "with PodPriorityClassRef": { + input: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassRef: &v1beta2.PriorityClassRef{ + Group: v1beta2.PodPriorityClassGroup, + Kind: v1beta2.PodPriorityClassKind, + Name: "low", + }, + }, + }, + expected: &Workload{ + ObjectMeta: defaultObjectMeta, + Spec: WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassSource: PodPriorityClassSource, + PriorityClassName: "low", + }, + }, + }, + "with WorkloadPriorityClassRef": { + input: &v1beta2.Workload{ + ObjectMeta: defaultObjectMeta, + Spec: v1beta2.WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassRef: &v1beta2.PriorityClassRef{ + Group: v1beta2.WorkloadPriorityClassGroup, + Kind: v1beta2.WorkloadPriorityClassKind, + Name: "low", + }, + }, + }, + expected: &Workload{ + ObjectMeta: defaultObjectMeta, + Spec: WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassSource: WorkloadPriorityClassSource, + PriorityClassName: "low", + }, + }, + }, } for name, tc := range testCases { @@ -217,15 +301,17 @@ func TestWorkloadConvertFrom(t *testing.T) { } func TestWorkloadConversion_RoundTrip(t *testing.T) { + defaultObjectMeta := metav1.ObjectMeta{ + Name: "test-workload", + Namespace: "default", + } + testCases := map[string]struct { v1beta1Obj *Workload }{ "complete Workload with AccumulatedPastExexcutionTimeSeconds": { v1beta1Obj: &Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-workload", - Namespace: "default", - }, + ObjectMeta: defaultObjectMeta, Status: WorkloadStatus{ Conditions: []metav1.Condition{ { @@ -240,10 +326,7 @@ func TestWorkloadConversion_RoundTrip(t *testing.T) { }, "Workload with nil AccumulatedPastExexcutionTimeSeconds": { v1beta1Obj: &Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "simple-workload", - Namespace: "test-ns", - }, + ObjectMeta: defaultObjectMeta, Status: WorkloadStatus{ AccumulatedPastExexcutionTimeSeconds: nil, }, @@ -251,15 +334,32 @@ func TestWorkloadConversion_RoundTrip(t *testing.T) { }, "Workload with zero AccumulatedPastExexcutionTimeSeconds": { v1beta1Obj: &Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: "zero-workload", - Namespace: "test-ns", - }, + ObjectMeta: defaultObjectMeta, Status: WorkloadStatus{ AccumulatedPastExexcutionTimeSeconds: ptr.To[int32](0), }, }, }, + "Workload with PodPriorityClassSource": { + v1beta1Obj: &Workload{ + ObjectMeta: defaultObjectMeta, + Spec: WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassSource: PodPriorityClassSource, + PriorityClassName: "low", + }, + }, + }, + "Workload with WorkloadPriorityClassSource": { + v1beta1Obj: &Workload{ + ObjectMeta: defaultObjectMeta, + Spec: WorkloadSpec{ + Priority: ptr.To[int32](100), + PriorityClassSource: WorkloadPriorityClassSource, + PriorityClassName: "low", + }, + }, + }, } for name, tc := range testCases { diff --git a/apis/kueue/v1beta1/workload_types.go b/apis/kueue/v1beta1/workload_types.go index 1c1d44696a6..86ab44baa3b 100644 --- a/apis/kueue/v1beta1/workload_types.go +++ b/apis/kueue/v1beta1/workload_types.go @@ -23,6 +23,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + WorkloadPriorityClassSource = "kueue.x-k8s.io/workloadpriorityclass" + PodPriorityClassSource = "scheduling.k8s.io/priorityclass" +) + // WorkloadSpec defines the desired state of Workload // +kubebuilder:validation:XValidation:rule="has(self.priorityClassName) ? has(self.priority) : true", message="priority should not be nil when priorityClassName is set" type WorkloadSpec struct { diff --git a/apis/kueue/v1beta1/zz_generated.conversion.go b/apis/kueue/v1beta1/zz_generated.conversion.go index a95e8f2194e..3ae4e022069 100644 --- a/apis/kueue/v1beta1/zz_generated.conversion.go +++ b/apis/kueue/v1beta1/zz_generated.conversion.go @@ -738,16 +738,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*WorkloadSpec)(nil), (*v1beta2.WorkloadSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(a.(*WorkloadSpec), b.(*v1beta2.WorkloadSpec), scope) - }); err != nil { - return err - } - if err := s.AddGeneratedConversionFunc((*v1beta2.WorkloadSpec)(nil), (*WorkloadSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(a.(*v1beta2.WorkloadSpec), b.(*WorkloadSpec), scope) - }); err != nil { - return err - } if err := s.AddConversionFunc((*AdmissionCheckSpec)(nil), (*v1beta2.AdmissionCheckSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_AdmissionCheckSpec_To_v1beta2_AdmissionCheckSpec(a.(*AdmissionCheckSpec), b.(*v1beta2.AdmissionCheckSpec), scope) }); err != nil { @@ -768,6 +758,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*WorkloadSpec)(nil), (*v1beta2.WorkloadSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(a.(*WorkloadSpec), b.(*v1beta2.WorkloadSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*WorkloadStatus)(nil), (*v1beta2.WorkloadStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_WorkloadStatus_To_v1beta2_WorkloadStatus(a.(*WorkloadStatus), b.(*v1beta2.WorkloadStatus), scope) }); err != nil { @@ -783,6 +778,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta2.WorkloadSpec)(nil), (*WorkloadSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(a.(*v1beta2.WorkloadSpec), b.(*WorkloadSpec), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta2.WorkloadStatus)(nil), (*WorkloadStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_WorkloadStatus_To_v1beta1_WorkloadStatus(a.(*v1beta2.WorkloadStatus), b.(*WorkloadStatus), scope) }); err != nil { @@ -2616,35 +2616,24 @@ func Convert_v1beta2_WorkloadSchedulingStatsEviction_To_v1beta1_WorkloadScheduli func autoConvert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(in *WorkloadSpec, out *v1beta2.WorkloadSpec, s conversion.Scope) error { out.PodSets = *(*[]v1beta2.PodSet)(unsafe.Pointer(&in.PodSets)) out.QueueName = v1beta2.LocalQueueName(in.QueueName) - out.PriorityClassName = in.PriorityClassName + // WARNING: in.PriorityClassName requires manual conversion: does not exist in peer-type out.Priority = (*int32)(unsafe.Pointer(in.Priority)) - out.PriorityClassSource = in.PriorityClassSource + // WARNING: in.PriorityClassSource requires manual conversion: does not exist in peer-type out.Active = (*bool)(unsafe.Pointer(in.Active)) out.MaximumExecutionTimeSeconds = (*int32)(unsafe.Pointer(in.MaximumExecutionTimeSeconds)) return nil } -// Convert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec is an autogenerated conversion function. -func Convert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(in *WorkloadSpec, out *v1beta2.WorkloadSpec, s conversion.Scope) error { - return autoConvert_v1beta1_WorkloadSpec_To_v1beta2_WorkloadSpec(in, out, s) -} - func autoConvert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(in *v1beta2.WorkloadSpec, out *WorkloadSpec, s conversion.Scope) error { out.PodSets = *(*[]PodSet)(unsafe.Pointer(&in.PodSets)) out.QueueName = LocalQueueName(in.QueueName) - out.PriorityClassName = in.PriorityClassName + // WARNING: in.PriorityClassRef requires manual conversion: does not exist in peer-type out.Priority = (*int32)(unsafe.Pointer(in.Priority)) - out.PriorityClassSource = in.PriorityClassSource out.Active = (*bool)(unsafe.Pointer(in.Active)) out.MaximumExecutionTimeSeconds = (*int32)(unsafe.Pointer(in.MaximumExecutionTimeSeconds)) return nil } -// Convert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec is an autogenerated conversion function. -func Convert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(in *v1beta2.WorkloadSpec, out *WorkloadSpec, s conversion.Scope) error { - return autoConvert_v1beta2_WorkloadSpec_To_v1beta1_WorkloadSpec(in, out, s) -} - func autoConvert_v1beta1_WorkloadStatus_To_v1beta2_WorkloadStatus(in *WorkloadStatus, out *v1beta2.WorkloadStatus, s conversion.Scope) error { out.Conditions = *(*[]v1.Condition)(unsafe.Pointer(&in.Conditions)) out.Admission = (*v1beta2.Admission)(unsafe.Pointer(in.Admission)) diff --git a/apis/kueue/v1beta2/workload_types.go b/apis/kueue/v1beta2/workload_types.go index 556e9151b17..825eea6a6c5 100644 --- a/apis/kueue/v1beta2/workload_types.go +++ b/apis/kueue/v1beta2/workload_types.go @@ -24,7 +24,7 @@ import ( ) // WorkloadSpec defines the desired state of Workload -// +kubebuilder:validation:XValidation:rule="has(self.priorityClassName) ? has(self.priority) : true", message="priority should not be nil when priorityClassName is set" +// +kubebuilder:validation:XValidation:rule="!has(self.priorityClassRef) || has(self.priority)", message="priority should not be nil when priorityClassRef is set" type WorkloadSpec struct { // podSets is a list of sets of homogeneous pods, each described by a Pod spec // and a count. @@ -43,35 +43,19 @@ type WorkloadSpec struct { // +optional QueueName LocalQueueName `json:"queueName,omitempty"` - // priorityClassName is the name of the PriorityClass the Workload is associated with. - // If specified, indicates the workload's priority. - // "system-node-critical" and "system-cluster-critical" are two special - // keywords which indicate the highest priorities with the former being - // the highest priority. Any other name must be defined by creating a - // PriorityClass object with that name. If not specified, the workload - // priority will be default or zero if there is no default. - // +kubebuilder:validation:MaxLength=253 - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + // priorityClassRef references a PriorityClass object that defines the workload's priority. + // // +optional - PriorityClassName string `json:"priorityClassName,omitempty"` + PriorityClassRef *PriorityClassRef `json:"priorityClassRef,omitempty"` // priority determines the order of access to the resources managed by the // ClusterQueue where the workload is queued. - // The priority value is populated from PriorityClassName. + // The priority value is populated from the referenced PriorityClass (via priorityClassRef). // The higher the value, the higher the priority. - // If priorityClassName is specified, priority must not be null. + // If priorityClassRef is specified, priority must not be null. // +optional Priority *int32 `json:"priority,omitempty"` - // priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. - // Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. - // When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. - // +kubebuilder:default="" - // +kubebuilder:validation:Enum=kueue.x-k8s.io/workloadpriorityclass;scheduling.k8s.io/priorityclass;"" - // +optional - PriorityClassSource string `json:"priorityClassSource,omitempty"` - // active determines if a workload can be admitted into a queue. // Changing active from true to false will evict any running workloads. // Possible values are: @@ -93,6 +77,81 @@ type WorkloadSpec struct { MaximumExecutionTimeSeconds *int32 `json:"maximumExecutionTimeSeconds,omitempty"` } +// PriorityClassGroup indicates the API group of the PriorityClass object. +// +// +enum +// +kubebuilder:validation:Enum=kueue.x-k8s.io;scheduling.k8s.io +type PriorityClassGroup string + +const ( + // PodPriorityClassGroup represents the core Kubernetes scheduling group. + PodPriorityClassGroup PriorityClassGroup = "scheduling.k8s.io" + + // WorkloadPriorityClassGroup represents the Kueue-specific priority group. + WorkloadPriorityClassGroup PriorityClassGroup = "kueue.x-k8s.io" +) + +// PriorityClassKind is the kind of the PriorityClass object. +// +// +enum +// +kubebuilder:validation:Enum=WorkloadPriorityClass;PriorityClass +type PriorityClassKind string + +const ( + // PodPriorityClassKind represents the core Kubernetes PriorityClass kind. + PodPriorityClassKind PriorityClassKind = "PriorityClass" + + // WorkloadPriorityClassKind represents the Kueue WorkloadPriorityClass kind. + WorkloadPriorityClassKind PriorityClassKind = "WorkloadPriorityClass" +) + +// NewPriorityClassRef creates a new PriorityClassRef with the specified group, kind, and name. +func NewPriorityClassRef(group PriorityClassGroup, kind PriorityClassKind, name string) *PriorityClassRef { + return &PriorityClassRef{Group: group, Kind: kind, Name: name} +} + +// NewWorkloadPriorityClassRef creates a PriorityClassRef for a WorkloadPriorityClass. +func NewWorkloadPriorityClassRef(name string) *PriorityClassRef { + return NewPriorityClassRef(WorkloadPriorityClassGroup, WorkloadPriorityClassKind, name) +} + +// NewPodPriorityClassRef creates a PriorityClassRef for a core Kubernetes PriorityClass. +func NewPodPriorityClassRef(name string) *PriorityClassRef { + return NewPriorityClassRef(PodPriorityClassGroup, PodPriorityClassKind, name) +} + +// PriorityClassRef references a PriorityClass in a specific API group. +// +// +kubebuilder:validation:XValidation:rule="(self.group == 'scheduling.k8s.io') ? self.kind == 'PriorityClass' : true", message="only the PriorityClass kind is allowed for the scheduling.k8s.io group" +// +kubebuilder:validation:XValidation:rule="(self.group == 'kueue.x-k8s.io') ? self.kind == 'WorkloadPriorityClass' : true", message="only the WorkloadPriorityClass kind is allowed for the kueue.x-k8s.io group" +type PriorityClassRef struct { + // group is the API group of the PriorityClass object. + // Use "kueue.x-k8s.io" for WorkloadPriorityClass. + // Use "scheduling.k8s.io" for Pod PriorityClass. + // + // +required + Group PriorityClassGroup `json:"group,omitempty"` + + // kind is the kind of the PriorityClass object. + // + // +required + Kind PriorityClassKind `json:"kind,omitempty"` + + // name is the name of the PriorityClass the Workload is associated with. + // If specified, indicates the workload's priority. + // "system-node-critical" and "system-cluster-critical" are two special + // keywords which indicate the highest priorities with the former being + // the highest priority. Any other name must be defined by creating a + // PriorityClass object with that name. If not specified, the workload + // priority will be default or zero if there is no default. + // + // +required + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + Name string `json:"name,omitempty"` +} + // PodSetTopologyRequest defines the topology request for a PodSet. type PodSetTopologyRequest struct { // required indicates the topology level required by the PodSet, as @@ -800,8 +859,10 @@ const ( // Workload is the Schema for the workloads API // +kubebuilder:validation:XValidation:rule="has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true", message="podSetAssignments must have the same number of podSets as the spec" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true", message="priorityClassSource is immutable while workload quota reserved" -// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && (self.spec.priorityClassSource != 'kueue.x-k8s.io/workloadpriorityclass') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true", message="priorityClassName is immutable while workload quota reserved and priorityClassSource is not equal to kueue.x-k8s.io/workloadpriorityclass" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) ? (!has(oldSelf.spec.priorityClassRef) && !has(self.spec.priorityClassRef)) || (has(oldSelf.spec.priorityClassRef) && has(self.spec.priorityClassRef)) : true",message="priorityClassRef is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassRef) && has(self.spec.priorityClassRef)) ? oldSelf.spec.priorityClassRef.group == self.spec.priorityClassRef.group : true",message="priorityClassRef.group is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassRef) && has(self.spec.priorityClassRef)) ? oldSelf.spec.priorityClassRef.kind == self.spec.priorityClassRef.kind : true",message="priorityClassRef.kind is immutable while workload quota reserved" +// +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True') && has(oldSelf.spec.priorityClassRef) && has(self.spec.priorityClassRef) && self.spec.priorityClassRef.group == 'scheduling.k8s.io' && self.spec.priorityClassRef.kind == 'PriorityClass') ? oldSelf.spec.priorityClassRef.name == self.spec.priorityClassRef.name : true",message="priorityClassRef.name is immutable for scheduling.k8s.io/priorityclass while workload quota reserved" // +kubebuilder:validation:XValidation:rule="(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'QuotaReserved' && c.status == 'True')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true", message="queueName is immutable while workload quota reserved" // +kubebuilder:validation:XValidation:rule="((has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == 'Admitted' && c.status == 'True')))?((has(oldSelf.spec.maximumExecutionTimeSeconds)?oldSelf.spec.maximumExecutionTimeSeconds:0) == (has(self.spec.maximumExecutionTimeSeconds)?self.spec.maximumExecutionTimeSeconds:0)):true", message="maximumExecutionTimeSeconds is immutable while workload quota reserved" type Workload struct { diff --git a/apis/kueue/v1beta2/zz_generated.deepcopy.go b/apis/kueue/v1beta2/zz_generated.deepcopy.go index b641161baff..77a57cd69d5 100644 --- a/apis/kueue/v1beta2/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta2/zz_generated.deepcopy.go @@ -1217,6 +1217,21 @@ func (in *PodSetUpdate) DeepCopy() *PodSetUpdate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PriorityClassRef) DeepCopyInto(out *PriorityClassRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PriorityClassRef. +func (in *PriorityClassRef) DeepCopy() *PriorityClassRef { + if in == nil { + return nil + } + out := new(PriorityClassRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProvisioningRequestConfig) DeepCopyInto(out *ProvisioningRequestConfig) { *out = *in @@ -1926,6 +1941,11 @@ func (in *WorkloadSpec) DeepCopyInto(out *WorkloadSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.PriorityClassRef != nil { + in, out := &in.PriorityClassRef, &out.PriorityClassRef + *out = new(PriorityClassRef) + **out = **in + } if in.Priority != nil { in, out := &in.Priority, &out.Priority *out = new(int32) diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml index 437d20e290e..aba2638267e 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_workloads.yaml @@ -17119,35 +17119,52 @@ spec: description: |- priority determines the order of access to the resources managed by the ClusterQueue where the workload is queued. - The priority value is populated from PriorityClassName. + The priority value is populated from the referenced PriorityClass (via priorityClassRef). The higher the value, the higher the priority. - If priorityClassName is specified, priority must not be null. + If priorityClassRef is specified, priority must not be null. format: int32 type: integer - priorityClassName: - description: |- - priorityClassName is the name of the PriorityClass the Workload is associated with. - If specified, indicates the workload's priority. - "system-node-critical" and "system-cluster-critical" are two special - keywords which indicate the highest priorities with the former being - the highest priority. Any other name must be defined by creating a - PriorityClass object with that name. If not specified, the workload - priority will be default or zero if there is no default. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - priorityClassSource: - default: "" - description: |- - priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. - Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. - When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. - enum: - - kueue.x-k8s.io/workloadpriorityclass - - scheduling.k8s.io/priorityclass - - "" - type: string + priorityClassRef: + description: priorityClassRef references a PriorityClass object that defines the workload's priority. + properties: + group: + description: |- + group is the API group of the PriorityClass object. + Use "kueue.x-k8s.io" for WorkloadPriorityClass. + Use "scheduling.k8s.io" for Pod PriorityClass. + enum: + - kueue.x-k8s.io + - scheduling.k8s.io + type: string + kind: + description: kind is the kind of the PriorityClass object. + enum: + - WorkloadPriorityClass + - PriorityClass + type: string + name: + description: |- + name is the name of the PriorityClass the Workload is associated with. + If specified, indicates the workload's priority. + "system-node-critical" and "system-cluster-critical" are two special + keywords which indicate the highest priorities with the former being + the highest priority. Any other name must be defined by creating a + PriorityClass object with that name. If not specified, the workload + priority will be default or zero if there is no default. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - group + - kind + - name + type: object + x-kubernetes-validations: + - message: only the PriorityClass kind is allowed for the scheduling.k8s.io group + rule: '(self.group == ''scheduling.k8s.io'') ? self.kind == ''PriorityClass'' : true' + - message: only the WorkloadPriorityClass kind is allowed for the kueue.x-k8s.io group + rule: '(self.group == ''kueue.x-k8s.io'') ? self.kind == ''WorkloadPriorityClass'' : true' queueName: description: |- queueName is the name of the LocalQueue the Workload is associated with. @@ -17157,8 +17174,8 @@ spec: type: string type: object x-kubernetes-validations: - - message: priority should not be nil when priorityClassName is set - rule: 'has(self.priorityClassName) ? has(self.priority) : true' + - message: priority should not be nil when priorityClassRef is set + rule: '!has(self.priorityClassRef) || has(self.priority)' status: description: status is the status of the Workload. properties: @@ -17700,10 +17717,14 @@ spec: x-kubernetes-validations: - message: podSetAssignments must have the same number of podSets as the spec rule: 'has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true' - - message: priorityClassSource is immutable while workload quota reserved - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? ((!has(self.spec.priorityClassSource) && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) && self.spec.priorityClassSource.size() == 0)) : true' - - message: priorityClassName is immutable while workload quota reserved and priorityClassSource is not equal to kueue.x-k8s.io/workloadpriorityclass - rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && (self.spec.priorityClassSource != ''kueue.x-k8s.io/workloadpriorityclass'') && has(oldSelf.spec.priorityClassName) && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName == self.spec.priorityClassName) : true' + - message: priorityClassRef is immutable while workload quota reserved + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) ? (!has(oldSelf.spec.priorityClassRef) && !has(self.spec.priorityClassRef)) || (has(oldSelf.spec.priorityClassRef) && has(self.spec.priorityClassRef)) : true' + - message: priorityClassRef.group is immutable while workload quota reserved + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassRef) && has(self.spec.priorityClassRef)) ? oldSelf.spec.priorityClassRef.group == self.spec.priorityClassRef.group : true' + - message: priorityClassRef.kind is immutable while workload quota reserved + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassRef) && has(self.spec.priorityClassRef)) ? oldSelf.spec.priorityClassRef.kind == self.spec.priorityClassRef.kind : true' + - message: priorityClassRef.name is immutable for scheduling.k8s.io/priorityclass while workload quota reserved + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassRef) && has(self.spec.priorityClassRef) && self.spec.priorityClassRef.group == ''scheduling.k8s.io'' && self.spec.priorityClassRef.kind == ''PriorityClass'') ? oldSelf.spec.priorityClassRef.name == self.spec.priorityClassRef.name : true' - message: queueName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) && has(self.status.conditions) && self.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && has(oldSelf.spec.queueName) && has(self.spec.queueName) ? oldSelf.spec.queueName == self.spec.queueName : true' - message: maximumExecutionTimeSeconds is immutable while workload quota reserved diff --git a/client-go/applyconfiguration/kueue/v1beta2/priorityclassref.go b/client-go/applyconfiguration/kueue/v1beta2/priorityclassref.go new file mode 100644 index 00000000000..6f31340a9de --- /dev/null +++ b/client-go/applyconfiguration/kueue/v1beta2/priorityclassref.go @@ -0,0 +1,60 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta2 + +import ( + kueuev1beta2 "sigs.k8s.io/kueue/apis/kueue/v1beta2" +) + +// PriorityClassRefApplyConfiguration represents a declarative configuration of the PriorityClassRef type for use +// with apply. +type PriorityClassRefApplyConfiguration struct { + Group *kueuev1beta2.PriorityClassGroup `json:"group,omitempty"` + Kind *kueuev1beta2.PriorityClassKind `json:"kind,omitempty"` + Name *string `json:"name,omitempty"` +} + +// PriorityClassRefApplyConfiguration constructs a declarative configuration of the PriorityClassRef type for use with +// apply. +func PriorityClassRef() *PriorityClassRefApplyConfiguration { + return &PriorityClassRefApplyConfiguration{} +} + +// WithGroup sets the Group field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Group field is set to the value of the last call. +func (b *PriorityClassRefApplyConfiguration) WithGroup(value kueuev1beta2.PriorityClassGroup) *PriorityClassRefApplyConfiguration { + b.Group = &value + return b +} + +// WithKind sets the Kind field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Kind field is set to the value of the last call. +func (b *PriorityClassRefApplyConfiguration) WithKind(value kueuev1beta2.PriorityClassKind) *PriorityClassRefApplyConfiguration { + b.Kind = &value + return b +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *PriorityClassRefApplyConfiguration) WithName(value string) *PriorityClassRefApplyConfiguration { + b.Name = &value + return b +} diff --git a/client-go/applyconfiguration/kueue/v1beta2/workloadspec.go b/client-go/applyconfiguration/kueue/v1beta2/workloadspec.go index ac14b47067b..ab7c0b9eab1 100644 --- a/client-go/applyconfiguration/kueue/v1beta2/workloadspec.go +++ b/client-go/applyconfiguration/kueue/v1beta2/workloadspec.go @@ -24,13 +24,12 @@ import ( // WorkloadSpecApplyConfiguration represents a declarative configuration of the WorkloadSpec type for use // with apply. type WorkloadSpecApplyConfiguration struct { - PodSets []PodSetApplyConfiguration `json:"podSets,omitempty"` - QueueName *kueuev1beta2.LocalQueueName `json:"queueName,omitempty"` - PriorityClassName *string `json:"priorityClassName,omitempty"` - Priority *int32 `json:"priority,omitempty"` - PriorityClassSource *string `json:"priorityClassSource,omitempty"` - Active *bool `json:"active,omitempty"` - MaximumExecutionTimeSeconds *int32 `json:"maximumExecutionTimeSeconds,omitempty"` + PodSets []PodSetApplyConfiguration `json:"podSets,omitempty"` + QueueName *kueuev1beta2.LocalQueueName `json:"queueName,omitempty"` + PriorityClassRef *PriorityClassRefApplyConfiguration `json:"priorityClassRef,omitempty"` + Priority *int32 `json:"priority,omitempty"` + Active *bool `json:"active,omitempty"` + MaximumExecutionTimeSeconds *int32 `json:"maximumExecutionTimeSeconds,omitempty"` } // WorkloadSpecApplyConfiguration constructs a declarative configuration of the WorkloadSpec type for use with @@ -60,11 +59,11 @@ func (b *WorkloadSpecApplyConfiguration) WithQueueName(value kueuev1beta2.LocalQ return b } -// WithPriorityClassName sets the PriorityClassName field in the declarative configuration to the given value +// WithPriorityClassRef sets the PriorityClassRef field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the PriorityClassName field is set to the value of the last call. -func (b *WorkloadSpecApplyConfiguration) WithPriorityClassName(value string) *WorkloadSpecApplyConfiguration { - b.PriorityClassName = &value +// If called multiple times, the PriorityClassRef field is set to the value of the last call. +func (b *WorkloadSpecApplyConfiguration) WithPriorityClassRef(value *PriorityClassRefApplyConfiguration) *WorkloadSpecApplyConfiguration { + b.PriorityClassRef = value return b } @@ -76,14 +75,6 @@ func (b *WorkloadSpecApplyConfiguration) WithPriority(value int32) *WorkloadSpec return b } -// WithPriorityClassSource sets the PriorityClassSource field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the PriorityClassSource field is set to the value of the last call. -func (b *WorkloadSpecApplyConfiguration) WithPriorityClassSource(value string) *WorkloadSpecApplyConfiguration { - b.PriorityClassSource = &value - return b -} - // WithActive sets the Active field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the Active field is set to the value of the last call. diff --git a/client-go/applyconfiguration/utils.go b/client-go/applyconfiguration/utils.go index ad820f3afd7..162c4b523e9 100644 --- a/client-go/applyconfiguration/utils.go +++ b/client-go/applyconfiguration/utils.go @@ -251,6 +251,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &kueuev1beta2.PodSetTopologyRequestApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("PodSetUpdate"): return &kueuev1beta2.PodSetUpdateApplyConfiguration{} + case v1beta2.SchemeGroupVersion.WithKind("PriorityClassRef"): + return &kueuev1beta2.PriorityClassRefApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("ProvisioningRequestConfig"): return &kueuev1beta2.ProvisioningRequestConfigApplyConfiguration{} case v1beta2.SchemeGroupVersion.WithKind("ProvisioningRequestConfigSpec"): diff --git a/cmd/importer/pod/import.go b/cmd/importer/pod/import.go index 6cadf7a7d37..8ffa694ee01 100644 --- a/cmd/importer/pod/import.go +++ b/cmd/importer/pod/import.go @@ -80,9 +80,8 @@ func Import(ctx context.Context, c client.Client, cache *util.ImportCache, jobs maps.Copy(wl.Labels, cache.AddLabels) if pc, found := cache.PriorityClasses[p.Spec.PriorityClassName]; found { - wl.Spec.PriorityClassName = pc.Name + wl.Spec.PriorityClassRef = kueuev1beta2.NewPodPriorityClassRef(pc.Name) wl.Spec.Priority = &pc.Value - wl.Spec.PriorityClassSource = constants.PodPriorityClassSource } wlv1beta1 := &kueue.Workload{ diff --git a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml index 60039d16bb9..5dfd7860f13 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_workloads.yaml @@ -18108,35 +18108,57 @@ spec: description: |- priority determines the order of access to the resources managed by the ClusterQueue where the workload is queued. - The priority value is populated from PriorityClassName. + The priority value is populated from the referenced PriorityClass (via priorityClassRef). The higher the value, the higher the priority. - If priorityClassName is specified, priority must not be null. + If priorityClassRef is specified, priority must not be null. format: int32 type: integer - priorityClassName: - description: |- - priorityClassName is the name of the PriorityClass the Workload is associated with. - If specified, indicates the workload's priority. - "system-node-critical" and "system-cluster-critical" are two special - keywords which indicate the highest priorities with the former being - the highest priority. Any other name must be defined by creating a - PriorityClass object with that name. If not specified, the workload - priority will be default or zero if there is no default. - maxLength: 253 - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ - type: string - priorityClassSource: - default: "" - description: |- - priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. - Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. - When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value. - enum: - - kueue.x-k8s.io/workloadpriorityclass - - scheduling.k8s.io/priorityclass - - "" - type: string + priorityClassRef: + description: priorityClassRef references a PriorityClass object that + defines the workload's priority. + properties: + group: + description: |- + group is the API group of the PriorityClass object. + Use "kueue.x-k8s.io" for WorkloadPriorityClass. + Use "scheduling.k8s.io" for Pod PriorityClass. + enum: + - kueue.x-k8s.io + - scheduling.k8s.io + type: string + kind: + description: kind is the kind of the PriorityClass object. + enum: + - WorkloadPriorityClass + - PriorityClass + type: string + name: + description: |- + name is the name of the PriorityClass the Workload is associated with. + If specified, indicates the workload's priority. + "system-node-critical" and "system-cluster-critical" are two special + keywords which indicate the highest priorities with the former being + the highest priority. Any other name must be defined by creating a + PriorityClass object with that name. If not specified, the workload + priority will be default or zero if there is no default. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - group + - kind + - name + type: object + x-kubernetes-validations: + - message: only the PriorityClass kind is allowed for the scheduling.k8s.io + group + rule: '(self.group == ''scheduling.k8s.io'') ? self.kind == ''PriorityClass'' + : true' + - message: only the WorkloadPriorityClass kind is allowed for the + kueue.x-k8s.io group + rule: '(self.group == ''kueue.x-k8s.io'') ? self.kind == ''WorkloadPriorityClass'' + : true' queueName: description: |- queueName is the name of the LocalQueue the Workload is associated with. @@ -18146,8 +18168,8 @@ spec: type: string type: object x-kubernetes-validations: - - message: priority should not be nil when priorityClassName is set - rule: 'has(self.priorityClassName) ? has(self.priority) : true' + - message: priority should not be nil when priorityClassRef is set + rule: '!has(self.priorityClassRef) || has(self.priority)' status: description: status is the status of the Workload. properties: @@ -18719,22 +18741,29 @@ spec: c.type == ''QuotaReserved'' && c.status == ''True'') && has(self.status.admission) ? size(self.spec.podSets) == size(self.status.admission.podSetAssignments) : true' - - message: priorityClassSource is immutable while workload quota reserved + - message: priorityClassRef is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'')) ? ((!has(self.spec.priorityClassSource) - && !has(oldSelf.spec.priorityClassSource)) || (has(self.spec.priorityClassSource) - && has(oldSelf.spec.priorityClassSource) && self.spec.priorityClassSource - == oldSelf.spec.priorityClassSource) || (!has(self.spec.priorityClassSource) - && has(oldSelf.spec.priorityClassSource) && oldSelf.spec.priorityClassSource.size() - == 0) || (!has(oldSelf.spec.priorityClassSource) && has(self.spec.priorityClassSource) - && self.spec.priorityClassSource.size() == 0)) : true' - - message: priorityClassName is immutable while workload quota reserved and - priorityClassSource is not equal to kueue.x-k8s.io/workloadpriorityclass + c.type == ''QuotaReserved'' && c.status == ''True'')) ? (!has(oldSelf.spec.priorityClassRef) + && !has(self.spec.priorityClassRef)) || (has(oldSelf.spec.priorityClassRef) + && has(self.spec.priorityClassRef)) : true' + - message: priorityClassRef.group is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, - c.type == ''QuotaReserved'' && c.status == ''True'') && (self.spec.priorityClassSource - != ''kueue.x-k8s.io/workloadpriorityclass'') && has(oldSelf.spec.priorityClassName) - && has(self.spec.priorityClassName)) ? (oldSelf.spec.priorityClassName - == self.spec.priorityClassName) : true' + c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassRef) + && has(self.spec.priorityClassRef)) ? oldSelf.spec.priorityClassRef.group + == self.spec.priorityClassRef.group : true' + - message: priorityClassRef.kind is immutable while workload quota reserved + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassRef) + && has(self.spec.priorityClassRef)) ? oldSelf.spec.priorityClassRef.kind + == self.spec.priorityClassRef.kind : true' + - message: priorityClassRef.name is immutable for scheduling.k8s.io/priorityclass + while workload quota reserved + rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, + c.type == ''QuotaReserved'' && c.status == ''True'') && has(oldSelf.spec.priorityClassRef) + && has(self.spec.priorityClassRef) && self.spec.priorityClassRef.group + == ''scheduling.k8s.io'' && self.spec.priorityClassRef.kind == ''PriorityClass'') + ? oldSelf.spec.priorityClassRef.name == self.spec.priorityClassRef.name + : true' - message: queueName is immutable while workload quota reserved rule: '(has(oldSelf.status) && has(oldSelf.status.conditions) && oldSelf.status.conditions.exists(c, c.type == ''QuotaReserved'' && c.status == ''True'')) && (has(self.status) diff --git a/pkg/cache/queue/cluster_queue_test.go b/pkg/cache/queue/cluster_queue_test.go index 5544fc3a912..2d79a05f67a 100644 --- a/pkg/cache/queue/cluster_queue_test.go +++ b/pkg/cache/queue/cluster_queue_test.go @@ -728,12 +728,12 @@ func TestStrictFIFO(t *testing.T) { name: "w1.priority is higher than w2.priority", w1: utiltestingapi.MakeWorkload("w1", ""). Creation(t1). - PriorityClass("highPriority"). + PodPriorityClassRef("highPriority"). Priority(highPriority). Obj(), w2: utiltestingapi.MakeWorkload("w2", ""). Creation(t2). - PriorityClass("lowPriority"). + PodPriorityClassRef("lowPriority"). Priority(lowPriority). Obj(), expected: "w1", @@ -789,12 +789,12 @@ func TestStrictFIFO(t *testing.T) { name: "p1.priority is lower than p2.priority and w1.create time is earlier than w2.create time", w1: utiltestingapi.MakeWorkload("w1", ""). Creation(t1). - PriorityClass("lowPriority"). + PodPriorityClassRef("lowPriority"). Priority(lowPriority). Obj(), w2: utiltestingapi.MakeWorkload("w2", ""). Creation(t2). - PriorityClass("highPriority"). + PodPriorityClassRef("highPriority"). Priority(highPriority). Obj(), expected: "w2", diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 856bbf038b5..b8b7507456b 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -32,12 +32,9 @@ const ( UpdatesBatchPeriod = time.Second // DefaultPriority is used to set priority of workloads - // that do not specify any priority class and there is no priority class + // that do not specify any priority class, and there is no priority class // marked as default. - DefaultPriority = 0 - - WorkloadPriorityClassSource = "kueue.x-k8s.io/workloadpriorityclass" - PodPriorityClassSource = "scheduling.k8s.io/priorityclass" + DefaultPriority int32 = 0 DefaultPendingWorkloadsLimit = 1000 diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go index c2ff65cc25d..c30e83470fd 100644 --- a/pkg/controller/core/workload_controller.go +++ b/pkg/controller/core/workload_controller.go @@ -53,7 +53,6 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" qcache "sigs.k8s.io/kueue/pkg/cache/queue" schdcache "sigs.k8s.io/kueue/pkg/cache/scheduler" - "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/dra" "sigs.k8s.io/kueue/pkg/features" @@ -410,17 +409,18 @@ func (r *WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c quotaReservedCondition := apimeta.FindStatusCondition(wl.Status.Conditions, kueue.WorkloadQuotaReserved) quotaReservedWaitTime := r.clock.Since(quotaReservedCondition.LastTransitionTime.Time) r.recorder.Eventf(&wl, corev1.EventTypeNormal, "Admitted", "Admitted by ClusterQueue %v, wait time since reservation was %.0fs", wl.Status.Admission.ClusterQueue, quotaReservedWaitTime.Seconds()) - metrics.AdmittedWorkload(cqName, wl.Spec.PriorityClassName, queuedWaitTime) - metrics.ReportAdmissionChecksWaitTime(cqName, wl.Spec.PriorityClassName, quotaReservedWaitTime) + priorityClassName := workload.PriorityClassName(&wl) + metrics.AdmittedWorkload(cqName, priorityClassName, queuedWaitTime) + metrics.ReportAdmissionChecksWaitTime(cqName, priorityClassName, quotaReservedWaitTime) if features.Enabled(features.LocalQueueMetrics) { metrics.LocalQueueAdmittedWorkload( metrics.LQRefFromWorkload(&wl), - wl.Spec.PriorityClassName, + priorityClassName, queuedWaitTime, ) metrics.ReportLocalQueueAdmissionChecksWaitTime( metrics.LQRefFromWorkload(&wl), - wl.Spec.PriorityClassName, + priorityClassName, quotaReservedWaitTime, ) } @@ -954,10 +954,9 @@ func (r *WorkloadReconciler) Update(e event.TypedUpdateEvent[*kueue.Workload]) b } func workloadPriorityClassChanged(old, new *kueue.Workload) bool { - return old.Spec.PriorityClassSource == constants.WorkloadPriorityClassSource && - new.Spec.PriorityClassSource == constants.WorkloadPriorityClassSource && - old.Spec.PriorityClassName != "" && new.Spec.PriorityClassName != "" && - old.Spec.PriorityClassName != new.Spec.PriorityClassName + return workload.IsWorkloadPriorityClass(old) && workload.IsWorkloadPriorityClass(new) && + workload.PriorityClassName(old) != "" && workload.PriorityClassName(new) != "" && + workload.PriorityClassName(old) != workload.PriorityClassName(new) } func (r *WorkloadReconciler) Generic(e event.TypedGenericEvent[*kueue.Workload]) bool { diff --git a/pkg/controller/jobframework/reconciler.go b/pkg/controller/jobframework/reconciler.go index c1830f7bc9a..3a3c8df7655 100644 --- a/pkg/controller/jobframework/reconciler.go +++ b/pkg/controller/jobframework/reconciler.go @@ -527,14 +527,15 @@ func (r *JobReconciler) ReconcileGenericJob(ctx context.Context, req ctrl.Reques // update the metrics only when PodsReady condition status is true if condition.Status == metav1.ConditionTrue { cqName := wl.Status.Admission.ClusterQueue + priorityClassName := workload.PriorityClassName(wl) queuedUntilReadyWaitTime := workload.QueuedWaitTime(wl, r.clock) - metrics.ReadyWaitTime(cqName, wl.Spec.PriorityClassName, queuedUntilReadyWaitTime) + metrics.ReadyWaitTime(cqName, priorityClassName, queuedUntilReadyWaitTime) admittedCond := apimeta.FindStatusCondition(wl.Status.Conditions, kueue.WorkloadAdmitted) admittedUntilReadyWaitTime := condition.LastTransitionTime.Sub(admittedCond.LastTransitionTime.Time) - metrics.ReportAdmittedUntilReadyWaitTime(cqName, wl.Spec.PriorityClassName, admittedUntilReadyWaitTime) + metrics.ReportAdmittedUntilReadyWaitTime(cqName, priorityClassName, admittedUntilReadyWaitTime) if features.Enabled(features.LocalQueueMetrics) { - metrics.LocalQueueReadyWaitTime(metrics.LQRefFromWorkload(wl), wl.Spec.PriorityClassName, queuedUntilReadyWaitTime) - metrics.ReportLocalQueueAdmittedUntilReadyWaitTime(metrics.LQRefFromWorkload(wl), wl.Spec.PriorityClassName, admittedUntilReadyWaitTime) + metrics.LocalQueueReadyWaitTime(metrics.LQRefFromWorkload(wl), priorityClassName, queuedUntilReadyWaitTime) + metrics.ReportLocalQueueAdmittedUntilReadyWaitTime(metrics.LQRefFromWorkload(wl), priorityClassName, admittedUntilReadyWaitTime) } } return ctrl.Result{}, nil @@ -945,7 +946,7 @@ func (r *JobReconciler) ensureOneWorkload(ctx context.Context, job GenericJob, o // UpdateWorkloadPriority updates workload priority if object's kueue.x-k8s.io/priority-class label changed. func UpdateWorkloadPriority(ctx context.Context, c client.Client, r record.EventRecorder, obj client.Object, wl *kueue.Workload, customPriorityClassFunc func() string) error { - if wl.Spec.PriorityClassSource == constants.WorkloadPriorityClassSource && WorkloadPriorityClassName(obj) != wl.Spec.PriorityClassName { + if workload.IsWorkloadPriorityClass(wl) && WorkloadPriorityClassName(obj) != workload.PriorityClassName(wl) { if err := PrepareWorkloadPriority(ctx, c, obj, wl, customPriorityClassFunc); err != nil { return fmt.Errorf("prepare workload priority: %w", err) } @@ -1294,14 +1295,13 @@ func getCustomPriorityClassFuncFromJob(job GenericJob) func() string { } func PrepareWorkloadPriority(ctx context.Context, c client.Client, obj client.Object, wl *kueue.Workload, customPriorityClassFunc func() string) error { - priorityClassName, source, p, err := ExtractPriority(ctx, c, obj, wl.Spec.PodSets, customPriorityClassFunc) + priorityClassRef, priority, err := ExtractPriority(ctx, c, obj, wl.Spec.PodSets, customPriorityClassFunc) if err != nil { return err } - wl.Spec.PriorityClassName = priorityClassName - wl.Spec.Priority = &p - wl.Spec.PriorityClassSource = source + wl.Spec.PriorityClassRef = priorityClassRef + wl.Spec.Priority = &priority return nil } @@ -1320,7 +1320,7 @@ func (r *JobReconciler) prepareWorkload(ctx context.Context, job GenericJob, wl return nil } -func ExtractPriority(ctx context.Context, c client.Client, obj client.Object, podSets []kueue.PodSet, customPriorityClassFunc func() string) (string, string, int32, error) { +func ExtractPriority(ctx context.Context, c client.Client, obj client.Object, podSets []kueue.PodSet, customPriorityClassFunc func() string) (*kueue.PriorityClassRef, int32, error) { if workloadPriorityClass := WorkloadPriorityClassName(obj); len(workloadPriorityClass) > 0 { return utilpriority.GetPriorityFromWorkloadPriorityClass(ctx, c, workloadPriorityClass) } diff --git a/pkg/controller/jobs/job/job_controller_test.go b/pkg/controller/jobs/job/job_controller_test.go index fc02ee2ba98..89115e213ec 100644 --- a/pkg/controller/jobs/job/job_controller_test.go +++ b/pkg/controller/jobs/job/job_controller_test.go @@ -38,7 +38,6 @@ import ( configapi "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" @@ -2689,8 +2688,7 @@ func TestReconciler(t *testing.T) { PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).Request(corev1.ResourceCPU, "1").Obj()). Queue("foo"). Priority(baseWPCWrapper.Value). - PriorityClassSource(constants.WorkloadPriorityClassSource). - PriorityClass(baseWPCWrapper.Name). + WorkloadPriorityClassRef(baseWPCWrapper.Name). Labels(map[string]string{ controllerconsts.JobUIDLabel: "test-uid", }). @@ -2702,8 +2700,7 @@ func TestReconciler(t *testing.T) { PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).Request(corev1.ResourceCPU, "1").Obj()). Queue("foo"). Priority(highWPCWrapper.Value). - PriorityClassSource(constants.WorkloadPriorityClassSource). - PriorityClass(highWPCWrapper.Name). + WorkloadPriorityClassRef(highWPCWrapper.Name). Labels(map[string]string{ controllerconsts.JobUIDLabel: "test-uid", }). @@ -2739,8 +2736,7 @@ func TestReconciler(t *testing.T) { PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).PriorityClass(basePCWrapper.Name).Request(corev1.ResourceCPU, "1").Obj()). Queue("foo"). Priority(basePCWrapper.Value). - PriorityClassSource(constants.PodPriorityClassSource). - PriorityClass(basePCWrapper.Name). + PodPriorityClassRef(basePCWrapper.Name). Labels(map[string]string{ controllerconsts.JobUIDLabel: "test-uid", }). @@ -2752,8 +2748,7 @@ func TestReconciler(t *testing.T) { PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).PriorityClass(basePCWrapper.Name).Request(corev1.ResourceCPU, "1").Obj()). Queue("foo"). Priority(basePCWrapper.Value). - PriorityClassSource(constants.PodPriorityClassSource). - PriorityClass(basePCWrapper.Name). + PodPriorityClassRef(basePCWrapper.Name). Labels(map[string]string{ controllerconsts.JobUIDLabel: "test-uid", }). @@ -3185,9 +3180,8 @@ func TestReconciler(t *testing.T) { Finalizers(kueue.ResourceInUseFinalizerName). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).Request(corev1.ResourceCPU, "1").Obj()). Queue("test-queue"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). Labels(map[string]string{ controllerconsts.JobUIDLabel: "test-uid", }). @@ -3230,9 +3224,8 @@ func TestReconciler(t *testing.T) { Finalizers(kueue.ResourceInUseFinalizerName). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).Request(corev1.ResourceCPU, "1").PriorityClass("test-pc").Obj()). Queue("test-queue"). - PriorityClass("test-pc"). + PodPriorityClassRef("test-pc"). Priority(200). - PriorityClassSource(constants.PodPriorityClassSource). Labels(map[string]string{ controllerconsts.JobUIDLabel: "test-uid", }). @@ -3277,9 +3270,8 @@ func TestReconciler(t *testing.T) { Finalizers(kueue.ResourceInUseFinalizerName). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).Request(corev1.ResourceCPU, "1").PriorityClass("test-pc").Obj()). Queue("test-queue"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). Labels(map[string]string{ controllerconsts.JobUIDLabel: "test-uid", }). @@ -3349,9 +3341,8 @@ func TestReconciler(t *testing.T) { Finalizers(kueue.ResourceInUseFinalizerName). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).Request(corev1.ResourceCPU, "1").PriorityClass("test-pc").Obj()). Queue("test-queue"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). Obj(), }, wantWorkloads: []kueue.Workload{ @@ -3359,9 +3350,8 @@ func TestReconciler(t *testing.T) { Finalizers(kueue.ResourceInUseFinalizerName). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).Request(corev1.ResourceCPU, "1").PriorityClass("test-pc").Obj()). Queue("test-queue"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). Labels(map[string]string{ controllerconsts.JobUIDLabel: "test-uid", }). @@ -3394,9 +3384,8 @@ func TestReconciler(t *testing.T) { Finalizers(kueue.ResourceInUseFinalizerName). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).Request(corev1.ResourceCPU, "1").PriorityClass("test-pc").Obj()). Queue("test-queue"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). ControllerReference(batchv1.SchemeGroupVersion.WithKind("Job"), "other-job", "other-uid"). Obj(), }, @@ -3405,9 +3394,8 @@ func TestReconciler(t *testing.T) { Finalizers(kueue.ResourceInUseFinalizerName). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 10).Request(corev1.ResourceCPU, "1").PriorityClass("test-pc").Obj()). Queue("test-queue"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). ControllerReference(batchv1.SchemeGroupVersion.WithKind("Job"), "other-job", "other-uid"). Obj(), }, @@ -3438,9 +3426,8 @@ func TestReconciler(t *testing.T) { Finalizers(kueue.ResourceInUseFinalizerName). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1).Request(corev1.ResourceCPU, "1").PriorityClass("test-pc").Obj()). Queue("test-queue"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). Obj(), }, wantWorkloads: []kueue.Workload{ @@ -3448,9 +3435,8 @@ func TestReconciler(t *testing.T) { Finalizers(kueue.ResourceInUseFinalizerName). PodSets(*utiltestingapi.MakePodSet(kueue.DefaultPodSetName, 1).Request(corev1.ResourceCPU, "1").PriorityClass("test-pc").Obj()). Queue("test-queue"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). Labels(map[string]string{ controllerconsts.JobUIDLabel: "test-uid", }). diff --git a/pkg/controller/jobs/jobset/jobset_controller_test.go b/pkg/controller/jobs/jobset/jobset_controller_test.go index b033317de8e..96dee2e6efd 100644 --- a/pkg/controller/jobs/jobset/jobset_controller_test.go +++ b/pkg/controller/jobs/jobset/jobset_controller_test.go @@ -32,7 +32,6 @@ import ( jobset "sigs.k8s.io/jobset/api/jobset/v1alpha2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/constants" controllerconsts "sigs.k8s.io/kueue/pkg/controller/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" @@ -473,9 +472,8 @@ func TestReconciler(t *testing.T) { ).WorkloadPriorityClass("test-wpc").Obj(), wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("jobset", "ns"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). PodSets( *utiltestingapi.MakePodSet("replicated-job-1", 1). PodIndexLabel(ptr.To("batch.kubernetes.io/job-completion-index")). @@ -512,9 +510,8 @@ func TestReconciler(t *testing.T) { ).PriorityClass("test-pc").Obj(), wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("jobset", "ns"). - PriorityClass("test-pc"). + PodPriorityClassRef("test-pc"). Priority(200). - PriorityClassSource(constants.PodPriorityClassSource). PodSets( *utiltestingapi.MakePodSet("replicated-job-1", 1). PodIndexLabel(ptr.To("batch.kubernetes.io/job-completion-index")). @@ -551,9 +548,8 @@ func TestReconciler(t *testing.T) { ).PriorityClass("test-pc").WorkloadPriorityClass("test-wpc").Obj(), wantWorkloads: []kueue.Workload{ *utiltestingapi.MakeWorkload("jobset", "ns"). - PriorityClass("test-wpc"). + WorkloadPriorityClassRef("test-wpc"). Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). PodSets( *utiltestingapi.MakePodSet("replicated-job-1", 1). PodIndexLabel(ptr.To("batch.kubernetes.io/job-completion-index")). diff --git a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler_test.go b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler_test.go index 2fbe671ebe5..63e5ed9df0f 100644 --- a/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler_test.go +++ b/pkg/controller/jobs/leaderworkerset/leaderworkerset_reconciler_test.go @@ -32,7 +32,6 @@ import ( leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/features" @@ -533,9 +532,8 @@ func TestReconciler(t *testing.T) { Count: 1, TopologyRequest: nil, }). - PriorityClass("high-priority"). + WorkloadPriorityClassRef("high-priority"). Priority(5000). - PriorityClassSource(constants.WorkloadPriorityClassSource). Obj(), }, wantEvents: []utiltesting.EventRecord{ diff --git a/pkg/controller/jobs/mpijob/mpijob_controller_test.go b/pkg/controller/jobs/mpijob/mpijob_controller_test.go index 2fb69af0579..1c752ada5b2 100644 --- a/pkg/controller/jobs/mpijob/mpijob_controller_test.go +++ b/pkg/controller/jobs/mpijob/mpijob_controller_test.go @@ -31,7 +31,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/controller/jobframework" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" @@ -408,8 +407,9 @@ func TestReconciler(t *testing.T) { *utiltestingapi.MakePodSet("worker", 2). PodIndexLabel(ptr.To(kfmpi.ReplicaIndexLabel)). Obj(), - ).PriorityClass("test-wpc").Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). + ). + WorkloadPriorityClassRef("test-wpc"). + Priority(100). Obj(), }, }, @@ -432,8 +432,9 @@ func TestReconciler(t *testing.T) { *utiltestingapi.MakePodSet("worker", 2). PodIndexLabel(ptr.To(kfmpi.ReplicaIndexLabel)). Obj(), - ).PriorityClass("test-pc").Priority(200). - PriorityClassSource(constants.PodPriorityClassSource). + ). + PodPriorityClassRef("test-pc"). + Priority(200). Obj(), }, }, @@ -458,8 +459,9 @@ func TestReconciler(t *testing.T) { *utiltestingapi.MakePodSet("worker", 2). PodIndexLabel(ptr.To(kfmpi.ReplicaIndexLabel)). Obj(), - ).PriorityClass("test-wpc").Priority(100). - PriorityClassSource(constants.WorkloadPriorityClassSource). + ). + WorkloadPriorityClassRef("test-wpc"). + Priority(100). Obj(), }, }, diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index f2ab75e6a07..d7c8cae8439 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -807,9 +807,10 @@ func (s *Scheduler) recordQuotaReservationMetrics(newWorkload, originalWorkload s.recorder.Eventf(newWorkload, corev1.EventTypeNormal, "QuotaReserved", "Quota reserved in ClusterQueue %v, wait time since queued was %.0fs", admission.ClusterQueue, waitTime.Seconds()) - metrics.QuotaReservedWorkload(admission.ClusterQueue, newWorkload.Spec.PriorityClassName, waitTime) + priorityClassName := workload.PriorityClassName(newWorkload) + metrics.QuotaReservedWorkload(admission.ClusterQueue, priorityClassName, waitTime) if features.Enabled(features.LocalQueueMetrics) { - metrics.LocalQueueQuotaReservedWorkload(metrics.LQRefFromWorkload(newWorkload), newWorkload.Spec.PriorityClassName, waitTime) + metrics.LocalQueueQuotaReservedWorkload(metrics.LQRefFromWorkload(newWorkload), priorityClassName, waitTime) } } @@ -820,20 +821,17 @@ func (s *Scheduler) recordWorkloadAdmissionEvents(newWorkload, originalWorkload } s.recorder.Eventf(newWorkload, corev1.EventTypeNormal, "Admitted", "Admitted by ClusterQueue %v, wait time since reservation was 0s", admission.ClusterQueue) - metrics.AdmittedWorkload(admission.ClusterQueue, newWorkload.Spec.PriorityClassName, waitTime) + priorityClassName := workload.PriorityClassName(newWorkload) + metrics.AdmittedWorkload(admission.ClusterQueue, priorityClassName, waitTime) if features.Enabled(features.LocalQueueMetrics) { - metrics.LocalQueueAdmittedWorkload( - metrics.LQRefFromWorkload(newWorkload), - newWorkload.Spec.PriorityClassName, - waitTime, - ) + metrics.LocalQueueAdmittedWorkload(metrics.LQRefFromWorkload(newWorkload), priorityClassName, waitTime) } if len(newWorkload.Status.AdmissionChecks) > 0 { - metrics.ReportAdmissionChecksWaitTime(admission.ClusterQueue, newWorkload.Spec.PriorityClassName, 0) + metrics.ReportAdmissionChecksWaitTime(admission.ClusterQueue, priorityClassName, 0) if features.Enabled(features.LocalQueueMetrics) { - metrics.ReportLocalQueueAdmissionChecksWaitTime(metrics.LQRefFromWorkload(newWorkload), newWorkload.Spec.PriorityClassName, 0) + metrics.ReportLocalQueueAdmissionChecksWaitTime(metrics.LQRefFromWorkload(newWorkload), priorityClassName, 0) } } } diff --git a/pkg/util/priority/priority.go b/pkg/util/priority/priority.go index 060d896d8dd..ee4894c3bcd 100644 --- a/pkg/util/priority/priority.go +++ b/pkg/util/priority/priority.go @@ -30,51 +30,49 @@ import ( // Priority returns priority of the given workload. func Priority(w *kueue.Workload) int32 { - // When priority of a running workload is nil, it means it was created at a time + // When the priority of a running workload is nil, it means it was created at a time // that there was no global default priority class and the priority class // name of the pod was empty. So, we resolve to the static default priority. return ptr.Deref(w.Spec.Priority, constants.DefaultPriority) } // GetPriorityFromPriorityClass returns the priority populated from -// priority class. If not specified, priority will be default or +// priority class. If not specified, the priority will be default or // zero if there is no default. -func GetPriorityFromPriorityClass(ctx context.Context, client client.Client, - priorityClass string) (string, string, int32, error) { +func GetPriorityFromPriorityClass(ctx context.Context, client client.Client, priorityClass string) (*kueue.PriorityClassRef, int32, error) { if len(priorityClass) == 0 { return getDefaultPriority(ctx, client) } pc := &schedulingv1.PriorityClass{} if err := client.Get(ctx, types.NamespacedName{Name: priorityClass}, pc); err != nil { - return "", "", 0, err + return nil, 0, err } - return pc.Name, constants.PodPriorityClassSource, pc.Value, nil + return kueue.NewPodPriorityClassRef(pc.Name), pc.Value, nil } // GetPriorityFromWorkloadPriorityClass returns the priority populated from // workload priority class. If not specified, returns 0. // DefaultPriority is not called within this function -// because k8s priority class should be checked next. -func GetPriorityFromWorkloadPriorityClass(ctx context.Context, client client.Client, - workloadPriorityClass string) (string, string, int32, error) { +// because k8s priority class should be checked next. +func GetPriorityFromWorkloadPriorityClass(ctx context.Context, client client.Client, workloadPriorityClass string) (*kueue.PriorityClassRef, int32, error) { wpc := &kueue.WorkloadPriorityClass{} if err := client.Get(ctx, types.NamespacedName{Name: workloadPriorityClass}, wpc); err != nil { - return "", "", 0, err + return nil, 0, err } - return wpc.Name, constants.WorkloadPriorityClassSource, wpc.Value, nil + return kueue.NewWorkloadPriorityClassRef(wpc.Name), wpc.Value, nil } -func getDefaultPriority(ctx context.Context, client client.Client) (string, string, int32, error) { +func getDefaultPriority(ctx context.Context, client client.Client) (*kueue.PriorityClassRef, int32, error) { dpc, err := getDefaultPriorityClass(ctx, client) if err != nil { - return "", "", 0, err + return nil, 0, err } if dpc != nil { - return dpc.Name, constants.PodPriorityClassSource, dpc.Value, nil + return kueue.NewPodPriorityClassRef(dpc.Name), dpc.Value, nil } - return "", "", int32(constants.DefaultPriority), nil + return nil, constants.DefaultPriority, nil } func getDefaultPriorityClass(ctx context.Context, client client.Client) (*schedulingv1.PriorityClass, error) { diff --git a/pkg/util/priority/priority_test.go b/pkg/util/priority/priority_test.go index 6d8f005d5f9..26ca76df6f4 100644 --- a/pkg/util/priority/priority_test.go +++ b/pkg/util/priority/priority_test.go @@ -66,12 +66,11 @@ func TestGetPriorityFromPriorityClass(t *testing.T) { } tests := map[string]struct { - priorityClassList *schedulingv1.PriorityClassList - priorityClassName string - wantPriorityClassName string - wantPriorityClassSource string - wantPriorityClassValue int32 - wantErr error + priorityClassList *schedulingv1.PriorityClassList + priorityClassName string + wantPriorityClassRef *kueue.PriorityClassRef + wantPriorityClassValue int32 + wantErr error }{ "priorityClass is specified and it exists": { priorityClassList: &schedulingv1.PriorityClassList{ @@ -82,10 +81,10 @@ func TestGetPriorityFromPriorityClass(t *testing.T) { }, }, }, - priorityClassName: "test", - wantPriorityClassSource: constants.PodPriorityClassSource, - wantPriorityClassName: "test", - wantPriorityClassValue: 50, + + priorityClassName: "test", + wantPriorityClassRef: kueue.NewPodPriorityClassRef("test"), + wantPriorityClassValue: 50, }, "priorityClass is specified and it does not exist": { priorityClassList: &schedulingv1.PriorityClassList{ @@ -104,9 +103,8 @@ func TestGetPriorityFromPriorityClass(t *testing.T) { }, }, }, - wantPriorityClassName: "globalDefault", - wantPriorityClassSource: constants.PodPriorityClassSource, - wantPriorityClassValue: 40, + wantPriorityClassRef: kueue.NewPodPriorityClassRef("globalDefault"), + wantPriorityClassValue: 40, }, "priorityClass is unspecified and multiple global defaults exist": { priorityClassList: &schedulingv1.PriorityClassList{ @@ -128,9 +126,8 @@ func TestGetPriorityFromPriorityClass(t *testing.T) { }, }, }, - wantPriorityClassName: "globalDefault2", - wantPriorityClassSource: constants.PodPriorityClassSource, - wantPriorityClassValue: 20, + wantPriorityClassRef: kueue.NewPodPriorityClassRef("globalDefault2"), + wantPriorityClassValue: 20, }, } @@ -142,17 +139,13 @@ func TestGetPriorityFromPriorityClass(t *testing.T) { client := builder.Build() ctx, _ := utiltesting.ContextWithLog(t) - name, source, value, err := GetPriorityFromPriorityClass(ctx, client, tt.priorityClassName) + priorityClassRef, value, err := GetPriorityFromPriorityClass(ctx, client, tt.priorityClassName) if diff := cmp.Diff(tt.wantErr, err); diff != "" { t.Errorf("unexpected error (-want,+got):\n%s", diff) } - if name != tt.wantPriorityClassName { - t.Errorf("unexpected name: got: %s, expected: %s", name, tt.wantPriorityClassName) - } - - if source != tt.wantPriorityClassSource { - t.Errorf("unexpected source: got: %s, expected: %s", source, tt.wantPriorityClassSource) + if diff := cmp.Diff(tt.wantPriorityClassRef, priorityClassRef); diff != "" { + t.Errorf("unexpected priortyClassRef (-want,+got):\n%s", diff) } if value != tt.wantPriorityClassValue { @@ -169,12 +162,11 @@ func TestGetPriorityFromWorkloadPriorityClass(t *testing.T) { } tests := map[string]struct { - workloadPriorityClassList *kueue.WorkloadPriorityClassList - workloadPriorityClassName string - wantWorkloadPriorityClassName string - wantWorkloadPriorityClassSource string - wantWorkloadPriorityClassValue int32 - wantErr error + workloadPriorityClassList *kueue.WorkloadPriorityClassList + workloadPriorityClassName string + wantPriorityClassRef *kueue.PriorityClassRef + wantPriorityClassValue int32 + wantErr error }{ "workloadPriorityClass is specified and it exists": { workloadPriorityClassList: &kueue.WorkloadPriorityClassList{ @@ -185,10 +177,9 @@ func TestGetPriorityFromWorkloadPriorityClass(t *testing.T) { }, }, }, - workloadPriorityClassName: "test", - wantWorkloadPriorityClassSource: constants.WorkloadPriorityClassSource, - wantWorkloadPriorityClassName: "test", - wantWorkloadPriorityClassValue: 50, + workloadPriorityClassName: "test", + wantPriorityClassRef: kueue.NewWorkloadPriorityClassRef("test"), + wantPriorityClassValue: 50, }, "workloadPriorityClass is specified and it does not exist": { workloadPriorityClassList: &kueue.WorkloadPriorityClassList{ @@ -206,21 +197,17 @@ func TestGetPriorityFromWorkloadPriorityClass(t *testing.T) { builder := fake.NewClientBuilder().WithScheme(scheme).WithLists(tt.workloadPriorityClassList) client := builder.Build() ctx, _ := utiltesting.ContextWithLog(t) - name, source, value, err := GetPriorityFromWorkloadPriorityClass(ctx, client, tt.workloadPriorityClassName) + priorityClassRef, value, err := GetPriorityFromWorkloadPriorityClass(ctx, client, tt.workloadPriorityClassName) if diff := cmp.Diff(tt.wantErr, err); diff != "" { t.Errorf("unexpected error (-want,+got):\n%s", diff) } - if name != tt.wantWorkloadPriorityClassName { - t.Errorf("unexpected name: got: %s, expected: %s", name, tt.wantWorkloadPriorityClassName) + if diff := cmp.Diff(tt.wantPriorityClassRef, priorityClassRef); diff != "" { + t.Errorf("unexpected priortyClassRef (-want,+got):\n%s", diff) } - if source != tt.wantWorkloadPriorityClassSource { - t.Errorf("unexpected source: got: %s, expected: %s", source, tt.wantWorkloadPriorityClassSource) - } - - if value != tt.wantWorkloadPriorityClassValue { - t.Errorf("unexpected value: got: %d, expected: %d", value, tt.wantWorkloadPriorityClassValue) + if value != tt.wantPriorityClassValue { + t.Errorf("unexpected value: got: %d, expected: %d", value, tt.wantPriorityClassValue) } }) } diff --git a/pkg/util/testing/v1beta2/wrappers.go b/pkg/util/testing/v1beta2/wrappers.go index 08c484e7ff9..bdff2997fc2 100644 --- a/pkg/util/testing/v1beta2/wrappers.go +++ b/pkg/util/testing/v1beta2/wrappers.go @@ -242,11 +242,6 @@ func (w *WorkloadWrapper) Creation(t time.Time) *WorkloadWrapper { return w } -func (w *WorkloadWrapper) PriorityClass(priorityClassName string) *WorkloadWrapper { - w.Spec.PriorityClassName = priorityClassName - return w -} - func (w *WorkloadWrapper) RuntimeClass(name string) *WorkloadWrapper { for i := range w.Spec.PodSets { w.Spec.PodSets[i].Template.Spec.RuntimeClassName = &name @@ -254,13 +249,21 @@ func (w *WorkloadWrapper) RuntimeClass(name string) *WorkloadWrapper { return w } -func (w *WorkloadWrapper) Priority(priority int32) *WorkloadWrapper { - w.Spec.Priority = &priority +func (w *WorkloadWrapper) PriorityClassRef(ref *kueue.PriorityClassRef) *WorkloadWrapper { + w.Spec.PriorityClassRef = ref return w } -func (w *WorkloadWrapper) PriorityClassSource(source string) *WorkloadWrapper { - w.Spec.PriorityClassSource = source +func (w *WorkloadWrapper) WorkloadPriorityClassRef(name string) *WorkloadWrapper { + return w.PriorityClassRef(kueue.NewWorkloadPriorityClassRef(name)) +} + +func (w *WorkloadWrapper) PodPriorityClassRef(name string) *WorkloadWrapper { + return w.PriorityClassRef(kueue.NewPodPriorityClassRef(name)) +} + +func (w *WorkloadWrapper) Priority(priority int32) *WorkloadWrapper { + w.Spec.Priority = &priority return w } diff --git a/pkg/workload/workload.go b/pkg/workload/workload.go index 5ef39aa8924..d465a11ac58 100644 --- a/pkg/workload/workload.go +++ b/pkg/workload/workload.go @@ -1296,11 +1296,24 @@ func Evict(ctx context.Context, c client.Client, recorder record.EventRecorder, } reportEvictedWorkload(recorder, wl, wl.Status.Admission.ClusterQueue, reason, msg, underlyingCause) if reportWorkloadEvictedOnce { - metrics.ReportEvictedWorkloadsOnce(wl.Status.Admission.ClusterQueue, reason, string(underlyingCause), wl.Spec.PriorityClassName) + metrics.ReportEvictedWorkloadsOnce(wl.Status.Admission.ClusterQueue, reason, string(underlyingCause), PriorityClassName(wl)) } return nil } +func PriorityClassName(wl *kueue.Workload) string { + if wl.Spec.PriorityClassRef != nil { + return wl.Spec.PriorityClassRef.Name + } + return "" +} + +func IsWorkloadPriorityClass(wl *kueue.Workload) bool { + return wl.Spec.PriorityClassRef != nil && + wl.Spec.PriorityClassRef.Kind == kueue.WorkloadPriorityClassKind && + wl.Spec.PriorityClassRef.Group == kueue.WorkloadPriorityClassGroup +} + func prepareForEviction(w *kueue.Workload, now time.Time, reason, message string) { SetEvictedCondition(w, now, reason, message) resetClusterNomination(w) @@ -1318,7 +1331,8 @@ func resetUnhealthyNodes(w *kueue.Workload) { } func reportEvictedWorkload(recorder record.EventRecorder, wl *kueue.Workload, cqName kueue.ClusterQueueReference, reason, message string, underlyingCause kueue.EvictionUnderlyingCause) { - metrics.ReportEvictedWorkloads(cqName, reason, string(underlyingCause), wl.Spec.PriorityClassName) + priorityClassName := PriorityClassName(wl) + metrics.ReportEvictedWorkloads(cqName, reason, string(underlyingCause), priorityClassName) if podsReadyToEvictionTime := workloadsWithPodsReadyToEvictedTime(wl); podsReadyToEvictionTime != nil { metrics.PodsReadyToEvictedTimeSeconds.WithLabelValues(string(cqName), reason, string(underlyingCause)).Observe(podsReadyToEvictionTime.Seconds()) } @@ -1327,7 +1341,7 @@ func reportEvictedWorkload(recorder record.EventRecorder, wl *kueue.Workload, cq metrics.LQRefFromWorkload(wl), reason, string(underlyingCause), - wl.Spec.PriorityClassName, + priorityClassName, ) } eventReason := ReasonWithCause(kueue.WorkloadEvicted, reason) diff --git a/site/content/en/docs/reference/kueue.v1beta2.md b/site/content/en/docs/reference/kueue.v1beta2.md index dc6b1ef176e..4a061fcd503 100644 --- a/site/content/en/docs/reference/kueue.v1beta2.md +++ b/site/content/en/docs/reference/kueue.v1beta2.md @@ -2200,6 +2200,82 @@ result in failure during workload admission.

    +## `PriorityClassGroup` {#kueue-x-k8s-io-v1beta2-PriorityClassGroup} + +(Alias of `string`) + +**Appears in:** + +- [PriorityClassRef](#kueue-x-k8s-io-v1beta2-PriorityClassRef) + + +

    PriorityClassGroup indicates the API group of the PriorityClass object.

    + + + + +## `PriorityClassKind` {#kueue-x-k8s-io-v1beta2-PriorityClassKind} + +(Alias of `string`) + +**Appears in:** + +- [PriorityClassRef](#kueue-x-k8s-io-v1beta2-PriorityClassRef) + + +

    PriorityClassKind is the kind of the PriorityClass object.

    + + + + +## `PriorityClassRef` {#kueue-x-k8s-io-v1beta2-PriorityClassRef} + + +**Appears in:** + +- [WorkloadSpec](#kueue-x-k8s-io-v1beta2-WorkloadSpec) + + +

    PriorityClassRef references a PriorityClass in a specific API group.

    + + + + + + + + + + + + + + + + + +
    FieldDescription
    group [Required]
    +PriorityClassGroup +
    +

    group is the API group of the PriorityClass object. +Use "kueue.x-k8s.io" for WorkloadPriorityClass. +Use "scheduling.k8s.io" for Pod PriorityClass.

    +
    kind [Required]
    +PriorityClassKind +
    +

    kind is the kind of the PriorityClass object.

    +
    name [Required]
    +string +
    +

    name is the name of the PriorityClass the Workload is associated with. +If specified, indicates the workload's priority. +"system-node-critical" and "system-cluster-critical" are two special +keywords which indicate the highest priorities with the former being +the highest priority. Any other name must be defined by creating a +PriorityClass object with that name. If not specified, the workload +priority will be default or zero if there is no default.

    +
    + ## `ProvisioningRequestConfigPodSetMergePolicy` {#kueue-x-k8s-io-v1beta2-ProvisioningRequestConfigPodSetMergePolicy} (Alias of `string`) @@ -2998,17 +3074,11 @@ podSets cannot be changed.

    queueName cannot be changed while .status.admission is not null.

    -priorityClassName
    -string +priorityClassRef
    +PriorityClassRef -

    priorityClassName is the name of the PriorityClass the Workload is associated with. -If specified, indicates the workload's priority. -"system-node-critical" and "system-cluster-critical" are two special -keywords which indicate the highest priorities with the former being -the highest priority. Any other name must be defined by creating a -PriorityClass object with that name. If not specified, the workload -priority will be default or zero if there is no default.

    +

    priorityClassRef references a PriorityClass object that defines the workload's priority.

    priority
    @@ -3017,18 +3087,9 @@ priority will be default or zero if there is no default.

    priority determines the order of access to the resources managed by the ClusterQueue where the workload is queued. -The priority value is populated from PriorityClassName. +The priority value is populated from the referenced PriorityClass (via priorityClassRef). The higher the value, the higher the priority. -If priorityClassName is specified, priority must not be null.

    - - -priorityClassSource
    -string - - -

    priorityClassSource determines whether the priorityClass field refers to a pod PriorityClass or kueue.x-k8s.io/workloadpriorityclass. -Workload's PriorityClass can accept the name of a pod priorityClass or a workloadPriorityClass. -When using pod PriorityClass, a priorityClassSource field has the scheduling.k8s.io/priorityclass value.

    +If priorityClassRef is specified, priority must not be null.

    active [Required]
    diff --git a/test/integration/singlecluster/controller/core/workload_controller_test.go b/test/integration/singlecluster/controller/core/workload_controller_test.go index 77ccc3b7e96..4887197db29 100644 --- a/test/integration/singlecluster/controller/core/workload_controller_test.go +++ b/test/integration/singlecluster/controller/core/workload_controller_test.go @@ -30,7 +30,6 @@ import ( config "sigs.k8s.io/kueue/apis/config/v1beta2" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/util/admissioncheck" "sigs.k8s.io/kueue/pkg/util/slices" @@ -414,7 +413,9 @@ var _ = ginkgo.Describe("Workload controller", ginkgo.Ordered, ginkgo.ContinueOn ginkgo.It("case of WorkloadPriorityClass", func() { ginkgo.By("creating workload") wl = utiltestingapi.MakeWorkload("wl", ns.Name).Queue("lq").Request(corev1.ResourceCPU, "1"). - PriorityClass("workload-priority-class").PriorityClassSource(constants.WorkloadPriorityClassSource).Priority(200).Obj() + WorkloadPriorityClassRef("workload-priority-class"). + Priority(200). + Obj() util.MustCreate(ctx, k8sClient, wl) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedQueueWorkload)).To(gomega.Succeed()) diff --git a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go index a04eef776da..c9c0097cd68 100644 --- a/test/integration/singlecluster/controller/jobs/job/job_controller_test.go +++ b/test/integration/singlecluster/controller/jobs/job/job_controller_test.go @@ -1394,8 +1394,6 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con ginkgo.By("checking that the second workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) - g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) - g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) g.Expect(createdWorkload2.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) @@ -1495,8 +1493,6 @@ var _ = ginkgo.Describe("Interacting with scheduler", ginkgo.Ordered, ginkgo.Con ginkgo.By("checking that the second workload is admitted", func() { gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, wlLookupKey2, createdWorkload2)).Should(gomega.Succeed()) - g.Expect(createdWorkload2.Spec.PriorityClassName).Should(gomega.Equal(highWorkloadPriorityClass.Name)) - g.Expect(*createdWorkload2.Spec.Priority).Should(gomega.Equal(highWorkloadPriorityClass.Value)) g.Expect(createdWorkload2.Status.Conditions).Should(utiltesting.HaveConditionStatusTrue(kueue.WorkloadAdmitted)) }, util.Timeout, util.Interval).Should(gomega.Succeed()) }) diff --git a/test/integration/singlecluster/webhook/core/workload_test.go b/test/integration/singlecluster/webhook/core/workload_test.go index 9ad6638c702..86994368ef1 100644 --- a/test/integration/singlecluster/webhook/core/workload_test.go +++ b/test/integration/singlecluster/webhook/core/workload_test.go @@ -33,7 +33,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/constants" "sigs.k8s.io/kueue/pkg/features" utiltesting "sigs.k8s.io/kueue/pkg/util/testing" utiltestingapi "sigs.k8s.io/kueue/pkg/util/testing/v1beta2" @@ -168,23 +167,71 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ).Obj() }, utiltesting.BeInvalidError()), - ginkgo.Entry("invalid priorityClassName", + ginkgo.Entry("invalid priorityClassRef.name", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClass("invalid_class"). + PodPriorityClassRef("invalid_class"). Priority(0). Obj() }, utiltesting.BeInvalidError()), - ginkgo.Entry("empty priorityClassName is valid", + ginkgo.Entry("invalid priorityClassRef.group", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassRef(&kueue.PriorityClassRef{ + Group: "test", + Kind: kueue.PodPriorityClassKind, + Name: "low", + }). + Priority(100). + Obj() + }, + utiltesting.BeInvalidError()), + ginkgo.Entry("invalid priorityClassRef.kind", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassRef(&kueue.PriorityClassRef{ + Group: kueue.PodPriorityClassGroup, + Kind: "test", + Name: "low", + }). + Priority(100). + Obj() + }, + utiltesting.BeInvalidError()), + ginkgo.Entry("incompatible priorityClassRef.kind (scheduling.k8s.io)", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassRef(&kueue.PriorityClassRef{ + Group: kueue.PodPriorityClassGroup, + Kind: kueue.WorkloadPriorityClassKind, + Name: "low", + }). + Priority(100). + Obj() + }, + utiltesting.BeInvalidError()), + ginkgo.Entry("incompatible priorityClassRef.kind (kueue.x-k8s.io)", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + PriorityClassRef(&kueue.PriorityClassRef{ + Group: kueue.WorkloadPriorityClassGroup, + Kind: kueue.PodPriorityClassKind, + Name: "low", + }). + Priority(100). + Obj() + }, + utiltesting.BeInvalidError()), + ginkgo.Entry("omitted priorityClassRef is valid", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name).Priority(0).Obj() }, gomega.Succeed()), - ginkgo.Entry("priority should not be nil when priorityClassName is set", + ginkgo.Entry("priority should not be nil when priorityClassRef is set", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClass("priority"). + PodPriorityClassRef("priority"). Obj() }, utiltesting.BeInvalidError()), @@ -638,34 +685,6 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }, gomega.Succeed(), ), - ginkgo.Entry("priorityClassSource should not be updated", - func() *kueue.Workload { - return utiltestingapi.MakeWorkload(workloadName, ns.Name). - Queue("q"). - PriorityClass("test-class").PriorityClassSource(constants.PodPriorityClassSource). - Priority(10). - Obj() - }, - true, - func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = constants.WorkloadPriorityClassSource - }, - utiltesting.BeInvalidError(), - ), - ginkgo.Entry("priorityClassName should not be updated", - func() *kueue.Workload { - return utiltestingapi.MakeWorkload(workloadName, ns.Name). - Queue("q"). - PriorityClass("test-class-1").PriorityClassSource(constants.PodPriorityClassSource). - Priority(10). - Obj() - }, - true, - func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassName = "test-class-2" - }, - utiltesting.BeInvalidError(), - ), ginkgo.Entry("should change other fields of admissionchecks when podSetUpdates is immutable", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). @@ -694,32 +713,6 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }, gomega.Succeed(), ), - ginkgo.Entry("updating priorityClassName before setting reserve quota for workload", - func() *kueue.Workload { - return utiltestingapi.MakeWorkload(workloadName, ns.Name). - Queue("q"). - PriorityClass("test-class-1").PriorityClassSource(constants.PodPriorityClassSource). - Priority(10).Obj() - }, - false, - func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassName = "test-class-2" - }, - gomega.Succeed(), - ), - ginkgo.Entry("updating priorityClassSource before setting reserve quota for workload", - func() *kueue.Workload { - return utiltestingapi.MakeWorkload(workloadName, ns.Name). - Queue("q"). - PriorityClass("test-class").PriorityClassSource(constants.PodPriorityClassSource). - Priority(10).Obj() - }, - false, - func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = constants.WorkloadPriorityClassSource - }, - gomega.Succeed(), - ), ginkgo.Entry("updating podSets before setting reserve quota for workload", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() @@ -747,16 +740,6 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }, gomega.Succeed(), ), - ginkgo.Entry("Should allow the change of priority", - func() *kueue.Workload { - return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() - }, - false, - func(newWL *kueue.Workload) { - newWL.Spec.Priority = ptr.To[int32](10) - }, - gomega.Succeed(), - ), ginkgo.Entry("Should forbid the change of spec.podSet", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() @@ -844,14 +827,59 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }, utiltesting.BeInvalidError(), ), - ginkgo.Entry("can set workload priority class when QuotaReserved=false", + ginkgo.Entry("Should allow to change priority when QuotaReserved=false", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.Priority = ptr.To[int32](10) + }, + gomega.Succeed(), + ), + ginkgo.Entry("Should allow to change priority when QuotaReserved=true", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.Priority = ptr.To[int32](10) + }, + gomega.Succeed(), + ), + ginkgo.Entry("Should allow to change priority with priorityClassRef when QuotaReserved=false", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + Priority(0). + PodPriorityClassRef("low"). + Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.Priority = ptr.To[int32](10) + }, + gomega.Succeed(), + ), + ginkgo.Entry("Should allow to change priority with priorityClassRef when QuotaReserved=true", + func() *kueue.Workload { + return utiltestingapi.MakeWorkload(workloadName, ns.Name). + Priority(0). + PodPriorityClassRef("low"). + Obj() + }, + false, + func(newWL *kueue.Workload) { + newWL.Spec.Priority = ptr.To[int32](10) + }, + gomega.Succeed(), + ), + ginkgo.Entry("can set workload priorityClassRef when QuotaReserved=false", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name).Obj() }, false, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = constants.WorkloadPriorityClassSource - newWL.Spec.PriorityClassName = "low" + newWL.Spec.PriorityClassRef = kueue.NewWorkloadPriorityClassRef("low") newWL.Spec.Priority = ptr.To[int32](100) }, gomega.Succeed(), @@ -862,8 +890,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }, true, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = constants.WorkloadPriorityClassSource - newWL.Spec.PriorityClassName = "low" + newWL.Spec.PriorityClassRef = kueue.NewWorkloadPriorityClassRef("low") newWL.Spec.Priority = ptr.To[int32](100) }, utiltesting.BeInvalidError(), @@ -871,14 +898,13 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.Entry("can update workload priority class when QuotaReserved=false", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClassSource(constants.WorkloadPriorityClassSource). - PriorityClass("high"). + WorkloadPriorityClassRef("high"). Priority(1000). Obj() }, false, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassName = "low" + newWL.Spec.PriorityClassRef.Name = "low" newWL.Spec.Priority = ptr.To[int32](100) }, gomega.Succeed(), @@ -886,14 +912,13 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.Entry("can update workload priority class when QuotaReserved=true", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClassSource(constants.WorkloadPriorityClassSource). - PriorityClass("high"). + WorkloadPriorityClassRef("high"). Priority(1000). Obj() }, true, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassName = "low" + newWL.Spec.PriorityClassRef.Name = "low" newWL.Spec.Priority = ptr.To[int32](100) }, gomega.Succeed(), @@ -901,15 +926,13 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.Entry("can delete workload priority class when QuotaReserved=false", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClassSource(constants.WorkloadPriorityClassSource). - PriorityClass("high"). + WorkloadPriorityClassRef("high"). Priority(1000). Obj() }, false, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = "" - newWL.Spec.PriorityClassName = "" + newWL.Spec.PriorityClassRef = nil newWL.Spec.Priority = nil }, gomega.Succeed(), @@ -917,15 +940,13 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.Entry("can't delete workload priority class when quota reserved when QuotaReserved=true", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClassSource(constants.WorkloadPriorityClassSource). - PriorityClass("high"). + WorkloadPriorityClassRef("high"). Priority(1000). Obj() }, true, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = "" - newWL.Spec.PriorityClassName = "" + newWL.Spec.PriorityClassRef = nil newWL.Spec.Priority = nil }, utiltesting.BeInvalidError(), @@ -936,8 +957,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }, false, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = constants.PodPriorityClassSource - newWL.Spec.PriorityClassName = "low" + newWL.Spec.PriorityClassRef = kueue.NewPodPriorityClassRef("low") newWL.Spec.Priority = ptr.To[int32](100) }, gomega.Succeed(), @@ -948,8 +968,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { }, true, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = constants.PodPriorityClassSource - newWL.Spec.PriorityClassName = "low" + newWL.Spec.PriorityClassRef = kueue.NewPodPriorityClassRef("low") newWL.Spec.Priority = ptr.To[int32](100) }, utiltesting.BeInvalidError(), @@ -957,14 +976,13 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.Entry("can update pod priority class when QuotaReserved=false", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClassSource(constants.PodPriorityClassSource). - PriorityClass("high"). + PodPriorityClassRef("high"). Priority(1000). Obj() }, false, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassName = "low" + newWL.Spec.PriorityClassRef.Name = "low" newWL.Spec.Priority = ptr.To[int32](100) }, gomega.Succeed(), @@ -972,14 +990,13 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.Entry("can't update pod priority class when QuotaReserved=true", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClassSource(constants.PodPriorityClassSource). - PriorityClass("high"). + PodPriorityClassRef("high"). Priority(1000). Obj() }, true, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassName = "low" + newWL.Spec.PriorityClassRef.Name = "low" newWL.Spec.Priority = ptr.To[int32](100) }, utiltesting.BeInvalidError(), @@ -987,15 +1004,13 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.Entry("can delete pod priority class when QuotaReserved=false", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClassSource(constants.PodPriorityClassSource). - PriorityClass("high"). + PodPriorityClassRef("high"). Priority(1000). Obj() }, false, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = "" - newWL.Spec.PriorityClassName = "" + newWL.Spec.PriorityClassRef = nil newWL.Spec.Priority = nil }, gomega.Succeed(), @@ -1003,15 +1018,13 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.Entry("can't delete pod priority class when QuotaReserved=true", func() *kueue.Workload { return utiltestingapi.MakeWorkload(workloadName, ns.Name). - PriorityClassSource(constants.PodPriorityClassSource). - PriorityClass("high"). + PodPriorityClassRef("high"). Priority(1000). Obj() }, true, func(newWL *kueue.Workload) { - newWL.Spec.PriorityClassSource = "" - newWL.Spec.PriorityClassName = "" + newWL.Spec.PriorityClassRef = nil newWL.Spec.Priority = nil }, utiltesting.BeInvalidError(), @@ -1055,7 +1068,7 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.It("Should have priority once priorityClassName is set", func() { ginkgo.By("Creating a new Workload") - workload := utiltestingapi.MakeWorkload(workloadName, ns.Name).PriorityClass("priority").Obj() + workload := utiltestingapi.MakeWorkload(workloadName, ns.Name).PodPriorityClassRef("priority").Obj() err := k8sClient.Create(ctx, workload) gomega.Expect(err).Should(utiltesting.BeInvalidError()) }) @@ -1063,7 +1076,9 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.It("workload's priority should be mutable when referencing WorkloadPriorityClass", func() { ginkgo.By("creating workload") wl := utiltestingapi.MakeWorkload("wl", ns.Name).Queue("lq").Request(corev1.ResourceCPU, "1"). - PriorityClass("workload-priority-class").PriorityClassSource(constants.WorkloadPriorityClassSource).Priority(200).Obj() + WorkloadPriorityClassRef("workload-priority-class"). + Priority(200). + Obj() util.MustCreate(ctx, k8sClient, wl) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedQueueWorkload)).To(gomega.Succeed()) @@ -1085,7 +1100,9 @@ var _ = ginkgo.Describe("Workload validating webhook", ginkgo.Ordered, func() { ginkgo.It("workload's priority should be mutable when referencing PriorityClass", func() { ginkgo.By("creating workload") wl := utiltestingapi.MakeWorkload("wl", ns.Name).Queue("lq").Request(corev1.ResourceCPU, "1"). - PriorityClass("priority-class").PriorityClassSource(constants.PodPriorityClassSource).Priority(100).Obj() + PodPriorityClassRef("priority-class"). + Priority(100). + Obj() util.MustCreate(ctx, k8sClient, wl) gomega.Eventually(func(g gomega.Gomega) { g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(wl), &updatedQueueWorkload)).To(gomega.Succeed()) diff --git a/test/util/util.go b/test/util/util.go index 981eeb10df0..68ac9828b3e 100644 --- a/test/util/util.go +++ b/test/util/util.go @@ -66,7 +66,6 @@ import ( leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - "sigs.k8s.io/kueue/pkg/constants" podconstants "sigs.k8s.io/kueue/pkg/controller/jobs/pod/constants" "sigs.k8s.io/kueue/pkg/metrics" "sigs.k8s.io/kueue/pkg/scheduler/preemption" @@ -369,22 +368,24 @@ func expectWorkloadsToBeAdmittedCountWithOffset(ctx context.Context, offset int, func ExpectWorkloadsWithWorkloadPriority(ctx context.Context, c client.Client, name string, value int32, wlKeys ...client.ObjectKey) { ginkgo.GinkgoHelper() - expectWorkloadsWithPriority(ctx, c, constants.WorkloadPriorityClassSource, name, value, wlKeys...) + expectWorkloadsWithPriority(ctx, c, kueue.WorkloadPriorityClassGroup, kueue.WorkloadPriorityClassKind, name, value, wlKeys...) } func ExpectWorkloadsWithPodPriority(ctx context.Context, c client.Client, name string, value int32, wlKeys ...client.ObjectKey) { ginkgo.GinkgoHelper() - expectWorkloadsWithPriority(ctx, c, constants.PodPriorityClassSource, name, value, wlKeys...) + expectWorkloadsWithPriority(ctx, c, kueue.PodPriorityClassGroup, kueue.PodPriorityClassKind, name, value, wlKeys...) } -func expectWorkloadsWithPriority(ctx context.Context, c client.Client, priorityClassSource, name string, value int32, wlKeys ...client.ObjectKey) { +func expectWorkloadsWithPriority(ctx context.Context, c client.Client, priorityClassGroup kueue.PriorityClassGroup, priorityClassKind kueue.PriorityClassKind, name string, value int32, wlKeys ...client.ObjectKey) { ginkgo.GinkgoHelper() createdWl := &kueue.Workload{} gomega.Eventually(func(g gomega.Gomega) { for _, wlKey := range wlKeys { g.Expect(c.Get(ctx, wlKey, createdWl)).To(gomega.Succeed()) - g.Expect(createdWl.Spec.PriorityClassSource).To(gomega.Equal(priorityClassSource)) - g.Expect(createdWl.Spec.PriorityClassName).To(gomega.Equal(name)) + g.Expect(createdWl.Spec.PriorityClassRef).ToNot(gomega.BeNil()) + g.Expect(createdWl.Spec.PriorityClassRef.Group).To(gomega.Equal(priorityClassGroup)) + g.Expect(createdWl.Spec.PriorityClassRef.Kind).To(gomega.Equal(priorityClassKind)) + g.Expect(createdWl.Spec.PriorityClassRef.Name).To(gomega.Equal(name)) g.Expect(createdWl.Spec.Priority).To(gomega.Equal(&value)) } }, Timeout, Interval).Should(gomega.Succeed())