add option to restart prowjob when node is terminated (enabled by default)

Signed-off-by: Tim Ramlot <[email protected]>

Author: inteon

pkg/apis/prowjobs/v1/types.go

@@ -182,6 +182,11 @@ type ProwJobSpec struct {
 	// If this field is unspecified or false, a new pod will be created to replace
 	// the evicted one.
 	ErrorOnEviction bool `json:"error_on_eviction,omitempty"`
+	// ErrorOnTermination indicates that the ProwJob should be completed and given
+	// the ErrorState status if the pod that is executing the job is terminated.
+	// If this field is unspecified or false, a new pod will be created to replace
+	// the terminated one.
+	ErrorOnTermination bool `json:"error_on_termination,omitempty"`
 
 	// PodSpec provides the basis for running the test under
 	// a Kubernetes agent

pkg/plank/reconciler.go

@@ -61,7 +61,8 @@ const ControllerName = "plank"
 
 // PodStatus constants
 const (
-	Evicted = "Evicted"
+	Evicted    = "Evicted"
+	Terminated = "Terminated"
 )
 
 // NodeStatus constants
@@ -480,6 +481,34 @@ func (r *reconciler) syncPendingJob(ctx context.Context, pj *prowv1.ProwJob) (*r
 			r.log.WithField("name", pj.ObjectMeta.Name).Debug("Delete Pod.")
 			return nil, ctrlruntimeclient.IgnoreNotFound(client.Delete(ctx, pod))
 		}
+	} else if pod.Status.Reason == Terminated {
+		// Pod was terminated.
+		if pj.Spec.ErrorOnTermination {
+			// ErrorOnTermination is enabled, complete the PJ and mark it as
+			// errored.
+			r.log.WithField("error-on-termination", true).WithFields(pjutil.ProwJobFields(pj)).Info("Pods Node got terminated, fail job.")
+			pj.SetComplete()
+			pj.Status.State = prowv1.ErrorState
+			pj.Status.Description = "Job pod's node was terminated."
+		} else {
+			// ErrorOnTermination is disabled. Delete the pod now and recreate it in
+			// the next resync.
+			r.log.WithFields(pjutil.ProwJobFields(pj)).Info("Pods Node got terminated, deleting & next sync loop will restart pod")
+			client, ok := r.buildClients[pj.ClusterAlias()]
+			if !ok {
+				return nil, TerminalError(fmt.Errorf("terminated pod %s: unknown cluster alias %q", pod.Name, pj.ClusterAlias()))
+			}
+			if finalizers := sets.New[string](pod.Finalizers...); finalizers.Has(kubernetesreporterapi.FinalizerName) {
+				// We want the end user to not see this, so we have to remove the finalizer, otherwise the pod hangs
+				oldPod := pod.DeepCopy()
+				pod.Finalizers = finalizers.Delete(kubernetesreporterapi.FinalizerName).UnsortedList()
+				if err := client.Patch(ctx, pod, ctrlruntimeclient.MergeFrom(oldPod)); err != nil {
+					return nil, fmt.Errorf("failed to patch pod trying to remove %s finalizer: %w", kubernetesreporterapi.FinalizerName, err)
+				}
+			}
+			r.log.WithField("name", pj.ObjectMeta.Name).Debug("Delete Pod.")
+			return nil, ctrlruntimeclient.IgnoreNotFound(client.Delete(ctx, pod))
+		}
 	} else if pod.DeletionTimestamp != nil && pod.Status.Reason == NodeUnreachablePodReason {
 		// This can happen in any phase and means the node got evicted after it became unresponsive. Delete the finalizer so the pod
 		// vanishes and we will silently re-create it in the next iteration.