controllers: added keep failed iterations

added the capability to keep resources on failed job execution with a timeout

Signed-off-by: Amit Berner <[email protected]>

controllers/storageclient_controller.go

@@ -433,14 +433,24 @@ func (s *StorageClientReconciler) reconcileClientStatusReporterJob(instance *v1a
 	cronJob.Namespace = s.OperatorNamespace
 	addLabel(cronJob, storageClientNameLabel, instance.Name)
 	addLabel(cronJob, storageClientNamespaceLabel, instance.Namespace)
+	var jobDeadLineSeconds int64 = 155
+	var podDeadLineSeconds int64 = 120
+	var keepJobResourceSeconds int32 = 600
+	var reducedKeptSuccecsful int32 = 1 
+
 
 	_, err := controllerutil.CreateOrUpdate(s.ctx, s.Client, cronJob, func() error {
 		cronJob.Spec = batchv1.CronJobSpec{
 			Schedule: "* * * * *",
+			ConcurrencyPolicy: batchv1.ForbidConcurrent,
+			SuccessfulJobsHistoryLimit: &reducedKeptSuccecsful,
 			JobTemplate: batchv1.JobTemplateSpec{
 				Spec: batchv1.JobSpec{
+					ActiveDeadlineSeconds: &jobDeadLineSeconds,
+					TTLSecondsAfterFinished: &keepJobResourceSeconds,
 					Template: corev1.PodTemplateSpec{
 						Spec: corev1.PodSpec{
+							ActiveDeadlineSeconds: &podDeadLineSeconds,
 							Containers: []corev1.Container{
 								{
 									Name:  "heartbeat",
@@ -470,7 +480,6 @@ func (s *StorageClientReconciler) reconcileClientStatusReporterJob(instance *v1a
 					},
 				},
 			},
-			ConcurrencyPolicy: "Forbid",
 		}
 		return nil
 	})