diff --git a/pkg/cache/stale.go b/pkg/cache/stale.go new file mode 100644 index 0000000000..0dffbeda7b --- /dev/null +++ b/pkg/cache/stale.go @@ -0,0 +1,79 @@ +/* +Copyright 2021 Red Hat Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "sync" + + "k8s.io/apimachinery/pkg/types" +) + +// Functions in this file solve the problem of duplicate reconciles when the cached +// client hasn't received the latest resource and feeds a stale resource to controller. +// We will reject reconciling a resource if its generation is older than one we've +// successfully reconciled. + +// UIDToGenerationMap keeps track of highest observed generation of a particular resource UID +type UIDToGenerationMap struct { + mutex sync.RWMutex + resourceUIDtoGeneration map[types.UID]int64 +} + +// CreateUIDToGenerationMap creates a new UID => generation map +func CreateUIDToGenerationMap() *UIDToGenerationMap { + uidGenMap := UIDToGenerationMap{} + uidGenMap.resourceUIDtoGeneration = make(map[types.UID]int64) + return &uidGenMap +} + +// getGenerationForUID returns the latest successfully reconciled resource generation +// returns -1 if not found. +func (u *UIDToGenerationMap) getGenerationForUID(resourceUID types.UID) int64 { + u.mutex.Lock() + defer u.mutex.Unlock() + generation, ok := u.resourceUIDtoGeneration[resourceUID] + if !ok { + return -1 + } + return generation +} + +// setGenerationForUID returns the latest successfully reconciled resource generation +func (u *UIDToGenerationMap) setGenerationForUID(resourceUID types.UID, generation int64) { + u.mutex.Lock() + defer u.mutex.Unlock() + u.resourceUIDtoGeneration[resourceUID] = generation +} + +// IsCacheStale checks if the cached client is providing a resource we've successfully reconciled. +func (u *UIDToGenerationMap) IsCacheStale(resourceUID types.UID, generation int64) bool { + reconciledGeneration := u.getGenerationForUID(resourceUID) + // Resource should be reconciled if we've never seen it + if generation == -1 { + return false + } + // Resource is stale if we've reconciled a newer generation + if generation < reconciledGeneration { + return true + } + return false +} + +// RecordReconciledGeneration records that a resource UID generation was pushed back to the APIserver +func (u *UIDToGenerationMap) RecordReconciledGeneration(resourceUID types.UID, generation int64) { + u.setGenerationForUID(resourceUID, generation) +} diff --git a/pkg/controller/directimagemigration/directimagemigration_controller.go b/pkg/controller/directimagemigration/directimagemigration_controller.go index e048c6fb3a..63b93d02e8 100644 --- a/pkg/controller/directimagemigration/directimagemigration_controller.go +++ b/pkg/controller/directimagemigration/directimagemigration_controller.go @@ -22,6 +22,7 @@ import ( "github.com/konveyor/controller/pkg/logging" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/cache" migref "github.com/konveyor/mig-controller/pkg/reference" "github.com/opentracing/opentracing-go" "k8s.io/apimachinery/pkg/api/errors" @@ -46,9 +47,10 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { return &ReconcileDirectImageMigration{ - Client: mgr.GetClient(), - scheme: mgr.GetScheme(), - EventRecorder: mgr.GetRecorder("directimagemigration_controller"), + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + EventRecorder: mgr.GetRecorder("directimagemigration_controller"), + uidGenerationMap: cache.CreateUIDToGenerationMap(), } } @@ -99,8 +101,9 @@ var _ reconcile.Reconciler = &ReconcileDirectImageMigration{} type ReconcileDirectImageMigration struct { client.Client record.EventRecorder - scheme *runtime.Scheme - tracer opentracing.Tracer + scheme *runtime.Scheme + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } // Reconcile reads that state of the cluster for a DirectImageMigration object and makes changes based on the state read @@ -127,6 +130,11 @@ func (r *ReconcileDirectImageMigration) Reconcile(request reconcile.Request) (re return reconcile.Result{Requeue: true}, err } + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(imageMigration.UID, imageMigration.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Set MigMigration name key on logger migration, err := imageMigration.GetMigrationForDIM(r) if migration != nil { @@ -183,6 +191,9 @@ func (r *ReconcileDirectImageMigration) Reconcile(request reconcile.Request) (re return reconcile.Result{Requeue: true}, nil } + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration(imageMigration.UID, imageMigration.Generation) + // Requeue if requeueAfter > 0 { return reconcile.Result{RequeueAfter: requeueAfter}, nil diff --git a/pkg/controller/directimagestreammigration/directimagestreammigration_controller.go b/pkg/controller/directimagestreammigration/directimagestreammigration_controller.go index f32ddee79e..de70ff1f6b 100644 --- a/pkg/controller/directimagestreammigration/directimagestreammigration_controller.go +++ b/pkg/controller/directimagestreammigration/directimagestreammigration_controller.go @@ -22,6 +22,7 @@ import ( "github.com/konveyor/controller/pkg/logging" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/cache" migref "github.com/konveyor/mig-controller/pkg/reference" "github.com/opentracing/opentracing-go" "k8s.io/apimachinery/pkg/api/errors" @@ -46,9 +47,10 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { return &ReconcileDirectImageStreamMigration{ - Client: mgr.GetClient(), - scheme: mgr.GetScheme(), - EventRecorder: mgr.GetRecorder("directimagestreammigration_controller"), + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + EventRecorder: mgr.GetRecorder("directimagestreammigration_controller"), + uidGenerationMap: cache.CreateUIDToGenerationMap(), } } @@ -90,8 +92,9 @@ var _ reconcile.Reconciler = &ReconcileDirectImageStreamMigration{} type ReconcileDirectImageStreamMigration struct { client.Client record.EventRecorder - scheme *runtime.Scheme - tracer opentracing.Tracer + scheme *runtime.Scheme + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } // Reconcile reads that state of the cluster for a DirectImageStreamMigration object and makes changes based on the state read @@ -118,6 +121,11 @@ func (r *ReconcileDirectImageStreamMigration) Reconcile(request reconcile.Reques return reconcile.Result{Requeue: true}, err } + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(imageStreamMigration.UID, imageStreamMigration.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Set MigMigration name key on logger migration, err := imageStreamMigration.GetMigrationForDISM(r) if migration != nil { @@ -174,6 +182,10 @@ func (r *ReconcileDirectImageStreamMigration) Reconcile(request reconcile.Reques return reconcile.Result{Requeue: true}, nil } + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration( + imageStreamMigration.UID, imageStreamMigration.Generation) + // Requeue if requeueAfter > 0 { return reconcile.Result{RequeueAfter: requeueAfter}, nil diff --git a/pkg/controller/directvolumemigration/directvolumemigration_controller.go b/pkg/controller/directvolumemigration/directvolumemigration_controller.go index a6c17c5f4c..7a3188914f 100644 --- a/pkg/controller/directvolumemigration/directvolumemigration_controller.go +++ b/pkg/controller/directvolumemigration/directvolumemigration_controller.go @@ -22,6 +22,7 @@ import ( "github.com/konveyor/controller/pkg/logging" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/cache" "github.com/opentracing/opentracing-go" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -43,7 +44,11 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileDirectVolumeMigration{Client: mgr.GetClient(), scheme: mgr.GetScheme()} + return &ReconcileDirectVolumeMigration{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + uidGenerationMap: cache.CreateUIDToGenerationMap(), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -79,8 +84,9 @@ var _ reconcile.Reconciler = &ReconcileDirectVolumeMigration{} // ReconcileDirectVolumeMigration reconciles a DirectVolumeMigration object type ReconcileDirectVolumeMigration struct { client.Client - scheme *runtime.Scheme - tracer opentracing.Tracer + scheme *runtime.Scheme + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } // Reconcile reads that state of the cluster for a DirectVolumeMigration object and makes changes based on the state read @@ -111,6 +117,11 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(request reconcile.Request) (r return reconcile.Result{Requeue: true}, err } + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(direct.UID, direct.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Set MigMigration name key on logger migration, err := direct.GetMigrationForDVM(r) if migration != nil { @@ -167,6 +178,9 @@ func (r *ReconcileDirectVolumeMigration) Reconcile(request reconcile.Request) (r return reconcile.Result{Requeue: true}, nil } + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration(direct.UID, direct.Generation) + // Requeue if requeueAfter > 0 { return reconcile.Result{RequeueAfter: requeueAfter}, nil diff --git a/pkg/controller/directvolumemigrationprogress/directvolumemigrationprogress_controller.go b/pkg/controller/directvolumemigrationprogress/directvolumemigrationprogress_controller.go index b738943994..edccce4a16 100644 --- a/pkg/controller/directvolumemigrationprogress/directvolumemigrationprogress_controller.go +++ b/pkg/controller/directvolumemigrationprogress/directvolumemigrationprogress_controller.go @@ -27,6 +27,7 @@ import ( "strings" "time" + "github.com/konveyor/mig-controller/pkg/cache" "github.com/konveyor/mig-controller/pkg/compat" "github.com/konveyor/mig-controller/pkg/errorutil" "github.com/opentracing/opentracing-go" @@ -72,7 +73,11 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileDirectVolumeMigrationProgress{Client: mgr.GetClient(), scheme: mgr.GetScheme()} + return &ReconcileDirectVolumeMigrationProgress{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + uidGenerationMap: cache.CreateUIDToGenerationMap(), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -97,8 +102,9 @@ var _ reconcile.Reconciler = &ReconcileDirectVolumeMigrationProgress{} // ReconcileDirectVolumeMigrationProgress reconciles a DirectVolumeMigrationProgress object type ReconcileDirectVolumeMigrationProgress struct { client.Client - scheme *runtime.Scheme - tracer opentracing.Tracer + scheme *runtime.Scheme + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } // Reconcile reads that state of the cluster for a DirectVolumeMigrationProgress object and makes changes based on the state read @@ -121,6 +127,11 @@ func (r *ReconcileDirectVolumeMigrationProgress) Reconcile(request reconcile.Req return reconcile.Result{Requeue: true}, err } + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(pvProgress.UID, pvProgress.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Set MigMigration name key on logger migration, err := pvProgress.GetMigrationforDVMP(r) if migration != nil { @@ -182,7 +193,10 @@ func (r *ReconcileDirectVolumeMigrationProgress) Reconcile(request reconcile.Req return reconcile.Result{Requeue: true}, nil } - // we will requeue this every 5 seconds + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration(pvProgress.UID, pvProgress.Generation) + + // Requeue every 5 seconds return reconcile.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil } diff --git a/pkg/controller/miganalytic/miganalytics_controller.go b/pkg/controller/miganalytic/miganalytics_controller.go index b7d02cf2f1..41862692d0 100644 --- a/pkg/controller/miganalytic/miganalytics_controller.go +++ b/pkg/controller/miganalytic/miganalytics_controller.go @@ -25,6 +25,7 @@ import ( "time" "github.com/konveyor/controller/pkg/logging" + "github.com/konveyor/mig-controller/pkg/cache" "github.com/konveyor/mig-controller/pkg/compat" "github.com/konveyor/mig-controller/pkg/errorutil" "github.com/konveyor/mig-controller/pkg/gvk" @@ -73,7 +74,12 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileMigAnalytic{Client: mgr.GetClient(), scheme: mgr.GetScheme(), EventRecorder: mgr.GetRecorder("miganalytic_controller")} + return &ReconcileMigAnalytic{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + EventRecorder: mgr.GetRecorder("miganalytic_controller"), + uidGenerationMap: cache.CreateUIDToGenerationMap(), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -108,8 +114,9 @@ type ReconcileMigAnalytic struct { client.Client record.EventRecorder - scheme *runtime.Scheme - tracer opentracing.Tracer + scheme *runtime.Scheme + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } // MigAnalyticPersistentVolumeDetails defines extended properties of a volume discovered by MigAnalytic @@ -141,6 +148,11 @@ func (r *ReconcileMigAnalytic) Reconcile(request reconcile.Request) (reconcile.R return reconcile.Result{Requeue: true}, nil } + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(analytic.UID, analytic.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Get jaeger span for reconcile, add to ctx reconcileSpan := r.initTracer(analytic) if reconcileSpan != nil { @@ -220,6 +232,9 @@ func (r *ReconcileMigAnalytic) Reconcile(request reconcile.Request) (reconcile.R return reconcile.Result{Requeue: true}, nil } + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration(analytic.UID, analytic.Generation) + // Done return reconcile.Result{Requeue: false}, nil } diff --git a/pkg/controller/migcluster/migcluster_controller.go b/pkg/controller/migcluster/migcluster_controller.go index 059aa3f4ff..41d77b84b9 100644 --- a/pkg/controller/migcluster/migcluster_controller.go +++ b/pkg/controller/migcluster/migcluster_controller.go @@ -20,6 +20,7 @@ import ( "context" "time" + "github.com/konveyor/mig-controller/pkg/cache" "github.com/konveyor/mig-controller/pkg/errorutil" "github.com/opentracing/opentracing-go" @@ -49,7 +50,12 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) *ReconcileMigCluster { - return &ReconcileMigCluster{Client: mgr.GetClient(), scheme: mgr.GetScheme(), EventRecorder: mgr.GetRecorder("migcluster_controller")} + return &ReconcileMigCluster{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + EventRecorder: mgr.GetRecorder("migcluster_controller"), + uidGenerationMap: cache.CreateUIDToGenerationMap(), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -102,9 +108,10 @@ type ReconcileMigCluster struct { k8sclient.Client record.EventRecorder - scheme *runtime.Scheme - Controller controller.Controller - tracer opentracing.Tracer + scheme *runtime.Scheme + Controller controller.Controller + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } func (r *ReconcileMigCluster) Reconcile(request reconcile.Request) (reconcile.Result, error) { @@ -124,6 +131,11 @@ func (r *ReconcileMigCluster) Reconcile(request reconcile.Request) (reconcile.Re return reconcile.Result{Requeue: true}, nil } + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(cluster.UID, cluster.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Get jaeger span for reconcile, add to ctx reconcileSpan := r.initTracer(cluster) if reconcileSpan != nil { @@ -189,6 +201,9 @@ func (r *ReconcileMigCluster) Reconcile(request reconcile.Request) (reconcile.Re return reconcile.Result{Requeue: true}, nil } + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration(cluster.UID, cluster.Generation) + // Done return reconcile.Result{Requeue: false}, nil } diff --git a/pkg/controller/mighook/mighook_controller.go b/pkg/controller/mighook/mighook_controller.go index 286420c1ab..30a0a55357 100644 --- a/pkg/controller/mighook/mighook_controller.go +++ b/pkg/controller/mighook/mighook_controller.go @@ -19,6 +19,7 @@ package mighook import ( "context" + "github.com/konveyor/mig-controller/pkg/cache" "github.com/konveyor/mig-controller/pkg/errorutil" "github.com/opentracing/opentracing-go" @@ -45,7 +46,12 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileMigHook{Client: mgr.GetClient(), scheme: mgr.GetScheme(), EventRecorder: mgr.GetRecorder("mighook_controller")} + return &ReconcileMigHook{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + EventRecorder: mgr.GetRecorder("mighook_controller"), + uidGenerationMap: cache.CreateUIDToGenerationMap(), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -75,8 +81,9 @@ type ReconcileMigHook struct { client.Client record.EventRecorder - scheme *runtime.Scheme - tracer opentracing.Tracer + scheme *runtime.Scheme + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } func (r *ReconcileMigHook) Reconcile(request reconcile.Request) (reconcile.Result, error) { @@ -96,6 +103,11 @@ func (r *ReconcileMigHook) Reconcile(request reconcile.Request) (reconcile.Resul return reconcile.Result{Requeue: true}, nil } + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(hook.UID, hook.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Get jaeger span for reconcile, add to ctx reconcileSpan := r.initTracer(hook) if reconcileSpan != nil { @@ -144,6 +156,9 @@ func (r *ReconcileMigHook) Reconcile(request reconcile.Request) (reconcile.Resul return reconcile.Result{Requeue: true}, nil } + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration(hook.UID, hook.Generation) + // Done return reconcile.Result{Requeue: false}, nil } diff --git a/pkg/controller/migmigration/migmigration_controller.go b/pkg/controller/migmigration/migmigration_controller.go index 38a18e4866..9a02f68058 100644 --- a/pkg/controller/migmigration/migmigration_controller.go +++ b/pkg/controller/migmigration/migmigration_controller.go @@ -24,6 +24,7 @@ import ( liberr "github.com/konveyor/controller/pkg/error" "github.com/konveyor/controller/pkg/logging" migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1" + "github.com/konveyor/mig-controller/pkg/cache" "github.com/konveyor/mig-controller/pkg/errorutil" migref "github.com/konveyor/mig-controller/pkg/reference" "github.com/opentracing/opentracing-go" @@ -50,7 +51,12 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileMigMigration{Client: mgr.GetClient(), scheme: mgr.GetScheme(), EventRecorder: mgr.GetRecorder("migmigration_controller")} + return &ReconcileMigMigration{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + EventRecorder: mgr.GetRecorder("migmigration_controller"), + uidGenerationMap: cache.CreateUIDToGenerationMap(), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -139,8 +145,9 @@ type ReconcileMigMigration struct { client.Client record.EventRecorder - scheme *runtime.Scheme - tracer opentracing.Tracer + scheme *runtime.Scheme + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } // Reconcile performs Migrations based on the data in MigMigration @@ -163,6 +170,12 @@ func (r *ReconcileMigMigration) Reconcile(request reconcile.Request) (reconcile. log.Trace(err) return reconcile.Result{Requeue: true}, nil } + + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(migration.UID, migration.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Get jaeger spans for migration and reconcile, add to ctx _, reconcileSpan := r.initTracer(migration) if reconcileSpan != nil { @@ -263,6 +276,9 @@ func (r *ReconcileMigMigration) Reconcile(request reconcile.Request) (reconcile. return reconcile.Result{Requeue: true}, nil } + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration(migration.UID, migration.Generation) + // Requeue if requeueAfter > 0 { return reconcile.Result{RequeueAfter: requeueAfter}, nil diff --git a/pkg/controller/migplan/migplan_controller.go b/pkg/controller/migplan/migplan_controller.go index 9f911cc9b5..4de7bcb92f 100644 --- a/pkg/controller/migplan/migplan_controller.go +++ b/pkg/controller/migplan/migplan_controller.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/konveyor/mig-controller/pkg/cache" "github.com/konveyor/mig-controller/pkg/errorutil" "github.com/opentracing/opentracing-go" @@ -67,7 +68,12 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileMigPlan{Client: mgr.GetClient(), scheme: mgr.GetScheme(), EventRecorder: mgr.GetRecorder("migplan_controller")} + return &ReconcileMigPlan{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + EventRecorder: mgr.GetRecorder("migplan_controller"), + uidGenerationMap: cache.CreateUIDToGenerationMap(), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -173,8 +179,9 @@ type ReconcileMigPlan struct { client.Client record.EventRecorder - scheme *runtime.Scheme - tracer opentracing.Tracer + scheme *runtime.Scheme + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } func (r *ReconcileMigPlan) Reconcile(request reconcile.Request) (reconcile.Result, error) { @@ -194,6 +201,11 @@ func (r *ReconcileMigPlan) Reconcile(request reconcile.Request) (reconcile.Resul return reconcile.Result{Requeue: true}, err } + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(plan.UID, plan.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Get jaeger span for reconcile, add to ctx reconcileSpan := r.initTracer(plan) if reconcileSpan != nil { @@ -326,6 +338,9 @@ func (r *ReconcileMigPlan) Reconcile(request reconcile.Request) (reconcile.Resul return reconcile.Result{Requeue: true}, nil } + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration(plan.UID, plan.Generation) + // Timed requeue on Plan conflict. if plan.Status.HasCondition(PlanConflict) { return reconcile.Result{RequeueAfter: time.Second * 10}, nil diff --git a/pkg/controller/migstorage/migstorage_controller.go b/pkg/controller/migstorage/migstorage_controller.go index e30ff3f1e7..779df5648b 100644 --- a/pkg/controller/migstorage/migstorage_controller.go +++ b/pkg/controller/migstorage/migstorage_controller.go @@ -20,6 +20,7 @@ import ( "context" "time" + "github.com/konveyor/mig-controller/pkg/cache" "github.com/konveyor/mig-controller/pkg/errorutil" "github.com/opentracing/opentracing-go" @@ -48,7 +49,12 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileMigStorage{Client: mgr.GetClient(), scheme: mgr.GetScheme(), EventRecorder: mgr.GetRecorder("migstorage_controller")} + return &ReconcileMigStorage{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + EventRecorder: mgr.GetRecorder("migstorage_controller"), + uidGenerationMap: cache.CreateUIDToGenerationMap(), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -100,9 +106,9 @@ var _ reconcile.Reconciler = &ReconcileMigStorage{} type ReconcileMigStorage struct { client.Client record.EventRecorder - - scheme *runtime.Scheme - tracer opentracing.Tracer + scheme *runtime.Scheme + tracer opentracing.Tracer + uidGenerationMap *cache.UIDToGenerationMap } func (r *ReconcileMigStorage) Reconcile(request reconcile.Request) (reconcile.Result, error) { @@ -122,6 +128,11 @@ func (r *ReconcileMigStorage) Reconcile(request reconcile.Request) (reconcile.Re return reconcile.Result{Requeue: true}, nil } + // Check if cache is still catching up + if r.uidGenerationMap.IsCacheStale(storage.UID, storage.Generation) { + return reconcile.Result{Requeue: true}, nil + } + // Get jaeger span for reconcile, add to ctx reconcileSpan := r.initTracer(storage) if reconcileSpan != nil { @@ -173,6 +184,9 @@ func (r *ReconcileMigStorage) Reconcile(request reconcile.Request) (reconcile.Re return reconcile.Result{Requeue: true}, nil } + // Record reconciled generation + r.uidGenerationMap.RecordReconciledGeneration(storage.UID, storage.Generation) + // Done return reconcile.Result{Requeue: false}, nil }