diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 5f7abe2f5ed..218f57effba 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -50,11 +50,12 @@ type frontend struct { baseLog *logrus.Entry env env.Interface - logMiddleware middleware.LogMiddleware - validateMiddleware middleware.ValidateMiddleware - m middleware.MetricsMiddleware - authMiddleware middleware.AuthMiddleware - apiVersionMiddleware middleware.ApiVersionValidator + logMiddleware middleware.LogMiddleware + validateMiddleware middleware.ValidateMiddleware + m middleware.MetricsMiddleware + authMiddleware middleware.AuthMiddleware + apiVersionMiddleware middleware.ApiVersionValidator + maintenanceMiddleware middleware.MaintenanceMiddleware dbAsyncOperations database.AsyncOperations dbClusterManagerConfiguration database.ClusterManagerConfigurations @@ -152,6 +153,7 @@ func NewFrontend(ctx context.Context, dbOpenShiftVersions: dbOpenShiftVersions, apis: apis, m: middleware.MetricsMiddleware{Emitter: m}, + maintenanceMiddleware: middleware.MaintenanceMiddleware{Emitter: m}, aead: aead, hiveClusterManager: hiveClusterManager, kubeActionsFactory: kubeActionsFactory, @@ -294,22 +296,17 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { }) r.Get("/supportedvmsizes", f.supportedvmsizes) - r.Route("/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/etcdrecovery", - func(r chi.Router) { - r.Post("/", f.postAdminOpenShiftClusterEtcdRecovery) - }) - - r.Route("/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/kubernetesobjects", - func(r chi.Router) { - r.Get("/", f.getAdminKubernetesObjects) - r.Post("/", f.postAdminKubernetesObjects) - r.Delete("/", f.deleteAdminKubernetesObjects) - }, - ) - r.Route("/subscriptions/{subscriptionId}", func(r chi.Router) { r.Route("/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}", func(r chi.Router) { - r.Post("/approvecsr", f.postAdminOpenShiftClusterApproveCSR) + // Etcd recovery + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/etcdrecovery", f.postAdminOpenShiftClusterEtcdRecovery) + + // Kubernetes objects + r.Get("/kubernetesobjects", f.getAdminKubernetesObjects) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/kubernetesobjects", f.postAdminKubernetesObjects) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Delete("/kubernetesobjects", f.deleteAdminKubernetesObjects) + + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/approvecsr", f.postAdminOpenShiftClusterApproveCSR) // Pod logs r.Get("/kubernetespodlogs", f.getAdminKubernetesPodLogs) @@ -320,23 +317,23 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { r.Get("/clusterdeployment", f.getAdminHiveClusterDeployment) - r.Post("/redeployvm", f.postAdminOpenShiftClusterRedeployVM) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/redeployvm", f.postAdminOpenShiftClusterRedeployVM) - r.Post("/stopvm", f.postAdminOpenShiftClusterStopVM) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/stopvm", f.postAdminOpenShiftClusterStopVM) - r.Post("/startvm", f.postAdminOpenShiftClusterStartVM) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/startvm", f.postAdminOpenShiftClusterStartVM) - r.Post("/upgrade", f.postAdminOpenShiftUpgrade) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/upgrade", f.postAdminOpenShiftUpgrade) r.Get("/skus", f.getAdminOpenShiftClusterVMResizeOptions) - r.Post("/resize", f.postAdminOpenShiftClusterVMResize) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/resize", f.postAdminOpenShiftClusterVMResize) - r.Post("/reconcilefailednic", f.postAdminReconcileFailedNIC) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/reconcilefailednic", f.postAdminReconcileFailedNIC) - r.Post("/cordonnode", f.postAdminOpenShiftClusterCordonNode) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/cordonnode", f.postAdminOpenShiftClusterCordonNode) - r.Post("/drainnode", f.postAdminOpenShiftClusterDrainNode) + r.With(f.maintenanceMiddleware.EmitUnplannedMaintenanceSignal).Post("/drainnode", f.postAdminOpenShiftClusterDrainNode) }) }) diff --git a/pkg/frontend/middleware/maintenance.go b/pkg/frontend/middleware/maintenance.go new file mode 100644 index 00000000000..4bb11ff94af --- /dev/null +++ b/pkg/frontend/middleware/maintenance.go @@ -0,0 +1,31 @@ +package middleware + +type MaintenanceMiddleware struct { + metrics.Emitter +} + +// Emit metric for unplanned maintenance +func (mm MaintenanceMiddleware) EmitUnplannedMaintenanceSignal(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + resourceID := strings.TrimPrefix(filepath.Dir(r.URL.Path), "/admin") + + go func(ctx context.Contetxt, resourceID string) { + for { + mm.EmitGauge("frontend.maintenance.unplanned", 1, map[string]string{ + "resource_id": resourceID, + }) + select { + case <-ctx.Done(): + return + default: + time.Sleep(1 * time.Minute) + } + } + }(ctx, resourceID) + + h.ServeHTTP(w, r) + }) +}