Skip to content

Commit

Permalink
Log abnormal events for resources we are waiting on (migtools#1052)
Browse files Browse the repository at this point in the history
* Log abnormal events for things we are waiting on
  • Loading branch information
Derek Whatley authored and pranavgaikwad committed Apr 8, 2021
1 parent 162427c commit 9d41095
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 0 deletions.
15 changes: 15 additions & 0 deletions pkg/controller/directvolumemigration/rsync.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
liberr "github.com/konveyor/controller/pkg/error"
migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1"
"github.com/konveyor/mig-controller/pkg/compat"
migevent "github.com/konveyor/mig-controller/pkg/event"
"github.com/konveyor/mig-controller/pkg/settings"
routev1 "github.com/openshift/api/route/v1"
"golang.org/x/crypto/ssh"
Expand Down Expand Up @@ -139,6 +140,13 @@ func (t *Task) areRsyncTransferPodsRunning() (bool, error) {
}
for _, pod := range pods.Items {
if pod.Status.Phase != corev1.PodRunning {
// Log abnormal events for Rsync transfer Pod if any are found
migevent.LogAbnormalEventsForResource(
destClient, t.Log,
"Found abnormal event for Rsync transfer Pod on destination cluster",
types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name},
"pod")

for _, podCond := range pod.Status.Conditions {
if podCond.Reason == corev1.PodReasonUnschedulable {
t.Log.Info("Found UNSCHEDULABLE Rsync Transfer Pod on destination cluster",
Expand Down Expand Up @@ -756,6 +764,13 @@ func (t *Task) areRsyncRoutesAdmitted() (bool, []string, error) {
if err != nil {
return false, messages, err
}
// Logs abnormal events related to route if any are found
migevent.LogAbnormalEventsForResource(
destClient, t.Log,
"Found abnormal event for Rsync Route on destination cluster",
types.NamespacedName{Namespace: route.Namespace, Name: route.Name},
"route")

admitted := false
message := "no status condition available for the route"
// Check if we can find the admitted condition for the route
Expand Down
9 changes: 9 additions & 0 deletions pkg/controller/directvolumemigration/stunnel.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"text/template"

liberr "github.com/konveyor/controller/pkg/error"
migevent "github.com/konveyor/mig-controller/pkg/event"
"github.com/konveyor/mig-controller/pkg/settings"
"gopkg.in/yaml.v2"

Expand All @@ -23,6 +24,7 @@ import (
"time"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"

//"k8s.io/apimachinery/pkg/types"
Expand Down Expand Up @@ -611,6 +613,13 @@ func (t *Task) areStunnelClientPodsRunning() (bool, error) {
}
for _, pod := range pods.Items {
if pod.Status.Phase != corev1.PodRunning {
// Logs abnormal events for Stunnel Pod if any are found
migevent.LogAbnormalEventsForResource(
srcClient, t.Log,
"Found abnormal event for Stunnel Client Pod on source cluster",
types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name},
"pod")

for _, podCond := range pod.Status.Conditions {
if podCond.Reason == corev1.PodReasonUnschedulable {
t.Log.Info("Found UNSCHEDULABLE Stunnel Client Pod "+
Expand Down
10 changes: 10 additions & 0 deletions pkg/controller/migmigration/hooks.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

liberr "github.com/konveyor/controller/pkg/error"
migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1"
migevent "github.com/konveyor/mig-controller/pkg/event"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -153,6 +154,15 @@ func (t *Task) stopHookJobs() (bool, error) {

func (t *Task) ensureJob(job *batchv1.Job, hook migapi.MigPlanHook, migHook migapi.MigHook, client k8sclient.Client) (bool, error) {
runningJob, err := migHook.GetPhaseJob(client, hook.Phase, string(t.Owner.UID))
if runningJob != nil {
// Logs abnormal events for Hook Jobs if any are found
migevent.LogAbnormalEventsForResource(
client, t.Log,
"Found abnormal event for Hook Job",
types.NamespacedName{Namespace: runningJob.Namespace, Name: runningJob.Name},
"job")
}

if runningJob == nil && err == nil {
err = client.Create(context.TODO(), job)
if err != nil {
Expand Down
15 changes: 15 additions & 0 deletions pkg/controller/migmigration/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
liberr "github.com/konveyor/controller/pkg/error"
migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1"
pvdr "github.com/konveyor/mig-controller/pkg/cloudprovider"
migevent "github.com/konveyor/mig-controller/pkg/event"
"github.com/konveyor/mig-controller/pkg/pods"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -147,6 +148,13 @@ func (t *Task) haveResticPodsStarted() (bool, error) {
}

for _, pod := range list.Items {
// Logs abnormal events for Restic Pods if any are found
migevent.LogAbnormalEventsForResource(
client, t.Log,
"Found abnormal event for Restic Pod",
types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name},
"pod")

if pod.DeletionTimestamp != nil {
t.Log.Info("Deletion timestamp found on Restic Pod, "+
"Pod is in the process of deleting. Requeuing and waiting for restart.",
Expand Down Expand Up @@ -265,6 +273,13 @@ func (t *Task) haveVeleroPodsStarted() (bool, error) {
}

for _, pod := range list.Items {
// Logs abnormal events for Velero Pod if any are found
migevent.LogAbnormalEventsForResource(
client, t.Log,
"Found abnormal event for Velero Pod",
types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name},
"pod")

if pod.DeletionTimestamp != nil {
t.Log.Info("Found Velero Pod with deletion timestamp."+
" Requeuing and waiting for Pod to finish deleting and restart.",
Expand Down
9 changes: 9 additions & 0 deletions pkg/controller/migmigration/stage.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

liberr "github.com/konveyor/controller/pkg/error"
migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1"
migevent "github.com/konveyor/mig-controller/pkg/event"
migpods "github.com/konveyor/mig-controller/pkg/pods"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -517,6 +518,14 @@ func (t *Task) stagePodReport(client k8sclient.Client) (report PodStartReport, e
for _, pod := range podList.Items {
t.Log.V(4).Info("Checking if Stage Pod is healthy.",
"pod", path.Join(pod.Namespace, pod.Name))

// Logs abnormal events for Stage Pods if any are found
migevent.LogAbnormalEventsForResource(
client, t.Log,
"Found abnormal event for Stage Pod",
types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name},
"pod")

initReady := true
for _, c := range pod.Status.InitContainerStatuses {
// If the init contianer is waiting, then nothing can happen.
Expand Down
11 changes: 11 additions & 0 deletions pkg/controller/migmigration/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@ import (
liberr "github.com/konveyor/controller/pkg/error"
migapi "github.com/konveyor/mig-controller/pkg/apis/migration/v1alpha1"
"github.com/konveyor/mig-controller/pkg/compat"
migevent "github.com/konveyor/mig-controller/pkg/event"
migref "github.com/konveyor/mig-controller/pkg/reference"
corev1 "k8s.io/api/core/v1"
k8sLabels "k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
)

Expand Down Expand Up @@ -298,6 +300,15 @@ func ensureRegistryHealth(c k8sclient.Client, migration *migapi.MigMigration) (i
return nEnsured, "", liberr.Wrap(err)
}

for _, registryPod := range registryPods.Items {
// Logs abnormal events for Registry Pods if any are found
migevent.LogAbnormalEventsForResource(
client, log,
"Found abnormal event for Registry Pod",
types.NamespacedName{Namespace: registryPod.Namespace, Name: registryPod.Name},
"pod")
}

registryPodCount := len(registryPods.Items)
if registryPodCount < 1 {
unHealthyClusterName = cluster.ObjectMeta.Name
Expand Down
87 changes: 87 additions & 0 deletions pkg/event/event.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package event

import (
"context"
"path"
"strings"

"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
)

// GetAbnormalEventsForResource gets unique events of non-normal type for
// a namespaced resource. Useful for logging the most relevant events
// related to a resource we're waiting on.
func GetAbnormalEventsForResource(client client.Client,
nsName types.NamespacedName, resourceKind string) ([]corev1.Event, error) {
uniqueEventMap := make(map[string]corev1.Event)

eList := corev1.EventList{}
options := k8sclient.InNamespace(nsName.Namespace)
err := client.List(context.TODO(), options, &eList)
if err != nil {
return nil, err
}
for _, event := range eList.Items {
// Only want events for the kind indicated
if strings.ToLower(event.InvolvedObject.Kind) != strings.ToLower(resourceKind) {
continue
}
// Only get events for the resource.name we're interested in
if event.InvolvedObject.Name != nsName.Name {
continue
}
// Only get abnormal events
if event.Type == "Normal" {
continue
}
// Check if same event reason has already been seen
eventFromMap, ok := uniqueEventMap[event.Reason]
if !ok {
uniqueEventMap[event.Reason] = event
continue
}
// Found event in map. Overwrite it if this one is newer.
if eventFromMap.ObjectMeta.CreationTimestamp.Time.
Before(event.ObjectMeta.CreationTimestamp.Time) {
uniqueEventMap[event.Reason] = event
}
}
// Turn map into slice of events
matchingEvents := []corev1.Event{}
for _, event := range uniqueEventMap {
matchingEvents = append(matchingEvents, event)
}

return matchingEvents, err
}

// LogAbnormalEventsForResource logs unique events of non-normal type for
// a namespaced resource. Useful for logging the most relevant events
// related to a resource we're waiting on.
// The message logged will match what is provided in 'message'
func LogAbnormalEventsForResource(
client client.Client, log logr.Logger, message string, nsName types.NamespacedName, resourceKind string) {

relevantEvents, err := GetAbnormalEventsForResource(client,
types.NamespacedName{Name: nsName.Name, Namespace: nsName.Namespace}, resourceKind)
if err != nil {
log.Info("Error getting events",
"kind", resourceKind,
"resource", path.Join(nsName.Namespace, nsName.Name),
"error", err)
return
}
for _, rEvent := range relevantEvents {
log.Info(message,
resourceKind, path.Join(nsName.Namespace, nsName.Name),
"eventType", rEvent.Type,
"eventReason", rEvent.Reason,
"eventMessage", rEvent.Message,
"eventFirstTimestamp", rEvent.FirstTimestamp)
}

}

0 comments on commit 9d41095

Please sign in to comment.