Set a maximum amount of time to wait for an image to be pulled (#550)

* Set a maximum amount of time to wait for an image to be pulled
after seeing ErrImagePull.

* Found the newline that gofmt didn't mind but golint did.
Looking at the "details" report of very little help.

pods/status.go

@@ -14,13 +14,18 @@ import (
 	log "k8s.io/klog/v2"
 )
 
+// MaxPullTime is how long we will give a pod to start after seeing
+// ErrImagePull.
+var MaxPullTime = time.Minute * 15
+
 // A Watcher watches pod and container updates.
 type Watcher struct {
 	ctx       context.Context
 	errCh     chan error
 	wstop     func()
 	cancel    func()
 	podStates map[types.UID]string
+	podStart  map[types.UID]time.Time
 	cStates   map[string]string
 	ch        chan *PodStatus
 	stdout    io.Writer
@@ -54,6 +59,7 @@ func newWatcher(ctx context.Context, cancel func(), ch chan *PodStatus, stop fun
 		cancel:    cancel,
 		stdout:    os.Stdout,
 		podStates: map[types.UID]string{},
+		podStart:  map[types.UID]time.Time{},
 		cStates:   map[string]string{},
 		warningf:  log.Warningf,
 	}
@@ -194,6 +200,17 @@ func (w *Watcher) updatePod(s *PodStatus) bool {
 			w.cancel()
 			return false
 		case c.Reason == "ErrImagePull":
+			if t, ok := w.podStart[s.UID]; ok {
+				if t.Add(MaxPullTime).Before(time.Now()) {
+					showContainer(s, &c, "PULL TIMED OUT")
+					w.warningf("%s: pull timed out after %v", fullName, MaxPullTime)
+					w.errCh <- fmt.Errorf("%s IMAGE:%s pull timed out after %v", fullName, c.Image, MaxPullTime)
+					w.cancel()
+					return false
+				}
+			} else {
+				w.podStart[s.UID] = time.Now()
+			}
 			showContainer(s, &c, c.Reason)
 			log.Infof("%s in ErrImagePull", fullName)
 		case c.Reason == "ImagePullBackOff":

pods/status_test.go

@@ -200,6 +200,50 @@ func TestUpdatePod(t *testing.T) {
 	}
 }
 
+func TestImagePullTimeout(t *testing.T) {
+	var buf strings.Builder
+
+	cancel := func() {}
+	stop := func() {}
+
+	w := newWatcher(context.TODO(), cancel, nil, stop)
+	w.stdout = &buf
+	w.SetProgress(true)
+	const uid = types.UID("uid1")
+
+	getErr := func() error {
+		select {
+		case err := <-w.errCh:
+			return err
+		default:
+			return nil
+		}
+	}
+
+	if tm, ok := w.podStart[uid]; ok {
+		t.Fatalf("w.podStart[%v] is %v, want unset", uid, tm)
+	}
+	ps := &PodStatus{Name: "pod", UID: uid, Namespace: "ns", Phase: PodPending,
+		Containers: []ContainerStatus{{Name: "cont", Reason: "ErrImagePull"}},
+	}
+	w.updatePod(ps)
+	if err := getErr(); err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	if _, ok := w.podStart[uid]; !ok {
+		t.Fatalf("w.podStart[%v] is not set", uid)
+	}
+	w.updatePod(ps)
+	if err := getErr(); err != nil {
+		t.Fatalf("unexpected error %v", err)
+	}
+	w.podStart[uid] = w.podStart[uid].Add(-(MaxPullTime + 1))
+	w.updatePod(ps)
+	if s := errdiff.Check(getErr(), "pull timed out after"); s != "" {
+		t.Error(s)
+	}
+}
+
 func TestStop(t *testing.T) {
 	canceled := false
 	cancel := func() { canceled = true }