Merge pull request #15 from vimeo/refresh_lock_abort_loop_leased

refreshLock: abort loop on term overruns

Author: dfinkel

.github/workflows/go.yml

@@ -8,17 +8,17 @@ jobs:
     strategy:
       matrix:
         os: [macOS-latest, ubuntu-latest]
-        goversion: [1.13, 1.14, 1.15]
+        goversion: [1.17, 1.18, 1.19, '1.20']
     steps:
 
     - name: Set up Go ${{matrix.goversion}} on ${{matrix.os}}
-      uses: actions/setup-go@v1
+      uses: actions/setup-go@v3
       with:
         go-version: ${{matrix.goversion}}
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
+      uses: actions/checkout@v3
 
     - name: gofmt
       run: |

campaign.go

@@ -145,6 +145,10 @@ func (c *campaign) manageWin(ctx context.Context, winningEntry *entry.RaceEntry,
 		defer func() { wg.Add(1); go func() { defer wg.Done(); c.c.OnOusting(ctx) }() }()
 	}
 	defer electedCancel()
+	// Wait for any outstanding goroutines before running OnElected (in particular, wait for
+	// previous OnOusting callbacks to complete.
+	wg.Wait()
+
 	wg.Add(1)
 	go func(ctx context.Context) { defer wg.Done(); c.c.OnElected(ctx, &tv) }(electedCtx)
 	finalEntry, refreshErr := c.refreshLock(ctx, winningEntry, &tv)
@@ -224,14 +228,27 @@ func (c *campaign) refreshLock(ctx context.Context, electedEntry *entry.RaceEntr
 	entry := &entryVal
 	// we have the lock, we should try to keep it
 	for {
+		// Check whether we still have some time left on the term before we try to do any
+		// real work (if we're past the end of the term, we've lost and need to reacquire
+		// anyway)
+		if c.clock.Until(entry.TermExpiry) < 0 {
+			return entry, context.DeadlineExceeded
+		}
 		newEntry, refreshErr := c.refreshLockOnce(ctx, entry)
 		switch refreshErr {
 		case nil:
 			tv.Set(newEntry.TermExpiry)
 			entry = newEntry
-		case context.DeadlineExceeded, context.Canceled, errAcquireFailed:
+		case context.Canceled, errAcquireFailed:
 			return entry, refreshErr
 		default:
+			if errors.Is(refreshErr, context.Canceled) {
+				return entry, context.Canceled
+			}
+			// Check whether we still have some time left on the term before we try to sleep
+			if c.clock.Until(entry.TermExpiry) < 0 {
+				return entry, context.DeadlineExceeded
+			}
 		}
 
 		// wake up in half the interval until the leadership term