fix, cli getChangeUuid add retry and improve lookup (#2575)

Failed 3 attempts NB DEBUG is on, hence we see the re-try attempts
<img width="1700" height="228" alt="image"
src="https://github.com/user-attachments/assets/85215031-3424-446c-a35c-231515a21b38"
/>

Succeeds after 1 failure
<img width="1715" height="394" alt="image"
src="https://github.com/user-attachments/assets/c617e5b8-5155-46ef-967e-92ceedbf5611"
/>
GitOrigin-RevId: 3fb1af3de3311da4d8b993db16460f29d85fb70d

Author: tphoney

cmd/changes_end_change.go

@@ -26,7 +26,7 @@ func EndChange(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	changeUuid, err := getChangeUuid(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_HAPPENING, viper.GetString("ticket-link"), true)
+	changeUuid, err := getChangeUUIDAndCheckStatus(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_HAPPENING, viper.GetString("ticket-link"), true)
 	if err != nil {
 		return loggedError{
 			err:     err,

cmd/changes_get_change.go

@@ -52,7 +52,7 @@ func GetChange(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	changeUuid, err := getChangeUuid(ctx, oi, sdp.ChangeStatus(sdp.ChangeStatus_value[viper.GetString("status")]), viper.GetString("ticket-link"), true)
+	changeUuid, err := getChangeUUIDAndCheckStatus(ctx, oi, sdp.ChangeStatus(sdp.ChangeStatus_value[viper.GetString("status")]), viper.GetString("ticket-link"), true)
 	if err != nil {
 		return loggedError{
 			err:     err,

cmd/changes_start_change.go

@@ -26,7 +26,7 @@ func StartChange(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	changeUuid, err := getChangeUuid(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, viper.GetString("ticket-link"), true)
+	changeUuid, err := getChangeUUIDAndCheckStatus(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, viper.GetString("ticket-link"), true)
 	if err != nil {
 		return loggedError{
 			err: err,

cmd/changes_submit_plan.go

@@ -135,7 +135,7 @@ func SubmitPlan(cmd *cobra.Command, args []string) error {
 	delete(lf, "file")
 
 	client := AuthenticatedChangesClient(ctx, oi)
-	changeUuid, err := getChangeUuid(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, viper.GetString("ticket-link"), false)
+	changeUuid, err := getChangeUUIDAndCheckStatus(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, viper.GetString("ticket-link"), false)
 	if err != nil {
 		return loggedError{
 			err:     err,

cmd/changes_submit_signal.go

@@ -38,7 +38,7 @@ func SubmitSignal(cmd *cobra.Command, args []string) error {
 	if viper.GetString("description") == "" {
 		return flagError{"--description is required"}
 	}
-	changeUUID, err := getChangeUuid(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, viper.GetString("ticket-link"), true)
+	changeUUID, err := getChangeUUIDAndCheckStatus(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, viper.GetString("ticket-link"), true)
 	if err != nil {
 		return loggedError{
 			err:     err,

cmd/root.go

@@ -183,9 +183,9 @@ Then enter the code:
 	%v
 `
 
-// getChangeUuid returns the UUID of a change, as selected by --uuid or --change, or a change with the specified status and having --ticket-link
-func getChangeUuid(ctx context.Context, oi sdp.OvermindInstance, expectedStatus sdp.ChangeStatus, ticketLink string, errNotFound bool) (uuid.UUID, error) {
-	var changeUuid uuid.UUID
+// getChangeUUIDAndCheckStatus returns the UUID of a change, as selected by --uuid or --change, or a change with the specified status and having --ticket-link
+func getChangeUUIDAndCheckStatus(ctx context.Context, oi sdp.OvermindInstance, expectedStatus sdp.ChangeStatus, ticketLink string, errorOnNotFound bool) (uuid.UUID, error) {
+	var changeUUID uuid.UUID
 	var err error
 
 	uuidString := viper.GetString("uuid")
@@ -198,47 +198,86 @@ func getChangeUuid(ctx context.Context, oi sdp.OvermindInstance, expectedStatus
 
 	// Check UUID first if more than one is set
 	if uuidString != "" {
-		changeUuid, err = uuid.Parse(uuidString)
+		changeUUID, err = uuid.Parse(uuidString)
 		if err != nil {
 			return uuid.Nil, fmt.Errorf("invalid --uuid value '%v', error: %w", uuidString, err)
 		}
 
-		return changeUuid, nil
+		return changeUUID, nil
 	}
 
 	// Then check for a change URL
 	if changeUrlString != "" {
 		return parseChangeUrl(changeUrlString)
 	}
 
-	// Finally look through all open changes to find one with a matching ticket link
-	client := AuthenticatedChangesClient(ctx, oi)
-
-	changesList, err := client.ListChangesByStatus(ctx, &connect.Request[sdp.ListChangesByStatusRequest]{
-		Msg: &sdp.ListChangesByStatusRequest{
-			Status: expectedStatus,
-		},
-	})
-	if err != nil {
-		return uuid.Nil, fmt.Errorf("failed to search for existing changes: %w", err)
+	// Finally look up by ticket link with retry
+	changeUUID, err = getChangeByTicketLinkWithRetry(ctx, oi, ticketLink, expectedStatus, errorOnNotFound)
+	if errorOnNotFound && err != nil {
+		return uuid.Nil, err
 	}
 
-	var maybeChangeUuid *uuid.UUID
-	for _, c := range changesList.Msg.GetChanges() {
-		if c.GetProperties().GetTicketLink() == ticketLink {
-			maybeChangeUuid = c.GetMetadata().GetUUIDParsed()
-			if maybeChangeUuid != nil {
-				changeUuid = *maybeChangeUuid
-				break
+	return changeUUID, nil
+}
+
+// getChangeByTicketLinkWithRetry performs the GetChangeByTicketLink API call with retry logic,
+// retrying both on error and when the status does not match the expected status.
+// NB api-server will only return the latest change with this ticket link.
+func getChangeByTicketLinkWithRetry(ctx context.Context, oi sdp.OvermindInstance, ticketLink string, expectedStatus sdp.ChangeStatus, errorOnNotFound bool) (uuid.UUID, error) {
+	client := AuthenticatedChangesClient(ctx, oi)
+
+	var change *connect.Response[sdp.GetChangeResponse]
+	var currentStatus sdp.ChangeStatus
+	var err error
+	maxRetries := 3
+	if !errorOnNotFound {
+		// If not erroring on not found, only attempt once.
+		maxRetries = 1
+	}
+	retryDelay := 3 * time.Second
+
+	for attempt := 1; attempt <= maxRetries; attempt++ {
+		change, err = client.GetChangeByTicketLink(ctx, &connect.Request[sdp.GetChangeByTicketLinkRequest]{
+			Msg: &sdp.GetChangeByTicketLinkRequest{
+				TicketLink: ticketLink,
+			},
+		})
+		if err == nil {
+			// change found
+			var uuidPtr *uuid.UUID
+			if change != nil && change.Msg != nil && change.Msg.GetChange() != nil && change.Msg.GetChange().GetMetadata() != nil {
+				uuidPtr = change.Msg.GetChange().GetMetadata().GetUUIDParsed()
+				currentStatus = change.Msg.GetChange().GetMetadata().GetStatus()
+				if uuidPtr != nil && (currentStatus == expectedStatus) {
+					// Success: we have a UUID and status matches the expected status
+					return *uuidPtr, nil
+				}
+			}
+		}
+		// Log the error and retry if not the last attempt
+		if attempt < maxRetries {
+			logFields := log.Fields{
+				"ovm.change.ticketLink": ticketLink,
+				"expectedStatus":        expectedStatus.String(),
+				"attempt":               attempt,
+				"maxRetries":            maxRetries,
+				"currentStatus":         currentStatus.String(),
+			}
+			if err != nil {
+				logFields["error"] = err.Error()
+				log.WithContext(ctx).WithFields(logFields).Debug("failed to get change by ticket link, retrying")
+			} else {
+				log.WithContext(ctx).WithFields(logFields).Debug("change found but status does not match, retrying")
 			}
+			time.Sleep(retryDelay)
 		}
 	}
-
-	if errNotFound && changeUuid == uuid.Nil {
-		return uuid.Nil, fmt.Errorf("no change found with ticket link %v and status %v", ticketLink, expectedStatus.String())
+	if err != nil {
+		// Final attempt failed with an error
+		return uuid.Nil, fmt.Errorf("error looking up change with ticket link %v after %d attempts: %w", ticketLink, maxRetries, err)
 	}
-
-	return changeUuid, nil
+	// Final attempt found a change but status did not match
+	return uuid.Nil, fmt.Errorf("change %s found with ticket link %v. Change status %v does not match expected status %v after %d attempts", change.Msg.GetChange().GetMetadata().GetUUIDParsed(), ticketLink, currentStatus.String(), expectedStatus.String(), maxRetries)
 }
 
 func parseChangeUrl(changeUrlString string) (uuid.UUID, error) {

cmd/terraform_apply.go

@@ -136,7 +136,7 @@ func TerraformApplyImpl(ctx context.Context, cmd *cobra.Command, oi sdp.Overmind
 			}
 		}
 
-		changeUuid, err := getChangeUuid(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, ticketLink, true)
+		changeUuid, err := getChangeUUIDAndCheckStatus(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, ticketLink, true)
 		if err != nil {
 			return uuid.Nil, fmt.Errorf("failed to identify change: %w", err)
 		}

cmd/terraform_plan.go

@@ -218,7 +218,7 @@ func TerraformPlanImpl(ctx context.Context, cmd *cobra.Command, oi sdp.OvermindI
 	}
 
 	client := AuthenticatedChangesClient(ctx, oi)
-	changeUuid, err := getChangeUuid(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, ticketLink, false)
+	changeUuid, err := getChangeUUIDAndCheckStatus(ctx, oi, sdp.ChangeStatus_CHANGE_STATUS_DEFINING, ticketLink, false)
 	if err != nil {
 		uploadChangesSpinner.Fail(fmt.Sprintf("Uploading planned changes: failed searching for existing changes: %v", err))
 		return nil