diff --git a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go index 6c73bb1d264..98d6fc499ca 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_slow_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_slow_test.go @@ -143,7 +143,12 @@ func TestEmergencyReparentShardSlow(t *testing.T) { }, "zone1-0000000200": { StopStatus: &replicationdatapb.StopReplicationStatus{ - Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)}, + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + IoState: int32(replication.ReplicationStateRunning), + SqlState: int32(replication.ReplicationStateRunning), + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", diff --git a/go/vt/vtctl/grpcvtctldserver/server_test.go b/go/vt/vtctl/grpcvtctldserver/server_test.go index 2a3f9a7d649..d4706424c8d 100644 --- a/go/vt/vtctl/grpcvtctldserver/server_test.go +++ b/go/vt/vtctl/grpcvtctldserver/server_test.go @@ -4786,7 +4786,12 @@ func TestEmergencyReparentShard(t *testing.T) { }, "zone1-0000000200": { StopStatus: &replicationdatapb.StopReplicationStatus{ - Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)}, + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + IoState: int32(replication.ReplicationStateRunning), + SqlState: int32(replication.ReplicationStateRunning), + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter.go b/go/vt/vtctl/reparentutil/emergency_reparenter.go index 12bb8bd7e2b..dd16d4fe88d 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter.go @@ -153,13 +153,13 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve shardInfo *topo.ShardInfo prevPrimary *topodatapb.Tablet tabletMap map[string]*topo.TabletInfo + candidateInfoMap map[string]*CandidateInfo validCandidates map[string]*RelayLogPositions intermediateSource *topodatapb.Tablet validCandidateTablets []*topodatapb.Tablet validReplacementCandidates []*topodatapb.Tablet betterCandidate *topodatapb.Tablet isIdeal bool - isGTIDBased bool ) shardInfo, err = erp.ts.GetShard(ctx, keyspace, shard) @@ -216,12 +216,15 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve } // find the positions of all the valid candidates. - validCandidates, isGTIDBased, err = FindPositionsOfAllCandidates(stoppedReplicationSnapshot.statusMap, stoppedReplicationSnapshot.primaryStatusMap) + validCandidates, candidateInfoMap, err = FindPositionsOfAllCandidates(stoppedReplicationSnapshot.statusMap, stoppedReplicationSnapshot.primaryStatusMap) if err != nil { return err } - // Restrict the valid candidates list. We remove any tablet which is of the type DRAINED, RESTORE or BACKUP. - validCandidates, err = restrictValidCandidates(validCandidates, tabletMap) + + // Restrict the valid candidates list. We remove any tablet which is of the type DRAINED, RESTORE or BACKUP, and any tablet + // using non-GTID replication. + isGTIDBased := isGTIDBasedShard(tabletMap, candidateInfoMap) + validCandidates, err = restrictValidCandidates(validCandidates, tabletMap, candidateInfoMap, erp.logger) if err != nil { return err } else if len(validCandidates) == 0 { diff --git a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go index b0133499347..7f69c32f063 100644 --- a/go/vt/vtctl/reparentutil/emergency_reparenter_test.go +++ b/go/vt/vtctl/reparentutil/emergency_reparenter_test.go @@ -735,7 +735,12 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }{ "zone1-0000000100": { StopStatus: &replicationdatapb.StopReplicationStatus{ - Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)}, + Before: &replicationdatapb.Status{ + IoState: int32(replication.ReplicationStateRunning), + SqlState: int32(replication.ReplicationStateRunning), + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", @@ -744,7 +749,12 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, "zone1-0000000101": { StopStatus: &replicationdatapb.StopReplicationStatus{ - Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)}, + Before: &replicationdatapb.Status{ + IoState: int32(replication.ReplicationStateRunning), + SqlState: int32(replication.ReplicationStateRunning), + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21", @@ -753,8 +763,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) { }, "zone1-0000000102": { StopStatus: &replicationdatapb.StopReplicationStatus{ - Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)}, - After: &replicationdatapb.Status{}, + Before: &replicationdatapb.Status{ + IoState: int32(replication.ReplicationStateRunning), + SqlState: int32(replication.ReplicationStateRunning), + }, + After: &replicationdatapb.Status{}, }, }, }, @@ -4734,18 +4747,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-99"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100"), SourceUuid: u1, @@ -4798,18 +4814,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-99", "1-30"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30"), SourceUuid: u1, @@ -4862,18 +4881,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-99", "1-30"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-101", "1-30"), SourceUuid: u1, @@ -4926,18 +4948,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-100"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-30", "1-50"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-30"), SourceUuid: u1, @@ -4990,18 +5015,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-30", "1-50"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-30", "1-50"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-30"), SourceUuid: u1, @@ -5054,12 +5082,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-90", "1-30", "1-50"), SourceUuid: u1, @@ -5117,18 +5147,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-20", "1-50"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-31", "1-50"), SourceUuid: u2, @@ -5181,12 +5214,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-31", "1-50"), SourceUuid: u2, @@ -5244,18 +5279,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-99", "1-31", "1-50"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"), SourceUuid: u1, @@ -5308,18 +5346,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-101", "1-31", "1-50"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"), SourceUuid: u1, @@ -5361,12 +5402,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-31", "1-50"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"), SourceUuid: u1, @@ -5419,18 +5462,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000102": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-31", "1-50"), SourceUuid: u1, }, }, "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-51"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-90", "1-30", "1-50"), SourceUuid: u1, @@ -5483,12 +5529,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-51"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-31", "1-50"), SourceUuid: u2, @@ -5542,12 +5590,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { }, statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "zone1-0000000103": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-51"), SourceUuid: u1, }, }, "zone1-0000000104": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: getRelayLogPosition("", "1-31", "1-50"), SourceUuid: u2, @@ -5567,9 +5617,9 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) { erp := &EmergencyReparenter{ tmc: tt.tmc, } - validCandidates, isGtid, err := FindPositionsOfAllCandidates(tt.statusMap, tt.primaryStatusMap) + validCandidates, isGTIDBasedMap, err := FindPositionsOfAllCandidates(tt.statusMap, tt.primaryStatusMap) require.NoError(t, err) - require.True(t, isGtid) + require.Len(t, isGTIDBasedMap, len(validCandidates)) candidates, err := erp.findErrantGTIDs(context.Background(), validCandidates, tt.statusMap, tt.tabletMap, 10*time.Second) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) diff --git a/go/vt/vtctl/reparentutil/replication.go b/go/vt/vtctl/reparentutil/replication.go index e13a3e0dd44..f5ba6fbb65b 100644 --- a/go/vt/vtctl/reparentutil/replication.go +++ b/go/vt/vtctl/reparentutil/replication.go @@ -82,76 +82,87 @@ func (rlp *RelayLogPositions) IsZero() bool { return rlp.Combined.IsZero() } +type CandidateInfo struct { + IsGTIDBased bool + IsSemiSyncReplica bool +} + // FindPositionsOfAllCandidates will find candidates for an emergency // reparent, and, if successful, return a mapping of those tablet aliases (as // raw strings) to their replication positions for later comparison. func FindPositionsOfAllCandidates( statusMap map[string]*replicationdatapb.StopReplicationStatus, primaryStatusMap map[string]*replicationdatapb.PrimaryStatus, -) (map[string]*RelayLogPositions, bool, error) { - replicationStatusMap := make(map[string]*replication.ReplicationStatus, len(statusMap)) +) (map[string]*RelayLogPositions, map[string]*CandidateInfo, error) { + replicationStatusMapBefore := make(map[string]*replication.ReplicationStatus, len(statusMap)) + replicationStatusMapAfter := make(map[string]*replication.ReplicationStatus, len(statusMap)) positionMap := make(map[string]*RelayLogPositions) // Build out replication status list from proto types. for alias, statuspb := range statusMap { - status := replication.ProtoToReplicationStatus(statuspb.After) - replicationStatusMap[alias] = &status + beforeStatus := replication.ProtoToReplicationStatus(statuspb.Before) + afterStatus := replication.ProtoToReplicationStatus(statuspb.After) + replicationStatusMapBefore[alias] = &beforeStatus + replicationStatusMapAfter[alias] = &afterStatus } // Determine if we're GTID-based. If we are, we'll need to look for errant // GTIDs below. var ( - isGTIDBased bool - isNonGTIDBased bool + candidateInfoMap = make(map[string]*CandidateInfo, len(replicationStatusMapAfter)) emptyRelayPosErrorRecorder concurrency.FirstErrorRecorder + isGTIDBasedShard bool ) - for alias, status := range replicationStatusMap { - if _, ok := status.RelayLogPosition.GTIDSet.(replication.Mysql56GTIDSet); ok { - isGTIDBased = true - } else { - isNonGTIDBased = true + for alias, beforeStatus := range replicationStatusMapBefore { + isSemiSyncReplica := beforeStatus.SemiSyncReplicaEnabled && beforeStatus.SemiSyncReplicaStatus + candidateInfoMap[alias] = &CandidateInfo{ + IsSemiSyncReplica: isSemiSyncReplica, } + if _, ok := beforeStatus.RelayLogPosition.GTIDSet.(replication.Mysql56GTIDSet); ok { + candidateInfoMap[alias].IsGTIDBased = true + isGTIDBasedShard = true + } + } - if status.RelayLogPosition.IsZero() { + for alias, afterStatus := range replicationStatusMapAfter { + candidateInfo, ok := candidateInfoMap[alias] + if ok && isGTIDBasedShard && candidateInfo.IsSemiSyncReplica && !candidateInfo.IsGTIDBased { + return nil, nil, vterrors.New(vtrpc.Code_FAILED_PRECONDITION, "semi-sync replica tablets without gtid positions is unsupported") + } + + if afterStatus.RelayLogPosition.IsZero() { // Potentially bail. If any other tablet is detected to have // GTID-based relay log positions, we will return the error recorded // here. emptyRelayPosErrorRecorder.RecordError(vterrors.Errorf(vtrpc.Code_UNAVAILABLE, "encountered tablet %v with no relay log position, when at least one other tablet in the status map has GTID based relay log positions", alias)) } - } - - if isGTIDBased && emptyRelayPosErrorRecorder.HasErrors() { - return nil, false, emptyRelayPosErrorRecorder.Error() - } - - if isGTIDBased && isNonGTIDBased { - return nil, false, vterrors.Errorf(vtrpc.Code_FAILED_PRECONDITION, "encountered mix of GTID-based and non GTID-based relay logs") - } - // Store the final positions in the map. - for alias, status := range replicationStatusMap { - if !isGTIDBased { - positionMap[alias] = &RelayLogPositions{Combined: status.Position} - - continue - } positionMap[alias] = &RelayLogPositions{ - Combined: status.RelayLogPosition, - Executed: status.Position, + Combined: afterStatus.RelayLogPosition, + Executed: afterStatus.Position, } } + if isGTIDBasedShard && emptyRelayPosErrorRecorder.HasErrors() { + return nil, nil, emptyRelayPosErrorRecorder.Error() + } + for alias, primaryStatus := range primaryStatusMap { executedPosition, err := replication.DecodePosition(primaryStatus.Position) if err != nil { - return nil, false, vterrors.Wrapf(err, "could not decode a primary status executed position for tablet %v: %v", alias, err) + return nil, nil, vterrors.Wrapf(err, "could not decode a primary status executed position for tablet %v: %v", alias, err) + } + + candidateInfoMap[alias] = &CandidateInfo{} + if _, ok := executedPosition.GTIDSet.(replication.Mysql56GTIDSet); ok { + candidateInfoMap[alias].IsGTIDBased = true } positionMap[alias] = &RelayLogPositions{Combined: executedPosition} } - return positionMap, isGTIDBased, nil + return positionMap, candidateInfoMap, nil } // ReplicaWasRunning returns true if a StopReplicationStatus indicates that the diff --git a/go/vt/vtctl/reparentutil/replication_test.go b/go/vt/vtctl/reparentutil/replication_test.go index d0fedde0f8a..38e9042a070 100644 --- a/go/vt/vtctl/reparentutil/replication_test.go +++ b/go/vt/vtctl/reparentutil/replication_test.go @@ -64,20 +64,28 @@ func TestFindPositionsOfAllCandidates(t *testing.T) { // point is, the combination of (1) whether the test should error and // (2) the set of keys we expect in the map is enough to fully assert on // the correctness of the behavior of this functional unit. - expected []string - expectedGTIDBased bool - shouldErr bool + expected []string + expectedCandidateInfoMap map[string]*CandidateInfo + shouldErr bool }{ { name: "success", statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "r1": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", }, }, "r2": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", @@ -89,84 +97,128 @@ func TestFindPositionsOfAllCandidates(t *testing.T) { Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", }, }, - expected: []string{"r1", "r2", "p1"}, - expectedGTIDBased: true, - shouldErr: false, + expected: []string{"r1", "r2", "p1"}, + expectedCandidateInfoMap: map[string]*CandidateInfo{ + "r1": {IsGTIDBased: true}, + "r2": {IsGTIDBased: true}, + "p1": {IsGTIDBased: true}, + }, + shouldErr: false, }, { name: "success for single tablet", statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "r1": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5,AAAAAAAA-71CA-11E1-9E33-C80AA9429562:1", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5,AAAAAAAA-71CA-11E1-9E33-C80AA9429562:1", }, }, }, - primaryStatusMap: map[string]*replicationdatapb.PrimaryStatus{}, - expected: []string{"r1"}, - expectedGTIDBased: true, - shouldErr: false, + primaryStatusMap: map[string]*replicationdatapb.PrimaryStatus{}, + expected: []string{"r1"}, + expectedCandidateInfoMap: map[string]*CandidateInfo{ + "r1": {IsGTIDBased: true}, + }, + shouldErr: false, }, { name: "mixed replication modes", statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "r1": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", }, }, "r2": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "FilePos/mysql-bin.0001:10", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "FilePos/mysql-bin.0001:10", }, }, }, - expected: nil, - shouldErr: true, + expected: []string{"r1", "r2"}, + expectedCandidateInfoMap: map[string]*CandidateInfo{ + "r1": {IsGTIDBased: true}, + "r2": {IsGTIDBased: false}, + }, + shouldErr: false, }, { name: "tablet without relay log position", statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "r1": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", }, }, "r2": { + Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false}, After: &replicationdatapb.Status{ RelayLogPosition: "", }, }, }, - expected: nil, - shouldErr: true, + expected: nil, + expectedCandidateInfoMap: nil, + shouldErr: true, }, { name: "non-GTID-based", statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "r1": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "FilePos/mysql-bin.0001:100", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "FilePos/mysql-bin.0001:100", }, }, "r2": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "FilePos/mysql-bin.0001:10", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "FilePos/mysql-bin.0001:10", }, }, }, - expected: []string{"r1", "r2"}, + expected: []string{"r1", "r2"}, + expectedCandidateInfoMap: map[string]*CandidateInfo{ + "r1": {IsGTIDBased: false}, + "r2": {IsGTIDBased: false}, + }, shouldErr: false, }, { name: "bad primary position fails the call", statusMap: map[string]*replicationdatapb.StopReplicationStatus{ "r1": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, After: &replicationdatapb.Status{ SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", @@ -178,8 +230,41 @@ func TestFindPositionsOfAllCandidates(t *testing.T) { Position: "InvalidFlavor/1234", }, }, - expected: nil, - shouldErr: true, + expected: nil, + expectedCandidateInfoMap: nil, + shouldErr: true, + }, + { + name: "unsupported replica with file positons in GTID-based shard and semi-sync", + statusMap: map[string]*replicationdatapb.StopReplicationStatus{ + "r1": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "FilePos/mysql-bin.0001:10", + SemiSyncReplicaEnabled: true, + SemiSyncReplicaStatus: true, + }, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "FilePos/mysql-bin.0001:10", + }, + }, + "r2": { + Before: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + SemiSyncReplicaEnabled: false, + SemiSyncReplicaStatus: false, + }, + After: &replicationdatapb.Status{ + SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562", + RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5", + }, + }, + }, + expected: nil, + expectedCandidateInfoMap: nil, + shouldErr: true, }, } @@ -187,8 +272,8 @@ func TestFindPositionsOfAllCandidates(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - actual, isGTIDBased, err := FindPositionsOfAllCandidates(tt.statusMap, tt.primaryStatusMap) - require.EqualValues(t, tt.expectedGTIDBased, isGTIDBased) + actual, isGTIDBasedMap, err := FindPositionsOfAllCandidates(tt.statusMap, tt.primaryStatusMap) + require.EqualValues(t, tt.expectedCandidateInfoMap, isGTIDBasedMap) if tt.shouldErr { assert.Error(t, err) return diff --git a/go/vt/vtctl/reparentutil/util.go b/go/vt/vtctl/reparentutil/util.go index dde181c7112..ccdfefeb7d5 100644 --- a/go/vt/vtctl/reparentutil/util.go +++ b/go/vt/vtctl/reparentutil/util.go @@ -315,16 +315,43 @@ func getValidCandidatesAndPositionsAsList(validCandidates map[string]*RelayLogPo return validTablets, tabletPositions, nil } +// isGTIDBasedShard uses a shard PRIMARY or REPLICA to determine if the shard is using GTID-based replication. +func isGTIDBasedShard(tabletMap map[string]*topo.TabletInfo, candidateInfoMap map[string]*CandidateInfo) bool { + for alias, tablet := range tabletMap { + switch tablet.Type { + case topodatapb.TabletType_PRIMARY, topodatapb.TabletType_REPLICA: + candidateInfo, ok := candidateInfoMap[alias] + if !ok { + continue + } + if candidateInfo.IsGTIDBased { + return true + } + } + } + return false +} + // restrictValidCandidates is used to restrict some candidates from being considered eligible for becoming the intermediate source or the final promotion candidate -func restrictValidCandidates(validCandidates map[string]*RelayLogPositions, tabletMap map[string]*topo.TabletInfo) (map[string]*RelayLogPositions, error) { +func restrictValidCandidates(validCandidates map[string]*RelayLogPositions, tabletMap map[string]*topo.TabletInfo, candidateInfoMap map[string]*CandidateInfo, + logger logutil.Logger) (map[string]*RelayLogPositions, error) { + isGTIDBasedShard := isGTIDBasedShard(tabletMap, candidateInfoMap) restrictedValidCandidates := make(map[string]*RelayLogPositions) for candidate, position := range validCandidates { - candidateInfo, ok := tabletMap[candidate] + topoTabletInfo, ok := tabletMap[candidate] + if !ok { + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this is an impossible situation", candidate) + } + candidateInfo, ok := candidateInfoMap[candidate] if !ok { - return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the tablet map; this an impossible situation", candidate) + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "candidate %v not found in the candidate info map; this is an impossible situation", candidate) + } + if isGTIDBasedShard && !candidateInfo.IsGTIDBased { + logger.Warningf("tablet %s is a member of a GTID-based shard, but it does not have GTID-based positions. Skipping", candidate) + continue } // We do not allow BACKUP, DRAINED or RESTORE type of tablets to be considered for being the replication source or the candidate for primary - if topoproto.IsTypeInList(candidateInfo.Type, []topodatapb.TabletType{topodatapb.TabletType_BACKUP, topodatapb.TabletType_RESTORE, topodatapb.TabletType_DRAINED}) { + if topoproto.IsTypeInList(topoTabletInfo.Type, []topodatapb.TabletType{topodatapb.TabletType_BACKUP, topodatapb.TabletType_RESTORE, topodatapb.TabletType_DRAINED}) { continue } restrictedValidCandidates[candidate] = position diff --git a/go/vt/vtctl/reparentutil/util_test.go b/go/vt/vtctl/reparentutil/util_test.go index d18c09e0075..eb5787da16d 100644 --- a/go/vt/vtctl/reparentutil/util_test.go +++ b/go/vt/vtctl/reparentutil/util_test.go @@ -1765,10 +1765,11 @@ func TestWaitForCatchUp(t *testing.T) { func TestRestrictValidCandidates(t *testing.T) { tests := []struct { - name string - validCandidates map[string]*RelayLogPositions - tabletMap map[string]*topo.TabletInfo - result map[string]*RelayLogPositions + name string + validCandidates map[string]*RelayLogPositions + tabletMap map[string]*topo.TabletInfo + candidateInfoMap map[string]*CandidateInfo + result map[string]*RelayLogPositions }{ { name: "remove invalid tablets", @@ -1836,17 +1837,105 @@ func TestRestrictValidCandidates(t *testing.T) { }, }, }, + candidateInfoMap: map[string]*CandidateInfo{ + "zone1-0000000100": {IsGTIDBased: true}, + "zone1-0000000101": {IsGTIDBased: true}, + "zone1-0000000102": {IsGTIDBased: true}, + "zone1-0000000103": {IsGTIDBased: true}, + "zone1-0000000104": {IsGTIDBased: true}, + "zone1-0000000105": {IsGTIDBased: true}, + }, result: map[string]*RelayLogPositions{ "zone1-0000000100": {}, "zone1-0000000101": {}, "zone1-0000000104": {}, }, }, + { + name: "remove invalid tablets with file-based async replica", + validCandidates: map[string]*RelayLogPositions{ + "zone1-0000000100": {}, + "zone1-0000000101": {}, + "zone1-0000000102": {}, + "zone1-0000000103": {}, + "zone1-0000000104": {}, + "zone1-0000000105": {}, + }, + tabletMap: map[string]*topo.TabletInfo{ + "zone1-0000000100": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 100, + }, + Type: topodatapb.TabletType_PRIMARY, + }, + }, + "zone1-0000000101": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 101, + }, + Type: topodatapb.TabletType_RDONLY, + }, + }, + "zone1-0000000102": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 102, + }, + Type: topodatapb.TabletType_RESTORE, + }, + }, + "zone1-0000000103": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 103, + }, + Type: topodatapb.TabletType_DRAINED, + }, + }, + "zone1-0000000104": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 104, + }, + Type: topodatapb.TabletType_SPARE, + }, + }, + "zone1-0000000105": { + Tablet: &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "zone1", + Uid: 103, + }, + Type: topodatapb.TabletType_BACKUP, + }, + }, + }, + candidateInfoMap: map[string]*CandidateInfo{ + "zone1-0000000100": {IsGTIDBased: true}, + "zone1-0000000101": {IsGTIDBased: true}, + "zone1-0000000102": {IsGTIDBased: true}, + "zone1-0000000103": {IsGTIDBased: true}, + "zone1-0000000104": {IsGTIDBased: false}, // file-based + "zone1-0000000105": {IsGTIDBased: true}, + }, + result: map[string]*RelayLogPositions{ + "zone1-0000000100": {}, + "zone1-0000000101": {}, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - res, err := restrictValidCandidates(test.validCandidates, test.tabletMap) + logger := logutil.NewMemoryLogger() + res, err := restrictValidCandidates(test.validCandidates, test.tabletMap, test.candidateInfoMap, logger) assert.NoError(t, err) assert.Equal(t, res, test.result) }) diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index d4b752c0882..9f34cdef64b 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -63,50 +63,32 @@ func TestEmergencyReparentShard(t *testing.T) { goodReplica2 := NewFakeTablet(t, wr, "cell2", 3, topodatapb.TabletType_REPLICA, nil) reparenttestutil.SetKeyspaceDurability(context.Background(), t, ts, "test_keyspace", policy.DurabilitySemiSync) + oldPrimaryPos, err := replication.ParseMysql56GTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:1-7") + require.NoError(t, err) oldPrimary.FakeMysqlDaemon.Replicating = false oldPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{ - GTIDSet: replication.MariadbGTIDSet{ - 2: replication.MariadbGTID{ - Domain: 2, - Server: 123, - Sequence: 456, - }, - }, + GTIDSet: oldPrimaryPos, }) - currentPrimaryFilePosition, _ := replication.ParseFilePosGTIDSet("mariadb-bin.000010:456") - oldPrimary.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{ - GTIDSet: currentPrimaryFilePosition, + oldPrimary.FakeMysqlDaemon.CurrentRelayLogPosition = replication.Position{ + GTIDSet: oldPrimaryPos, } - // new primary + // new primary (equal GTID to old primary) newPrimary.FakeMysqlDaemon.ReadOnly = true newPrimary.FakeMysqlDaemon.Replicating = true newPrimary.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{ - GTIDSet: replication.MariadbGTIDSet{ - 2: replication.MariadbGTID{ - Domain: 2, - Server: 123, - Sequence: 456, - }, - }, + GTIDSet: oldPrimaryPos, }) - newPrimaryRelayLogPos, _ := replication.ParseFilePosGTIDSet("relay-bin.000004:456") - newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{ - GTIDSet: newPrimaryRelayLogPos, + newPrimary.FakeMysqlDaemon.CurrentRelayLogPosition = replication.Position{ + GTIDSet: oldPrimaryPos, } - newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = append(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions, newPrimary.FakeMysqlDaemon.CurrentSourceFilePosition) + newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = append(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions, newPrimary.FakeMysqlDaemon.CurrentPrimaryPosition) newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP REPLICA IO_THREAD", "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, primary_alias, replication_position) VALUES", } newPrimary.FakeMysqlDaemon.PromoteResult = replication.Position{ - GTIDSet: replication.MariadbGTIDSet{ - 2: replication.MariadbGTID{ - Domain: 2, - Server: 123, - Sequence: 456, - }, - }, + GTIDSet: oldPrimaryPos, } newPrimary.StartActionLoop(t, wr) defer newPrimary.StopActionLoop(t) @@ -122,23 +104,17 @@ func TestEmergencyReparentShard(t *testing.T) { defer oldPrimary.StopActionLoop(t) // good replica 1 is replicating + goodReplica1Pos, err := replication.ParseMysql56GTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:1-6") + require.NoError(t, err) goodReplica1.FakeMysqlDaemon.ReadOnly = true goodReplica1.FakeMysqlDaemon.Replicating = true goodReplica1.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{ - GTIDSet: replication.MariadbGTIDSet{ - 2: replication.MariadbGTID{ - Domain: 2, - Server: 123, - Sequence: 455, - }, - }, + GTIDSet: goodReplica1Pos, }) - goodReplica1RelayLogPos, err := replication.ParseFilePosGTIDSet("relay-bin.003222:18321744073709551612") // Requires all 64 bits or uint64 - require.NoError(t, err) - goodReplica1.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{ - GTIDSet: goodReplica1RelayLogPos, + goodReplica1.FakeMysqlDaemon.CurrentRelayLogPosition = replication.Position{ + GTIDSet: goodReplica1Pos, } - goodReplica1.FakeMysqlDaemon.WaitPrimaryPositions = append(goodReplica1.FakeMysqlDaemon.WaitPrimaryPositions, goodReplica1.FakeMysqlDaemon.CurrentSourceFilePosition) + goodReplica1.FakeMysqlDaemon.WaitPrimaryPositions = append(goodReplica1.FakeMysqlDaemon.WaitPrimaryPositions, goodReplica1.FakeMysqlDaemon.CurrentPrimaryPosition) goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ // These 3 statements come from tablet startup @@ -154,22 +130,17 @@ func TestEmergencyReparentShard(t *testing.T) { defer goodReplica1.StopActionLoop(t) // good replica 2 is not replicating + goodReplica2Pos, err := replication.ParseMysql56GTIDSet("3E11FA47-71CA-11E1-9E33-C80AA9429562:1") + require.NoError(t, err) goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.Replicating = false goodReplica2.FakeMysqlDaemon.SetPrimaryPositionLocked(replication.Position{ - GTIDSet: replication.MariadbGTIDSet{ - 2: replication.MariadbGTID{ - Domain: 2, - Server: 123, - Sequence: 454, - }, - }, + GTIDSet: goodReplica2Pos, }) - goodReplica2RelayLogPos, _ := replication.ParseFilePosGTIDSet("relay-bin.000004:454") - goodReplica2.FakeMysqlDaemon.CurrentSourceFilePosition = replication.Position{ - GTIDSet: goodReplica2RelayLogPos, + goodReplica2.FakeMysqlDaemon.CurrentRelayLogPosition = replication.Position{ + GTIDSet: goodReplica2Pos, } - goodReplica2.FakeMysqlDaemon.WaitPrimaryPositions = append(goodReplica2.FakeMysqlDaemon.WaitPrimaryPositions, goodReplica2.FakeMysqlDaemon.CurrentSourceFilePosition) + goodReplica2.FakeMysqlDaemon.WaitPrimaryPositions = append(goodReplica2.FakeMysqlDaemon.WaitPrimaryPositions, goodReplica2.FakeMysqlDaemon.CurrentPrimaryPosition) goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ // These 3 statements come from tablet startup