Skip to content

EmergencyReparentShard: support ignored file-based replicas #19069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
timvaillancourt wants to merge 11 commits into
base: main
Choose a base branch
from
Draft

EmergencyReparentShard: support ignored file-based replicas #19069

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
400e043
`EmergencyReparentShard`: support ignored file-based replicas
timvaillancourt Dec 9, 2025
e337d02
return semi-sync state in status RPC
timvaillancourt Dec 10, 2025
efa41ca
fixes
timvaillancourt Dec 10, 2025
75b3a21
fixes
timvaillancourt Dec 10, 2025
6d0f421
WIP
timvaillancourt Dec 10, 2025
b0b9dd2
Merge remote-tracking branch 'origin/main' into dont-reparent-file-based
timvaillancourt Dec 23, 2025
6b2ea89
fix tests
timvaillancourt Dec 23, 2025
755d90d
support file-based primary
timvaillancourt Dec 23, 2025
975ecf5
update test
timvaillancourt Dec 23, 2025
38ee64d
Merge remote-tracking branch 'origin/main' into dont-reparent-file-based
timvaillancourt Jan 7, 2026
1518183
Merge remote-tracking branch 'origin/main' into dont-reparent-file-based
timvaillancourt Jan 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion go/vt/vtctl/grpcvtctldserver/server_slow_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,12 @@ func TestEmergencyReparentShardSlow(t *testing.T) {
},
"zone1-0000000200": {
StopStatus: &replicationdatapb.StopReplicationStatus{
Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)},
Before: &replicationdatapb.Status{
SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562",
RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5",
IoState: int32(replication.ReplicationStateRunning),
SqlState: int32(replication.ReplicationStateRunning),
},
After: &replicationdatapb.Status{
SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562",
RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5",
Expand Down
7 changes: 6 additions & 1 deletion go/vt/vtctl/grpcvtctldserver/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4786,7 +4786,12 @@ func TestEmergencyReparentShard(t *testing.T) {
},
"zone1-0000000200": {
StopStatus: &replicationdatapb.StopReplicationStatus{
Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)},
Before: &replicationdatapb.Status{
SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562",
RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5",
IoState: int32(replication.ReplicationStateRunning),
SqlState: int32(replication.ReplicationStateRunning),
},
After: &replicationdatapb.Status{
SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562",
RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5",
Expand Down
11 changes: 7 additions & 4 deletions go/vt/vtctl/reparentutil/emergency_reparenter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,13 +153,13 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve
shardInfo *topo.ShardInfo
prevPrimary *topodatapb.Tablet
tabletMap map[string]*topo.TabletInfo
candidateInfoMap map[string]*CandidateInfo
validCandidates map[string]*RelayLogPositions
intermediateSource *topodatapb.Tablet
validCandidateTablets []*topodatapb.Tablet
validReplacementCandidates []*topodatapb.Tablet
betterCandidate *topodatapb.Tablet
isIdeal bool
isGTIDBased bool
)

shardInfo, err = erp.ts.GetShard(ctx, keyspace, shard)
Expand Down Expand Up @@ -216,12 +216,15 @@ func (erp *EmergencyReparenter) reparentShardLocked(ctx context.Context, ev *eve
}

// find the positions of all the valid candidates.
validCandidates, isGTIDBased, err = FindPositionsOfAllCandidates(stoppedReplicationSnapshot.statusMap, stoppedReplicationSnapshot.primaryStatusMap)
validCandidates, candidateInfoMap, err = FindPositionsOfAllCandidates(stoppedReplicationSnapshot.statusMap, stoppedReplicationSnapshot.primaryStatusMap)
if err != nil {
return err
}
// Restrict the valid candidates list. We remove any tablet which is of the type DRAINED, RESTORE or BACKUP.
validCandidates, err = restrictValidCandidates(validCandidates, tabletMap)

// Restrict the valid candidates list. We remove any tablet which is of the type DRAINED, RESTORE or BACKUP, and any tablet
// using non-GTID replication.
isGTIDBased := isGTIDBasedShard(tabletMap, candidateInfoMap)
validCandidates, err = restrictValidCandidates(validCandidates, tabletMap, candidateInfoMap, erp.logger)
if err != nil {
return err
} else if len(validCandidates) == 0 {
Expand Down
62 changes: 56 additions & 6 deletions go/vt/vtctl/reparentutil/emergency_reparenter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -735,7 +735,12 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) {
}{
"zone1-0000000100": {
StopStatus: &replicationdatapb.StopReplicationStatus{
Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)},
Before: &replicationdatapb.Status{
IoState: int32(replication.ReplicationStateRunning),
SqlState: int32(replication.ReplicationStateRunning),
SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562",
RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21",
},
After: &replicationdatapb.Status{
SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562",
RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21",
Expand All @@ -744,7 +749,12 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) {
},
"zone1-0000000101": {
StopStatus: &replicationdatapb.StopReplicationStatus{
Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)},
Before: &replicationdatapb.Status{
IoState: int32(replication.ReplicationStateRunning),
SqlState: int32(replication.ReplicationStateRunning),
SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562",
RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21",
},
After: &replicationdatapb.Status{
SourceUuid: "3E11FA47-71CA-11E1-9E33-C80AA9429562",
RelayLogPosition: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-21",
Expand All @@ -753,8 +763,11 @@ func TestEmergencyReparenter_reparentShardLocked(t *testing.T) {
},
"zone1-0000000102": {
StopStatus: &replicationdatapb.StopReplicationStatus{
Before: &replicationdatapb.Status{IoState: int32(replication.ReplicationStateRunning), SqlState: int32(replication.ReplicationStateRunning)},
After: &replicationdatapb.Status{},
Before: &replicationdatapb.Status{
IoState: int32(replication.ReplicationStateRunning),
SqlState: int32(replication.ReplicationStateRunning),
},
After: &replicationdatapb.Status{},
},
},
},
Expand Down Expand Up @@ -4734,18 +4747,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-99"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100"),
SourceUuid: u1,
Expand Down Expand Up @@ -4798,18 +4814,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-99", "1-30"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30"),
SourceUuid: u1,
Expand Down Expand Up @@ -4862,18 +4881,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-99", "1-30"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-101", "1-30"),
SourceUuid: u1,
Expand Down Expand Up @@ -4926,18 +4948,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-100"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-30", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-30"),
SourceUuid: u1,
Expand Down Expand Up @@ -4990,18 +5015,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-30", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-30", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-30"),
SourceUuid: u1,
Expand Down Expand Up @@ -5054,12 +5082,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-90", "1-30", "1-50"),
SourceUuid: u1,
Expand Down Expand Up @@ -5117,18 +5147,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-20", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-31", "1-50"),
SourceUuid: u2,
Expand Down Expand Up @@ -5181,12 +5214,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-31", "1-50"),
SourceUuid: u2,
Expand Down Expand Up @@ -5244,18 +5279,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-99", "1-31", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"),
SourceUuid: u1,
Expand Down Expand Up @@ -5308,18 +5346,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-101", "1-31", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"),
SourceUuid: u1,
Expand Down Expand Up @@ -5361,12 +5402,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-31", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-50"),
SourceUuid: u1,
Expand Down Expand Up @@ -5419,18 +5462,21 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000102": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-31", "1-50"),
SourceUuid: u1,
},
},
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-51"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-90", "1-30", "1-50"),
SourceUuid: u1,
Expand Down Expand Up @@ -5483,12 +5529,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-51"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-31", "1-50"),
SourceUuid: u2,
Expand Down Expand Up @@ -5542,12 +5590,14 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
},
statusMap: map[string]*replicationdatapb.StopReplicationStatus{
"zone1-0000000103": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("1-100", "1-30", "1-51"),
SourceUuid: u1,
},
},
"zone1-0000000104": {
Before: &replicationdatapb.Status{SemiSyncReplicaStatus: false},
After: &replicationdatapb.Status{
RelayLogPosition: getRelayLogPosition("", "1-31", "1-50"),
SourceUuid: u2,
Expand All @@ -5567,9 +5617,9 @@ func TestEmergencyReparenterFindErrantGTIDs(t *testing.T) {
erp := &EmergencyReparenter{
tmc: tt.tmc,
}
validCandidates, isGtid, err := FindPositionsOfAllCandidates(tt.statusMap, tt.primaryStatusMap)
validCandidates, isGTIDBasedMap, err := FindPositionsOfAllCandidates(tt.statusMap, tt.primaryStatusMap)
require.NoError(t, err)
require.True(t, isGtid)
require.Len(t, isGTIDBasedMap, len(validCandidates))
candidates, err := erp.findErrantGTIDs(context.Background(), validCandidates, tt.statusMap, tt.tabletMap, 10*time.Second)
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr)
Expand Down
Loading
Loading