Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 57 additions & 1 deletion go/test/endtoend/backup/vtctlbackup/backup_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp
return 1, err
}
newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql")
err = os.WriteFile(newInitDBFile, []byte(sql), 0666)
err = os.WriteFile(newInitDBFile, []byte(sql), 0o666)
if err != nil {
return 1, err
}
Expand Down Expand Up @@ -381,6 +381,10 @@ func TestBackup(t *testing.T, setupType int, streamMode string, stripes int, cDe
name: "TestPrimaryBackup",
method: primaryBackup,
},
{
name: "TestPrimaryRestoreSidecarReplication",
method: primaryRestoreSidecarReplication,
},
{
name: "TestPrimaryReplicaSameBackup",
method: primaryReplicaSameBackup,
Expand Down Expand Up @@ -563,6 +567,58 @@ func primaryBackup(t *testing.T) {
require.NoError(t, err)
}

// primaryRestoreSidecarReplication ensures replication remains healthy after restoring a primary from a backup that
// has a mismatch in _vt.tables relative to existing replicas.
func primaryRestoreSidecarReplication(t *testing.T) {
localCluster.DisableVTOrcRecoveries(t)
defer localCluster.EnableVTOrcRecoveries(t)

// Step 1: create the table and get initial replication.
verifyInitialReplication(t)

// Step 2: take a backup before the primary has recorded _vt.tables.
err := localCluster.VtctldClientProcess.ExecuteCommand("Backup", "--allow-primary", primary.Alias)
require.NoError(t, err)
firstBackupTimestamp := time.Now().UTC().Format(mysqlctl.BackupTimestampFormat)

// // Step 3: make the primary advance so the replica diverges.
// _, err = primary.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true)
// require.NoError(t, err)

// Step 3: bring up a second replica
restoreWaitForBackup(t, "replica", nil, true)
err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout)
require.NoError(t, err)

// Step 4: Promote the second replica, which initiates a schema reload. A table is now created in _vt.tables.
err = localCluster.VtctldClientProcess.ExecuteCommand("PlannedReparentShard", "--new-primary", replica2.Alias, shardKsName)
require.NoError(t, err)

// Replica 2 isn't needed anymore
err = localCluster.VtctldClientProcess.ExecuteCommand("DeleteTablets", "--allow-primary", replica2.Alias)
require.NoError(t, err)
err = replica2.VttabletProcess.TearDown()
require.NoError(t, err)

// Step 5: Restore the primary from the backup we took
err = localCluster.VtctldClientProcess.ExecuteCommand("RestoreFromBackup", "--backup-timestamp", firstBackupTimestamp, primary.Alias)
require.NoError(t, err)

// Step 6: make the old primary the primary again. This will initiate a schema reload, but since the backup is missing the table
// in _vt.tables, it will create it and replicate it to the replica, which already has the table in its _vt.tables from replica 2.
err = localCluster.VtctldClientProcess.InitShardPrimary(keyspaceName, shardName, cell, primary.TabletUID)
require.NoError(t, err)

// Validate replication still works
_, err = primary.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test_after_init')", keyspaceName, true)
require.NoError(t, err)

require.Eventually(t, func() bool {
res, err := replica1.VttabletProcess.QueryTablet("select msg from vt_insert_test where msg = 'test_after_init'", keyspaceName, true)
return err == nil && len(res.Rows) == 1
}, 1*time.Second, 10*time.Millisecond, "expected replication to be working")
}

// Test a primary and replica from the same backup.
//
// Check that a replica and primary both restored from the same backup
Expand Down
4 changes: 2 additions & 2 deletions go/vt/vttablet/grpctmclient/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -671,7 +671,7 @@ func (client *Client) ExecuteFetchAsDba(ctx context.Context, tablet *topodatapb.
DbName: topoproto.TabletDbName(tablet),
MaxRows: req.MaxRows,
DisableBinlogs: req.DisableBinlogs,
ReloadSchema: req.DisableBinlogs,
ReloadSchema: req.ReloadSchema,
DisableForeignKeyChecks: req.DisableForeignKeyChecks,
})
if err != nil {
Expand Down Expand Up @@ -707,7 +707,7 @@ func (client *Client) ExecuteMultiFetchAsDba(ctx context.Context, tablet *topoda
DbName: topoproto.TabletDbName(tablet),
MaxRows: req.MaxRows,
DisableBinlogs: req.DisableBinlogs,
ReloadSchema: req.DisableBinlogs,
ReloadSchema: req.ReloadSchema,
DisableForeignKeyChecks: req.DisableForeignKeyChecks,
})
if err != nil {
Expand Down
6 changes: 6 additions & 0 deletions go/vt/vttablet/tabletmanager/rpc_backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ func (tm *TabletManager) Backup(ctx context.Context, logger logutil.Logger, req
return vterrors.Wrap(err, "failed to execute backup init SQL queries")
}

// Reload the schema so that backup takes most up-to-date snapshot of the sidecar database (for example,
// to capture a new table that has been created since the last schema reload).
if err := tm.QueryServiceControl.ReloadSchema(ctx); err != nil {
return vterrors.Wrap(err, "failed to reload schema before backup")
}

// Prevent concurrent backups, and record stats
backupMode := backupModeOnline
if engine.ShouldDrainForBackup(req) {
Expand Down
Loading