Skip to content

feat: full backup w/ indexes for backup collections #27691

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 29, 2025
Merged

feat: full backup w/ indexes for backup collections #27691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ec5ba68
WIP
Enjection Oct 22, 2025
fbc61db
address PR issues
Enjection Oct 28, 2025
edc86f6
fix
Enjection Oct 28, 2025
17a5085
fix
Enjection Oct 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ydb/core/protos/flat_scheme_op.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2326,6 +2326,8 @@ message TBackupCollectionDescription {
oneof Storage {
google.protobuf.Empty Cluster = 7;
}

optional bool OmitIndexes = 9 [default = false];
}

message TBackupBackupCollection {
Expand Down
14 changes: 13 additions & 1 deletion ydb/core/tx/schemeshard/schemeshard__operation_backup_backup_collection.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ TVector<ISubOperation::TPtr> CreateBackupBackupCollection(TOperationId opId, con
bool incrBackupEnabled = bc->Description.HasIncrementalBackupConfig();
TString streamName = NBackup::ToX509String(TlsActivationContext->AsActorContext().Now()) + "_continuousBackupImpl";

bool omitIndexes = bc->Description.HasOmitIndexes()
? bc->Description.GetOmitIndexes()
: false;
Comment on lines +69 to +71
Copy link
Member

@CyberROFL CyberROFL Oct 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
bool omitIndexes = bc->Description.HasOmitIndexes()
? bc->Description.GetOmitIndexes()
: false;
const bool omitIndexes = bc->Description.HasOmitIndexes() && bc->Description.GetOmitIndexes();


for (const auto& item : bc->Description.GetExplicitEntryList().GetEntries()) {
auto& desc = *copyTables.Add();
desc.SetSrcPath(item.GetPath());
Expand All @@ -77,7 +81,15 @@ TVector<ISubOperation::TPtr> CreateBackupBackupCollection(TOperationId opId, con
}
auto& relativeItemPath = paths.second;
desc.SetDstPath(JoinPath({tx.GetWorkingDir(), tx.GetBackupBackupCollection().GetName(), tx.GetBackupBackupCollection().GetTargetDir(), relativeItemPath}));
desc.SetOmitIndexes(true);

// For incremental backups, always omit indexes from table copy (backed up separately via CDC)
// For full backups, respect the OmitIndexes configuration
if (incrBackupEnabled) {
desc.SetOmitIndexes(true);
} else {
desc.SetOmitIndexes(omitIndexes);
}

desc.SetOmitFollowers(true);
desc.SetAllowUnderSameOperation(true);

Expand Down
7 changes: 0 additions & 7 deletions ydb/core/tx/schemeshard/schemeshard_path.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1723,13 +1723,6 @@ bool TPath::IsInsideTableIndexPath(bool failOnUnresolved) const {
return false;
}

++item;
for (; item != Elements.rend(); ++item) {
if (!(*item)->IsDirectory() && !(*item)->IsSubDomainRoot()) {
return false;
}
}

return true;
}

Expand Down
235 changes: 235 additions & 0 deletions ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1699,4 +1699,239 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) {
// Verify it was created
TestLs(runtime, "/MyRoot/.backups/collections/TestCollection/__ydb_backup_meta", false, NLs::PathExist);
}

Y_UNIT_TEST(BackupWithIndexes) {
TTestBasicRuntime runtime;
TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true));
SetupLogging(runtime);
ui64 txId = 100;

// Create table with index
TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"(
TableDescription {
Name: "TableWithIndex"
Columns { Name: "key" Type: "Uint64" }
Columns { Name: "value" Type: "Utf8" }
KeyColumnNames: ["key"]
}
IndexDescription {
Name: "ValueIndex"
KeyColumnNames: ["value"]
}
)");
env.TestWaitNotification(runtime, txId);

// Verify source table has the index
TestDescribeResult(DescribePath(runtime, "/MyRoot/TableWithIndex"), {
NLs::PathExist,
NLs::IndexesCount(1)
});

PrepareDirs(runtime, env, txId);

// Create backup collection with OmitIndexes = false (explicitly request indexes)
TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"(
Name: "CollectionWithIndex"
ExplicitEntryList {
Entries {
Type: ETypeTable
Path: "/MyRoot/TableWithIndex"
}
}
OmitIndexes: false
)");
env.TestWaitNotification(runtime, txId);

// Backup the table (indexes should be included)
TestBackupBackupCollection(runtime, ++txId, "/MyRoot",
R"(Name: ".backups/collections/CollectionWithIndex")");
env.TestWaitNotification(runtime, txId);

// Verify backup collection has children (the backup directory)
TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithIndex"), {
NLs::PathExist,
NLs::IsBackupCollection,
NLs::ChildrenCount(1)
});

// Get the backup directory and verify its structure contains index
auto backupDesc = DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithIndex");
UNIT_ASSERT(backupDesc.GetPathDescription().ChildrenSize() == 1);
TString backupDirName = backupDesc.GetPathDescription().GetChildren(0).GetName();

// Verify backup directory has the table (indexes are stored under the table)
TString backupPath = "/MyRoot/.backups/collections/CollectionWithIndex/" + backupDirName;
auto backupContentDesc = DescribePath(runtime, backupPath);

// The backup should contain 1 child (the table; indexes are children of the table)
UNIT_ASSERT_C(backupContentDesc.GetPathDescription().ChildrenSize() == 1,
"Backup should contain 1 table, got " << backupContentDesc.GetPathDescription().ChildrenSize());

// Verify the table HAS indexes in the backup (check via TableIndexesSize)
UNIT_ASSERT_VALUES_EQUAL(backupContentDesc.GetPathDescription().GetChildren(0).GetName(), "TableWithIndex");

auto tableDesc = DescribePath(runtime, backupPath + "/TableWithIndex");
UNIT_ASSERT(tableDesc.GetPathDescription().HasTable());
UNIT_ASSERT_VALUES_EQUAL(tableDesc.GetPathDescription().GetTable().TableIndexesSize(), 1);

// Verify ChildrenExist flag is set (index exists as child, even if not in Children list)
UNIT_ASSERT(tableDesc.GetPathDescription().GetSelf().GetChildrenExist());
}

Y_UNIT_TEST(BackupWithIndexesOmit) {
TTestBasicRuntime runtime;
TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true));
SetupLogging(runtime);
ui64 txId = 100;

// Create table with index
TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"(
TableDescription {
Name: "TableWithIndex"
Columns { Name: "key" Type: "Uint64" }
Columns { Name: "value" Type: "Utf8" }
KeyColumnNames: ["key"]
}
IndexDescription {
Name: "ValueIndex"
KeyColumnNames: ["value"]
}
)");
env.TestWaitNotification(runtime, txId);

// Verify source table has the index
TestDescribeResult(DescribePath(runtime, "/MyRoot/TableWithIndex"), {
NLs::PathExist,
NLs::IndexesCount(1)
});

PrepareDirs(runtime, env, txId);

// Create backup collection with OmitIndexes = true (at collection level)
TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"(
Name: "CollectionWithoutIndex"
ExplicitEntryList {
Entries {
Type: ETypeTable
Path: "/MyRoot/TableWithIndex"
}
}
OmitIndexes: true
)");
env.TestWaitNotification(runtime, txId);

// Backup the table (indexes should be omitted)
TestBackupBackupCollection(runtime, ++txId, "/MyRoot",
R"(Name: ".backups/collections/CollectionWithoutIndex")");
env.TestWaitNotification(runtime, txId);

// Verify backup collection has children (the backup directory)
TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithoutIndex"), {
NLs::PathExist,
NLs::IsBackupCollection,
NLs::ChildrenCount(1)
});

// Get the backup directory and verify its structure does NOT contain index
auto backupDesc = DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithoutIndex");
UNIT_ASSERT(backupDesc.GetPathDescription().ChildrenSize() == 1);
TString backupDirName = backupDesc.GetPathDescription().GetChildren(0).GetName();

// Verify backup directory has only the table (no index children when omitted)
TString backupPath = "/MyRoot/.backups/collections/CollectionWithoutIndex/" + backupDirName;
auto backupContentDesc = DescribePath(runtime, backupPath);

// The backup should contain 1 child (the table), without index children
UNIT_ASSERT_C(backupContentDesc.GetPathDescription().ChildrenSize() == 1,
"Backup should contain only table without index, got " << backupContentDesc.GetPathDescription().ChildrenSize());

// Verify the table exists but has NO indexes (omitted via OmitIndexes: true)
UNIT_ASSERT_VALUES_EQUAL(backupContentDesc.GetPathDescription().GetChildren(0).GetName(), "TableWithIndex");

auto tableDesc = DescribePath(runtime, backupPath + "/TableWithIndex");
UNIT_ASSERT(tableDesc.GetPathDescription().HasTable());

// When indexes are omitted, TableIndexesSize should be 0
UNIT_ASSERT_VALUES_EQUAL(tableDesc.GetPathDescription().GetTable().TableIndexesSize(), 0);

// Verify ChildrenExist is false (no index children)
UNIT_ASSERT(!tableDesc.GetPathDescription().GetSelf().GetChildrenExist());
}

Y_UNIT_TEST(BackupWithIndexesDefault) {
TTestBasicRuntime runtime;
TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true));
SetupLogging(runtime);
ui64 txId = 100;

// Create table with index
TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"(
TableDescription {
Name: "TableWithIndex"
Columns { Name: "key" Type: "Uint64" }
Columns { Name: "value" Type: "Utf8" }
KeyColumnNames: ["key"]
}
IndexDescription {
Name: "ValueIndex"
KeyColumnNames: ["value"]
}
)");
env.TestWaitNotification(runtime, txId);

// Verify source table has the index
TestDescribeResult(DescribePath(runtime, "/MyRoot/TableWithIndex"), {
NLs::PathExist,
NLs::IndexesCount(1)
});

PrepareDirs(runtime, env, txId);

// Create backup collection without specifying OmitIndexes (default behavior)
TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"(
Name: "CollectionDefaultBehavior"
ExplicitEntryList {
Entries {
Type: ETypeTable
Path: "/MyRoot/TableWithIndex"
}
}
)");
env.TestWaitNotification(runtime, txId);

// Backup the table (default behavior: OmitIndexes not specified, should default to false)
TestBackupBackupCollection(runtime, ++txId, "/MyRoot",
R"(Name: ".backups/collections/CollectionDefaultBehavior")");
env.TestWaitNotification(runtime, txId);

// Verify backup collection has children (the backup directory)
TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/CollectionDefaultBehavior"), {
NLs::PathExist,
NLs::IsBackupCollection,
NLs::ChildrenCount(1)
});

// Get the backup directory and verify its structure
auto backupDesc = DescribePath(runtime, "/MyRoot/.backups/collections/CollectionDefaultBehavior");
UNIT_ASSERT(backupDesc.GetPathDescription().ChildrenSize() == 1);
TString backupDirName = backupDesc.GetPathDescription().GetChildren(0).GetName();

// Verify backup directory structure
TString backupPath = "/MyRoot/.backups/collections/CollectionDefaultBehavior/" + backupDirName;
auto backupContentDesc = DescribePath(runtime, backupPath);

// The backup should contain 1 child (the table; indexes are children of the table)
UNIT_ASSERT_C(backupContentDesc.GetPathDescription().ChildrenSize() == 1,
"Backup should contain 1 table, got " << backupContentDesc.GetPathDescription().ChildrenSize());

// Verify the table HAS indexes in the backup by default (check via TableIndexesSize)
UNIT_ASSERT_VALUES_EQUAL(backupContentDesc.GetPathDescription().GetChildren(0).GetName(), "TableWithIndex");

auto tableDesc = DescribePath(runtime, backupPath + "/TableWithIndex");
UNIT_ASSERT(tableDesc.GetPathDescription().HasTable());
UNIT_ASSERT_VALUES_EQUAL(tableDesc.GetPathDescription().GetTable().TableIndexesSize(), 1);

// Verify ChildrenExist flag is set by default (index exists as child)
UNIT_ASSERT(tableDesc.GetPathDescription().GetSelf().GetChildrenExist());
}
} // TBackupCollectionTests
Loading