Skip to content

Commit 9eb6468

Browse files
EnjectionEnjection
authored
feat: full backup w/ indexes for backup collections (#27691)
1 parent 1ca26e1 commit 9eb6468

File tree

4 files changed

+250
-8
lines changed

4 files changed

+250
-8
lines changed

‎ydb/core/protos/flat_scheme_op.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2326,6 +2326,8 @@ message TBackupCollectionDescription {
23262326
oneof Storage {
23272327
google.protobuf.Empty Cluster = 7;
23282328
}
2329+
2330+
optional bool OmitIndexes = 9 [default = false];
23292331
}
23302332

23312333
message TBackupBackupCollection {

‎ydb/core/tx/schemeshard/schemeshard__operation_backup_backup_collection.cpp‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ TVector<ISubOperation::TPtr> CreateBackupBackupCollection(TOperationId opId, con
6666
bool incrBackupEnabled = bc->Description.HasIncrementalBackupConfig();
6767
TString streamName = NBackup::ToX509String(TlsActivationContext->AsActorContext().Now()) + "_continuousBackupImpl";
6868

69+
bool omitIndexes = bc->Description.HasOmitIndexes()
70+
? bc->Description.GetOmitIndexes()
71+
: false;
72+
6973
for (const auto& item : bc->Description.GetExplicitEntryList().GetEntries()) {
7074
auto& desc = *copyTables.Add();
7175
desc.SetSrcPath(item.GetPath());
@@ -77,7 +81,15 @@ TVector<ISubOperation::TPtr> CreateBackupBackupCollection(TOperationId opId, con
7781
}
7882
auto& relativeItemPath = paths.second;
7983
desc.SetDstPath(JoinPath({tx.GetWorkingDir(), tx.GetBackupBackupCollection().GetName(), tx.GetBackupBackupCollection().GetTargetDir(), relativeItemPath}));
80-
desc.SetOmitIndexes(true);
84+
85+
// For incremental backups, always omit indexes from table copy (backed up separately via CDC)
86+
// For full backups, respect the OmitIndexes configuration
87+
if (incrBackupEnabled) {
88+
desc.SetOmitIndexes(true);
89+
} else {
90+
desc.SetOmitIndexes(omitIndexes);
91+
}
92+
8193
desc.SetOmitFollowers(true);
8294
desc.SetAllowUnderSameOperation(true);
8395

‎ydb/core/tx/schemeshard/schemeshard_path.cpp‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1723,13 +1723,6 @@ bool TPath::IsInsideTableIndexPath(bool failOnUnresolved) const {
17231723
return false;
17241724
}
17251725

1726-
++item;
1727-
for (; item != Elements.rend(); ++item) {
1728-
if (!(*item)->IsDirectory() && !(*item)->IsSubDomainRoot()) {
1729-
return false;
1730-
}
1731-
}
1732-
17331726
return true;
17341727
}
17351728

‎ydb/core/tx/schemeshard/ut_backup_collection/ut_backup_collection.cpp‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1699,4 +1699,239 @@ Y_UNIT_TEST_SUITE(TBackupCollectionTests) {
16991699
// Verify it was created
17001700
TestLs(runtime, "/MyRoot/.backups/collections/TestCollection/__ydb_backup_meta", false, NLs::PathExist);
17011701
}
1702+
1703+
Y_UNIT_TEST(BackupWithIndexes) {
1704+
TTestBasicRuntime runtime;
1705+
TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true));
1706+
SetupLogging(runtime);
1707+
ui64 txId = 100;
1708+
1709+
// Create table with index
1710+
TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"(
1711+
TableDescription {
1712+
Name: "TableWithIndex"
1713+
Columns { Name: "key" Type: "Uint64" }
1714+
Columns { Name: "value" Type: "Utf8" }
1715+
KeyColumnNames: ["key"]
1716+
}
1717+
IndexDescription {
1718+
Name: "ValueIndex"
1719+
KeyColumnNames: ["value"]
1720+
}
1721+
)");
1722+
env.TestWaitNotification(runtime, txId);
1723+
1724+
// Verify source table has the index
1725+
TestDescribeResult(DescribePath(runtime, "/MyRoot/TableWithIndex"), {
1726+
NLs::PathExist,
1727+
NLs::IndexesCount(1)
1728+
});
1729+
1730+
PrepareDirs(runtime, env, txId);
1731+
1732+
// Create backup collection with OmitIndexes = false (explicitly request indexes)
1733+
TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"(
1734+
Name: "CollectionWithIndex"
1735+
ExplicitEntryList {
1736+
Entries {
1737+
Type: ETypeTable
1738+
Path: "/MyRoot/TableWithIndex"
1739+
}
1740+
}
1741+
OmitIndexes: false
1742+
)");
1743+
env.TestWaitNotification(runtime, txId);
1744+
1745+
// Backup the table (indexes should be included)
1746+
TestBackupBackupCollection(runtime, ++txId, "/MyRoot",
1747+
R"(Name: ".backups/collections/CollectionWithIndex")");
1748+
env.TestWaitNotification(runtime, txId);
1749+
1750+
// Verify backup collection has children (the backup directory)
1751+
TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithIndex"), {
1752+
NLs::PathExist,
1753+
NLs::IsBackupCollection,
1754+
NLs::ChildrenCount(1)
1755+
});
1756+
1757+
// Get the backup directory and verify its structure contains index
1758+
auto backupDesc = DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithIndex");
1759+
UNIT_ASSERT(backupDesc.GetPathDescription().ChildrenSize() == 1);
1760+
TString backupDirName = backupDesc.GetPathDescription().GetChildren(0).GetName();
1761+
1762+
// Verify backup directory has the table (indexes are stored under the table)
1763+
TString backupPath = "/MyRoot/.backups/collections/CollectionWithIndex/" + backupDirName;
1764+
auto backupContentDesc = DescribePath(runtime, backupPath);
1765+
1766+
// The backup should contain 1 child (the table; indexes are children of the table)
1767+
UNIT_ASSERT_C(backupContentDesc.GetPathDescription().ChildrenSize() == 1,
1768+
"Backup should contain 1 table, got " << backupContentDesc.GetPathDescription().ChildrenSize());
1769+
1770+
// Verify the table HAS indexes in the backup (check via TableIndexesSize)
1771+
UNIT_ASSERT_VALUES_EQUAL(backupContentDesc.GetPathDescription().GetChildren(0).GetName(), "TableWithIndex");
1772+
1773+
auto tableDesc = DescribePath(runtime, backupPath + "/TableWithIndex");
1774+
UNIT_ASSERT(tableDesc.GetPathDescription().HasTable());
1775+
UNIT_ASSERT_VALUES_EQUAL(tableDesc.GetPathDescription().GetTable().TableIndexesSize(), 1);
1776+
1777+
// Verify ChildrenExist flag is set (index exists as child, even if not in Children list)
1778+
UNIT_ASSERT(tableDesc.GetPathDescription().GetSelf().GetChildrenExist());
1779+
}
1780+
1781+
Y_UNIT_TEST(BackupWithIndexesOmit) {
1782+
TTestBasicRuntime runtime;
1783+
TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true));
1784+
SetupLogging(runtime);
1785+
ui64 txId = 100;
1786+
1787+
// Create table with index
1788+
TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"(
1789+
TableDescription {
1790+
Name: "TableWithIndex"
1791+
Columns { Name: "key" Type: "Uint64" }
1792+
Columns { Name: "value" Type: "Utf8" }
1793+
KeyColumnNames: ["key"]
1794+
}
1795+
IndexDescription {
1796+
Name: "ValueIndex"
1797+
KeyColumnNames: ["value"]
1798+
}
1799+
)");
1800+
env.TestWaitNotification(runtime, txId);
1801+
1802+
// Verify source table has the index
1803+
TestDescribeResult(DescribePath(runtime, "/MyRoot/TableWithIndex"), {
1804+
NLs::PathExist,
1805+
NLs::IndexesCount(1)
1806+
});
1807+
1808+
PrepareDirs(runtime, env, txId);
1809+
1810+
// Create backup collection with OmitIndexes = true (at collection level)
1811+
TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"(
1812+
Name: "CollectionWithoutIndex"
1813+
ExplicitEntryList {
1814+
Entries {
1815+
Type: ETypeTable
1816+
Path: "/MyRoot/TableWithIndex"
1817+
}
1818+
}
1819+
OmitIndexes: true
1820+
)");
1821+
env.TestWaitNotification(runtime, txId);
1822+
1823+
// Backup the table (indexes should be omitted)
1824+
TestBackupBackupCollection(runtime, ++txId, "/MyRoot",
1825+
R"(Name: ".backups/collections/CollectionWithoutIndex")");
1826+
env.TestWaitNotification(runtime, txId);
1827+
1828+
// Verify backup collection has children (the backup directory)
1829+
TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithoutIndex"), {
1830+
NLs::PathExist,
1831+
NLs::IsBackupCollection,
1832+
NLs::ChildrenCount(1)
1833+
});
1834+
1835+
// Get the backup directory and verify its structure does NOT contain index
1836+
auto backupDesc = DescribePath(runtime, "/MyRoot/.backups/collections/CollectionWithoutIndex");
1837+
UNIT_ASSERT(backupDesc.GetPathDescription().ChildrenSize() == 1);
1838+
TString backupDirName = backupDesc.GetPathDescription().GetChildren(0).GetName();
1839+
1840+
// Verify backup directory has only the table (no index children when omitted)
1841+
TString backupPath = "/MyRoot/.backups/collections/CollectionWithoutIndex/" + backupDirName;
1842+
auto backupContentDesc = DescribePath(runtime, backupPath);
1843+
1844+
// The backup should contain 1 child (the table), without index children
1845+
UNIT_ASSERT_C(backupContentDesc.GetPathDescription().ChildrenSize() == 1,
1846+
"Backup should contain only table without index, got " << backupContentDesc.GetPathDescription().ChildrenSize());
1847+
1848+
// Verify the table exists but has NO indexes (omitted via OmitIndexes: true)
1849+
UNIT_ASSERT_VALUES_EQUAL(backupContentDesc.GetPathDescription().GetChildren(0).GetName(), "TableWithIndex");
1850+
1851+
auto tableDesc = DescribePath(runtime, backupPath + "/TableWithIndex");
1852+
UNIT_ASSERT(tableDesc.GetPathDescription().HasTable());
1853+
1854+
// When indexes are omitted, TableIndexesSize should be 0
1855+
UNIT_ASSERT_VALUES_EQUAL(tableDesc.GetPathDescription().GetTable().TableIndexesSize(), 0);
1856+
1857+
// Verify ChildrenExist is false (no index children)
1858+
UNIT_ASSERT(!tableDesc.GetPathDescription().GetSelf().GetChildrenExist());
1859+
}
1860+
1861+
Y_UNIT_TEST(BackupWithIndexesDefault) {
1862+
TTestBasicRuntime runtime;
1863+
TTestEnv env(runtime, TTestEnvOptions().EnableBackupService(true));
1864+
SetupLogging(runtime);
1865+
ui64 txId = 100;
1866+
1867+
// Create table with index
1868+
TestCreateIndexedTable(runtime, ++txId, "/MyRoot", R"(
1869+
TableDescription {
1870+
Name: "TableWithIndex"
1871+
Columns { Name: "key" Type: "Uint64" }
1872+
Columns { Name: "value" Type: "Utf8" }
1873+
KeyColumnNames: ["key"]
1874+
}
1875+
IndexDescription {
1876+
Name: "ValueIndex"
1877+
KeyColumnNames: ["value"]
1878+
}
1879+
)");
1880+
env.TestWaitNotification(runtime, txId);
1881+
1882+
// Verify source table has the index
1883+
TestDescribeResult(DescribePath(runtime, "/MyRoot/TableWithIndex"), {
1884+
NLs::PathExist,
1885+
NLs::IndexesCount(1)
1886+
});
1887+
1888+
PrepareDirs(runtime, env, txId);
1889+
1890+
// Create backup collection without specifying OmitIndexes (default behavior)
1891+
TestCreateBackupCollection(runtime, ++txId, "/MyRoot/.backups/collections", R"(
1892+
Name: "CollectionDefaultBehavior"
1893+
ExplicitEntryList {
1894+
Entries {
1895+
Type: ETypeTable
1896+
Path: "/MyRoot/TableWithIndex"
1897+
}
1898+
}
1899+
)");
1900+
env.TestWaitNotification(runtime, txId);
1901+
1902+
// Backup the table (default behavior: OmitIndexes not specified, should default to false)
1903+
TestBackupBackupCollection(runtime, ++txId, "/MyRoot",
1904+
R"(Name: ".backups/collections/CollectionDefaultBehavior")");
1905+
env.TestWaitNotification(runtime, txId);
1906+
1907+
// Verify backup collection has children (the backup directory)
1908+
TestDescribeResult(DescribePath(runtime, "/MyRoot/.backups/collections/CollectionDefaultBehavior"), {
1909+
NLs::PathExist,
1910+
NLs::IsBackupCollection,
1911+
NLs::ChildrenCount(1)
1912+
});
1913+
1914+
// Get the backup directory and verify its structure
1915+
auto backupDesc = DescribePath(runtime, "/MyRoot/.backups/collections/CollectionDefaultBehavior");
1916+
UNIT_ASSERT(backupDesc.GetPathDescription().ChildrenSize() == 1);
1917+
TString backupDirName = backupDesc.GetPathDescription().GetChildren(0).GetName();
1918+
1919+
// Verify backup directory structure
1920+
TString backupPath = "/MyRoot/.backups/collections/CollectionDefaultBehavior/" + backupDirName;
1921+
auto backupContentDesc = DescribePath(runtime, backupPath);
1922+
1923+
// The backup should contain 1 child (the table; indexes are children of the table)
1924+
UNIT_ASSERT_C(backupContentDesc.GetPathDescription().ChildrenSize() == 1,
1925+
"Backup should contain 1 table, got " << backupContentDesc.GetPathDescription().ChildrenSize());
1926+
1927+
// Verify the table HAS indexes in the backup by default (check via TableIndexesSize)
1928+
UNIT_ASSERT_VALUES_EQUAL(backupContentDesc.GetPathDescription().GetChildren(0).GetName(), "TableWithIndex");
1929+
1930+
auto tableDesc = DescribePath(runtime, backupPath + "/TableWithIndex");
1931+
UNIT_ASSERT(tableDesc.GetPathDescription().HasTable());
1932+
UNIT_ASSERT_VALUES_EQUAL(tableDesc.GetPathDescription().GetTable().TableIndexesSize(), 1);
1933+
1934+
// Verify ChildrenExist flag is set by default (index exists as child)
1935+
UNIT_ASSERT(tableDesc.GetPathDescription().GetSelf().GetChildrenExist());
1936+
}
17021937
} // TBackupCollectionTests

0 commit comments

Comments
 (0)