Added persistent BucketList tests

src/bucket/Bucket.cpp

@@ -136,14 +136,23 @@ Bucket::apply(Application& app) const
 {
     ZoneScoped;
 
+    auto filter = [&](LedgerEntryType t) {
+        if (app.getConfig().isUsingBucketListDB())
+        {
+            return t == OFFER;
+        }
+
+        return true;
+    };
+
     std::unordered_set<LedgerKey> emptySet;
     BucketApplicator applicator(
         app, app.getConfig().LEDGER_PROTOCOL_VERSION,
         0 /*set to 0 so we always load from the parent to check state*/,
         0 /*set to a level that's not the bottom so we don't treat live entries
              as init*/
         ,
-        shared_from_this(), [](LedgerEntryType) { return true; }, emptySet);
+        shared_from_this(), filter, emptySet);
     BucketApplicator::Counters counters(app.getClock().now());
     while (applicator)
     {

src/bucket/test/BucketIndexTests.cpp

@@ -604,7 +604,7 @@ TEST_CASE("ContractData key with same ScVal", "[bucket][bucketindex]")
 
 TEST_CASE("serialize bucket indexes", "[bucket][bucketindex]")
 {
-    Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
 
     // All levels use range config
     cfg.BUCKETLIST_DB_INDEX_CUTOFF = 0;

src/bucket/test/BucketListTests.cpp

@@ -669,7 +669,7 @@ TEST_CASE("BucketList check bucket sizes", "[bucket][bucketlist][count]")
 TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY));
     cfg.USE_CONFIG_FOR_GENESIS = true;
 
     auto app = createTestApplication<BucketTestApplication>(clock, cfg);
@@ -756,7 +756,7 @@ TEST_CASE_VERSIONS("network config snapshots BucketList size", "[bucketlist]")
 TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig());
     cfg.USE_CONFIG_FOR_GENESIS = true;
 
     auto test = [&](bool backgroundScan) {
@@ -1231,7 +1231,7 @@ TEST_CASE_VERSIONS("eviction scan", "[bucketlist]")
 TEST_CASE_VERSIONS("Searchable BucketListDB snapshots", "[bucketlist]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig());
     cfg.DEPRECATED_SQL_LEDGER_STATE = false;
 
     auto app = createTestApplication<BucketTestApplication>(clock, cfg);

src/bucket/test/BucketManagerTests.cpp

@@ -303,7 +303,7 @@ TEST_CASE_VERSIONS("bucketmanager ownership", "[bucket][bucketmanager]")
 
 TEST_CASE("bucketmanager missing buckets fail", "[bucket][bucketmanager]")
 {
-    Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
     std::string someBucketFileName;
     {
         VirtualClock clock;
@@ -318,7 +318,10 @@ TEST_CASE("bucketmanager missing buckets fail", "[bucket][bucketmanager]")
         {
             ++ledger;
             lm.setNextLedgerEntryBatchForBucketTesting(
-                {}, LedgerTestUtils::generateValidUniqueLedgerEntries(10), {});
+                {},
+                LedgerTestUtils::generateValidUniqueLedgerEntriesWithExclusions(
+                    {CONFIG_SETTING}, 10),
+                {});
             closeLedger(*app);
         } while (!BucketList::levelShouldSpill(ledger, level - 1));
         auto someBucket = bl.getLevel(1).getCurr();
@@ -341,7 +344,7 @@ TEST_CASE_VERSIONS("bucketmanager reattach to finished merge",
                    "[bucket][bucketmanager]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig());
     cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
     cfg.MANUAL_CLOSE = false;
 
@@ -403,7 +406,7 @@ TEST_CASE_VERSIONS("bucketmanager reattach to running merge",
                    "[bucket][bucketmanager]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
     cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
     cfg.MANUAL_CLOSE = false;
 
@@ -482,9 +485,10 @@ TEST_CASE("bucketmanager do not leak empty-merge futures",
     // The point of this test is to confirm that
     // BucketManager::noteEmptyMergeOutput is being called properly from merges
     // that produce empty outputs, and that the input buckets to those merges
-    // are thereby not leaking.
+    // are thereby not leaking. Disable BucketListDB so that snapshots do not
+    // hold persist buckets, complicating bucket counting.
     VirtualClock clock;
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY));
     cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
     cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION =
         static_cast<uint32_t>(
@@ -1055,7 +1059,7 @@ class StopAndRestartBucketMergesTest
     collectControlSurveys()
     {
         VirtualClock clock;
-        Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+        Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
         cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
         cfg.ARTIFICIALLY_REDUCE_MERGE_COUNTS_FOR_TESTING = true;
         cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = mProtocol;
@@ -1166,7 +1170,7 @@ class StopAndRestartBucketMergesTest
     runStopAndRestartTest(uint32_t firstProtocol, uint32_t secondProtocol)
     {
         std::unique_ptr<VirtualClock> clock = std::make_unique<VirtualClock>();
-        Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
+        Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
         cfg.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;
         cfg.ARTIFICIALLY_REDUCE_MERGE_COUNTS_FOR_TESTING = true;
         cfg.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION = firstProtocol;
@@ -1358,11 +1362,11 @@ TEST_CASE_VERSIONS("bucket persistence over app restart",
     std::vector<stellar::LedgerKey> emptySet;
     std::vector<stellar::LedgerEntry> emptySetEntry;
 
-    Config cfg0(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
+    Config cfg0(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
     cfg0.MANUAL_CLOSE = false;
 
     for_versions_with_differing_bucket_logic(cfg0, [&](Config const& cfg0) {
-        Config cfg1(getTestConfig(1, Config::TESTDB_ON_DISK_SQLITE));
+        Config cfg1(getTestConfig(1, Config::TESTDB_BUCKET_DB_PERSISTENT));
         cfg1.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION =
             cfg0.TESTING_UPGRADE_LEDGER_PROTOCOL_VERSION;
         cfg1.ARTIFICIALLY_PESSIMIZE_MERGES_FOR_TESTING = true;

src/bucket/test/BucketTests.cpp

@@ -910,10 +910,10 @@ TEST_CASE_VERSIONS("merging bucket entries with initentry with shadows",
     });
 }
 
-TEST_CASE_VERSIONS("bucket apply", "[bucket]")
+TEST_CASE_VERSIONS("legacy bucket apply", "[bucket]")
 {
     VirtualClock clock;
-    Config cfg(getTestConfig());
+    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
     for_versions_with_differing_bucket_logic(cfg, [&](Config const& cfg) {
         Application::pointer app = createTestApplication(clock, cfg);
 
@@ -983,14 +983,5 @@ TEST_CASE("bucket apply bench", "[bucketbench][!hide]")
         birth->apply(*app);
     };
 
-    SECTION("sqlite")
-    {
-        runtest(Config::TESTDB_ON_DISK_SQLITE);
-    }
-#ifdef USE_POSTGRES
-    SECTION("postgresql")
-    {
-        runtest(Config::TESTDB_POSTGRESQL);
-    }
-#endif
+    runtest(Config::TESTDB_BUCKET_DB_PERSISTENT);
 }

src/herder/test/HerderTests.cpp

@@ -2567,7 +2567,11 @@ TEST_CASE("SCP State", "[herder]")
             configure(Config::TestDbMode::TESTDB_ON_DISK_SQLITE);
         }
 
-        // TODO: Persist BucketList
+        SECTION("bucketlistDB")
+        {
+            configure(Config::TestDbMode::TESTDB_BUCKET_DB_PERSISTENT);
+        }
+
 #ifdef USE_POSTGRES
         SECTION("postgres")
         {
@@ -2932,7 +2936,7 @@ TEST_CASE("tx queue source account limit", "[herder][transactionqueue]")
         auto networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
         simulation = std::make_shared<Simulation>(
             Simulation::OVER_LOOPBACK, networkID, [](int i) {
-                auto cfg = getTestConfig(i, Config::TESTDB_ON_DISK_SQLITE);
+                auto cfg = getTestConfig(i, Config::TESTDB_DEFAULT);
                 return cfg;
             });
 
@@ -3049,7 +3053,7 @@ TEST_CASE("soroban txs each parameter surge priced", "[soroban][herder]")
             std::function<void(Config & appCfg)> tweakAppCfg) {
             auto simulation = Topologies::core(
                 4, 1, Simulation::OVER_LOOPBACK, networkID, [&](int i) {
-                    auto cfg = getTestConfig(i, Config::TESTDB_ON_DISK_SQLITE);
+                    auto cfg = getTestConfig(i, Config::TESTDB_DEFAULT);
                     auto mid = std::numeric_limits<uint32_t>::max() / 2;
                     cfg.LOADGEN_INSTRUCTIONS_FOR_TESTING = {mid};
                     cfg.LOADGEN_INSTRUCTIONS_FOR_TESTING = {1};
@@ -3422,7 +3426,7 @@ TEST_CASE("soroban txs accepted by the network",
     // Set threshold to 1 so all have to vote
     auto simulation =
         Topologies::core(4, 1, Simulation::OVER_LOOPBACK, networkID, [](int i) {
-            auto cfg = getTestConfig(i, Config::TESTDB_ON_DISK_SQLITE);
+            auto cfg = getTestConfig(i, Config::TESTDB_DEFAULT);
             cfg.TESTING_UPGRADE_MAX_TX_SET_SIZE = 100;
             return cfg;
         });
@@ -4284,7 +4288,7 @@ TEST_CASE("In quorum filtering", "[quorum][herder][acceptance]")
     auto networkID = sha256(getTestConfig().NETWORK_PASSPHRASE);
 
     auto sim = Topologies::core(4, 0.75, mode, networkID, [](int i) {
-        return getTestConfig(i, Config::TESTDB_ON_DISK_SQLITE);
+        return getTestConfig(i, Config::TESTDB_DEFAULT);
     });
 
     sim->startAllNodes();

src/herder/test/QuorumTrackerTests.cpp

@@ -17,7 +17,7 @@ using namespace stellar;
 void
 testQuorumTracker()
 {
-    Config cfg(getTestConfig(0, Config::TESTDB_ON_DISK_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_BUCKET_DB_PERSISTENT));
     cfg.MANUAL_CLOSE = false;
 
     std::vector<SecretKey> otherKeys;
@@ -185,7 +185,7 @@ TEST_CASE("quorum tracker", "[quorum][herder]")
 
 TEST_CASE("quorum tracker closest validators", "[quorum][herder]")
 {
-    Config cfg(getTestConfig(0, Config::TESTDB_IN_MEMORY_SQLITE));
+    Config cfg(getTestConfig(0, Config::TESTDB_DEFAULT));
 
     std::vector<PublicKey> otherKeys;
     int const kKeysCount = 7;