fix

Author: hx235

db/error_handler.cc

@@ -423,14 +423,17 @@ void ErrorHandler::SetBGError(const Status& bg_status,
            reason == BackgroundErrorReason::kMemTable ||
            reason == BackgroundErrorReason::kFlush);
   }
-  if (db_options_.manual_wal_flush && wal_related && bg_io_err.IsIOError()) {
-    // With manual_wal_flush, a WAL write failure can drop buffered WAL writes.
-    // Memtables and WAL then become inconsistent. A successful memtable flush
-    // on one CF can cause CFs to be inconsistent upon restart. Before we fix
-    // the bug in auto recovery from WAL write failures that can flush one CF
-    // at a time, we set the error severity to fatal to disallow auto recovery.
-    // TODO: remove parameter `wal_related` once we can automatically recover
-    //  from WAL write failures.
+
+  // When `atomic_flush = false` with multiple column families, when
+  // encountering WAL related IO error, individual CF flushing during auto
+  // recovery can create data inconsistencies where some column families advance
+  // past the corruption point while others remain behind, preventing successful
+  // database restart. Therefore we disable auto recovery by setting a higher
+  // severity `Status::Severity::kFatalError`.
+  bool has_multiple_cfs =
+      db_->versions_->GetColumnFamilySet()->NumberOfColumnFamilies() > 1;
+  if (!db_options_.atomic_flush && has_multiple_cfs && wal_related &&
+      bg_io_err.IsIOError()) {
     bool auto_recovery = false;
     Status bg_err(new_bg_io_err, Status::Severity::kFatalError);
     CheckAndSetRecoveryAndBGError(bg_err);

db_stress_tool/db_stress_test_base.h

@@ -354,7 +354,8 @@ class StressTest {
     assert(!error_s.ok());
     return error_s.getState() &&
            FaultInjectionTestFS::IsInjectedError(error_s) &&
-           !status_to_io_status(Status(error_s)).GetDataLoss();
+           !status_to_io_status(Status(error_s)).GetDataLoss() &&
+           error_s.severity() <= Status::Severity::kHardError;
   }
 
   void ProcessStatus(SharedState* shared, std::string msg, const Status& s,

tools/db_crashtest.py

@@ -994,7 +994,7 @@ def finalize_and_sanitize(src_params):
         if (
             dest_params.get("reopen") > 0
             or (
-                dest_params.get("manual_wal_flush_one_in")
+                dest_params.get("atomic_flush") != 1
                 and dest_params.get("column_families") != 1
             )
             or (
@@ -1008,10 +1008,10 @@ def finalize_and_sanitize(src_params):
             # To simplify, we disable any WAL write error injection.
             # TODO(hx235): support WAL write error injection with reopen
             #
-            # 2. WAL write failure can drop buffered WAL data. This can cause
-            # inconsistency when one CF has a successful flush during auto
-            # recovery. Disable the fault injection in this path for now until
-            # we have a fix that allows auto recovery.
+            # 2.  When `atomic_flush = false` with multiple column families, when encountering WAL related IO error,
+            # individual CF flushing during auto recovery can create data inconsistencies where some column families advance
+            # past the corruption point while others remain behind, preventing successful database restart.
+            # Therefore we disable auto recovery and testing for this case in crash test
             #
             # 3. Pessimistic transactions use 2PC, which can't auto-recover from WAL write errors.
             # This is because RocksDB cannot easily discard the corrupted WAL without risking the