Count level merge metrics (#262)

* count level merge metrics

Signed-off-by: Connor1996 <zbk602423539@gmail.com>

include/titan/statistics.h

@@ -21,25 +21,38 @@ enum TickerType : uint32_t {
 
   TITAN_GC_NUM_FILES,
   TITAN_GC_NUM_NEW_FILES,
+
+  // the number of keys overwritten by foreground within the span of GC
   TITAN_GC_NUM_KEYS_OVERWRITTEN,
+  // the number of keys relocated to new blob file by GC
   TITAN_GC_NUM_KEYS_RELOCATED,
+  // the number of keys rewritten to LSM tree due to fallback
+  TITAN_GC_NUM_KEYS_FALLBACK,
   TITAN_GC_BYTES_OVERWRITTEN,
   TITAN_GC_BYTES_RELOCATED,
+  TITAN_GC_BYTES_FALLBACK,
+
   TITAN_GC_BYTES_WRITTEN,
   TITAN_GC_BYTES_READ,
 
   TITAN_BLOB_CACHE_HIT,
   TITAN_BLOB_CACHE_MISS,
 
-  TITAN_GC_NO_NEED,
-  TITAN_GC_REMAIN,
-
+  // the count of blob file gced due to discardable ratio hit the threshold
   TITAN_GC_DISCARDABLE,
-  TITAN_GC_SAMPLE,
+  // the count of blob file gced due to small file size
   TITAN_GC_SMALL_FILE,
+  // the count of blob file marked to be merged by level merge
+  TITAN_GC_LEVEL_MERGE_MARK,
+  // the count of blob file deleted directly by level merge
+  TITAN_GC_LEVEL_MERGE_DELETE,
 
+  TITAN_GC_NO_NEED,
+  // the times of still has blob files remained to be gced after one round of gc
+  TITAN_GC_REMAIN,
   TITAN_GC_FAILURE,
   TITAN_GC_SUCCESS,
+  // the times of triggering next round of GC actively
   TITAN_GC_TRIGGER_NEXT,
 
   TITAN_TICKER_ENUM_MAX,
@@ -61,17 +74,18 @@ const std::vector<std::pair<TickerType, std::string>> TitanTickersNameMap = {
     {TITAN_GC_NUM_NEW_FILES, "titandb.gc.num.new.files"},
     {TITAN_GC_NUM_KEYS_OVERWRITTEN, "titandb.gc.num.keys.overwritten"},
     {TITAN_GC_NUM_KEYS_RELOCATED, "titandb.gc.num.keys.relocated"},
+    {TITAN_GC_NUM_KEYS_FALLBACK, "titandb.gc.num.keys.fallback"},
     {TITAN_GC_BYTES_OVERWRITTEN, "titandb.gc.bytes.overwritten"},
     {TITAN_GC_BYTES_RELOCATED, "titandb.gc.bytes.relocated"},
+    {TITAN_GC_BYTES_FALLBACK, "titandb.gc.bytes.fallback"},
     {TITAN_GC_BYTES_WRITTEN, "titandb.gc.bytes.written"},
     {TITAN_GC_BYTES_READ, "titandb.gc.bytes.read"},
     {TITAN_BLOB_CACHE_HIT, "titandb.blob.cache.hit"},
     {TITAN_BLOB_CACHE_MISS, "titandb.blob.cache.miss"},
-    {TITAN_GC_NO_NEED, "titandb.gc.no.need"},
-    {TITAN_GC_REMAIN, "titandb.gc.remain"},
     {TITAN_GC_DISCARDABLE, "titandb.gc.discardable"},
-    {TITAN_GC_SAMPLE, "titandb.gc.sample"},
     {TITAN_GC_SMALL_FILE, "titandb.gc.small.file"},
+    {TITAN_GC_NO_NEED, "titandb.gc.no.need"},
+    {TITAN_GC_REMAIN, "titandb.gc.remain"},
     {TITAN_GC_FAILURE, "titandb.gc.failure"},
     {TITAN_GC_SUCCESS, "titandb.gc.success"},
     {TITAN_GC_TRIGGER_NEXT, "titandb.gc.trigger.next"},

src/blob_gc_job.cc

@@ -118,12 +118,13 @@ BlobGCJob::~BlobGCJob() {
              metrics_.gc_num_keys_relocated);
   RecordTick(statistics(stats_), TITAN_GC_BYTES_RELOCATED,
              metrics_.gc_bytes_relocated);
+  RecordTick(statistics(stats_), TITAN_GC_NUM_KEYS_FALLBACK,
+             metrics_.gc_num_keys_fallback);
+  RecordTick(statistics(stats_), TITAN_GC_BYTES_FALLBACK,
+             metrics_.gc_bytes_fallback);
   RecordTick(statistics(stats_), TITAN_GC_NUM_NEW_FILES,
              metrics_.gc_num_new_files);
   RecordTick(statistics(stats_), TITAN_GC_NUM_FILES, metrics_.gc_num_files);
-  RecordTick(statistics(stats_), TITAN_GC_DISCARDABLE, metrics_.gc_discardable);
-  RecordTick(statistics(stats_), TITAN_GC_SMALL_FILE, metrics_.gc_small_file);
-  RecordTick(statistics(stats_), TITAN_GC_SAMPLE, metrics_.gc_sample);
 }
 
 Status BlobGCJob::Prepare() {
@@ -517,9 +518,9 @@ Status BlobGCJob::RewriteValidKeyToLSM() {
         metrics_.gc_num_keys_relocated++;
         metrics_.gc_bytes_relocated += write_batch.second.blob_record_size();
       } else {
-        // Rewritten as inline value.
-        metrics_.gc_num_keys_overwritten++;
-        metrics_.gc_bytes_overwritten += write_batch.second.blob_record_size();
+        // Rewritten as inline value due to fallback mode.
+        metrics_.gc_num_keys_fallback++;
+        metrics_.gc_bytes_fallback += write_batch.second.blob_record_size();
       }
     } else if (s.IsBusy()) {
       metrics_.gc_num_keys_overwritten++;
@@ -629,8 +630,6 @@ void BlobGCJob::UpdateInternalOpStats() {
            metrics_.gc_num_files);
   AddStats(internal_op_stats, InternalOpStatsType::OUTPUT_FILE_NUM,
            metrics_.gc_num_new_files);
-  AddStats(internal_op_stats, InternalOpStatsType::GC_SAMPLING_MICROS,
-           metrics_.gc_sampling_micros);
   AddStats(internal_op_stats, InternalOpStatsType::GC_READ_LSM_MICROS,
            metrics_.gc_read_lsm_micros);
   AddStats(internal_op_stats, InternalOpStatsType::GC_UPDATE_LSM_MICROS,

src/blob_gc_job.h

@@ -71,12 +71,10 @@ class BlobGCJob {
     uint64_t gc_bytes_overwritten = 0;
     uint64_t gc_num_keys_relocated = 0;
     uint64_t gc_bytes_relocated = 0;
+    uint64_t gc_num_keys_fallback = 0;
+    uint64_t gc_bytes_fallback = 0;
     uint64_t gc_num_new_files = 0;
     uint64_t gc_num_files = 0;
-    uint64_t gc_small_file = 0;
-    uint64_t gc_discardable = 0;
-    uint64_t gc_sample = 0;
-    uint64_t gc_sampling_micros = 0;
     uint64_t gc_read_lsm_micros = 0;
     uint64_t gc_update_lsm_micros = 0;
   } metrics_;

src/blob_gc_picker.cc

@@ -42,6 +42,11 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickBlobGC(
     }
     if (!stop_picking) {
       blob_files.emplace_back(blob_file);
+      if (blob_file->file_size() <= cf_options_.merge_small_file_threshold) {
+        RecordTick(statistics(stats_), TITAN_GC_SMALL_FILE, 1);
+      } else {
+        RecordTick(statistics(stats_), TITAN_GC_DISCARDABLE, 1);
+      }
       batch_size += blob_file->file_size();
       estimate_output_size += blob_file->live_data_size();
       if (batch_size >= cf_options_.max_gc_batch_size ||

src/db_impl.cc

@@ -951,32 +951,33 @@ Status TitanDBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
     uint64_t file_number = file_size.first;
     int64_t delta = file_size.second;
     auto file = bs->FindFile(file_number).lock();
-    if (!file) {
+    if (!file || file->is_obsolete()) {
       // file has been gc out
       continue;
     }
-    if (!file->is_obsolete()) {
-      auto before = file->GetDiscardableRatioLevel();
-      bool ok = file->UpdateLiveDataSize(delta);
-      if (!ok) {
-        TITAN_LOG_WARN(db_options_.info_log,
-                       "During DeleteFilesInRanges: blob file %" PRIu64
-                       " live size below zero.",
-                       file_number);
-        assert(false);
-      }
-      auto after = file->GetDiscardableRatioLevel();
-      if (before != after) {
-        AddStats(stats_.get(), cf_id, after, 1);
-        SubStats(stats_.get(), cf_id, before, 1);
-      }
+    auto before = file->GetDiscardableRatioLevel();
+    bool ok = file->UpdateLiveDataSize(delta);
+    if (!ok) {
+      TITAN_LOG_WARN(db_options_.info_log,
+                     "During DeleteFilesInRanges: blob file %" PRIu64
+                     " live size below zero.",
+                     file_number);
+      assert(false);
+    }
+    auto after = file->GetDiscardableRatioLevel();
+    if (before != after) {
+      AddStats(stats_.get(), cf_id, after, 1);
+      SubStats(stats_.get(), cf_id, before, 1);
     }
     if (cf_options.level_merge) {
       if (file->NoLiveData()) {
+        RecordTick(statistics(stats_.get()), TITAN_GC_NUM_FILES, 1);
+        RecordTick(statistics(stats_.get()), TITAN_GC_LEVEL_MERGE_DELETE, 1);
         edit.DeleteBlobFile(file->file_number(),
                             db_impl_->GetLatestSequenceNumber());
       } else if (file->GetDiscardableRatio() >
                  cf_options.blob_file_discardable_ratio) {
+        RecordTick(statistics(stats_.get()), TITAN_GC_LEVEL_MERGE_MARK, 1);
         file->FileStateTransit(BlobFileMeta::FileEvent::kNeedMerge);
       }
     }
@@ -1034,6 +1035,7 @@ void TitanDBImpl::MarkFileIfNeedMerge(
       set.insert(end.first);
       if (set.size() > static_cast<size_t>(max_sorted_runs)) {
         for (auto file : set) {
+          RecordTick(statistics(stats_.get()), TITAN_GC_LEVEL_MERGE_MARK, 1);
           file->FileStateTransit(BlobFileMeta::FileEvent::kNeedMerge);
         }
       }
@@ -1307,12 +1309,14 @@ void TitanDBImpl::OnCompactionCompleted(
                       compaction_job_info.job_id, compaction_job_info.cf_id);
       return;
     }
+
     VersionEdit edit;
     auto cf_options = bs->cf_options();
     std::vector<std::shared_ptr<BlobFileMeta>> to_merge_candidates;
     bool count_sorted_run =
         cf_options.level_merge && cf_options.range_merge &&
         cf_options.num_levels - 1 == compaction_job_info.output_level;
+
     for (const auto& file_diff : blob_file_size_diff) {
       uint64_t file_number = file_diff.first;
       int64_t delta = file_diff.second;
@@ -1356,21 +1360,26 @@ void TitanDBImpl::OnCompactionCompleted(
         if (file->live_data_size() != 0) {
           // It records the delta as a positive number if any later compaction
           // is trigger before previous `OnCompactionCompleted()` is called, so
-          // here subtract it
-          // from the total live data size.
+          // here subtract it from the total live data size.
           delta -= file->live_data_size();
         }
         file->set_live_data_size(static_cast<uint64_t>(delta));
         AddStats(stats_.get(), compaction_job_info.cf_id,
                  file->GetDiscardableRatioLevel(), 1);
         file->FileStateTransit(BlobFileMeta::FileEvent::kCompactionCompleted);
+        if (file->NoLiveData()) {
+          RecordTick(statistics(stats_.get()), TITAN_GC_NUM_FILES, 1);
+          RecordTick(statistics(stats_.get()), TITAN_GC_LEVEL_MERGE_DELETE, 1);
+          edit.DeleteBlobFile(file->file_number(),
+                              db_impl_->GetLatestSequenceNumber());
+        }
         to_merge_candidates.push_back(file);
         TITAN_LOG_INFO(
             db_options_.info_log,
             "OnCompactionCompleted[%d]: compaction output blob file %" PRIu64
-            ", live data size %" PRIu64 ".",
+            ", live data size %" PRIu64 ", discardable ratio %lf.",
             compaction_job_info.job_id, file->file_number(),
-            file->live_data_size());
+            file->live_data_size(), file->GetDiscardableRatio());
       } else if (file->file_state() == BlobFileMeta::FileState::kNormal ||
                  file->file_state() == BlobFileMeta::FileState::kToMerge) {
         if (delta > 0) {
@@ -1398,17 +1407,19 @@ void TitanDBImpl::OnCompactionCompleted(
           // Mark last two level blob files to merge in next compaction if
           // discardable size reached GC threshold
           if (file->NoLiveData()) {
+            RecordTick(statistics(stats_.get()), TITAN_GC_NUM_FILES, 1);
+            RecordTick(statistics(stats_.get()), TITAN_GC_LEVEL_MERGE_DELETE,
+                       1);
             edit.DeleteBlobFile(file->file_number(),
                                 db_impl_->GetLatestSequenceNumber());
           } else if (static_cast<int>(file->file_level()) >=
                          cf_options.num_levels - 2 &&
                      file->GetDiscardableRatio() >
                          cf_options.blob_file_discardable_ratio) {
+            RecordTick(statistics(stats_.get()), TITAN_GC_LEVEL_MERGE_MARK, 1);
             file->FileStateTransit(BlobFileMeta::FileEvent::kNeedMerge);
-          } else {
-            if (count_sorted_run) {
-              to_merge_candidates.push_back(file);
-            }
+          } else if (count_sorted_run) {
+            to_merge_candidates.push_back(file);
           }
         }
       }

src/db_impl_gc.cc

@@ -166,7 +166,6 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
 
   std::unique_ptr<BlobGC> blob_gc;
   std::unique_ptr<ColumnFamilyHandle> cfh;
-  Status s;
 
   std::shared_ptr<BlobStorage> blob_storage;
   // Skip CFs that have been dropped.
@@ -191,8 +190,8 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
     }
   }
 
+  Status s;
   // TODO(@DorianZheng) Make sure enough room for GC
-
   if (UNLIKELY(!blob_gc)) {
     RecordTick(statistics(stats_.get()), TITAN_GC_NO_NEED, 1);
     // Nothing to do
@@ -226,16 +225,16 @@ Status TitanDBImpl::BackgroundGC(LogBuffer* log_buffer,
       // then put this cf to GC queue for next GC
       AddToGCQueue(blob_gc->column_family_handle()->GetID());
     }
-  }
 
-  if (s.ok()) {
-    RecordTick(statistics(stats_.get()), TITAN_GC_SUCCESS, 1);
-    // Done
-  } else {
-    SetBGError(s);
-    RecordTick(statistics(stats_.get()), TITAN_GC_FAILURE, 1);
-    TITAN_LOG_WARN(db_options_.info_log, "Titan GC error: %s",
-                   s.ToString().c_str());
+    if (s.ok()) {
+      RecordTick(statistics(stats_.get()), TITAN_GC_SUCCESS, 1);
+      // Done
+    } else {
+      SetBGError(s);
+      RecordTick(statistics(stats_.get()), TITAN_GC_FAILURE, 1);
+      TITAN_LOG_WARN(db_options_.info_log, "Titan GC error: %s",
+                     s.ToString().c_str());
+    }
   }
 
   TEST_SYNC_POINT("TitanDBImpl::BackgroundGC:Finish");

src/table_builder.cc

@@ -97,6 +97,8 @@ void TitanTableBuilder::Add(const Slice& key, const Slice& value) {
       // If not ok, write original blob index as compaction output without
       // doing level merge.
       if (get_status.ok()) {
+        gc_num_keys_relocated_++;
+        gc_bytes_relocated_ += record.value.size();
         AddBlob(ikey, record.value);
         if (ok()) return;
       } else {
@@ -241,6 +243,16 @@ void TitanTableBuilder::FinishBlobFile() {
           blob_builder_->GetSmallestKey(), blob_builder_->GetLargestKey());
       file->FileStateTransit(BlobFileMeta::FileEvent::kFlushOrCompactionOutput);
       finished_blobs_.push_back({file, std::move(blob_handle_)});
+      // level merge is performed
+      if (gc_num_keys_relocated_ != 0) {
+        RecordTick(statistics(stats_), TITAN_GC_NUM_NEW_FILES, 1);
+        RecordTick(statistics(stats_), TITAN_GC_NUM_KEYS_RELOCATED,
+                   gc_num_keys_relocated_);
+        RecordTick(statistics(stats_), TITAN_GC_BYTES_RELOCATED,
+                   gc_bytes_relocated_);
+        gc_num_keys_relocated_ = 0;
+        gc_bytes_relocated_ = 0;
+      }
       blob_builder_.reset();
     } else {
       TITAN_LOG_WARN(

src/table_builder.h

@@ -109,6 +109,8 @@ class TitanTableBuilder : public TableBuilder {
   uint64_t bytes_written_ = 0;
   uint64_t io_bytes_read_ = 0;
   uint64_t io_bytes_written_ = 0;
+  uint64_t gc_num_keys_relocated_ = 0;
+  uint64_t gc_bytes_relocated_ = 0;
   uint64_t error_read_cnt_ = 0;
 };
 

src/titan_stats.cc

@@ -165,10 +165,9 @@ uint64_t TitanInternalStats::HandleNumBlobFilesAtLevel(Slice arg) const {
 void TitanInternalStats::DumpAndResetInternalOpStats(LogBuffer* log_buffer) {
   constexpr double GB = 1.0 * 1024 * 1024 * 1024;
   constexpr double SECOND = 1.0 * 1000000;
-  LogToBuffer(
-      log_buffer,
-      "OP           COUNT READ(GB)  WRITE(GB) IO_READ(GB) IO_WRITE(GB) "
-      " FILE_IN FILE_OUT GC_SAMPLE(MICROS) GC_READ(MICROS) GC_UPDATE(MICROS)");
+  LogToBuffer(log_buffer,
+              "OP           COUNT READ(GB)  WRITE(GB) IO_READ(GB) IO_WRITE(GB) "
+              " FILE_IN FILE_OUT GC_READ(MICROS) GC_UPDATE(MICROS)");
   LogToBuffer(log_buffer,
               "----------------------------------------------------------------"
               "-----------------");
@@ -195,9 +194,6 @@ void TitanInternalStats::DumpAndResetInternalOpStats(LogBuffer* log_buffer) {
                          InternalOpStatsType::INPUT_FILE_NUM),
         GetAndResetStats(&internal_op_stats_[op],
                          InternalOpStatsType::OUTPUT_FILE_NUM),
-        GetAndResetStats(&internal_op_stats_[op],
-                         InternalOpStatsType::GC_SAMPLING_MICROS) /
-            SECOND,
         GetAndResetStats(&internal_op_stats_[op],
                          InternalOpStatsType::GC_READ_LSM_MICROS) /
             SECOND,

src/titan_stats.h

@@ -29,7 +29,6 @@ enum class InternalOpStatsType : int {
   IO_BYTES_WRITTEN,
   INPUT_FILE_NUM,
   OUTPUT_FILE_NUM,
-  GC_SAMPLING_MICROS,
   GC_READ_LSM_MICROS,
   // Update lsm and write callback
   GC_UPDATE_LSM_MICROS,