Cleaning up

Signed-off-by: v01dstar <[email protected]>

src/blob_file_builder.cc

@@ -165,8 +165,7 @@ void BlobFileBuilder::WriteEncoderData(BlobHandle* handle) {
   handle->offset = file_->GetFileSize();
   handle->size = encoder_.GetEncodedSize();
   if (block_size_ > 0) {
-    live_data_size_ +=
-        (handle->size + block_size_ - 1) / block_size_ * block_size_;
+    live_data_size_ += Roundup(handle->size, block_size_);
   } else {
     live_data_size_ += handle->size;
   }

src/blob_file_iterator.h

@@ -35,8 +35,6 @@ class BlobFileIterator {
   Slice value() const;
   Status status() const { return status_; }
   uint64_t header_size() const { return header_size_; }
-  // Returns the size of the "footer", this includes the meta blocks.
-  uint64_t footer_size() const { return file_size_ - end_of_blob_record_; }
 
   void IterateForPrev(uint64_t);
 

src/blob_format.cc

@@ -142,7 +142,6 @@ void BlobFileMeta::EncodeTo(std::string* dst) const {
   PutVarint64(dst, block_size_);
   PutLengthPrefixedSlice(dst, smallest_key_);
   PutLengthPrefixedSlice(dst, largest_key_);
-  PutVarint64(dst, effective_file_size_);
 }
 
 Status BlobFileMeta::DecodeFromV1(Slice* src) {
@@ -190,12 +189,6 @@ Status BlobFileMeta::DecodeFrom(Slice* src) {
   } else {
     return Status::Corruption("BlobLargestKey decode failed");
   }
-  uint64_t effective_file_size;
-  if (!GetVarint64(src, &effective_file_size)) {
-    return Status::Corruption(
-        "BlobFileMeta hole_punchable_size_ decode failed");
-  }
-  effective_file_size_ = effective_file_size;
   return Status::OK();
 }
 

src/blob_gc_picker.cc

@@ -38,7 +38,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickRegularBlobGC(
   uint64_t batch_size = 0;
   uint64_t estimate_output_size = 0;
   bool stop_picking = false;
-  bool maybe_continue_next_time = false;
+  bool need_trigger_next = false;
   uint64_t next_gc_size = 0;
   bool in_fallback = cf_options_.blob_run_mode == TitanBlobRunMode::kFallback;
 
@@ -71,14 +71,14 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickRegularBlobGC(
       estimate_output_size += blob_file->live_data_size();
       if (batch_size >= cf_options_.max_gc_batch_size ||
           estimate_output_size >= cf_options_.blob_file_target_size) {
-        // Stop pick file for this gc, but still check file for whether need
-        // trigger gc after this
+        // Stop picking file for this gc, but still check file for whether
+        // another round of gc is needed.
         stop_picking = true;
       }
     } else {
       next_gc_size += blob_file->file_size();
       if (next_gc_size > cf_options_.min_gc_batch_size || in_fallback) {
-        maybe_continue_next_time = true;
+        need_trigger_next = true;
         RecordTick(statistics(stats_), TITAN_GC_REMAIN, 1);
         TITAN_LOG_INFO(db_options_.info_log,
                        "remain more than %" PRIu64
@@ -112,7 +112,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickRegularBlobGC(
   }
 
   return std::unique_ptr<BlobGC>(new BlobGC(
-      std::move(blob_files), std::move(cf_options_), maybe_continue_next_time));
+      std::move(blob_files), std::move(cf_options_), need_trigger_next));
 }
 
 std::unique_ptr<BlobGC> BasicBlobGCPicker::PickPunchHoleGC(
@@ -123,7 +123,7 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickPunchHoleGC(
   uint64_t batch_size = 0;
   uint64_t estimate_output_size = 0;
   bool stop_picking = false;
-  bool maybe_continue_next_time = false;
+  bool need_trigger_next = false;
   uint64_t next_gc_size = 0;
 
   for (auto& gc_score : blob_storage->punch_hole_score()) {
@@ -146,13 +146,13 @@ std::unique_ptr<BlobGC> BasicBlobGCPicker::PickPunchHoleGC(
       }
     } else {
       // TODO: add a batch threshold for punch hole gc.
-      maybe_continue_next_time = true;
+      need_trigger_next = true;
       break;
     }
   }
   if (blob_files.empty()) return nullptr;
   return std::unique_ptr<BlobGC>(new BlobGC(
-      std::move(blob_files), std::move(cf_options_), maybe_continue_next_time,
+      std::move(blob_files), std::move(cf_options_), need_trigger_next,
       /*punch_hole_gc=*/true));
 }
 

src/punch_hole_gc_job.cc

@@ -1,6 +1,7 @@
 #include "punch_hole_gc_job.h"
 
 #include <fcntl.h>
+#include <sys/stat.h>
 #include <unistd.h>
 
 #include "db/db_impl/db_impl.h"
@@ -44,14 +45,14 @@ Status PunchHoleGCJob::HolePunchBlobFiles() {
 Status PunchHoleGCJob::HolePunchSingleBlobFile(
     std::shared_ptr<BlobFileMeta> file) {
   Status s;
-  auto fd = open(BlobFileName(db_options_.dirname, file->file_number()).c_str(),
-                 O_WRONLY);
   std::unique_ptr<RandomAccessFileReader> file_reader;
   s = NewBlobFileReader(file->file_number(), 0, db_options_, env_options_, env_,
                         &file_reader);
   if (!s.ok()) {
     return s;
   }
+  auto fd = open(BlobFileName(db_options_.dirname, file->file_number()).c_str(),
+                 O_WRONLY);
   uint64_t effective_file_size = 0;
   uint64_t aligned_data_size = 0;
   std::unique_ptr<BlobFileIterator> iter(
@@ -74,8 +75,7 @@ Status PunchHoleGCJob::HolePunchSingleBlobFile(
       return s;
     }
 
-    aligned_data_size = (blob_index.blob_handle.size + block_size - 1) /
-                        block_size * block_size;
+    aligned_data_size = Roundup(blob_index.blob_handle.size, block_size);
 
     if (!discardable) {
       effective_file_size += aligned_data_size;
@@ -97,6 +97,12 @@ Status PunchHoleGCJob::HolePunchSingleBlobFile(
   if (!iter->status().ok()) {
     return iter->status();
   }
+  struct stat st;
+  if (fstat(fd, &st) != 0) {
+    // Do nothing, so far, this is only for stats.
+  }
+  close(fd);
+  disk_usage_map_[file->file_number()] = st.st_blocks * 512;
   effective_file_size_map_[file->file_number()] = effective_file_size;
   return Status::OK();
 }
@@ -147,6 +153,7 @@ void PunchHoleGCJob::UpdateBlobFilesMeta() {
       continue;
     }
     file->set_effective_file_size(it->second);
+    file->set_disk_uage(disk_usage_map_[file->file_number()]);
   }
 }
 

src/punch_hole_gc_job.h

@@ -49,6 +49,7 @@ class PunchHoleGCJob {
   std::atomic_bool* shuting_down_{nullptr};
 
   std::unordered_map<uint64_t, uint64_t> effective_file_size_map_;
+  std::unordered_map<uint64_t, int64_t> disk_usage_map_;
 
   // TODO: Add more stats
 

src/titan_stats.h

@@ -68,9 +68,6 @@ class TitanInternalStats {
     NUM_DISCARDABLE_RATIO_LE80,
     NUM_DISCARDABLE_RATIO_LE100,
 
-    PENDING_PUNCH_HOLE_SIZE,
-    EFFECTIVE_BLOB_FILE_SIZE,
-
     INTERNAL_STATS_ENUM_MAX,
   };