make use of batch inserter in test case

Author: tjhu

format.sh

@@ -6,6 +6,6 @@ for folder in examples unittests include/input_reader src/input_reader; do
 done 
 
 # Format files
-for file in include/tests/HashjoinTest.hpp src/tests/hashjoin_test.cpp; do
+for file in include/tests/HashjoinTest.hpp src/tests/hashjoin_test.cpp include/hashtables/batch_inserter.hpp; do
   clang-format -style=file -i $file
 done 

include/hashtables/batch_inserter.hpp

@@ -2,16 +2,16 @@
 #define HASHTABLES_BATCH_INSERTER_HPP
 
 #include "base_kht.hpp"
+#include "constants.hpp"
 #include "types.hpp"
 
 namespace kmercounter {
-template <size_t N>
+template <size_t N = HT_TESTS_BATCH_LENGTH>
 class HTBatchInserter {
  public:
-  HTBatchInserter(BaseHashtable* ht) : ht_(ht), buffer_(), buffer_size_(0) {}
-  ~HTBatchInserter() {
-    flush();
-  }
+  HTBatchInserter() : HTBatchInserter(nullptr) {}
+  HTBatchInserter(BaseHashTable* ht) : ht_(ht), buffer_(), buffer_size_(0) {}
+  ~HTBatchInserter() { flush(); }
 
   // Insert one kv pair.
   void insert(const uint64_t key, const uint64_t value) {
@@ -22,34 +22,43 @@ class HTBatchInserter {
 
     // Flush if `buffer_` is full.
     if (buffer_size_ >= N) {
-      flush_();
+      flush_buffer_();
     }
   }
 
-  // Flush if there's anything in the buffer.
+  // Flush everything to the hashtable and flush the hashtable insert queue.
   void flush() {
     if (buffer_size_ > 0) {
-      flush_();
+      flush_buffer_();
     }
+    flush_ht_();
   }
 
+  // Returns the number of elements flushed.
+  size_t num_flushed() { return num_flushed_; }
+
  private:
-  // Flush without checking `buffer_size_`.
-  void flush_() {
-    InsertFindArguments kp = std::make_pair(buffer_size_, buffer_);
-    ht->insert_batch(kp);
+  // Flush the insertion buffer without checking `buffer_size_`.
+  void flush_buffer_() {
+    ht_->insert_batch(InsertFindArguments(buffer_, buffer_size_));
+    num_flushed_ += buffer_size_;
     buffer_size_ = 0;
   }
 
+  // Issue a flush to the hashtable.
+  void flush_ht_() { ht_->flush_insert_queue(); }
+
   // Target hashtable.
   BaseHashTable* ht_;
   // Buffer to hold the arguments for batch insertion.
   __attribute__((aligned(64))) InsertFindArgument buffer_[N];
   // Current size of the buffer.
   size_t buffer_size_;
+  // Total number of elements flushed.
+  size_t num_flushed_;
 
   // Sanity checks
   static_assert(N > 0);
 };
-} // namespace kmercounter
+}  // namespace kmercounter
 #endif  // HASHTABLES_BATCH_INSERTER_HPP

include/types.hpp

@@ -205,6 +205,7 @@ struct InsertFindArgument {
   uint64_t value;
   /// A user-provided value for the user to keep track of this operation.
   /// This is returned as `FindResult::id`.
+  /// In aggregation mode, this is the "key". Don't ask why.
   uint64_t id;
   /// The id of the partition that will be handling this operation.
   /// Might not be used depends on the configuration/kind of operation. 
@@ -221,6 +222,7 @@ struct FindResult {
   /// This matches the `InsertFindArgument::id`.
   uint64_t id;
   /// The value of the key of the find operation.
+  /// This is the number of occurrences in aggregation mode. 
   uint64_t value;
 
   constexpr FindResult() = default;

unittests/hashmap_test.cpp

@@ -11,6 +11,7 @@
 #include <unordered_map>
 
 #include "hashtable.h"
+#include "hashtables/batch_inserter.hpp"
 #include "hashtables/cas_kht.hpp"
 #include "hashtables/simple_kht.hpp"
 #include "test_lib.hpp"
@@ -47,9 +48,11 @@ class HashtableTest : public ::testing::TestWithParam<const char*> {
             return nullptr;
         }());
     ASSERT_NE(ht_, nullptr) << "Invalid hashtable type: " << ht_name;
+    inserter_ = HTBatchInserter<>(ht_.get());
   }
 
   std::unique_ptr<kmercounter::BaseHashTable> ht_;
+  HTBatchInserter<> inserter_;
 };
 
 /// Correctness test for insertion and lookup without prefetch.
@@ -76,19 +79,17 @@ TEST_P(HashtableTest, NO_PREFETCH_TEST) {
 
 TEST_P(HashtableTest, SIMPLE_BATCH_INSERT_TEST) {
   // Insertion.
-  std::array<InsertFindArgument, 2> arguments{
-      InsertFindArgument{key : 12, value : 128},
-      InsertFindArgument{key : 23, value : 256}};
-  InsertFindArguments keypairs(arguments);
-  ht_->insert_batch(keypairs);
-  ht_->flush_insert_queue();
+  inserter_.insert(12, 128);
+  inserter_.insert(23, 256);
+  inserter_.flush();
 
   // Look up.
-  arguments[0] = InsertFindArgument{key : 12, id : 123};
-  arguments[1] = InsertFindArgument{key : 23, id : 321};
+  std::array<InsertFindArgument, 2> arguments{
+      InsertFindArgument{key : 12, id : 123},
+      InsertFindArgument{key : 23, id : 321}};
   std::array<FindResult, HT_TESTS_BATCH_LENGTH> results{};
   ValuePairs valuepairs{0, results.data()};
-  ht_->find_batch(keypairs, valuepairs);
+  ht_->find_batch(arguments, valuepairs);
   ht_->flush_find_queue(valuepairs);
 
   // Check for correctness.
@@ -101,24 +102,22 @@ TEST_P(HashtableTest, SIMPLE_BATCH_INSERT_TEST) {
 
 TEST_P(HashtableTest, SIMPLE_BATCH_UPDATE_TEST) {
   // Insertion.
-  std::array<InsertFindArgument, 2> arguments{
-      InsertFindArgument{key : 12, value : 128, part_id : 0},
-      InsertFindArgument{key : 23, value : 256, part_id : 0}};
-  InsertFindArguments keypairs(arguments);
-  ht_->insert_batch(InsertFindArguments(arguments));
+  inserter_.insert(12, 128);
+  inserter_.insert(23, 256);
+  inserter_.flush();
 
   // Update.
-  arguments[0].value = 1025;
-  arguments[1].value = 4097;
-  ht_->insert_batch(keypairs);
-  ht_->flush_insert_queue();
+  inserter_.insert(12, 1025);
+  inserter_.insert(23, 4097);
+  inserter_.flush();
 
   // Look up.
-  arguments[0].id = 123;
-  arguments[1].id = 321;
+  std::array<InsertFindArgument, 2> arguments{
+      InsertFindArgument{key : 12, id : 123},
+      InsertFindArgument{key : 23, id : 321}};
   std::array<FindResult, HT_TESTS_BATCH_LENGTH> values{};
   ValuePairs valuepairs{0, values.data()};
-  ht_->find_batch(keypairs, valuepairs);
+  ht_->find_batch(arguments, valuepairs);
   ht_->flush_find_queue(valuepairs);
 
   // Check for correctness.
@@ -141,20 +140,10 @@ TEST_P(HashtableTest, BATCH_QUERY_TEST) {
     const uint64_t key = i;
     const uint64_t value = i * i;
     const uint64_t id = 2 * i;
-    arguments[k].key = key;
-    arguments[k].value = value;
-    arguments[k].part_id = 0;
+    inserter_.insert(key, value);
     reference_map[value] = id;
-    if (++k == HT_TESTS_BATCH_LENGTH) {
-      ht_->insert_batch(InsertFindArguments(arguments));
-      k = 0;
-    }
-  }
-  if (k != 0) {
-    ht_->insert_batch(InsertFindArguments(arguments, k));
-    k = 0;
   }
-  ht_->flush_insert_queue();
+  inserter_.flush();
 
   // Helper function for checking the result of the batch finds.
   auto check_valuepairs = [&reference_map](const ValuePairs& vp) {
@@ -175,7 +164,6 @@ TEST_P(HashtableTest, BATCH_QUERY_TEST) {
   for (uint64_t i = 0; i < test_size; i++) {
     arguments[k].key = i;
     arguments[k].id = 2 * i;
-    arguments[k].part_id = 0;
     if (++k == HT_TESTS_BATCH_LENGTH) {
       ValuePairs valuepairs{0, values};
       ht_->find_batch(InsertFindArguments(arguments), valuepairs);