Merge remote-tracking branch 'origin/jian.z/fix-test'

Author: zz-jason

CMakeLists.txt

@@ -1,9 +1,10 @@
+cmake_minimum_required(VERSION 3.7)
+
 # ------------------------------------------------------------------------------
 # Project settings
 # ------------------------------------------------------------------------------
 
 project(leanstoredb)
-cmake_minimum_required(VERSION 3.7)
 if (NOT UNIX)
     message(SEND_ERROR "unsupported platform")
 endif ()
@@ -12,20 +13,21 @@ enable_language(ASM)
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -rdynamic -fno-omit-frame-pointer -g3 -static-libstdc++")
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wunused -Wunreachable-code")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wunused -Wunreachable-code")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
-  ADD_COMPILE_OPTIONS(-Wall -Wextra -mavx2 -mcx16 -m64) # -fno-elide-constructors no need for now
+  add_compile_options(-Wall -Wextra -mavx2 -mcx16 -m64) # -fno-elide-constructors no need for now
 else()
-  ADD_COMPILE_OPTIONS(-Wall -Wextra -march=native)
+  add_compile_options(-Wall -Wextra -march=native)
 endif()
 
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3") # because we had weird behavior on aws ARM
 
 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND CMAKE_BUILD_TYPE MATCHES Debug)
-    ADD_COMPILE_OPTIONS(-fstandalone-debug)
+    add_compile_options(-fstandalone-debug)
 endif ()
 
 if (APPLE)
@@ -46,6 +48,10 @@ include(${CMAKE_SOURCE_DIR}/third-party/tabluate.cmake)
 include(${CMAKE_SOURCE_DIR}/third-party/rapidjson.cmake)
 include(${CMAKE_SOURCE_DIR}/third-party/gflags.cmake)
 include(${CMAKE_SOURCE_DIR}/third-party/glog.cmake)
+include(${CMAKE_SOURCE_DIR}/third-party/gtest.cmake)
+include(${CMAKE_SOURCE_DIR}/third-party/gbench.cmake)
+include(${CMAKE_SOURCE_DIR}/third-party/prometheus-cpp.cmake)
+include(${CMAKE_SOURCE_DIR}/third-party/onetbb.cmake)
 
 
 # ------------------------------------------------------------------------------

source/CMakeLists.txt

@@ -11,7 +11,7 @@ file(GLOB_RECURSE LEANSTORE_SRC **.cpp **/**.cpp **.cc **/**.cc **.hpp **/**.hpp
 add_library(leanstore STATIC ${LEANSTORE_SRC})
 
 # dependencies
-target_link_libraries(leanstore aio tbb tabulate rapidjson gflags glog)
+target_link_libraries(leanstore aio onetbb tabulate rapidjson gflags glog)
 
 # include dirs
 target_include_directories(leanstore PUBLIC ${CMAKE_SOURCE_DIR}/include/shared-headers)

source/LeanStore.hpp

@@ -77,6 +77,9 @@ class LeanStore {
   /**
    * @brief Unregister a BTreeLL
    *
+   * TODO(jian.z): reclaim all the buffer frames in memory and pages in disk
+   * occupied by the tree.
+   *
    * @param name The unique name of the btree
    */
   void UnRegisterBTreeLL(const std::string& name) {
@@ -136,10 +139,22 @@ class LeanStore {
   /**
    * @brief Unregister a BTreeVI
    *
+   * TODO(jian.z): reclaim all the buffer frames in memory and pages in disk
+   * occupied by the tree.
+   *
    * @param name The unique name of the btree
    */
   void UnRegisterBTreeVI(const std::string& name) {
+    auto btree = dynamic_cast<storage::btree::BTreeGeneric*>(
+        storage::TreeRegistry::sInstance->GetTree(name));
+    leanstore::storage::btree::BTreeGeneric::FreeAndReclaim(*btree);
     storage::TreeRegistry::sInstance->UnregisterTree(name);
+
+    auto graveyardName = "_" + name + "_graveyard";
+    btree = dynamic_cast<storage::btree::BTreeGeneric*>(
+        storage::TreeRegistry::sInstance->GetTree(graveyardName));
+    leanstore::storage::btree::BTreeGeneric::FreeAndReclaim(*btree);
+    storage::TreeRegistry::sInstance->UnregisterTree(graveyardName);
   }
 
 private:

source/concurrency-recovery/ConcurrencyControl.cpp

@@ -1,3 +1,4 @@
+#include "CRMG.hpp"
 #include "Worker.hpp"
 #include "storage/buffer-manager/TreeRegistry.hpp"
 
@@ -145,7 +146,7 @@ synclwm : {
   if (local_all_lwm > cleaned_untill_oltp_lwm) {
     utils::Timer timer(CRCounters::myCounters().cc_ms_gc_history_tree);
     // PURGE!
-    mHistoryTree.purgeVersions(
+    CRManager::sInstance->mHistoryTreePtr->purgeVersions(
         Worker::my().mWorkerId, 0, local_all_lwm - 1,
         [&](const TXID tx_id, const TREEID treeId, const u8* version_payload,
             [[maybe_unused]] u64 version_payload_length,
@@ -167,7 +168,7 @@ synclwm : {
       // MOVE deletes to the graveyard
       const u64 fromTxId =
           cleaned_untill_oltp_lwm > 0 ? cleaned_untill_oltp_lwm : 0;
-      mHistoryTree.visitRemoveVersions(
+      CRManager::sInstance->mHistoryTreePtr->visitRemoveVersions(
           Worker::my().mWorkerId, fromTxId, local_oltp_lwm - 1,
           [&](const TXID tx_id, const TREEID treeId, const u8* version_payload,
               [[maybe_unused]] u64 version_payload_length,

source/storage/btree/core/BTreeGeneric.hpp

@@ -244,6 +244,21 @@ class BTreeGeneric : public leanstore::storage::BufferManagedTree {
     return findParent<false>(btree, bfToFind);
   }
 
+  /// Removes a btree from disk, reclaim all the buffer frames in memory and
+  /// pages in disk used by it.
+  ///
+  /// @param btree The tree to free.
+  static void FreeAndReclaim(BTreeGeneric& btree) {
+    HybridPageGuard<BTreeNode> guardedMetaNode(btree.mMetaNodeSwip);
+    HybridPageGuard<BTreeNode> guardedRootNode(
+        guardedMetaNode, guardedMetaNode->mRightMostChildSwip);
+    BTreeGeneric::freeBTreeNodesRecursive(guardedRootNode);
+
+    auto exclusiveGuardedMetaNode =
+        ExclusivePageGuard(std::move(guardedMetaNode));
+    exclusiveGuardedMetaNode.reclaim();
+  }
+
   static rapidjson::Document ToJSON(BTreeGeneric& btree) {
     HybridPageGuard<BTreeNode> guardedMetaNode(btree.mMetaNodeSwip);
     HybridPageGuard<BTreeNode> guardedRootNode(
@@ -252,6 +267,8 @@ class BTreeGeneric : public leanstore::storage::BufferManagedTree {
   }
 
 private:
+  static void freeBTreeNodesRecursive(HybridPageGuard<BTreeNode>& guardedNode);
+
   static rapidjson::Document ToJSONRecursive(
       HybridPageGuard<BTreeNode>& guardedNode) {
     auto bfDoc = guardedNode.mBf->ToJSON();
@@ -282,6 +299,20 @@ class BTreeGeneric : public leanstore::storage::BufferManagedTree {
   static constexpr std::string META_PAGE_ID = "metaPageId";
 };
 
+inline void BTreeGeneric::freeBTreeNodesRecursive(
+    HybridPageGuard<BTreeNode>& guardedNode) {
+  if (!guardedNode->mIsLeaf) {
+    for (auto i = 0u; i <= guardedNode->mNumSeps; ++i) {
+      auto childSwip = guardedNode->GetChildIncludingRightMost(i);
+      HybridPageGuard<BTreeNode> guardedChild(guardedNode, childSwip);
+      freeBTreeNodesRecursive(guardedChild);
+    }
+  }
+
+  auto exclusiveGuardedNode = ExclusivePageGuard(std::move(guardedNode));
+  exclusiveGuardedNode.reclaim();
+}
+
 inline void BTreeGeneric::IterateChildSwips(
     BufferFrame& bf, std::function<bool(Swip<BufferFrame>&)> callback) {
   // Pre: bf is read locked

source/storage/btree/core/BTreePessimisticIterator.hpp

@@ -63,41 +63,41 @@ class BTreePessimisticIterator : public BTreePessimisticIteratorInterface {
 protected:
   // We need a custom findLeafAndLatch to track the position in parent node
   template <LATCH_FALLBACK_MODE mode = LATCH_FALLBACK_MODE::SHARED>
-  void findLeafAndLatch(HybridPageGuard<BTreeNode>& childGuard, Slice key) {
+  void findLeafAndLatch(HybridPageGuard<BTreeNode>& guardedChild, Slice key) {
     while (true) {
       mLeafPosInParent = -1;
       JUMPMU_TRY() {
         mGuardedParent = HybridPageGuard<BTreeNode>(mBTree.mMetaNodeSwip);
-        childGuard.unlock();
+        guardedChild.unlock();
 
         // it's the root node right now.
-        childGuard = HybridPageGuard<BTreeNode>(
+        guardedChild = HybridPageGuard<BTreeNode>(
             mGuardedParent, mGuardedParent->mRightMostChildSwip);
 
-        for (u16 level = 0; !childGuard->mIsLeaf; level++) {
+        for (u16 level = 0; !guardedChild->mIsLeaf; level++) {
           COUNTERS_BLOCK() {
             WorkerCounters::myCounters().dt_inner_page[mBTree.mTreeId]++;
           }
-          mLeafPosInParent = mGuardedLeaf->lowerBound<false>(key);
+          mLeafPosInParent = guardedChild->lowerBound<false>(key);
           auto childSwip =
-              &childGuard->GetChildIncludingRightMost(mLeafPosInParent);
-          mGuardedParent = std::move(childGuard);
+              &guardedChild->GetChildIncludingRightMost(mLeafPosInParent);
+          mGuardedParent = std::move(guardedChild);
           if (level == mBTree.mHeight - 1) {
-            childGuard = HybridPageGuard(mGuardedParent, *childSwip, mode);
+            guardedChild = HybridPageGuard(mGuardedParent, *childSwip, mode);
           } else {
-            childGuard = HybridPageGuard(mGuardedParent, *childSwip);
+            guardedChild = HybridPageGuard(mGuardedParent, *childSwip);
           }
         }
 
         mGuardedParent.unlock();
         if (mode == LATCH_FALLBACK_MODE::EXCLUSIVE) {
-          childGuard.toExclusive();
+          guardedChild.toExclusive();
         } else {
-          childGuard.toShared();
+          guardedChild.toShared();
         }
         mIsPrefixCopied = false;
         if (mFuncEnterLeaf != nullptr) {
-          mFuncEnterLeaf(childGuard);
+          mFuncEnterLeaf(guardedChild);
         }
         JUMPMU_RETURN;
       }

source/sync-primitives/PageGuard.hpp

@@ -270,7 +270,6 @@ template <typename T> class ExclusivePageGuard {
   }
 
   template <typename WT, typename... Args>
-  // template <> void init() {
   cr::WALPayloadHandler<WT> ReserveWALPayload(u64 payloadSize, Args&&... args) {
     return mRefGuard.template ReserveWALPayload<WT>(
         payloadSize, std::forward<Args>(args)...);