chore: use PerfCounters to replace counters & metrics (#129)

Author: zz-jason

CMakeLists.txt

@@ -76,7 +76,6 @@ find_package(spdlog CONFIG REQUIRED)
 find_package(GTest CONFIG REQUIRED)
 find_package(benchmark CONFIG REQUIRED)
 find_package(Crc32c CONFIG REQUIRED)
-find_package(prometheus-cpp CONFIG REQUIRED)
 find_package(httplib CONFIG REQUIRED)
 find_package(PkgConfig)
 pkg_check_modules(libunwind REQUIRED IMPORTED_TARGET GLOBAL libunwind)

CMakePresets.json

@@ -10,7 +10,8 @@
                 "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
                 "CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
                 "BUILD_SHARED_LIBS": "ON",
-                "ENABLE_PROFILING": "ON"
+                "ENABLE_PROFILING": "ON",
+                "ENABLE_PERF_COUNTERS": "ON"
             }
         },
         {
@@ -26,8 +27,7 @@
             "inherits": "base",
             "hidden": true,
             "cacheVariables": {
-                "CMAKE_BUILD_TYPE": "Release",
-                "COUNTERS_LEVEL": "none"
+                "CMAKE_BUILD_TYPE": "Release"
             }
         },
         {
@@ -68,4 +68,4 @@
             }
         }
     ]
-}
+}

benchmarks/ycsb/YcsbLeanStore.hpp

@@ -1,5 +1,7 @@
 #include "Ycsb.hpp"
+#include "leanstore-c/PerfCounters.h"
 #include "leanstore-c/StoreOption.h"
+#include "leanstore-c/leanstore-c.h"
 #include "leanstore/KVInterface.hpp"
 #include "leanstore/LeanStore.hpp"
 #include "leanstore/btree/BasicKV.hpp"
@@ -17,14 +19,20 @@
 
 #include <atomic>
 #include <chrono>
+#include <cstdint>
 #include <cstdlib>
 #include <cstring>
 #include <format>
+#include <fstream>
 #include <iostream>
 #include <memory>
 #include <string>
+#include <thread>
 #include <vector>
 
+#include <sys/types.h>
+#include <unistd.h>
+
 namespace leanstore::ycsb {
 
 constexpr std::string kTableName = "ycsb_leanstore";
@@ -44,8 +52,6 @@ class YcsbLeanStore : public YcsbExecutor {
     option->mEnableEagerGc = true;
     option->mWorkerThreads = FLAGS_ycsb_threads;
     option->mBufferPoolSize = FLAGS_ycsb_mem_kb * 1024;
-    option->mEnableMetrics = true;
-    option->mMetricsPort = 8080;
 
     auto res = LeanStore::Open(option);
     if (!res) {
@@ -55,6 +61,9 @@ class YcsbLeanStore : public YcsbExecutor {
     }
 
     mStore = std::move(res.value());
+
+    // start metrics http exposer for cpu/mem profiling
+    StartMetricsHttpExposer(8080);
   }
 
   ~YcsbLeanStore() override {
@@ -164,6 +173,12 @@ class YcsbLeanStore : public YcsbExecutor {
       a = 0;
     }
 
+    std::vector<PerfCounters*> workerPerfCounters;
+    for (auto i = 0u; i < mStore->mStoreOption->mWorkerThreads; i++) {
+      mStore->ExecSync(
+          i, [&]() { workerPerfCounters.push_back(cr::WorkerContext::My().GetPerfCounters()); });
+    }
+
     for (uint64_t workerId = 0; workerId < mStore->mStoreOption->mWorkerThreads; workerId++) {
       mStore->ExecAsync(workerId, [&]() {
         uint8_t key[FLAGS_ycsb_key_size];
@@ -239,11 +254,45 @@ class YcsbLeanStore : public YcsbExecutor {
       a = 0;
     }
 
+    std::thread perfContextReporter([&]() {
+      auto reportPeriod = 1;
+      const char* counterFilePath = "/tmp/leanstore/worker-counters.txt";
+      std::ofstream ost;
+
+      while (keepRunning) {
+        sleep(reportPeriod);
+        uint64_t txWithRemoteDependencies = 0;
+        uint64_t lcbExecuted = 0;
+        uint64_t lcbTotalLatNs [[maybe_unused]] = 0;
+        uint64_t gcExecuted = 0;
+        uint64_t gcTotalLatNs [[maybe_unused]] = 0;
+        uint64_t txCommitWait = 0;
+
+        // collect counters
+        for (auto* perfCounters : workerPerfCounters) {
+          txWithRemoteDependencies += atomic_exchange(&perfCounters->mTxWithRemoteDependencies, 0);
+          txCommitWait += atomic_exchange(&perfCounters->mTxCommitWait, 0);
+
+          lcbExecuted += atomic_exchange(&perfCounters->mLcbExecuted, 0);
+          lcbTotalLatNs += atomic_exchange(&perfCounters->mLcbTotalLatNs, 0);
+
+          gcExecuted += atomic_exchange(&perfCounters->mGcExecuted, 0);
+          gcTotalLatNs += atomic_exchange(&perfCounters->mGcTotalLatNs, 0);
+        }
+        ost.open(counterFilePath, std::ios_base::app);
+        ost << std::format("TxWithDep: {}, txCommitWait: {}, LcbExec: {}, GcExec: {}",
+                           txWithRemoteDependencies, txCommitWait, lcbExecuted, gcExecuted)
+            << std::endl;
+        ost.close();
+      }
+    });
+
     printTpsSummary(1, FLAGS_ycsb_run_for_seconds, mStore->mStoreOption->mWorkerThreads,
                     threadCommitted, threadAborted);
 
     // Shutdown threads
     keepRunning = false;
+    perfContextReporter.join();
     mStore->WaitAll();
   }
 };

benchmarks/ycsb/YcsbWiredTiger.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "Ycsb.hpp"
+#include "leanstore-c/leanstore-c.h"
 #include "leanstore/utils/Defer.hpp"
 #include "leanstore/utils/Log.hpp"
 #include "leanstore/utils/Parallelize.hpp"
@@ -26,6 +27,7 @@ class YcsbWiredTiger : public YcsbExecutor {
 
 public:
   YcsbWiredTiger() : mConn(nullptr) {
+    StartMetricsHttpExposer(8080);
   }
 
   ~YcsbWiredTiger() override {
@@ -195,7 +197,8 @@ class YcsbWiredTiger : public YcsbExecutor {
     }
 
     std::string configString(
-        "create, direct_io=[data, log, checkpoint], log=(enabled=false), statistics_log=(wait=1), "
+        "create, direct_io=[data, log, checkpoint], "
+        "log=(enabled=true,archive=true), statistics_log=(wait=1), "
         "statistics=(all, clear), session_max=2000, eviction=(threads_max=4), cache_size=" +
         std::to_string(FLAGS_ycsb_mem_kb / 1024) + "M");
     int ret = wiredtiger_open(dataDir.c_str(), nullptr, configString.c_str(), &mConn);

docker/monitor-compose/docker-compose.yml

@@ -0,0 +1,83 @@
+#ifndef LEANSTORE_PERF_COUNTERS_H
+#define LEANSTORE_PERF_COUNTERS_H
+
+#include <stdatomic.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//! The counter type.
+typedef atomic_ullong CounterType;
+
+//! The performance counters for each worker.
+typedef struct PerfCounters {
+
+  // ---------------------------------------------------------------------------
+  // Transaction related counters
+  // ---------------------------------------------------------------------------
+
+  //! The number of transactions committed.
+  CounterType mTxCommitted;
+
+  //! The number of transactions commit wait.
+  CounterType mTxCommitWait;
+
+  //! The number of transactions aborted.
+  CounterType mTxAborted;
+
+  ///! The number of transactions with remote dependencies.
+  CounterType mTxWithRemoteDependencies;
+
+  //! The number of transactions without remote dependencies.
+  CounterType mTxWithoutRemoteDependencies;
+
+  //! The number of short running transactions.
+  CounterType mTxShortRunning;
+
+  //! The number of long running transactions.
+  CounterType mTxLongRunning;
+
+  // ---------------------------------------------------------------------------
+  // MVCC concurrency control related counters
+  // ---------------------------------------------------------------------------
+
+  //! The number of LCB query executed.
+  CounterType mLcbExecuted;
+
+  //! The total latency of LCB query in nanoseconds.
+  CounterType mLcbTotalLatNs;
+
+  // ---------------------------------------------------------------------------
+  // MVCC garbage collection related counters
+  // ---------------------------------------------------------------------------
+
+  //! The number of MVCC garbage collection executed.
+  CounterType mGcExecuted;
+
+  //! The total latency of MVCC garbage collection in nanoseconds.
+  CounterType mGcTotalLatNs;
+
+  // ---------------------------------------------------------------------------
+  // Contention split related counters
+  // ---------------------------------------------------------------------------
+
+  //! The number of contention split succeed.
+  CounterType mContentionSplitSucceed;
+
+  //! The number of contention split failed.
+  CounterType mContentionSplitFailed;
+
+  //! The number of normal split succeed.
+  CounterType mSplitSucceed;
+
+  //! The number of normal split failed.
+  CounterType mSplitFailed;
+
+} PerfCounters;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif