From 585aeaa032fbd7e386d0ab3ac35cd3e776946063 Mon Sep 17 00:00:00 2001 From: qicosmos Date: Tue, 25 Jun 2024 18:02:12 +0800 Subject: [PATCH] add capacity of metrics --- include/ylt/metric/counter.hpp | 24 ++++------ include/ylt/metric/gauge.hpp | 4 +- include/ylt/metric/histogram.hpp | 6 +++ include/ylt/metric/metric.hpp | 63 +++++++++++++++++++++++-- include/ylt/metric/system_metric.hpp | 3 +- src/metric/tests/test_metric.cpp | 69 +++++++++++++++++++++++++++- 6 files changed, 147 insertions(+), 22 deletions(-) diff --git a/include/ylt/metric/counter.hpp b/include/ylt/metric/counter.hpp index 48fbaf824..594297057 100644 --- a/include/ylt/metric/counter.hpp +++ b/include/ylt/metric/counter.hpp @@ -51,7 +51,7 @@ class counter_t : public metric_t { virtual ~counter_t() { g_user_metric_count--; } - double value() { return default_lable_value_; } + double value() { return default_label_value_; } double value(const std::vector &labels_value) { if (use_atomic_) { @@ -78,7 +78,7 @@ class counter_t : public metric_t { void serialize(std::string &str) override { if (labels_name_.empty()) { - if (default_lable_value_ == 0) { + if (default_label_value_ == 0) { return; } serialize_head(str); @@ -103,11 +103,11 @@ class counter_t : public metric_t { void serialize_to_json(std::string &str) override { std::string s; if (labels_name_.empty()) { - if (default_lable_value_ == 0) { + if (default_label_value_ == 0) { return; } json_counter_t counter{name_, help_, std::string(metric_name())}; - int64_t value = default_lable_value_; + int64_t value = default_label_value_; counter.metrics.push_back({{}, value}); iguana::to_json(counter, str); return; @@ -144,9 +144,9 @@ class counter_t : public metric_t { } #ifdef __APPLE__ - mac_os_atomic_fetch_add(&default_lable_value_, val); + mac_os_atomic_fetch_add(&default_label_value_, val); #else - default_lable_value_ += val; + default_label_value_ += val; #endif } @@ -172,15 +172,11 @@ class counter_t : public metric_t { void stat_metric(const std::vector &labels_value) { if (!value_map_.contains(labels_value)) { - for (auto &key : labels_value) { - g_user_metric_memory->inc(key.size()); - } - g_user_metric_memory->inc(8); g_user_metric_labels->inc(); } } - void update(double value) { default_lable_value_ = value; } + void update(double value) { default_label_value_ = value; } void update(const std::vector &labels_value, double value) { if (labels_value.empty() || labels_name_.size() != labels_value.size()) { @@ -212,10 +208,10 @@ class counter_t : public metric_t { } if (type_ == MetricType::Counter) { - str.append(std::to_string((int64_t)default_lable_value_)); + str.append(std::to_string((int64_t)default_label_value_)); } else { - str.append(std::to_string(default_lable_value_)); + str.append(std::to_string(default_label_value_)); } str.append("\n"); @@ -305,7 +301,7 @@ class counter_t : public metric_t { } metric_hash_map> atomic_value_map_; - std::atomic default_lable_value_ = 0; + std::atomic default_label_value_ = 0; std::mutex mtx_; metric_hash_map value_map_; diff --git a/include/ylt/metric/gauge.hpp b/include/ylt/metric/gauge.hpp index 8ab33f143..6261ce07d 100644 --- a/include/ylt/metric/gauge.hpp +++ b/include/ylt/metric/gauge.hpp @@ -24,9 +24,9 @@ class gauge_t : public counter_t { void dec(double value = 1) { #ifdef __APPLE__ - mac_os_atomic_fetch_sub(&default_lable_value_, value); + mac_os_atomic_fetch_sub(&default_label_value_, value); #else - default_lable_value_ -= value; + default_label_value_ -= value; #endif } diff --git a/include/ylt/metric/histogram.hpp b/include/ylt/metric/histogram.hpp index a15a350ed..69f8c3cb3 100644 --- a/include/ylt/metric/histogram.hpp +++ b/include/ylt/metric/histogram.hpp @@ -36,6 +36,8 @@ class histogram_t : public metric_t { throw std::invalid_argument("Bucket Boundaries must be strictly sorted"); } + g_user_metric_count++; + for (size_t i = 0; i < buckets.size() + 1; i++) { bucket_counts_.push_back(std::make_shared("", "")); } @@ -51,6 +53,8 @@ class histogram_t : public metric_t { throw std::invalid_argument("Bucket Boundaries must be strictly sorted"); } + g_user_metric_count++; + for (size_t i = 0; i < buckets.size() + 1; i++) { bucket_counts_.push_back( std::make_shared(name, help, labels_name)); @@ -66,6 +70,8 @@ class histogram_t : public metric_t { throw std::invalid_argument("Bucket Boundaries must be strictly sorted"); } + g_user_metric_count++; + for (size_t i = 0; i < buckets.size() + 1; i++) { bucket_counts_.push_back(std::make_shared(name, help, labels)); } diff --git a/include/ylt/metric/metric.hpp b/include/ylt/metric/metric.hpp index 73424c094..bfd3de1d7 100644 --- a/include/ylt/metric/metric.hpp +++ b/include/ylt/metric/metric.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "async_simple/coro/Lazy.h" @@ -208,14 +209,18 @@ struct ylt_system_tag_t {}; using system_metric_manager = metric_manager_t; class counter_t; -inline auto g_user_metric_memory = - std::make_shared("ylt_user_metric_memory", ""); inline auto g_user_metric_labels = std::make_shared("ylt_user_metric_labels", ""); inline auto g_summary_failed_count = std::make_shared("ylt_summary_failed_count", ""); inline std::atomic g_user_metric_count = 0; +inline std::atomic ylt_metric_capacity = 10000000; + +inline void set_metric_capacity(int64_t max_count) { + ylt_metric_capacity = max_count; +} + template struct metric_manager_t { struct null_mutex_t { @@ -264,6 +269,31 @@ struct metric_manager_t { return remove_metric_impl(name); } + static bool remove_metric_dynamic(std::shared_ptr metric) { + return remove_metric_impl(std::string(metric->name())); + } + + static void remove_metric_dynamic(const std::vector& names) { + if (names.empty()) { + return; + } + auto lock = get_lock(); + for (auto& name : names) { + metric_map_.erase(name); + } + } + + static void remove_metric_dynamic( + std::vector> metrics) { + if (metrics.empty()) { + return; + } + auto lock = get_lock(); + for (auto& metric : metrics) { + metric_map_.erase(std::string(metric->name())); + } + } + template static bool register_metric_dynamic(Metrics... metrics) { bool r = true; @@ -271,6 +301,27 @@ struct metric_manager_t { return r; } + static bool register_metric_dynamic( + std::vector> metrics) { + bool r = true; + std::vector> vec; + for (auto& metric : metrics) { + r = register_metric_impl(metric); + if (!r) { + r = false; + break; + } + + vec.push_back(metric); + } + + if (!r) { + remove_metric_dynamic(vec); + } + + return r; + } + template static bool register_metric_static(Metrics... metrics) { bool r = true; @@ -476,6 +527,11 @@ struct metric_manager_t { std::string name(metric->name()); auto lock = get_lock(); + if (g_user_metric_count > ylt_metric_capacity) { + CINATRA_LOG_ERROR << "metric count at capacity size: " + << g_user_metric_count; + return false; + } bool r = metric_map_.emplace(name, std::move(metric)).second; if (!r) { CINATRA_LOG_ERROR << "duplicate registered metric name: " << name; @@ -597,7 +653,8 @@ struct metric_manager_t { } static inline std::mutex mtx_; - static inline std::map> metric_map_; + static inline std::unordered_map> + metric_map_; static inline null_mutex_t null_mtx_; static inline std::atomic_bool need_lock_ = true; diff --git a/include/ylt/metric/system_metric.hpp b/include/ylt/metric/system_metric.hpp index 81d1e1b40..95c52c7d2 100644 --- a/include/ylt/metric/system_metric.hpp +++ b/include/ylt/metric/system_metric.hpp @@ -109,7 +109,7 @@ inline void stat_memory() { file >> virtual_size >> resident >> share; static long page_size = sysconf(_SC_PAGE_SIZE); - process_memory_virtual->update(virtual_size); + process_memory_virtual->update(virtual_size * page_size); process_memory_resident->update(resident * page_size); process_memory_shared->update(share * page_size); } @@ -324,7 +324,6 @@ inline bool start_system_metric() { system_metric_manager::create_metric_static( "ylt_process_io_write_second", ""); - system_metric_manager::register_metric_static(g_user_metric_memory); system_metric_manager::register_metric_static(g_user_metric_labels); system_metric_manager::register_metric_static(g_summary_failed_count); diff --git a/src/metric/tests/test_metric.cpp b/src/metric/tests/test_metric.cpp index 6242e285d..f820eb946 100644 --- a/src/metric/tests/test_metric.cpp +++ b/src/metric/tests/test_metric.cpp @@ -338,7 +338,7 @@ TEST_CASE("test remove metric and serialize metrics") { auto s = metric_mgr2::serialize_to_json_static(); std::cout << s << "\n"; auto s1 = metric_mgr2::serialize_to_json({c, c2}); - CHECK(s == s1); + CHECK(s.size() == s1.size()); #endif CHECK_THROWS_AS(metric_mgr2::metric_count_dynamic(), std::invalid_argument); count = metric_mgr2::metric_count_static(); @@ -973,6 +973,73 @@ TEST_CASE("test system metric") { } #endif +TEST_CASE("test metric capacity") { + std::cout << g_user_metric_count << "\n"; + using test_metric_manager = metric_manager_t>; + set_metric_capacity(g_user_metric_count + 2); + auto c = test_metric_manager::create_metric_dynamic("counter", ""); + CHECK(c != nullptr); + auto c1 = + test_metric_manager::create_metric_dynamic("counter1", ""); + CHECK(c1 != nullptr); + auto c2 = + test_metric_manager::create_metric_dynamic("counter2", ""); + CHECK(c2 == nullptr); + set_metric_capacity(10000000); + + auto process_memory_resident = + system_metric_manager::get_metric_static( + "ylt_process_memory_resident"); + std::cout << (int64_t)process_memory_resident->value() << "\n"; + + auto process_memory_virtual = + system_metric_manager::get_metric_static( + "ylt_process_memory_virtual"); + std::cout << (int64_t)process_memory_virtual->value() << "\n"; +} + +TEST_CASE("test remove dynamic metric") { + using test_metric_manager = metric_manager_t>; + auto c = test_metric_manager::create_metric_dynamic("counter", ""); + CHECK(c != nullptr); + auto c1 = + test_metric_manager::create_metric_dynamic("counter1", ""); + CHECK(c1 != nullptr); + auto c2 = + test_metric_manager::create_metric_dynamic("counter2", ""); + CHECK(c2 != nullptr); + + test_metric_manager::remove_metric_dynamic(c); + CHECK(test_metric_manager::metric_count_dynamic() == 2); + test_metric_manager::remove_metric_dynamic(c1); + CHECK(test_metric_manager::metric_count_dynamic() == 1); + test_metric_manager::remove_metric_dynamic(c2); + CHECK(test_metric_manager::metric_count_dynamic() == 0); + + test_metric_manager::register_metric_dynamic(c, c1, c2); + CHECK(test_metric_manager::metric_count_dynamic() == 3); + test_metric_manager::remove_metric_dynamic("counter"); + CHECK(test_metric_manager::metric_count_dynamic() == 2); + test_metric_manager::remove_metric_dynamic( + std::vector{"counter1", "counter2"}); + CHECK(test_metric_manager::metric_count_dynamic() == 0); + + test_metric_manager::register_metric_dynamic( + std::vector>{c, c1, c2}); + CHECK(test_metric_manager::metric_count_dynamic() == 3); + test_metric_manager::remove_metric_dynamic({c1, c2}); + CHECK(test_metric_manager::metric_count_dynamic() == 1); + auto r = test_metric_manager::register_metric_dynamic( + std::vector>{c, c1}); + CHECK(!r); + CHECK(test_metric_manager::metric_count_dynamic() == 1); + + r = test_metric_manager::register_metric_dynamic( + std::vector>{c1, c}); + CHECK(!r); + CHECK(test_metric_manager::metric_count_dynamic() == 1); +} + DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4007) int main(int argc, char** argv) { return doctest::Context(argc, argv).run(); } DOCTEST_MSVC_SUPPRESS_WARNING_POP \ No newline at end of file