From c465b6c8f06cf8137aeb4f138487777fe04159bb Mon Sep 17 00:00:00 2001 From: qicosmos Date: Sun, 11 Aug 2024 22:12:23 +0800 Subject: [PATCH] [metric]Fix and improve (#743) --- include/ylt/metric/counter.hpp | 76 ++++++++++++++++++++++++++- include/ylt/metric/gauge.hpp | 3 ++ include/ylt/metric/metric.hpp | 46 +++++++++------- include/ylt/metric/metric_manager.hpp | 61 ++++++++++++++++++--- src/metric/tests/test_metric.cpp | 25 ++++++++- 5 files changed, 184 insertions(+), 27 deletions(-) diff --git a/include/ylt/metric/counter.hpp b/include/ylt/metric/counter.hpp index 0c1a24f37..5fd62aace 100644 --- a/include/ylt/metric/counter.hpp +++ b/include/ylt/metric/counter.hpp @@ -32,7 +32,7 @@ inline void set_value(T &label_val, value_type value, op_type_t type) { case op_type_t::INC: { #ifdef __APPLE__ if constexpr (std::is_floating_point_v) { - mac_os_atomic_fetch_add(&label_val.local_value(), value); + mac_os_atomic_fetch_add(&label_val, value); } else { label_val += value; @@ -211,6 +211,9 @@ class basic_dynamic_counter : public dynamic_metric { } std::lock_guard lock(mtx_); + if (value_map_.size() > ylt_label_capacity) { + return; + } auto [it, r] = value_map_.try_emplace( labels_value, thread_local_value(dupli_count_)); if (r) { @@ -222,6 +225,9 @@ class basic_dynamic_counter : public dynamic_metric { value_type update(const std::array &labels_value, value_type value) { std::lock_guard lock(mtx_); + if (value_map_.size() > ylt_label_capacity) { + return value_type{}; + } auto [it, r] = value_map_.try_emplace( labels_value, thread_local_value(dupli_count_)); return it->second.update(value); @@ -257,6 +263,66 @@ class basic_dynamic_counter : public dynamic_metric { return map; } + size_t label_value_count() override { + std::lock_guard lock(mtx_); + return value_map_.size(); + } + + void remove_label_value( + const std::map &labels) override { + { + std::lock_guard lock(mtx_); + if (value_map_.empty()) { + return; + } + } + + const auto &labels_name = this->labels_name(); + if (labels.size() > labels_name.size()) { + return; + } + + if (labels.size() == labels_name.size()) { + std::vector label_value; + for (auto &lb_name : labels_name) { + if (auto i = labels.find(lb_name); i != labels.end()) { + label_value.push_back(i->second); + } + } + + std::lock_guard lock(mtx_); + std::erase_if(value_map_, [&, this](auto &pair) { + return equal(label_value, pair.first); + }); + return; + } + else { + std::vector vec; + for (auto &lb_name : labels_name) { + if (auto i = labels.find(lb_name); i != labels.end()) { + vec.push_back(i->second); + } + else { + vec.push_back(""); + } + } + if (vec.empty()) { + return; + } + + std::lock_guard lock(mtx_); + std::erase_if(value_map_, [&](auto &pair) { + auto &[arr, _] = pair; + for (size_t i = 0; i < vec.size(); i++) { + if (!vec[i].empty() && vec[i] != arr[i]) { + return false; + } + } + return true; + }); + } + } + bool has_label_value(const std::string &value) override { auto map = value_map(); for (auto &[label_value, _] : map) { @@ -361,6 +427,14 @@ class basic_dynamic_counter : public dynamic_metric { } } + template + bool equal(const std::vector &v, const std::array &a) { + if (v.size() != N) + return false; + + return std::equal(v.begin(), v.end(), a.begin()); + } + void build_string(std::string &str, const std::vector &v1, const auto &v2) { for (size_t i = 0; i < v1.size(); i++) { diff --git a/include/ylt/metric/gauge.hpp b/include/ylt/metric/gauge.hpp index 80c7913d5..6e626a597 100644 --- a/include/ylt/metric/gauge.hpp +++ b/include/ylt/metric/gauge.hpp @@ -67,6 +67,9 @@ class basic_dynamic_gauge : public basic_dynamic_counter { } std::lock_guard lock(mtx_); + if (value_map_.size() > ylt_label_capacity) { + return; + } auto [it, r] = value_map_.try_emplace( labels_value, thread_local_value(dupli_count_)); if (r) { diff --git a/include/ylt/metric/metric.hpp b/include/ylt/metric/metric.hpp index 923fca18a..93fb51248 100644 --- a/include/ylt/metric/metric.hpp +++ b/include/ylt/metric/metric.hpp @@ -48,6 +48,24 @@ struct metric_filter_options { bool is_white = true; }; +#ifdef __APPLE__ +inline double mac_os_atomic_fetch_add(std::atomic* obj, double arg) { + double v; + do { + v = obj->load(); + } while (!std::atomic_compare_exchange_weak(obj, &v, v + arg)); + return v; +} + +inline double mac_os_atomic_fetch_sub(std::atomic* obj, double arg) { + double v; + do { + v = obj->load(); + } while (!std::atomic_compare_exchange_weak(obj, &v, v - arg)); + return v; +} +#endif + class metric_t { public: metric_t() = default; @@ -109,6 +127,8 @@ class metric_t { return static_labels_; } + virtual size_t label_value_count() { return 0; } + virtual bool has_label_value(const std::string& label_value) { return std::find(labels_value_.begin(), labels_value_.end(), label_value) != labels_value_.end(); @@ -133,6 +153,9 @@ class metric_t { labels_name_.end(); } + virtual void remove_label_value( + const std::map& labels) {} + virtual void serialize(std::string& str) {} #ifdef CINATRA_ENABLE_METRIC_JSON @@ -181,24 +204,6 @@ class metric_t { str.pop_back(); } -#ifdef __APPLE__ - double mac_os_atomic_fetch_add(std::atomic* obj, double arg) { - double v; - do { - v = obj->load(); - } while (!std::atomic_compare_exchange_weak(obj, &v, v + arg)); - return v; - } - - double mac_os_atomic_fetch_sub(std::atomic* obj, double arg) { - double v; - do { - v = obj->load(); - } while (!std::atomic_compare_exchange_weak(obj, &v, v - arg)); - return v; - } -#endif - MetricType type_ = MetricType::Nil; std::string name_; std::string help_; @@ -221,8 +226,13 @@ inline std::atomic g_summary_failed_count = 0; inline std::atomic g_user_metric_count = 0; inline std::atomic ylt_metric_capacity = 10000000; +inline int64_t ylt_label_capacity = 20000000; inline void set_metric_capacity(int64_t max_count) { ylt_metric_capacity = max_count; } + +inline void set_label_capacity(int64_t max_label_count) { + ylt_label_capacity = max_label_count; +} } // namespace ylt::metric \ No newline at end of file diff --git a/include/ylt/metric/metric_manager.hpp b/include/ylt/metric/metric_manager.hpp index d198bbc38..e128e7d38 100644 --- a/include/ylt/metric/metric_manager.hpp +++ b/include/ylt/metric/metric_manager.hpp @@ -350,14 +350,55 @@ class dynamic_metric_manager { } } - void remove_metric_by_label( - const std::vector>& labels) { - std::vector label_value; - for (auto& [k, v] : labels) { - label_value.push_back(v); + void remove_label_value(const std::map& labels) { + std::unique_lock lock(mtx_); + for (auto& [_, m] : metric_map_) { + m->remove_label_value(labels); } + } - remove_metric_by_label_value(label_value); + void remove_metric_by_label( + const std::map& labels) { + std::unique_lock lock(mtx_); + for (auto it = metric_map_.begin(); it != metric_map_.end();) { + auto& m = it->second; + const auto& labels_name = m->labels_name(); + if (labels.size() > labels_name.size()) { + continue; + } + + if (labels.size() == labels_name.size()) { + std::vector label_value; + for (auto& lb_name : labels_name) { + if (auto i = labels.find(lb_name); i != labels.end()) { + label_value.push_back(i->second); + } + } + + std::erase_if(metric_map_, [&](auto& pair) { + return pair.second->has_label_value(label_value); + }); + if (m->has_label_value(label_value)) { + metric_map_.erase(it); + } + break; + } + else { + bool need_erase = false; + for (auto& lb_name : labels_name) { + if (auto i = labels.find(lb_name); i != labels.end()) { + if (m->has_label_value(i->second)) { + it = metric_map_.erase(it); + need_erase = true; + break; + } + } + } + + if (!need_erase) + ++it; + } + } } void remove_metric_by_label_name( @@ -517,9 +558,15 @@ struct metric_collector_t { auto vec = get_all_metrics(); return manager_helper::serialize_to_json(vec); } + + static std::string serialize_to_json( + const std::vector>& metrics) { + return manager_helper::serialize_to_json(metrics); + } #endif - static std::string serialize(std::vector> metrics) { + static std::string serialize( + const std::vector>& metrics) { return manager_helper::serialize(metrics); } diff --git a/src/metric/tests/test_metric.cpp b/src/metric/tests/test_metric.cpp index 6e7814de1..6cafe9f9c 100644 --- a/src/metric/tests/test_metric.cpp +++ b/src/metric/tests/test_metric.cpp @@ -94,10 +94,28 @@ TEST_CASE("test metric manager") { CHECK(inst_d.metric_count() == 0); inst_d.register_metric(dc); - inst_d.remove_metric_by_label({{"url", "/"}, {"code", "200"}}); + inst_d.remove_metric_by_label({{"code", "400"}}); + CHECK(inst_d.metric_count() == 1); + inst_d.remove_metric_by_label({{"code", "200"}}); CHECK(inst_d.metric_count() == 0); inst_d.register_metric(dc); + inst_d.remove_label_value({{"code", "400"}}); + CHECK(inst_d.metric_count() == 1); + inst_d.remove_label_value({{"code", "200"}}); + CHECK(dc->label_value_count() == 0); + dc->inc({"/", "200"}); + + CHECK(dc->label_value_count() == 1); + inst_d.remove_label_value({{"url", "/"}}); + CHECK(dc->label_value_count() == 0); + dc->inc({"/", "200"}); + + CHECK(dc->label_value_count() == 1); + inst_d.remove_label_value({{"url", "/"}, {"code", "200"}}); + CHECK(dc->label_value_count() == 0); + dc->inc({"/", "200"}); + inst_d.remove_metric_by_label_name(std::vector{"url", "code"}); CHECK(inst_d.metric_count() == 0); inst_d.register_metric(dc); @@ -1260,6 +1278,11 @@ TEST_CASE("test serialize with emptry metrics") { } TEST_CASE("test serialize with multiple threads") { + { + dynamic_histogram_d h("test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, + std::array{"url", "code"}); + h.observe({"/", "code"}, 23); + } auto c = std::make_shared( std::string("get_count"), std::string("get counter"), std::array{"method"});