[metric][feat]Improve metric some (#746)

include/ylt/metric/counter.hpp

@@ -217,7 +217,10 @@ class basic_dynamic_counter : public dynamic_metric {
     auto [it, r] = value_map_.try_emplace(
         labels_value, thread_local_value<value_type>(dupli_count_));
     if (r) {
-      g_user_metric_label_count++;  // TODO: use thread local
+      g_user_metric_label_count.local_value()++;
+      if (ylt_label_max_age.count()) {
+        it->second.set_created_time(std::chrono::system_clock::now());
+      }
     }
     set_value(it->second.local_value(), value, op_type_t::INC);
   }
@@ -230,6 +233,12 @@ class basic_dynamic_counter : public dynamic_metric {
     }
     auto [it, r] = value_map_.try_emplace(
         labels_value, thread_local_value<value_type>(dupli_count_));
+    if (r) {
+      g_user_metric_label_count.local_value()++;
+      if (ylt_label_max_age.count()) {
+        it->second.set_created_time(std::chrono::system_clock::now());
+      }
+    }
     return it->second.update(value);
   }
 
@@ -268,6 +277,21 @@ class basic_dynamic_counter : public dynamic_metric {
     return value_map_.size();
   }
 
+  void clean_expired_label() override {
+    if (ylt_label_max_age.count() == 0) {
+      return;
+    }
+
+    auto now = std::chrono::system_clock::now();
+    std::lock_guard lock(mtx_);
+    std::erase_if(value_map_, [&now](auto &pair) {
+      bool r = std::chrono::duration_cast<std::chrono::seconds>(
+                   now - pair.second.get_created_time())
+                   .count() >= ylt_label_max_age.count();
+      return r;
+    });
+  }
+
   void remove_label_value(
       const std::map<std::string, std::string> &labels) override {
     {

include/ylt/metric/gauge.hpp

@@ -73,7 +73,10 @@ class basic_dynamic_gauge : public basic_dynamic_counter<value_type, N> {
     auto [it, r] = value_map_.try_emplace(
         labels_value, thread_local_value<value_type>(dupli_count_));
     if (r) {
-      g_user_metric_label_count++;  // TODO: use thread local
+      g_user_metric_label_count.local_value()++;
+      if (ylt_label_max_age.count()) {
+        it->second.set_created_time(std::chrono::system_clock::now());
+      }
     }
 
     set_value(it->second.local_value(), value, op_type_t::DEC);

include/ylt/metric/metric.hpp

@@ -15,6 +15,7 @@
 #include "async_simple/coro/Lazy.h"
 #include "async_simple/coro/SyncAwait.h"
 #include "cinatra/cinatra_log_wrapper.hpp"
+#include "thread_local_value.hpp"
 #if __has_include("ylt/coro_io/coro_io.hpp")
 #include "ylt/coro_io/coro_io.hpp"
 #else
@@ -45,6 +46,7 @@ enum class MetricType {
 struct metric_filter_options {
   std::optional<std::regex> name_regex{};
   std::optional<std::regex> label_regex{};
+  std::optional<std::regex> label_value_regex{};
   bool is_white = true;
 };
 
@@ -134,6 +136,8 @@ class metric_t {
            labels_value_.end();
   }
 
+  virtual void clean_expired_label() {}
+
   virtual bool has_label_value(const std::vector<std::string>& label_value) {
     return labels_value_ == label_value;
   }
@@ -221,18 +225,28 @@ class dynamic_metric : public metric_t {
   using metric_t::metric_t;
 };
 
-inline std::atomic<int64_t> g_user_metric_label_count = 0;
+inline thread_local_value<int64_t> g_user_metric_label_count{2};
 inline std::atomic<int64_t> g_summary_failed_count = 0;
 inline std::atomic<int64_t> g_user_metric_count = 0;
 
 inline std::atomic<int64_t> ylt_metric_capacity = 10000000;
 inline int64_t ylt_label_capacity = 20000000;
 
+inline std::chrono::seconds ylt_label_max_age{0};
+inline std::chrono::seconds ylt_label_check_expire_duration{0};
+
 inline void set_metric_capacity(int64_t max_count) {
   ylt_metric_capacity = max_count;
 }
 
 inline void set_label_capacity(int64_t max_label_count) {
   ylt_label_capacity = max_label_count;
 }
+
+inline void set_label_max_age(
+    std::chrono::seconds max_age,
+    std::chrono::seconds check_duration = std::chrono::seconds(60 * 10)) {
+  ylt_label_max_age = max_age;
+  ylt_label_check_expire_duration = check_duration;
+}
 }  // namespace ylt::metric

include/ylt/metric/metric_manager.hpp

@@ -69,63 +69,74 @@ class manager_helper {
   }
 #endif
 
-  static std::vector<std::shared_ptr<metric_t>> filter_metrics_by_label_value(
-      auto& metrics, const std::regex& label_regex, bool is_white) {
+  static std::vector<std::shared_ptr<metric_t>> filter_metrics_by_name(
+      auto& metrics, const std::regex& name_regex) {
     std::vector<std::shared_ptr<metric_t>> filtered_metrics;
-    std::vector<size_t> indexs;
-    size_t index = 0;
     for (auto& m : metrics) {
-      if (m->has_label_value(label_regex)) {
-        if (is_white) {
+      if (std::regex_match(m->str_name(), name_regex)) {
+        filtered_metrics.push_back(m);
+      }
+    }
+    return filtered_metrics;
+  }
+
+  static std::vector<std::shared_ptr<metric_t>> filter_metrics_by_label_name(
+      auto& metrics, const std::regex& label_name_regex) {
+    std::vector<std::shared_ptr<metric_t>> filtered_metrics;
+    for (auto& m : metrics) {
+      const auto& labels_name = m->labels_name();
+      for (auto& label_name : labels_name) {
+        if (std::regex_match(label_name, label_name_regex)) {
           filtered_metrics.push_back(m);
         }
-        else {
-          indexs.push_back(index);
-        }
       }
-
-      index++;
     }
+    return filtered_metrics;
+  }
 
-    if (!is_white) {
-      for (size_t i : indexs) {
-        metrics.erase(std::next(metrics.begin(), i));
+  static std::vector<std::shared_ptr<metric_t>> filter_metrics_by_label_value(
+      auto& metrics, const std::regex& label_value_regex) {
+    std::vector<std::shared_ptr<metric_t>> filtered_metrics;
+    for (auto& m : metrics) {
+      if (m->has_label_value(label_value_regex)) {
+        filtered_metrics.push_back(m);
       }
-      return metrics;
     }
-
     return filtered_metrics;
   }
 
   static std::vector<std::shared_ptr<metric_t>> filter_metrics(
       auto& metrics, const metric_filter_options& options) {
-    if (!options.name_regex && !options.label_regex) {
+    if (!(options.name_regex || options.label_regex ||
+          options.label_value_regex)) {
       return metrics;
     }
 
-    std::vector<std::shared_ptr<metric_t>> filtered_metrics;
-    for (auto& m : metrics) {
-      if (options.name_regex &&
-          !options.label_regex) {  // only check metric name
-        if (std::regex_match(std::string(m->name()), *options.name_regex)) {
-          filtered_metrics.push_back(m);
-        }
+    std::vector<std::shared_ptr<metric_t>> filtered_metrics = metrics;
+    if (options.name_regex) {
+      filtered_metrics = filter_metrics_by_name(metrics, *options.name_regex);
+      if (filtered_metrics.empty()) {
+        return {};
       }
-      else if (options.label_regex &&
-               !options.name_regex) {  // only check label name
-        filter_by_label_name(filtered_metrics, m, options);
+    }
+
+    if (options.label_regex) {
+      filtered_metrics =
+          filter_metrics_by_label_name(filtered_metrics, *options.label_regex);
+      if (filtered_metrics.empty()) {
+        return {};
       }
-      else {
-        if (std::regex_match(
-                std::string(m->name()),
-                *options.name_regex)) {  // check label name and label value
-          filter_by_label_name(filtered_metrics, m, options);
-        }
+    }
+
+    if (options.label_value_regex) {
+      filtered_metrics = filter_metrics_by_label_value(
+          filtered_metrics, *options.label_value_regex);
+      if (filtered_metrics.empty()) {
+        return {};
       }
     }
 
     if (!options.is_white) {
-      auto it = metrics.begin();
       for (auto& m : filtered_metrics) {
         std::erase_if(metrics, [&](auto t) {
           return t == m;
@@ -239,19 +250,18 @@ class static_metric_manager {
     return metrics;
   }
 
-  std::vector<std::shared_ptr<metric_t>> filter_metrics_by_label_value(
-      const std::regex& label_regex, bool is_white = true) {
-    auto metrics = collect();
-    return manager_helper::filter_metrics_by_label_value(metrics, label_regex,
-                                                         is_white);
-  }
-
   std::vector<std::shared_ptr<metric_t>> filter_metrics_static(
       const metric_filter_options& options) {
     auto metrics = collect();
     return manager_helper::filter_metrics(metrics, options);
   }
 
+  std::vector<std::shared_ptr<metric_t>> filter_metrics_by_label_value(
+      const std::regex& label_regex) {
+    auto metrics = collect();
+    return manager_helper::filter_metrics_by_label_value(metrics, label_regex);
+  }
+
  private:
   static_metric_manager() = default;
 
@@ -498,14 +508,47 @@ class dynamic_metric_manager {
   }
 
   std::vector<std::shared_ptr<metric_t>> filter_metrics_by_label_value(
-      const std::regex& label_regex, bool is_white = true) {
+      const std::regex& label_regex) {
     auto metrics = collect();
-    return manager_helper::filter_metrics_by_label_value(metrics, label_regex,
-                                                         is_white);
+    return manager_helper::filter_metrics_by_label_value(metrics, label_regex);
   }
 
  private:
-  dynamic_metric_manager() = default;
+  void clean_label_expired() {
+    executor_ = coro_io::create_io_context_pool(1);
+    timer_ = std::make_shared<coro_io::period_timer>(executor_->get_executor());
+    check_label_expired(timer_)
+        .via(executor_->get_executor())
+        .start([](auto&&) {
+        });
+  }
+
+  async_simple::coro::Lazy<void> check_label_expired(
+      std::weak_ptr<coro_io::period_timer> weak) {
+    while (true) {
+      auto timer = weak.lock();
+      if (timer == nullptr) {
+        co_return;
+      }
+
+      timer->expires_after(ylt_label_check_expire_duration);
+      bool r = co_await timer->async_await();
+      if (!r) {
+        co_return;
+      }
+
+      std::unique_lock lock(mtx_);
+      for (auto& [_, m] : metric_map_) {
+        m->clean_expired_label();
+      }
+    }
+  }
+
+  dynamic_metric_manager() {
+    if (ylt_label_max_age.count() > 0) {
+      clean_label_expired();
+    }
+  }
 
   std::vector<std::shared_ptr<dynamic_metric>> get_metric_by_label_value(
       const std::vector<std::string>& label_value) {
@@ -532,6 +575,8 @@ class dynamic_metric_manager {
 
   std::shared_mutex mtx_;
   std::unordered_map<std::string, std::shared_ptr<dynamic_metric>> metric_map_;
+  std::shared_ptr<coro_io::period_timer> timer_ = nullptr;
+  std::shared_ptr<coro_io::io_context_pool> executor_ = nullptr;
 };
 
 struct ylt_default_metric_tag_t {};
@@ -582,13 +627,6 @@ struct metric_collector_t {
     return manager_helper::filter_metrics(vec, options);
   }
 
-  static std::vector<std::shared_ptr<metric_t>> filter_metrics_by_label_value(
-      const std::regex& label_regex, bool is_white = true) {
-    auto vec = get_all_metrics();
-    return manager_helper::filter_metrics_by_label_value(vec, label_regex,
-                                                         is_white);
-  }
-
  private:
   template <typename T>
   static void append_vector(std::vector<std::shared_ptr<metric_t>>& vec) {

include/ylt/metric/system_metric.hpp

@@ -362,7 +362,7 @@ inline void stat_metric() {
   static auto user_metric_label_count =
       system_metric_manager::instance().get_metric_static<gauge_t>(
           "ylt_user_metric_labels");
-  user_metric_label_count->update(g_user_metric_label_count);
+  user_metric_label_count->update(g_user_metric_label_count.value());
 
   static auto ylt_summary_failed_count =
       system_metric_manager::instance().get_metric_static<gauge_t>(

include/ylt/metric/thread_local_value.hpp

@@ -1,5 +1,6 @@
 #pragma once
 #include <atomic>
+#include <chrono>
 #include <cstddef>
 #include <thread>
 #include <vector>
@@ -98,7 +99,14 @@ class thread_local_value {
     return val;
   }
 
+  void set_created_time(std::chrono::system_clock::time_point tm) {
+    created_time_ = tm;
+  }
+
+  auto get_created_time() { return created_time_; }
+
  private:
   std::vector<std::atomic<std::atomic<value_type> *>> duplicates_;
+  std::chrono::system_clock::time_point created_time_{};
 };
 }  // namespace ylt::metric