[metric][feat and fix]add inner metrics (alibaba#697)

include/ylt/metric.hpp

@@ -17,6 +17,6 @@
 #define CINATRA_ENABLE_METRIC_JSON
 #include "metric/gauge.hpp"
 #include "metric/histogram.hpp"
-#include "metric/metric.hpp"
 #include "metric/summary.hpp"
+#include "metric/system_metric.hpp"
 #include "ylt/struct_json/json_writer.h"

include/ylt/metric/counter.hpp

@@ -28,6 +28,7 @@ class counter_t : public metric_t {
   counter_t(std::string name, std::string help)
       : metric_t(MetricType::Counter, std::move(name), std::move(help)) {
     use_atomic_ = true;
+    g_user_metric_count++;
   }
 
   // static labels value, contains a map with atomic value.
@@ -36,16 +37,19 @@ class counter_t : public metric_t {
       : metric_t(MetricType::Counter, std::move(name), std::move(help),
                  std::move(labels)) {
     atomic_value_map_.emplace(labels_value_, 0);
+    g_user_metric_count++;
     use_atomic_ = true;
   }
 
   // dynamic labels value
   counter_t(std::string name, std::string help,
             std::vector<std::string> labels_name)
       : metric_t(MetricType::Counter, std::move(name), std::move(help),
-                 std::move(labels_name)) {}
+                 std::move(labels_name)) {
+    g_user_metric_count++;
+  }
 
-  virtual ~counter_t() {}
+  virtual ~counter_t() { g_user_metric_count--; }
 
   double value() { return default_lable_value_; }
 
@@ -82,20 +86,16 @@ class counter_t : public metric_t {
       return;
     }
 
-    serialize_head(str);
-    std::string s;
-    if (use_atomic_) {
-      serialize_map(atomic_value_map_, s);
-    }
-    else {
-      serialize_map(value_map_, s);
+    auto map = value_map();
+    if (map.empty()) {
+      return;
     }
 
-    if (s.empty()) {
-      str.clear();
-    }
-    else {
-      str.append(s);
+    std::string value_str;
+    serialize_map(map, value_str);
+    if (!value_str.empty()) {
+      serialize_head(str);
+      str.append(value_str);
     }
   }
 
@@ -113,18 +113,17 @@ class counter_t : public metric_t {
       return;
     }
 
+    auto map = value_map();
     json_counter_t counter{name_, help_, std::string(metric_name())};
-    if (use_atomic_) {
-      to_json(counter, atomic_value_map_, str);
-    }
-    else {
-      to_json(counter, value_map_, str);
-    }
+    to_json(counter, map, str);
   }
 
   template <typename T>
   void to_json(json_counter_t &counter, T &map, std::string &str) {
     for (auto &[k, v] : map) {
+      if (v == 0) {
+        continue;
+      }
       json_counter_metric_t metric;
       size_t index = 0;
       for (auto &label_value : k) {
@@ -133,7 +132,9 @@ class counter_t : public metric_t {
       metric.value = (int64_t)v;
       counter.metrics.push_back(std::move(metric));
     }
-    iguana::to_json(counter, str);
+    if (!counter.metrics.empty()) {
+      iguana::to_json(counter, str);
+    }
   }
 #endif
 
@@ -164,10 +165,21 @@ class counter_t : public metric_t {
     }
     else {
       std::lock_guard lock(mtx_);
+      stat_metric(labels_value);
       set_value<false>(value_map_[labels_value], value, op_type_t::INC);
     }
   }
 
+  void stat_metric(const std::vector<std::string> &labels_value) {
+    if (!value_map_.contains(labels_value)) {
+      for (auto &key : labels_value) {
+        g_user_metric_memory->inc(key.size());
+      }
+      g_user_metric_memory->inc(8);
+      g_user_metric_labels->inc();
+    }
+  }
+
   void update(double value) { default_lable_value_ = value; }
 
   void update(const std::vector<std::string> &labels_value, double value) {

include/ylt/metric/gauge.hpp

@@ -45,6 +45,7 @@ class gauge_t : public counter_t {
     }
     else {
       std::lock_guard lock(mtx_);
+      stat_metric(labels_value);
       set_value<false>(value_map_[labels_value], value, op_type_t::DEC);
     }
   }

include/ylt/metric/histogram.hpp

@@ -108,6 +108,10 @@ class histogram_t : public metric_t {
       return;
     }
 
+    if (sum_->value() == 0) {
+      return;
+    }
+
     serialize_head(str);
     double count = 0;
     auto bucket_counts = get_bucket_counts();
@@ -146,6 +150,10 @@ class histogram_t : public metric_t {
       return;
     }
 
+    if (sum_->value() == 0) {
+      return;
+    }
+
     json_histogram_t hist{name_, help_, std::string(metric_name())};
 
     double count = 0;
@@ -183,11 +191,13 @@ class histogram_t : public metric_t {
   }
 
   void serialize_with_labels(std::string &str) {
-    serialize_head(str);
+    auto value_map = sum_->value_map();
+    if (value_map.empty()) {
+      return;
+    }
 
+    std::string value_str;
     auto bucket_counts = get_bucket_counts();
-
-    auto value_map = sum_->value_map();
     for (auto &[labels_value, value] : value_map) {
       if (value == 0) {
         continue;
@@ -196,24 +206,31 @@ class histogram_t : public metric_t {
       double count = 0;
       for (size_t i = 0; i < bucket_counts.size(); i++) {
         auto counter = bucket_counts[i];
-        str.append(name_).append("_bucket{");
+        value_str.append(name_).append("_bucket{");
         build_label_string(str, sum_->labels_name(), labels_value);
-        str.append(",");
+        value_str.append(",");
 
         if (i == bucket_boundaries_.size()) {
-          str.append("le=\"").append("+Inf").append("\"} ");
+          value_str.append("le=\"").append("+Inf").append("\"} ");
         }
         else {
-          str.append("le=\"")
+          value_str.append("le=\"")
               .append(std::to_string(bucket_boundaries_[i]))
               .append("\"} ");
         }
 
         count += counter->value(labels_value);
-        str.append(std::to_string(count));
-        str.append("\n");
+        value_str.append(std::to_string(count));
+        value_str.append("\n");
+      }
+
+      if (value_str.empty()) {
+        return;
       }
 
+      serialize_head(str);
+      str.append(value_str);
+
       str.append(name_);
       str.append("_sum{");
       build_label_string(str, sum_->labels_name(), labels_value);
@@ -237,10 +254,14 @@ class histogram_t : public metric_t {
 
 #ifdef CINATRA_ENABLE_METRIC_JSON
   void serialize_to_json_with_labels(std::string &str) {
+    auto value_map = sum_->value_map();
+    if (value_map.empty()) {
+      return;
+    }
+
     json_histogram_t hist{name_, help_, std::string(metric_name())};
     auto bucket_counts = get_bucket_counts();
 
-    auto value_map = sum_->value_map();
     for (auto &[labels_value, value] : value_map) {
       if (value == 0) {
         continue;
@@ -272,7 +293,9 @@ class histogram_t : public metric_t {
       hist.metrics.push_back(std::move(metric));
     }
 
-    iguana::to_json(hist, str);
+    if (!hist.metrics.empty()) {
+      iguana::to_json(hist, str);
+    }
   }
 #endif
 

include/ylt/metric/metric.hpp

@@ -201,7 +201,22 @@ class metric_t {
   std::chrono::system_clock::time_point metric_created_time_{};
 };
 
-template <size_t ID = 0>
+template <typename Tag>
+struct metric_manager_t;
+
+struct ylt_system_tag_t {};
+using system_metric_manager = metric_manager_t<ylt_system_tag_t>;
+
+class counter_t;
+inline auto g_user_metric_memory =
+    std::make_shared<counter_t>("ylt_user_metric_memory", "");
+inline auto g_user_metric_labels =
+    std::make_shared<counter_t>("ylt_user_metric_labels", "");
+inline auto g_summary_failed_count =
+    std::make_shared<counter_t>("ylt_summary_failed_count", "");
+inline std::atomic<int64_t> g_user_metric_count = 0;
+
+template <typename Tag>
 struct metric_manager_t {
   struct null_mutex_t {
     void lock() {}
@@ -263,6 +278,15 @@ struct metric_manager_t {
     return r;
   }
 
+  static auto get_metrics() {
+    if (need_lock_) {
+      return collect<true>();
+    }
+    else {
+      return collect<false>();
+    }
+  }
+
   static auto metric_map_static() { return metric_map_impl<false>(); }
   static auto metric_map_dynamic() { return metric_map_impl<true>(); }
 
@@ -580,5 +604,34 @@ struct metric_manager_t {
   static inline std::once_flag flag_;
 };
 
-using default_metric_manager = metric_manager_t<0>;
+struct ylt_default_metric_tag_t {};
+using default_metric_manager = metric_manager_t<ylt_default_metric_tag_t>;
+
+template <typename... Args>
+struct metric_collector_t {
+  static std::string serialize() {
+    auto vec = get_all_metrics();
+    return default_metric_manager::serialize(vec);
+  }
+
+#ifdef CINATRA_ENABLE_METRIC_JSON
+  static std::string serialize_to_json() {
+    auto vec = get_all_metrics();
+    return default_metric_manager::serialize_to_json(vec);
+  }
+#endif
+
+  static std::vector<std::shared_ptr<metric_t>> get_all_metrics() {
+    std::vector<std::shared_ptr<metric_t>> vec;
+    (append_vector<Args>(vec), ...);
+    return vec;
+  }
+
+ private:
+  template <typename T>
+  static void append_vector(std::vector<std::shared_ptr<metric_t>>& vec) {
+    auto v = T::get_metrics();
+    vec.insert(vec.end(), v.begin(), v.end());
+  }
+};
 }  // namespace ylt::metric

include/ylt/metric/summary.hpp

@@ -109,7 +109,7 @@ class summary_t : public metric_t {
       throw std::invalid_argument("not a default label metric");
     }
     if (block_->sample_queue_.size_approx() >= 20000000) {
-      // TODO: record failed count.
+      g_summary_failed_count->inc();
       return;
     }
     block_->sample_queue_.enqueue(value);
@@ -131,7 +131,7 @@ class summary_t : public metric_t {
       }
     }
     if (labels_block_->sample_queue_.size_approx() >= 20000000) {
-      // TODO: record failed count.
+      g_summary_failed_count->inc();
       return;
     }
     labels_block_->sample_queue_.enqueue({std::move(labels_value), value});