diff --git a/include/ylt/metric.hpp b/include/ylt/metric.hpp new file mode 100644 index 000000000..89279894d --- /dev/null +++ b/include/ylt/metric.hpp @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024, Alibaba Group Holding Limited; + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#define CINATRA_ENABLE_METRIC_JSON +#include "metric/gauge.hpp" +#include "metric/histogram.hpp" +#include "metric/metric.hpp" +#include "metric/summary.hpp" +#include "ylt/struct_json/json_writer.h" diff --git a/include/ylt/metric/counter.hpp b/include/ylt/metric/counter.hpp index a252106d1..3c339b078 100644 --- a/include/ylt/metric/counter.hpp +++ b/include/ylt/metric/counter.hpp @@ -4,13 +4,23 @@ #include "metric.hpp" -namespace ylt { +namespace ylt::metric { enum class op_type_t { INC, DEC, SET }; -struct counter_sample { - op_type_t op_type; - std::vector labels_value; - double value; + +#ifdef CINATRA_ENABLE_METRIC_JSON +struct json_counter_metric_t { + std::unordered_multimap labels; + int64_t value; +}; +REFLECTION(json_counter_metric_t, labels, value); +struct json_counter_t { + std::string name; + std::string help; + std::string type; + std::vector metrics; }; +REFLECTION(json_counter_t, name, help, type, metrics); +#endif class counter_t : public metric_t { public: @@ -23,12 +33,8 @@ class counter_t : public metric_t { // static labels value, contains a map with atomic value. counter_t(std::string name, std::string help, std::map labels) - : metric_t(MetricType::Counter, std::move(name), std::move(help)) { - for (auto &[k, v] : labels) { - labels_name_.push_back(k); - labels_value_.push_back(v); - } - + : metric_t(MetricType::Counter, std::move(name), std::move(help), + std::move(labels)) { atomic_value_map_.emplace(labels_value_, 0); use_atomic_ = true; } @@ -56,7 +62,7 @@ class counter_t : public metric_t { std::map, double, std::less>> - value_map() { + value_map() override { std::map, double, std::less>> map; @@ -97,6 +103,44 @@ class counter_t : public metric_t { } } +#ifdef CINATRA_ENABLE_METRIC_JSON + void serialize_to_json(std::string &str) override { + std::string s; + if (labels_name_.empty()) { + if (default_lable_value_ == 0) { + return; + } + json_counter_t counter{name_, help_, std::string(metric_name())}; + int64_t value = default_lable_value_; + counter.metrics.push_back({{}, value}); + iguana::to_json(counter, str); + return; + } + + json_counter_t counter{name_, help_, std::string(metric_name())}; + if (use_atomic_) { + to_json(counter, atomic_value_map_, str); + } + else { + to_json(counter, value_map_, str); + } + } + + template + void to_json(json_counter_t &counter, T &map, std::string &str) { + for (auto &[k, v] : map) { + json_counter_metric_t metric; + size_t index = 0; + for (auto &label_value : k) { + metric.labels.emplace(labels_name_[index++], label_value); + } + metric.value = (int64_t)v; + counter.metrics.push_back(std::move(metric)); + } + iguana::to_json(counter, str); + } +#endif + void inc(double val = 1) { if (val < 0) { throw std::invalid_argument("the value is less than zero"); @@ -255,4 +299,4 @@ class counter_t : public metric_t { std::less>> value_map_; }; -} // namespace ylt \ No newline at end of file +} // namespace ylt::metric \ No newline at end of file diff --git a/include/ylt/metric/detail/ckms_quantiles.hpp b/include/ylt/metric/detail/ckms_quantiles.hpp index cdaf1db92..fd42a8c24 100644 --- a/include/ylt/metric/detail/ckms_quantiles.hpp +++ b/include/ylt/metric/detail/ckms_quantiles.hpp @@ -5,7 +5,7 @@ // https://github.com/jupp0r/prometheus-cpp/blob/master/core/include/prometheus/detail/ckms_quantiles.h -namespace ylt { +namespace ylt::metric { class CKMSQuantiles { public: struct Quantile { @@ -172,4 +172,4 @@ class CKMSQuantiles { std::array buffer_; std::size_t buffer_count_; }; -} // namespace ylt \ No newline at end of file +} // namespace ylt::metric \ No newline at end of file diff --git a/include/ylt/metric/detail/time_window_quantiles.hpp b/include/ylt/metric/detail/time_window_quantiles.hpp index fd7df105f..6b81179f0 100644 --- a/include/ylt/metric/detail/time_window_quantiles.hpp +++ b/include/ylt/metric/detail/time_window_quantiles.hpp @@ -2,7 +2,7 @@ #include "ckms_quantiles.hpp" // https://github.com/jupp0r/prometheus-cpp/blob/master/core/include/prometheus/detail/time_window_quantiles.h -namespace ylt { +namespace ylt::metric { class TimeWindowQuantiles { using Clock = std::chrono::steady_clock; @@ -49,4 +49,4 @@ class TimeWindowQuantiles { mutable Clock::time_point last_rotation_; const Clock::duration rotation_interval_; }; -} // namespace ylt \ No newline at end of file +} // namespace ylt::metric \ No newline at end of file diff --git a/include/ylt/metric/gauge.hpp b/include/ylt/metric/gauge.hpp index 19c4b65c2..18693f1af 100644 --- a/include/ylt/metric/gauge.hpp +++ b/include/ylt/metric/gauge.hpp @@ -3,7 +3,7 @@ #include "counter.hpp" -namespace ylt { +namespace ylt::metric { class gauge_t : public counter_t { public: gauge_t(std::string name, std::string help) @@ -49,4 +49,4 @@ class gauge_t : public counter_t { } } }; -} // namespace ylt \ No newline at end of file +} // namespace ylt::metric \ No newline at end of file diff --git a/include/ylt/metric/histogram.hpp b/include/ylt/metric/histogram.hpp index dfd2ca131..d85225a5b 100644 --- a/include/ylt/metric/histogram.hpp +++ b/include/ylt/metric/histogram.hpp @@ -8,7 +8,24 @@ #include "counter.hpp" #include "metric.hpp" -namespace ylt { +namespace ylt::metric { +#ifdef CINATRA_ENABLE_METRIC_JSON +struct json_histogram_metric_t { + std::map labels; + std::map quantiles; + int64_t count; + double sum; +}; +REFLECTION(json_histogram_metric_t, labels, quantiles, count, sum); +struct json_histogram_t { + std::string name; + std::string help; + std::string type; + std::vector metrics; +}; +REFLECTION(json_histogram_t, name, help, type, metrics); +#endif + class histogram_t : public metric_t { public: histogram_t(std::string name, std::string help, std::vector buckets) @@ -25,7 +42,41 @@ class histogram_t : public metric_t { use_atomic_ = true; } + histogram_t(std::string name, std::string help, std::vector buckets, + std::vector labels_name) + : bucket_boundaries_(buckets), + metric_t(MetricType::Histogram, name, help, labels_name), + sum_(std::make_shared(name, help, labels_name)) { + if (!is_strict_sorted(begin(bucket_boundaries_), end(bucket_boundaries_))) { + throw std::invalid_argument("Bucket Boundaries must be strictly sorted"); + } + + for (size_t i = 0; i < buckets.size() + 1; i++) { + bucket_counts_.push_back( + std::make_shared(name, help, labels_name)); + } + } + + histogram_t(std::string name, std::string help, std::vector buckets, + std::map labels) + : bucket_boundaries_(buckets), + metric_t(MetricType::Histogram, name, help, labels), + sum_(std::make_shared(name, help, labels)) { + if (!is_strict_sorted(begin(bucket_boundaries_), end(bucket_boundaries_))) { + throw std::invalid_argument("Bucket Boundaries must be strictly sorted"); + } + + for (size_t i = 0; i < buckets.size() + 1; i++) { + bucket_counts_.push_back(std::make_shared(name, help, labels)); + } + use_atomic_ = true; + } + void observe(double value) { + if (!use_atomic_ || !labels_name_.empty()) { + throw std::invalid_argument("not a default label metric"); + } + const auto bucket_index = static_cast( std::distance(bucket_boundaries_.begin(), std::lower_bound(bucket_boundaries_.begin(), @@ -34,9 +85,33 @@ class histogram_t : public metric_t { bucket_counts_[bucket_index]->inc(); } + void observe(const std::vector &labels_value, double value) { + if (sum_->labels_name().empty()) { + throw std::invalid_argument("not a label metric"); + } + + const auto bucket_index = static_cast( + std::distance(bucket_boundaries_.begin(), + std::lower_bound(bucket_boundaries_.begin(), + bucket_boundaries_.end(), value))); + sum_->inc(labels_value, value); + bucket_counts_[bucket_index]->inc(labels_value); + } + auto get_bucket_counts() { return bucket_counts_; } - void serialize(std::string& str) override { + std::map, double, + std::less>> + value_map() override { + return sum_->value_map(); + } + + void serialize(std::string &str) override { + if (!sum_->labels_name().empty()) { + serialize_with_labels(str); + return; + } + serialize_head(str); double count = 0; auto bucket_counts = get_bucket_counts(); @@ -68,6 +143,41 @@ class histogram_t : public metric_t { .append("\n"); } +#ifdef CINATRA_ENABLE_METRIC_JSON + void serialize_to_json(std::string &str) override { + if (!sum_->labels_name().empty()) { + serialize_to_json_with_labels(str); + return; + } + + json_histogram_t hist{name_, help_, std::string(metric_name())}; + + double count = 0; + auto bucket_counts = get_bucket_counts(); + json_histogram_metric_t metric{}; + for (size_t i = 0; i < bucket_counts.size(); i++) { + auto counter = bucket_counts[i]; + + count += counter->value(); + + if (i == bucket_boundaries_.size()) { + metric.quantiles.emplace(std::numeric_limits::max(), + (int64_t)count); + } + else { + metric.quantiles.emplace(bucket_boundaries_[i], + (int64_t)counter->value()); + } + } + metric.count = (int64_t)count; + metric.sum = sum_->value(); + + hist.metrics.push_back(std::move(metric)); + + iguana::to_json(hist, str); + } +#endif + private: template bool is_strict_sorted(ForwardIterator first, ForwardIterator last) { @@ -76,8 +186,102 @@ class histogram_t : public metric_t { ForwardIterator>::value_type>()) == last; } + void serialize_with_labels(std::string &str) { + serialize_head(str); + + auto bucket_counts = get_bucket_counts(); + + auto value_map = sum_->value_map(); + for (auto &[labels_value, value] : value_map) { + if (value == 0) { + continue; + } + + double count = 0; + for (size_t i = 0; i < bucket_counts.size(); i++) { + auto counter = bucket_counts[i]; + str.append(name_).append("_bucket{"); + build_label_string(str, sum_->labels_name(), labels_value); + str.append(","); + + if (i == bucket_boundaries_.size()) { + str.append("le=\"").append("+Inf").append("\"} "); + } + else { + str.append("le=\"") + .append(std::to_string(bucket_boundaries_[i])) + .append("\"} "); + } + + count += counter->value(labels_value); + str.append(std::to_string(count)); + str.append("\n"); + } + + str.append(name_); + str.append("_sum{"); + build_label_string(str, sum_->labels_name(), labels_value); + str.append("} "); + + if (type_ == MetricType::Counter) { + str.append(std::to_string((int64_t)value)); + } + else { + str.append(std::to_string(value)); + } + str.append("\n"); + + str.append(name_).append("_count{"); + build_label_string(str, sum_->labels_name(), labels_value); + str.append("} "); + str.append(std::to_string(count)); + str.append("\n"); + } + } + +#ifdef CINATRA_ENABLE_METRIC_JSON + void serialize_to_json_with_labels(std::string &str) { + json_histogram_t hist{name_, help_, std::string(metric_name())}; + auto bucket_counts = get_bucket_counts(); + + auto value_map = sum_->value_map(); + for (auto &[labels_value, value] : value_map) { + if (value == 0) { + continue; + } + + size_t count = 0; + json_histogram_metric_t metric{}; + for (size_t i = 0; i < bucket_counts.size(); i++) { + auto counter = bucket_counts[i]; + + count += counter->value(labels_value); + + if (i == bucket_boundaries_.size()) { + metric.quantiles.emplace(std::numeric_limits::max(), + (int64_t)count); + } + else { + metric.quantiles.emplace(bucket_boundaries_[i], + (int64_t)counter->value(labels_value)); + } + } + metric.count = (int64_t)count; + metric.sum = sum_->value(labels_value); + + for (size_t i = 0; i < labels_value.size(); i++) { + metric.labels[sum_->labels_name()[i]] = labels_value[i]; + } + + hist.metrics.push_back(std::move(metric)); + } + + iguana::to_json(hist, str); + } +#endif + std::vector bucket_boundaries_; std::vector> bucket_counts_; // readonly std::shared_ptr sum_; }; -} // namespace ylt \ No newline at end of file +} // namespace ylt::metric \ No newline at end of file diff --git a/include/ylt/metric/metric.hpp b/include/ylt/metric/metric.hpp index f93896b6c..ea103e59f 100644 --- a/include/ylt/metric/metric.hpp +++ b/include/ylt/metric/metric.hpp @@ -1,17 +1,33 @@ #pragma once +#include #include #include #include #include #include +#include +#include #include #include #include #include "async_simple/coro/Lazy.h" +#include "async_simple/coro/SyncAwait.h" #include "cinatra/cinatra_log_wrapper.hpp" -namespace ylt { +#ifdef CINATRA_ENABLE_METRIC_JSON +namespace iguana { + +template +inline char* to_chars_float(T value, char* buffer) { + return buffer + snprintf(buffer, 65, "%g", value); +} + +} // namespace iguana + +#include +#endif +namespace ylt::metric { enum class MetricType { Counter, Gauge, @@ -20,15 +36,32 @@ enum class MetricType { Nil, }; +struct metric_filter_options { + std::optional name_regex{}; + std::optional label_regex{}; + bool is_white = true; +}; + class metric_t { public: metric_t() = default; + metric_t(MetricType type, std::string name, std::string help) + : type_(type), name_(std::move(name)), help_(std::move(help)) {} metric_t(MetricType type, std::string name, std::string help, - std::vector labels_name = {}) - : type_(type), - name_(std::move(name)), - help_(std::move(help)), - labels_name_(std::move(labels_name)) {} + std::vector labels_name) + : metric_t(type, std::move(name), std::move(help)) { + labels_name_ = std::move(labels_name); + } + + metric_t(MetricType type, std::string name, std::string help, + std::map static_labels) + : metric_t(type, std::move(name), std::move(help)) { + static_labels_ = std::move(static_labels); + for (auto& [k, v] : static_labels_) { + labels_name_.push_back(k); + labels_value_.push_back(v); + } + } virtual ~metric_t() {} std::string_view name() { return name_; } @@ -55,13 +88,35 @@ class metric_t { const std::vector& labels_name() { return labels_name_; } + const std::map& get_static_labels() { + return static_labels_; + } + + virtual std::map, double, + std::less>> + value_map() { + return {}; + } + virtual void serialize(std::string& str) {} +#ifdef CINATRA_ENABLE_METRIC_JSON + virtual void serialize_to_json(std::string& str) {} +#endif + // only for summary virtual async_simple::coro::Lazy serialize_async(std::string& out) { co_return; } +#ifdef CINATRA_ENABLE_METRIC_JSON + // only for summary + virtual async_simple::coro::Lazy serialize_to_json_async( + std::string& out) { + co_return; + } +#endif + bool is_atomic() const { return use_atomic_; } template @@ -80,6 +135,19 @@ class metric_t { .append("\n"); } + void build_label_string(std::string& str, + const std::vector& label_name, + const std::vector& label_value) { + for (size_t i = 0; i < label_name.size(); i++) { + str.append(label_name[i]) + .append("=\"") + .append(label_value[i]) + .append("\"") + .append(","); + } + str.pop_back(); + } + #ifdef __APPLE__ double mac_os_atomic_fetch_add(std::atomic* obj, double arg) { double v; @@ -101,6 +169,7 @@ class metric_t { MetricType type_ = MetricType::Nil; std::string name_; std::string help_; + std::map static_labels_; std::vector labels_name_; // read only std::vector labels_value_; // read only bool use_atomic_ = false; @@ -138,12 +207,20 @@ struct metric_manager_t { return m; } + static bool register_metric_static(std::shared_ptr metric) { + return register_metric_impl(metric); + } + static bool register_metric_dynamic(std::shared_ptr metric) { return register_metric_impl(metric); } - static bool register_metric_static(std::shared_ptr metric) { - return register_metric_impl(metric); + static bool remove_metric_static(const std::string& name) { + return remove_metric_impl(name); + } + + static bool remove_metric_dynamic(const std::string& name) { + return remove_metric_impl(name); } template @@ -175,30 +252,144 @@ struct metric_manager_t { return metric_keys_impl(); } + // static labels: {{"method", "GET"}, {"url", "/"}} + static std::vector> get_metric_by_labels_static( + const std::map& labels) { + std::vector> vec; + auto map = metric_map_static(); + for (auto& [name, m] : map) { + const auto& static_labels = m->get_static_labels(); + if (static_labels == labels) { + vec.push_back(m); + } + } + return vec; + } + + // static label: {"method", "GET"} + static std::vector> get_metric_by_label_static( + const std::pair& label) { + std::vector> vec; + auto map = metric_map_static(); + for (auto& [name, t] : map) { + const auto& static_labels = t->get_static_labels(); + for (const auto& pair : static_labels) { + if (pair.first == label.first && pair.second == label.second) { + vec.push_back(t); + } + } + } + return vec; + } + + // labels: {{"method", "POST"}, {"code", "200"}} + static std::vector> get_metric_by_labels_dynamic( + const std::map& labels) { + std::vector> vec; + auto map = metric_map_dynamic(); + for (auto& [name, t] : map) { + auto val_map = t->value_map(); + auto labels_name = t->labels_name(); + + for (auto& [k, v] : labels) { + if (auto it = std::find(labels_name.begin(), labels_name.end(), k); + it != labels_name.end()) { + if (auto it = std::find_if(val_map.begin(), val_map.end(), + [label_val = v](auto& pair) { + auto& key = pair.first; + return std::find(key.begin(), key.end(), + label_val) != key.end(); + }); + it != val_map.end()) { + vec.push_back(t); + } + } + } + } + + return vec; + } + template - static T* get_metric_static(const std::string& name) { + static std::shared_ptr get_metric_static(const std::string& name) { auto m = get_metric_impl(name); if (m == nullptr) { return nullptr; } - return m->template as(); + return std::dynamic_pointer_cast(m); } template - static T* get_metric_dynamic(const std::string& name) { + static std::shared_ptr get_metric_dynamic(const std::string& name) { auto m = get_metric_impl(name); if (m == nullptr) { return nullptr; } - return m->template as(); + return std::dynamic_pointer_cast(m); } - static async_simple::coro::Lazy serialize_static() { - return serialize_impl(); + static std::string serialize( + const std::vector>& metrics) { + std::string str; + for (auto& m : metrics) { + if (m->metric_type() == MetricType::Summary) { + async_simple::coro::syncAwait(m->serialize_async(str)); + } + else { + m->serialize(str); + } + } + + return str; + } + + static std::string serialize_static() { return serialize(collect()); } + + static std::string serialize_dynamic() { return serialize(collect()); } + +#ifdef CINATRA_ENABLE_METRIC_JSON + static std::string serialize_to_json_static() { + auto metrics = collect(); + return serialize_to_json(metrics); + } + + static std::string serialize_to_json_dynamic() { + auto metrics = collect(); + return serialize_to_json(metrics); } - static async_simple::coro::Lazy serialize_dynamic() { - return serialize_impl(); + static std::string serialize_to_json( + const std::vector>& metrics) { + if (metrics.empty()) { + return ""; + } + std::string str; + str.append("["); + for (auto& m : metrics) { + size_t start = str.size(); + if (m->metric_type() == MetricType::Summary) { + async_simple::coro::syncAwait(m->serialize_to_json_async(str)); + } + else { + m->serialize_to_json(str); + } + + if (str.size() > start) + str.append(","); + } + str.back() = ']'; + return str; + } +#endif + + static std::vector> filter_metrics_static( + const metric_filter_options& options) { + return filter_metrics(options); + } + + static std::vector> filter_metrics_dynamic( + const metric_filter_options& options) { + return filter_metrics(options); } private: @@ -242,6 +433,12 @@ struct metric_manager_t { return r; } + template + static size_t remove_metric_impl(const std::string& name) { + auto lock = get_lock(); + return metric_map_.erase(name); + } + template static auto metric_map_impl() { auto lock = get_lock(); @@ -289,19 +486,64 @@ struct metric_manager_t { return metrics; } - template - static async_simple::coro::Lazy serialize_impl() { - std::string str; + static void filter_by_label_name( + std::vector>& filtered_metrics, + std::shared_ptr m, const metric_filter_options& options, + std::vector& indexs, size_t index) { + const auto& labels_name = m->labels_name(); + for (auto& label_name : labels_name) { + if (std::regex_match(label_name, *options.label_regex)) { + if (options.is_white) { + filtered_metrics.push_back(m); + } + else { + indexs.push_back(index); + } + } + } + } + + template + static std::vector> filter_metrics( + const metric_filter_options& options) { auto metrics = collect(); + if (!options.name_regex && !options.label_regex) { + return metrics; + } + + std::vector> filtered_metrics; + std::vector indexs; + size_t index = 0; for (auto& m : metrics) { - if (m->metric_type() == MetricType::Summary) { - co_await m->serialize_async(str); + if (options.name_regex && !options.label_regex) { + if (std::regex_match(std::string(m->name()), *options.name_regex)) { + if (options.is_white) { + filtered_metrics.push_back(m); + } + else { + indexs.push_back(index); + } + } + } + else if (options.label_regex && !options.name_regex) { + filter_by_label_name(filtered_metrics, m, options, indexs, index); } else { - m->serialize(str); + if (std::regex_match(std::string(m->name()), *options.name_regex)) { + filter_by_label_name(filtered_metrics, m, options, indexs, index); + } } + index++; } - co_return str; + + if (!options.is_white) { + for (size_t i : indexs) { + metrics.erase(std::next(metrics.begin(), i)); + } + return metrics; + } + + return filtered_metrics; } static inline std::mutex mtx_; @@ -313,4 +555,4 @@ struct metric_manager_t { }; using default_metric_manager = metric_manager_t<0>; -} // namespace ylt \ No newline at end of file +} // namespace ylt::metric \ No newline at end of file diff --git a/include/ylt/metric/summary.hpp b/include/ylt/metric/summary.hpp index 415d6b322..bad8a132a 100644 --- a/include/ylt/metric/summary.hpp +++ b/include/ylt/metric/summary.hpp @@ -6,7 +6,29 @@ #include "ylt/coro_io/coro_io.hpp" #include "ylt/util/concurrentqueue.h" -namespace ylt { +namespace ylt::metric { +#ifdef CINATRA_ENABLE_METRIC_JSON +struct json_summary_metric_t { + std::map labels; + std::map quantiles; + int64_t count; + double sum; +}; +REFLECTION(json_summary_metric_t, labels, quantiles, count, sum); +struct json_summary_t { + std::string name; + std::string help; + std::string type; + std::vector metrics; +}; +REFLECTION(json_summary_t, name, help, type, metrics); +#endif + +struct summary_label_sample { + std::vector labels_value; + double value; +}; + class summary_t : public metric_t { public: using Quantiles = std::vector; @@ -14,22 +36,56 @@ class summary_t : public metric_t { std::chrono::milliseconds max_age = std::chrono::seconds{60}, int age_buckets = 5) : quantiles_{std::move(quantiles)}, - metric_t(MetricType::Summary, std::move(name), std::move(help)) { - work_ = std::make_shared(ctx_); - thd_ = std::thread([this] { - ctx_.run(); - }); - excutor_ = - std::make_unique>(ctx_.get_executor()); - block_ = std::make_shared(); + metric_t(MetricType::Summary, std::move(name), std::move(help)), + max_age_(max_age), + age_buckets_(age_buckets) { + init_executor(); + init_block(block_); block_->quantile_values_ = std::make_shared(quantiles_, max_age, age_buckets); - start_timer(block_).via(excutor_.get()).start([](auto &&) { - }); + use_atomic_ = true; + } + + summary_t(std::string name, std::string help, Quantiles quantiles, + std::vector labels_name, + std::chrono::milliseconds max_age = std::chrono::seconds{60}, + int age_buckets = 5) + : quantiles_{std::move(quantiles)}, + metric_t(MetricType::Summary, std::move(name), std::move(help), + std::move(labels_name)), + max_age_(max_age), + age_buckets_(age_buckets) { + init_executor(); + init_block(labels_block_); + } + + summary_t(std::string name, std::string help, Quantiles quantiles, + std::map static_labels, + std::chrono::milliseconds max_age = std::chrono::seconds{60}, + int age_buckets = 5) + : quantiles_{std::move(quantiles)}, + metric_t(MetricType::Summary, std::move(name), std::move(help), + std::move(static_labels)), + max_age_(max_age), + age_buckets_(age_buckets) { + init_executor(); + init_block(labels_block_); + labels_block_->label_quantile_values_[labels_value_] = + std::make_shared(quantiles_, max_age, age_buckets); + labels_block_->label_count_.emplace(labels_value_, 0); + labels_block_->label_sum_.emplace(labels_value_, 0); + use_atomic_ = true; } ~summary_t() { - block_->stop_ = true; + if (block_) { + block_->stop_ = true; + } + + if (labels_block_) { + labels_block_->stop_ = true; + } + work_ = nullptr; thd_.join(); } @@ -42,7 +98,39 @@ class summary_t : public metric_t { double sum_; }; - void observe(double value) { block_->sample_queue_.enqueue(value); } + struct labels_block_t { + std::atomic stop_ = false; + moodycamel::ConcurrentQueue sample_queue_; + + std::map, std::shared_ptr, + std::less>> + label_quantile_values_; + std::map, std::uint64_t, + std::less>> + label_count_; + std::map, double, + std::less>> + label_sum_; + }; + + void observe(double value) { + if (!labels_name_.empty()) { + throw std::invalid_argument("not a default label metric"); + } + block_->sample_queue_.enqueue(value); + } + + void observe(std::vector labels_value, double value) { + if (labels_value.empty()) { + throw std::invalid_argument("not a label metric"); + } + if (use_atomic_) { + if (labels_value != labels_value_) { + throw std::invalid_argument("not equal with static label"); + } + } + labels_block_->sample_queue_.enqueue({std::move(labels_value), value}); + } async_simple::coro::Lazy> get_rates(double &sum, uint64_t &count) { @@ -51,34 +139,86 @@ class summary_t : public metric_t { co_return std::vector{}; } - co_await coro_io::post([this, &vec, &sum, &count] { - sum = block_->sum_; - count = block_->count_; - for (const auto &quantile : quantiles_) { - vec.push_back(block_->quantile_values_->get(quantile.quantile)); + co_await coro_io::post( + [this, &vec, &sum, &count] { + sum = block_->sum_; + count = block_->count_; + for (const auto &quantile : quantiles_) { + vec.push_back(block_->quantile_values_->get(quantile.quantile)); + } + }, + excutor_.get()); + + co_return vec; + } + + async_simple::coro::Lazy> get_rates( + const std::vector &labels_value, double &sum, + uint64_t &count) { + std::vector vec; + if (quantiles_.empty()) { + co_return std::vector{}; + } + + if (use_atomic_) { + if (labels_value != labels_value_) { + throw std::invalid_argument("not equal with static label"); } - }); + } + + co_await coro_io::post( + [this, &vec, &sum, &count, &labels_value] { + auto it = labels_block_->label_quantile_values_.find(labels_value); + if (it == labels_block_->label_quantile_values_.end()) { + return; + } + sum = labels_block_->label_sum_[labels_value]; + count = labels_block_->label_count_[labels_value]; + for (const auto &quantile : quantiles_) { + vec.push_back(it->second->get(quantile.quantile)); + } + }, + excutor_.get()); co_return vec; } + std::map, double, + std::less>> + value_map() override { + auto ret = async_simple::coro::syncAwait(coro_io::post( + [this] { + return labels_block_->label_sum_; + }, + excutor_.get())); + return ret.value(); + } + async_simple::coro::Lazy get_sum() { - auto ret = co_await coro_io::post([this] { - return block_->sum_; - }); + auto ret = co_await coro_io::post( + [this] { + return block_->sum_; + }, + excutor_.get()); co_return ret.value(); } async_simple::coro::Lazy get_count() { - auto ret = co_await coro_io::post([this] { - return block_->count_; - }); + auto ret = co_await coro_io::post( + [this] { + return block_->count_; + }, + excutor_.get()); co_return ret.value(); } size_t size_approx() { return block_->sample_queue_.size_approx(); } async_simple::coro::Lazy serialize_async(std::string &str) override { + if (block_ == nullptr) { + co_await serialize_async_with_label(str); + co_return; + } if (quantiles_.empty()) { co_return; } @@ -103,8 +243,55 @@ class summary_t : public metric_t { .append("\n"); } +#ifdef CINATRA_ENABLE_METRIC_JSON + async_simple::coro::Lazy serialize_to_json_async( + std::string &str) override { + if (block_ == nullptr) { + co_await serialize_to_json_with_label_async(str); + co_return; + } + + if (quantiles_.empty()) { + co_return; + } + + json_summary_t summary{name_, help_, std::string(metric_name())}; + double sum = 0; + uint64_t count = 0; + auto rates = co_await get_rates(sum, count); + + json_summary_metric_t metric; + + for (size_t i = 0; i < quantiles_.size(); i++) { + metric.quantiles.emplace(quantiles_[i].quantile, rates[i]); + } + + metric.sum = sum; + metric.count = count; + + summary.metrics.push_back(std::move(metric)); + + iguana::to_json(summary, str); + } +#endif private: - async_simple::coro::Lazy start_timer(std::shared_ptr block) { + void init_executor() { + work_ = std::make_shared(ctx_); + thd_ = std::thread([this] { + ctx_.run(); + }); + excutor_ = + std::make_unique>(ctx_.get_executor()); + } + + template + void init_block(std::shared_ptr &block) { + block = std::make_shared(); + start(block).via(excutor_.get()).start([](auto &&) { + }); + } + + async_simple::coro::Lazy start(std::shared_ptr block) { double sample; size_t count = 1000000; while (!block->stop_) { @@ -122,18 +309,132 @@ class summary_t : public metric_t { co_await async_simple::coro::Yield{}; if (block->sample_queue_.size_approx() == 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(5)); + co_await coro_io::sleep_for(std::chrono::milliseconds(5), + excutor_.get()); } } co_return; } + async_simple::coro::Lazy start(std::shared_ptr block) { + summary_label_sample sample; + size_t count = 1000000; + while (!block->stop_) { + size_t index = 0; + while (block->sample_queue_.try_dequeue(sample)) { + auto &ptr = block->label_quantile_values_[sample.labels_value]; + + if (ptr == nullptr) { + ptr = std::make_shared(quantiles_, max_age_, + age_buckets_); + } + + ptr->insert(sample.value); + + block->label_count_[sample.labels_value] += 1; + block->label_sum_[sample.labels_value] += sample.value; + index++; + if (index == count) { + break; + } + } + + co_await async_simple::coro::Yield{}; + + if (block->sample_queue_.size_approx() == 0) { + co_await coro_io::sleep_for(std::chrono::milliseconds(5), + excutor_.get()); + } + } + + co_return; + } + + async_simple::coro::Lazy serialize_async_with_label(std::string &str) { + if (quantiles_.empty()) { + co_return; + } + + serialize_head(str); + + auto sum_map = co_await coro_io::post( + [this] { + return labels_block_->label_sum_; + }, + excutor_.get()); + + for (auto &[labels_value, sum_val] : sum_map.value()) { + double sum = 0; + uint64_t count = 0; + auto rates = co_await get_rates(labels_value, sum, count); + for (size_t i = 0; i < quantiles_.size(); i++) { + str.append(name_); + str.append("{"); + build_label_string(str, labels_name_, labels_value); + str.append(","); + str.append("quantile=\""); + str.append(std::to_string(quantiles_[i].quantile)).append("\"} "); + str.append(std::to_string(rates[i])).append("\n"); + } + + str.append(name_).append("_sum "); + str.append("{"); + build_label_string(str, labels_name_, labels_value); + str.append("} "); + str.append(std::to_string(sum)).append("\n"); + + str.append(name_).append("_count "); + str.append("{"); + build_label_string(str, labels_name_, labels_value); + str.append("} "); + str.append(std::to_string((uint64_t)count)).append("\n"); + } + } + +#ifdef CINATRA_ENABLE_METRIC_JSON + async_simple::coro::Lazy serialize_to_json_with_label_async( + std::string &str) { + if (quantiles_.empty()) { + co_return; + } + + auto sum_map = co_await coro_io::post( + [this] { + return labels_block_->label_sum_; + }, + excutor_.get()); + + json_summary_t summary{name_, help_, std::string(metric_name())}; + + for (auto &[labels_value, sum_val] : sum_map.value()) { + json_summary_metric_t metric; + double sum = 0; + uint64_t count = 0; + auto rates = co_await get_rates(labels_value, sum, count); + metric.count = count; + metric.sum = sum; + for (size_t i = 0; i < quantiles_.size(); i++) { + for (size_t i = 0; i < labels_value.size(); i++) { + metric.labels[labels_name_[i]] = labels_value[i]; + } + metric.quantiles.emplace(quantiles_[i].quantile, rates[i]); + } + + summary.metrics.push_back(std::move(metric)); + } + iguana::to_json(summary, str); + } +#endif + Quantiles quantiles_; // readonly std::shared_ptr block_; + std::shared_ptr labels_block_; std::unique_ptr> excutor_ = nullptr; std::shared_ptr work_; asio::io_context ctx_; std::thread thd_; + std::chrono::milliseconds max_age_; + int age_buckets_; }; -} // namespace ylt \ No newline at end of file +} // namespace ylt::metric \ No newline at end of file diff --git a/include/ylt/standalone/cinatra/coro_http_server.hpp b/include/ylt/standalone/cinatra/coro_http_server.hpp index 29031eae1..17a057fdb 100644 --- a/include/ylt/standalone/cinatra/coro_http_server.hpp +++ b/include/ylt/standalone/cinatra/coro_http_server.hpp @@ -182,12 +182,23 @@ class coro_http_server { } } - void use_metrics(std::string url_path = "/metrics") { + void use_metrics(bool enable_json = false, + std::string url_path = "/metrics") { init_metrics(); set_http_handler( - url_path, [](coro_http_request &req, coro_http_response &res) { - std::string str = async_simple::coro::syncAwait( - ylt::default_metric_manager::serialize_static()); + url_path, + [enable_json](coro_http_request &req, coro_http_response &res) { + std::string str; +#ifdef CINATRA_ENABLE_METRIC_JSON + if (enable_json) { + str = + ylt::metric::default_metric_manager::serialize_to_json_static(); + res.set_content_type(); + } + else +#endif + str = ylt::metric::default_metric_manager::serialize_static(); + res.set_status_and_content(status_type::ok, std::move(str)); }); } @@ -900,7 +911,7 @@ class coro_http_server { private: void init_metrics() { - using namespace ylt; + using namespace ylt::metric; cinatra_metric_conf::enable_metric = true; default_metric_manager::create_metric_static( diff --git a/include/ylt/standalone/cinatra/metric_conf.hpp b/include/ylt/standalone/cinatra/metric_conf.hpp index 549c93cd0..4800481ad 100644 --- a/include/ylt/standalone/cinatra/metric_conf.hpp +++ b/include/ylt/standalone/cinatra/metric_conf.hpp @@ -25,9 +25,8 @@ struct cinatra_metric_conf { return; } - static auto m = - ylt::default_metric_manager::get_metric_static( - server_total_req); + static auto m = ylt::metric::default_metric_manager::get_metric_static< + ylt::metric::counter_t>(server_total_req); if (m == nullptr) { return; } @@ -38,9 +37,8 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = - ylt::default_metric_manager::get_metric_static( - server_failed_req); + static auto m = ylt::metric::default_metric_manager::get_metric_static< + ylt::metric::counter_t>(server_failed_req); if (m == nullptr) { return; } @@ -51,9 +49,8 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = - ylt::default_metric_manager::get_metric_static( - server_total_fd); + static auto m = ylt::metric::default_metric_manager::get_metric_static< + ylt::metric::gauge_t>(server_total_fd); if (m == nullptr) { return; } @@ -64,9 +61,8 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = - ylt::default_metric_manager::get_metric_static( - server_total_fd); + static auto m = ylt::metric::default_metric_manager::get_metric_static< + ylt::metric::gauge_t>(server_total_fd); if (m == nullptr) { return; } @@ -77,9 +73,8 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = - ylt::default_metric_manager::get_metric_static( - server_total_recv_bytes); + static auto m = ylt::metric::default_metric_manager::get_metric_static< + ylt::metric::counter_t>(server_total_recv_bytes); if (m == nullptr) { return; } @@ -90,9 +85,8 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = - ylt::default_metric_manager::get_metric_static( - server_total_send_bytes); + static auto m = ylt::metric::default_metric_manager::get_metric_static< + ylt::metric::counter_t>(server_total_send_bytes); if (m == nullptr) { return; } @@ -103,9 +97,8 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = - ylt::default_metric_manager::get_metric_static( - server_req_latency); + static auto m = ylt::metric::default_metric_manager::get_metric_static< + ylt::metric::histogram_t>(server_req_latency); if (m == nullptr) { return; } @@ -116,9 +109,8 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = - ylt::default_metric_manager::get_metric_static( - server_read_latency); + static auto m = ylt::metric::default_metric_manager::get_metric_static< + ylt::metric::histogram_t>(server_read_latency); if (m == nullptr) { return; } diff --git a/src/metric/tests/test_metric.cpp b/src/metric/tests/test_metric.cpp index ed5d38220..5608460c8 100644 --- a/src/metric/tests/test_metric.cpp +++ b/src/metric/tests/test_metric.cpp @@ -1,12 +1,11 @@ -#include "ylt/metric/gauge.hpp" #define DOCTEST_CONFIG_IMPLEMENT #include #include "doctest.h" -#include "ylt/metric/counter.hpp" -#include "ylt/metric/histogram.hpp" -#include "ylt/metric/summary.hpp" +#include "ylt/metric.hpp" + using namespace ylt; +using namespace ylt::metric; TEST_CASE("test no lable") { { @@ -139,6 +138,13 @@ TEST_CASE("test gauge") { CHECK(g.value() == 2); g.inc(0); +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string str_json; + g.serialize_to_json(str_json); + std::cout << str_json << "\n"; + CHECK(str_json.find("\"value\":2") != std::string::npos); +#endif + g.dec(); CHECK(g.value() == 1); g.dec(); @@ -156,6 +162,15 @@ TEST_CASE("test gauge") { values = g.value_map(); CHECK(values[{"GET", "200", "/"}] == 3); + g.inc({"POST", "200", "/"}, 4); + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string str_json; + g.serialize_to_json(str_json); + std::cout << str_json << "\n"; + CHECK(str_json.find("\"code\":\"200\"") != std::string::npos); +#endif + std::string str; g.serialize(str); std::cout << str; @@ -175,7 +190,7 @@ TEST_CASE("test gauge") { } TEST_CASE("test histogram") { - histogram_t h("test", "help", {5.0, 10.0, 20.0, 50.0, 100.0}); + histogram_t h("test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}); h.observe(23); auto counts = h.get_bucket_counts(); CHECK(counts[3]->value() == 1); @@ -189,11 +204,18 @@ TEST_CASE("test histogram") { CHECK(counts[0]->value() == 1); std::string str; h.serialize(str); - std::cout << str; + std::cout << str << "\n"; CHECK(str.find("test_count") != std::string::npos); CHECK(str.find("test_sum") != std::string::npos); - CHECK(str.find("test_bucket{le=\"5") != std::string::npos); + CHECK(str.find("test_bucket{le=\"5.23") != std::string::npos); CHECK(str.find("test_bucket{le=\"+Inf\"}") != std::string::npos); + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string str_json; + h.serialize_to_json(str_json); + std::cout << str_json << "\n"; + CHECK(str_json.find("\"5.23\":1") != std::string::npos); +#endif } TEST_CASE("test summary") { @@ -217,6 +239,13 @@ TEST_CASE("test summary") { CHECK(str.find("test_summary_count") != std::string::npos); CHECK(str.find("test_summary_sum") != std::string::npos); CHECK(str.find("test_summary{quantile=\"") != std::string::npos); + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string str_json; + async_simple::coro::syncAwait(summary.serialize_to_json_async(str_json)); + std::cout << str_json << "\n"; + CHECK(str_json.find("\"0.9\":") != std::string::npos); +#endif } TEST_CASE("test register metric") { @@ -245,8 +274,7 @@ TEST_CASE("test register metric") { CHECK(map["get_count"]->as()->value() == 1); CHECK(map["get_guage_count"]->as()->value() == 1); - auto s = - async_simple::coro::syncAwait(default_metric_manager::serialize_static()); + auto s = default_metric_manager::serialize_static(); std::cout << s << "\n"; CHECK(s.find("get_count 1") != std::string::npos); CHECK(s.find("get_guage_count 1") != std::string::npos); @@ -275,6 +303,474 @@ TEST_CASE("test register metric") { } } +TEST_CASE("test remove metric and serialize metrics") { + using metric_mgr = metric_manager_t<1>; + metric_mgr::create_metric_dynamic("test_counter", ""); + metric_mgr::create_metric_dynamic("test_counter2", ""); + + size_t count = metric_mgr::metric_count_dynamic(); + CHECK(count == 2); + + metric_mgr::remove_metric_dynamic("test_counter"); + count = metric_mgr::metric_count_dynamic(); + CHECK(count == 1); + + metric_mgr::remove_metric_dynamic("test_counter2"); + count = metric_mgr::metric_count_dynamic(); + CHECK(count == 0); + + CHECK_THROWS_AS( + metric_mgr::create_metric_static("test_static_counter", ""), + std::invalid_argument); + + using metric_mgr2 = metric_manager_t<2>; + auto c = + metric_mgr2::create_metric_static("test_static_counter", ""); + auto c2 = + metric_mgr2::create_metric_static("test_static_counter2", ""); + c->inc(); + c2->inc(); + +#ifdef CINATRA_ENABLE_METRIC_JSON + auto s = metric_mgr2::serialize_to_json_static(); + std::cout << s << "\n"; + auto s1 = metric_mgr2::serialize_to_json({c, c2}); + CHECK(s == s1); +#endif + CHECK_THROWS_AS(metric_mgr2::metric_count_dynamic(), std::invalid_argument); + count = metric_mgr2::metric_count_static(); + CHECK(count == 2); + CHECK_THROWS_AS(metric_mgr2::remove_metric_dynamic("test_static_counter"), + std::invalid_argument); + + metric_mgr2::remove_metric_static("test_static_counter"); + count = metric_mgr2::metric_count_static(); + CHECK(count == 1); +} + +TEST_CASE("test filter metrics static") { + using metric_mgr = metric_manager_t<3>; + auto c = metric_mgr::create_metric_static( + "test_static_counter", "", + std::map{{"method", "GET"}}); + auto c2 = metric_mgr::create_metric_static( + "test_static_counter2", "", + std::map{{"url", "/"}}); + c->inc({"GET"}); + c2->inc({"/"}); + + metric_filter_options options; + options.name_regex = ".*counter.*"; + { + auto metrics = metric_mgr::filter_metrics_static(options); + CHECK(metrics.size() == 2); + + auto s = metric_mgr::serialize(metrics); + CHECK(s.find("test_static_counter") != std::string::npos); + std::cout << s << "\n"; + } + + options.label_regex = ".*ur.*"; + { + auto metrics = metric_mgr::filter_metrics_static(options); + CHECK(metrics.size() == 1); + auto s = metric_mgr::serialize(metrics); + CHECK(s.find("test_static_counter2") != std::string::npos); + std::cout << s << "\n"; + } + + options.name_regex = "no_such_name"; + { + auto metrics = metric_mgr::filter_metrics_static(options); + CHECK(metrics.empty()); + auto s = metric_mgr::serialize(metrics); + CHECK(s.empty()); + } + + options = {}; + options.label_regex = "no_such_label"; + { + auto metrics = metric_mgr::filter_metrics_static(options); + CHECK(metrics.empty()); + auto s = metric_mgr::serialize(metrics); + CHECK(s.empty()); + } + + // don't filter + options = {}; + { + auto metrics = metric_mgr::filter_metrics_static(options); + CHECK(metrics.size() == 2); + } + + // black + options.label_regex = ".*ur.*"; + options.is_white = false; + { + auto metrics = metric_mgr::filter_metrics_static(options); + CHECK(metrics.size() == 1); + auto s = metric_mgr::serialize(metrics); + CHECK(s.find("test_static_counter") != std::string::npos); + CHECK(s.find("test_static_counter2") == std::string::npos); + } + + options = {}; + options.label_regex = ".*ur.*"; + options.is_white = false; + { + auto metrics = metric_mgr::filter_metrics_static(options); + CHECK(metrics.size() == 1); + auto s = metric_mgr::serialize(metrics); + CHECK(s.find("test_static_counter") != std::string::npos); + CHECK(s.find("method") != std::string::npos); + CHECK(s.find("test_static_counter2") == std::string::npos); + CHECK(s.find("url") == std::string::npos); + } +} + +TEST_CASE("test filter metrics dynamic") { + using metric_mgr = metric_manager_t<4>; + auto c = metric_mgr::create_metric_dynamic( + "test_dynamic_counter", "", std::vector{{"method"}}); + auto c2 = metric_mgr::create_metric_dynamic( + "test_dynamic_counter2", "", std::vector{{"url"}}); + c->inc({"GET"}); + c->inc({"POST"}); + c2->inc({"/"}); + c2->inc({"/test"}); + + metric_filter_options options; + options.name_regex = ".*counter.*"; + { + auto metrics = metric_mgr::filter_metrics_dynamic(options); + CHECK(metrics.size() == 2); + + auto s = metric_mgr::serialize(metrics); + CHECK(s.find("test_dynamic_counter") != std::string::npos); + std::cout << s << "\n"; + } + + options.label_regex = ".*ur.*"; + { + auto metrics = metric_mgr::filter_metrics_dynamic(options); + CHECK(metrics.size() == 1); + auto s = metric_mgr::serialize(metrics); + CHECK(s.find("test_dynamic_counter2") != std::string::npos); + std::cout << s << "\n"; + } + + options.name_regex = "no_such_name"; + { + auto metrics = metric_mgr::filter_metrics_dynamic(options); + CHECK(metrics.empty()); + auto s = metric_mgr::serialize(metrics); + CHECK(s.empty()); + } + + options = {}; + options.label_regex = "no_such_label"; + { + auto metrics = metric_mgr::filter_metrics_dynamic(options); + CHECK(metrics.empty()); + auto s = metric_mgr::serialize(metrics); + CHECK(s.empty()); + } + + // don't filter + options = {}; + { + auto metrics = metric_mgr::filter_metrics_dynamic(options); + CHECK(metrics.size() == 2); + } + + // black + options.label_regex = ".*ur.*"; + options.is_white = false; + { + auto metrics = metric_mgr::filter_metrics_dynamic(options); + CHECK(metrics.size() == 1); + auto s = metric_mgr::serialize(metrics); + CHECK(s.find("test_dynamic_counter") != std::string::npos); + CHECK(s.find("test_dynamic_counter2") == std::string::npos); + } + + options = {}; + options.label_regex = ".*ur.*"; + options.is_white = false; + { + auto metrics = metric_mgr::filter_metrics_dynamic(options); + CHECK(metrics.size() == 1); + auto s = metric_mgr::serialize(metrics); + CHECK(s.find("test_dynamic_counter") != std::string::npos); + CHECK(s.find("method") != std::string::npos); + CHECK(s.find("test_dynamic_counter2") == std::string::npos); + CHECK(s.find("url") == std::string::npos); + } +} + +TEST_CASE("test get metric by static labels and label") { + using metric_mgr = metric_manager_t<9>; + metric_mgr::create_metric_static( + "http_req_test", "", + std::map{{"method", "GET"}, {"url", "/"}}); + metric_mgr::create_metric_static( + "http_req_test1", "", + std::map{{"method", "POST"}, {"url", "/"}}); + metric_mgr::create_metric_static( + "http_req_test2", "", + std::map{{"method", "GET"}, {"url", "/test"}}); + + auto v = metric_mgr::get_metric_by_labels_static( + std::map{{"method", "GET"}, {"url", "/test"}}); + CHECK(v[0]->name() == "http_req_test2"); + + v = metric_mgr::get_metric_by_labels_static( + std::map{{"method", "GET"}, {"url", "/"}}); + CHECK(v[0]->name() == "http_req_test"); + + auto h1 = metric_mgr::create_metric_static( + "http_req_static_hist", "help", + std::vector{5.23, 10.54, 20.0, 50.0, 100.0}, + std::map{{"method", "GET"}, {"url", "/"}}); + + h1->observe({"GET", "/"}, 23); + + auto s1 = metric_mgr::create_metric_static( + "http_req_static_summary", "help", + summary_t::Quantiles{ + {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, + std::map{{"method", "GET"}, {"url", "/"}}); + s1->observe({"GET", "/"}, 23); + + auto vec = metric_mgr::get_metric_by_label_static({"method", "GET"}); + CHECK(vec.size() == 4); + + vec = metric_mgr::get_metric_by_label_static({"url", "/"}); + CHECK(vec.size() == 4); + + vec = metric_mgr::get_metric_by_label_static({"url", "/test"}); + CHECK(vec.size() == 1); + + vec = metric_mgr::get_metric_by_label_static({"method", "POST"}); + CHECK(vec.size() == 1); + + vec = metric_mgr::get_metric_by_labels_static( + std::map{{"method", "HEAD"}, {"url", "/test"}}); + CHECK(vec.empty()); + + vec = metric_mgr::get_metric_by_labels_static( + std::map{{"method", "GET"}}); + CHECK(vec.empty()); + + vec = metric_mgr::get_metric_by_label_static({"url", "/index"}); + CHECK(vec.empty()); +} + +TEST_CASE("test get metric by dynamic labels") { + using metric_mgr = metric_manager_t<10>; + auto c = metric_mgr::create_metric_dynamic( + "http_req_static", "", std::vector{"method", "code"}); + + auto c1 = metric_mgr::create_metric_dynamic( + "http_req_static1", "", std::vector{"method", "code"}); + + auto c2 = metric_mgr::create_metric_dynamic( + "http_req_static2", "", std::vector{"method", "code"}); + + auto c3 = metric_mgr::create_metric_dynamic( + "http_req_static3", "", std::vector{"method", "code"}); + + c->inc({"POST", "200"}); + c1->inc({"GET", "200"}); + c2->inc({"POST", "301"}); + c3->inc({"POST", "400"}); + + auto c4 = metric_mgr::create_metric_dynamic( + "http_req_static4", "", std::vector{"host", "url"}); + + auto c5 = metric_mgr::create_metric_dynamic( + "http_req_static5", "", std::vector{"host", "url"}); + + c4->inc({"shanghai", "/"}); + c5->inc({"shanghai", "/test"}); + + auto vec = metric_mgr::get_metric_by_labels_dynamic({{"method", "POST"}}); + CHECK(vec.size() == 3); + + vec = metric_mgr::get_metric_by_labels_dynamic({{"method", "GET"}}); + CHECK(vec.size() == 1); + + vec = metric_mgr::get_metric_by_labels_dynamic({{"host", "shanghai"}}); + CHECK(vec.size() == 2); + + vec = metric_mgr::get_metric_by_labels_dynamic({{"url", "/"}}); + CHECK(vec.size() == 1); + + vec = metric_mgr::get_metric_by_labels_dynamic({{"url", "/test"}}); + CHECK(vec.size() == 1); + + vec = metric_mgr::get_metric_by_labels_dynamic({{"url", "/none"}}); + CHECK(vec.size() == 0); + + vec = metric_mgr::get_metric_by_labels_dynamic({{"method", "HEAD"}}); + CHECK(vec.size() == 0); + + auto h1 = metric_mgr::create_metric_dynamic( + "http_req_static_hist", "help", + std::vector{5.23, 10.54, 20.0, 50.0, 100.0}, + std::vector{"method", "url"}); + + h1->observe({"GET", "/"}, 23); + + auto s1 = metric_mgr::create_metric_dynamic( + "http_req_static_summary", "help", + summary_t::Quantiles{ + {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, + std::vector{"method", "url"}); + s1->observe({"GET", "/"}, 23); + + vec = metric_mgr::get_metric_by_labels_dynamic({{"method", "GET"}}); + CHECK(vec.size() >= 2); + + auto str = metric_mgr::serialize(vec); + std::cout << str; + +#ifdef CINATRA_ENABLE_METRIC_JSON + auto json_str = metric_mgr::serialize_to_json(vec); + std::cout << json_str << "\n"; +#endif +} + +TEST_CASE("test histogram serialize with dynamic labels") { + histogram_t h("test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, + std::vector{"method", "url"}); + h.observe({"GET", "/"}, 23); + auto counts = h.get_bucket_counts(); + CHECK(counts[3]->value({"GET", "/"}) == 1); + h.observe({"GET", "/"}, 42); + CHECK(counts[3]->value({"GET", "/"}) == 2); + h.observe({"GET", "/"}, 60); + CHECK(counts[4]->value({"GET", "/"}) == 1); + h.observe({"GET", "/"}, 120); + CHECK(counts[5]->value({"GET", "/"}) == 1); + h.observe({"GET", "/"}, 1); + CHECK(counts[0]->value({"GET", "/"}) == 1); + + h.observe({"POST", "/"}, 23); + CHECK(counts[3]->value({"POST", "/"}) == 1); + h.observe({"POST", "/"}, 42); + CHECK(counts[3]->value({"POST", "/"}) == 2); + h.observe({"POST", "/"}, 60); + CHECK(counts[4]->value({"POST", "/"}) == 1); + h.observe({"POST", "/"}, 120); + CHECK(counts[5]->value({"POST", "/"}) == 1); + h.observe({"POST", "/"}, 1); + CHECK(counts[0]->value({"POST", "/"}) == 1); + + std::string str; + h.serialize(str); + std::cout << str; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string str_json; + h.serialize_to_json(str_json); + std::cout << str_json << "\n"; +#endif +} + +TEST_CASE("test histogram serialize with static labels") { + histogram_t h( + "test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, + std::map{{"method", "GET"}, {"url", "/"}}); + h.observe({"GET", "/"}, 23); + auto counts = h.get_bucket_counts(); + CHECK(counts[3]->value({"GET", "/"}) == 1); + h.observe({"GET", "/"}, 42); + CHECK(counts[3]->value({"GET", "/"}) == 2); + h.observe({"GET", "/"}, 60); + CHECK(counts[4]->value({"GET", "/"}) == 1); + h.observe({"GET", "/"}, 120); + CHECK(counts[5]->value({"GET", "/"}) == 1); + h.observe({"GET", "/"}, 1); + CHECK(counts[0]->value({"GET", "/"}) == 1); + + std::string str; + h.serialize(str); + std::cout << str; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string str_json; + h.serialize_to_json(str_json); + std::cout << str_json << "\n"; +#endif +} + +TEST_CASE("test summary with dynamic labels") { + summary_t summary{"test_summary", + "summary help", + {{0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, + std::vector{"method", "url"}}; + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> distr(1, 100); + for (int i = 0; i < 50; i++) { + summary.observe({"GET", "/"}, distr(gen)); + summary.observe({"POST", "/test"}, distr(gen)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + double sum; + uint64_t count; + auto rates = async_simple::coro::syncAwait( + summary.get_rates({"GET", "/"}, sum, count)); + std::cout << rates.size() << "\n"; + + std::string str; + async_simple::coro::syncAwait(summary.serialize_async(str)); + std::cout << str; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json_str; + async_simple::coro::syncAwait(summary.serialize_to_json_async(json_str)); + std::cout << json_str << "\n"; +#endif +} + +TEST_CASE("test summary with static labels") { + summary_t summary{ + "test_summary", + "summary help", + {{0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, + std::map{{"method", "GET"}, {"url", "/"}}}; + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> distr(1, 100); + for (int i = 0; i < 50; i++) { + summary.observe({"GET", "/"}, distr(gen)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + CHECK_THROWS_AS(summary.observe({"POST", "/"}, 1), std::invalid_argument); + + double sum; + uint64_t count; + auto rates = async_simple::coro::syncAwait( + summary.get_rates({"GET", "/"}, sum, count)); + std::cout << rates.size() << "\n"; + + std::string str; + async_simple::coro::syncAwait(summary.serialize_async(str)); + std::cout << str; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json_str; + async_simple::coro::syncAwait(summary.serialize_to_json_async(json_str)); + std::cout << json_str << "\n"; +#endif +} + DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4007) int main(int argc, char** argv) { return doctest::Context(argc, argv).run(); } DOCTEST_MSVC_SUPPRESS_WARNING_POP \ No newline at end of file diff --git a/website/docs/zh/metric/metrict_introduction.md b/website/docs/zh/metric/metrict_introduction.md index 8d1ce4581..a26d75bac 100644 --- a/website/docs/zh/metric/metrict_introduction.md +++ b/website/docs/zh/metric/metrict_introduction.md @@ -148,18 +148,20 @@ some_counter.inc({"GET", "/"}, 1); 构造函数: ```cpp -// 无标签,调用inc时不带标签,如c.inc() +// 无标签,调用inc时不带标签,如c.inc(),调用此函数则metric 为静态标签的metric // name: 指标对象的名称,注册到指标管理器时会使用这个名称 // help: 指标对象的帮助信息 counter_t(std::string name, std::string help); // labels: 静态标签,构造时需要将标签键值都填完整,如:{{"method", "GET"}, {"url", "/"}} +// 调用此函数则metric 为静态标签的metric // 调用inc时必须带静态标签的值,如:c.inc({"GET", "/"}, 1); counter_t(std::string name, std::string help, std::map labels); // labels_name: 动态标签的键名称,因为标签的值是动态的,而键的名称是固定的,所以这里只需要填键名称,如: {"method", "url"} // 调用时inc时必须带动态标签的值,如:c.inc({method, url}, 1); +// 调用此函数则metric 为动态标签的metric counter_t(std::string name, std::string help, std::vector labels_name); ``` @@ -230,12 +232,18 @@ class metric_t { // 获取标签的键,如{"method", "url"} const std::vector& labels_name(); + // 获取静态标签,如{{"method", "GET"}, {"code", "200"}} + const std::map& get_static_labels(); + // 序列化,调用派生类实现序列化 virtual void serialize(std::string& str); // 给summary专用的api,序列化,调用派生类实现序列化 virtual async_simple::coro::Lazy serialize_async(std::string& out); + // 序列化到json + void serialize_to_json(std::string& str); + // 将基类指针向下转换到派生类指针,如: // std::shared_ptr c = std::make_shared("test", "test"); // counter_t* t = c->as(); @@ -302,6 +310,10 @@ struct metric_manager_t { static bool register_metric_static(std::shared_ptr metric); static bool register_metric_dynamic(std::shared_ptr metric); + // 根据metric名称删除metric + static bool remove_metric_static(const std::string& name); + static bool remove_metric_dynamic(const std::string& name); + // 获取注册的所有指标对象 static std::map> metric_map_static(); static std::map> metric_map_dynamic(); @@ -324,9 +336,43 @@ struct metric_manager_t { static std::shared_ptr get_metric_static(const std::string& name); static std::shared_ptr get_metric_dynamic(const std::string& name); + // 根据静态标签获取所有的指标, 如{{"method", "GET"}, {"url", "/"}} + static std::vector> get_metric_by_labels_static( + const std::map& labels); + + // 根据标签值获取所有的静态标签的指标, 如{"method", "GET"} + static std::vector> get_metric_by_label_static( + const std::pair& label); + + // 根据标签值获取所有动态标签的指标, 如{"method", "GET"} + static std::vector> get_metric_by_labels_dynamic( + const std::map& labels); + // 序列化 static async_simple::coro::Lazy serialize_static(); static async_simple::coro::Lazy serialize_dynamic(); + + // 序列化静态标签的指标到json + static std::string serialize_to_json_static(); + // 序列化动态标签的指标到json + static std::string serialize_to_json_dynamic(); + // 序列化metric集合到json + static std::string serialize_to_json( + const std::vector>& metrics); + + // 过滤配置选项,如果name_regex和label_regex都设置了,则会检查这两个条件,如果只设置了一个则只检查设置过的条件 + struct metric_filter_options { + std::optional name_regex{}; // metric 名称的过滤正则表达式 + std::optional label_regex{};// metric label名称的过滤正则表达式 + bool is_white = true; //true: 白名单,包括语义;false: 黑名单,排除语义 + }; + + // 过滤静态标签的指标 + static std::vector> filter_metrics_static( + const metric_filter_options& options); + // 过滤动态标签的指标 + static std::vector> filter_metrics_dynamic( + const metric_filter_options& options); }; using default_metric_manager = metric_manager_t<0>; ``` @@ -347,11 +393,23 @@ using my_metric_manager = metric_manager_t; // 内部还有一个+Inf 默认的桶,当输入的数据不在前面设置这些桶中,则会落到+Inf 默认桶中。 // 实际上桶的总数为 buckets.size() + 1 // 每个bucket 实际上对应了一个counter指标 +// 调用此函数,则metric为静态metric指标 histogram_t(std::string name, std::string help, std::vector buckets); +// labels_value: 标签key,后面可以使用动态标签值去observe,调用此函数则metric为动态metric 指标 +histogram_t(std::string name, std::string help, std::vector buckets, + std::vector labels_name); + +// labels: 静态标签,调用此函数则metric为静态metric指标 +histogram_t(std::string name, std::string help, std::vector buckets, + std::map labels); + // 往histogram_t 中插入数据,内部会自动增加对应桶的计数 void observe(double value); +// 根据标签值插入数据,可以是动态标签值也可以是静态标签值。如果是静态标签,会做额外的检车,检查传入的labels_value是否和注册时的静态标签值是否相同,不相同会抛异常; +void observe(const std::vector &labels_value, double value); + // 获取所有桶对应的counter指标对象 std::vector> get_bucket_counts(); @@ -416,13 +474,28 @@ test_count 3.000000 ```cpp // Quantiles: 百分位和误差, 如:{{0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}} +// 调用此函数则metric为静态metric 指标 summary_t(std::string name, std::string help, Quantiles quantiles); +// labels_name: 标签名,调用此函数则metric为动态metric 指标 +summary_t(std::string name, std::string help, Quantiles quantiles, std::vector labels_name); + +// static_labels:静态标签,调用此函数则metric为静态metric 指标 +summary_t(std::string name, std::string help, Quantiles quantiles, std::map static_labels); + // 往summary_t插入数据,会自动计算百分位的数量 void observe(double value); -// 获取百分位结果 -async_simple::coro::Lazy> get_rates(); +// 根据标签值(动态或静态的标签值,依据构造函数决定是动态还是静态metric),往summary_t插入数据,会自动计算百分位的数量 +void observe(std::vector labels_value, double value); + +// 获取分位数结果, sum 和count +async_simple::coro::Lazy> get_rates(double &sum, + uint64_t &count) +// 根据标签获取分位数结果, sum 和count +async_simple::coro::Lazy> get_rates( + const std::vector &labels_value, double &sum, + uint64_t &count); // 获取总和 async_simple::coro::Lazy get_sum();