From 6323967e92efdec1c7fca65ea2a4c687c92d1b23 Mon Sep 17 00:00:00 2001 From: qicosmos Date: Wed, 21 Aug 2024 14:51:22 +0800 Subject: [PATCH] Sync code (#624) --- example/main.cpp | 39 +- include/cinatra/coro_http_client.hpp | 4 +- include/cinatra/coro_http_server.hpp | 46 +- include/cinatra/metric_conf.hpp | 42 +- include/cinatra/ylt/coro_io/client_pool.hpp | 97 +- .../ylt/coro_io/detail/client_queue.hpp | 1 + .../coro_io/{channel.hpp => load_blancer.hpp} | 64 +- include/cinatra/ylt/metric/counter.hpp | 593 ++++++--- .../ylt/metric/detail/ckms_quantiles.hpp | 8 +- .../metric/detail/time_window_quantiles.hpp | 4 +- include/cinatra/ylt/metric/gauge.hpp | 109 +- include/cinatra/ylt/metric/histogram.hpp | 236 ++-- include/cinatra/ylt/metric/metric.hpp | 598 ++------- include/cinatra/ylt/metric/metric_manager.hpp | 637 +++++++++ include/cinatra/ylt/metric/summary.hpp | 342 +++-- include/cinatra/ylt/metric/system_metric.hpp | 253 +++- .../cinatra/ylt/metric/thread_local_value.hpp | 112 ++ include/cinatra/ylt/util/concurrentqueue.h | 32 +- tests/test_metric.cpp | 1184 +++++++++++++---- 19 files changed, 2936 insertions(+), 1465 deletions(-) rename include/cinatra/ylt/coro_io/{channel.hpp => load_blancer.hpp} (81%) create mode 100644 include/cinatra/ylt/metric/metric_manager.hpp create mode 100644 include/cinatra/ylt/metric/thread_local_value.hpp diff --git a/example/main.cpp b/example/main.cpp index 0468b051..f9cc8e75 100644 --- a/example/main.cpp +++ b/example/main.cpp @@ -430,12 +430,9 @@ async_simple::coro::Lazy basic_usage() { void use_metric() { using namespace ylt::metric; - auto c = - std::make_shared("request_count", "request count", - std::vector{"method", "url"}); - auto failed = std::make_shared( - "not_found_request_count", "not found request count", - std::vector{"method", "code", "url"}); + auto c = std::make_shared("request_count", "request count"); + auto failed = std::make_shared("not_found_request_count", + "not found request count"); auto total = std::make_shared("total_request_count", "total request count"); @@ -448,11 +445,11 @@ void use_metric() { summary_t::Quantiles{ {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}); - default_metric_manager::register_metric_dynamic(c); - default_metric_manager::register_metric_dynamic(total); - default_metric_manager::register_metric_dynamic(failed); - default_metric_manager::register_metric_dynamic(h); - default_metric_manager::register_metric_dynamic(summary); + default_static_metric_manager::instance().register_metric(c); + default_static_metric_manager::instance().register_metric(total); + default_static_metric_manager::instance().register_metric(failed); + default_static_metric_manager::instance().register_metric(h); + default_static_metric_manager::instance().register_metric(summary); std::random_device rd; std::mt19937 gen(rd()); @@ -460,7 +457,7 @@ void use_metric() { std::thread thd([&] { while (true) { - c->inc({"GET", "/test"}); + c->inc(); total->inc(); h->observe(distr(gen)); summary->observe(distr(gen)); @@ -473,9 +470,7 @@ void use_metric() { server.set_default_handler( [&](coro_http_request &req, coro_http_response &resp) -> async_simple::coro::Lazy { - failed->inc({std::string(req.get_method()), - std::to_string((int)status_type::not_found), - std::string(req.get_url())}); + failed->inc(); total->inc(); resp.set_status_and_content(status_type::not_found, "not found"); co_return; @@ -484,14 +479,14 @@ void use_metric() { server.set_http_handler( "/get", [&](coro_http_request &req, coro_http_response &resp) { resp.set_status_and_content(status_type::ok, "ok"); - c->inc({std::string(req.get_method()), std::string(req.get_url())}); + c->inc(); total->inc(); }); server.set_http_handler( "/test", [&](coro_http_request &req, coro_http_response &resp) { resp.set_status_and_content(status_type::ok, "ok"); - c->inc({std::string(req.get_method()), std::string(req.get_url())}); + c->inc(); total->inc(); }); @@ -514,13 +509,13 @@ void metrics_example() { "get_req_count", "get req count", std::map{{"url", "/get"}}); auto get_req_qps = std::make_shared("get_req_qps", "get req qps"); - // default_metric_manager::register_metric_static(get_req_counter, + // default_static_metric_manager::instance().register_metric_static(get_req_counter, // get_req_qps); int64_t last = 0; std::thread thd([&] { while (true) { std::this_thread::sleep_for(1s); - auto value = get_req_counter->value({"/get"}); + auto value = get_req_counter->value(); get_req_qps->update(value - last); last = value; } @@ -551,8 +546,8 @@ async_simple::coro::Lazy use_channel() { server.async_start(); std::this_thread::sleep_for(100ms); - auto channel = std::make_shared>( - coro_io::channel::create( + auto channel = std::make_shared>( + coro_io::load_blancer::create( {"127.0.0.1:9001"}, {.lba = coro_io::load_blance_algorithm::random})); std::string url = "http://127.0.0.1:9001/"; co_await channel->send_request( @@ -573,7 +568,7 @@ async_simple::coro::Lazy use_pool() { server.use_metrics(); server.async_start(); - auto map = default_metric_manager::metric_map_static(); + auto map = default_static_metric_manager::instance().metric_map(); for (auto &[k, m] : map) { std::cout << k << ", "; std::cout << m->help() << "\n"; diff --git a/include/cinatra/coro_http_client.hpp b/include/cinatra/coro_http_client.hpp index 66e7e892..a8a6f668 100644 --- a/include/cinatra/coro_http_client.hpp +++ b/include/cinatra/coro_http_client.hpp @@ -884,7 +884,7 @@ class coro_http_client : public std::enable_shared_from_this { co_return; } std::string file_data; - detail::resize(file_data, std::min(max_single_part_size_, length)); + detail::resize(file_data, (std::min)(max_single_part_size_, length)); coro_io::coro_file file{}; file.open(source, std::ios::in); file.seek(offset, std::ios::cur); @@ -895,7 +895,7 @@ class coro_http_client : public std::enable_shared_from_this { std::size_t size; while (length > 0) { if (std::tie(ec, size) = co_await file.async_read( - file_data.data(), std::min(file_data.size(), length)); + file_data.data(), (std::min)(file_data.size(), length)); ec) { // bad request, file may smaller than content-length break; diff --git a/include/cinatra/coro_http_server.hpp b/include/cinatra/coro_http_server.hpp index bbdfbead..1bc287b0 100644 --- a/include/cinatra/coro_http_server.hpp +++ b/include/cinatra/coro_http_server.hpp @@ -7,10 +7,10 @@ #include "cinatra/mime_types.hpp" #include "cinatra_log_wrapper.hpp" #include "coro_http_connection.hpp" -#include "ylt/coro_io/channel.hpp" #include "ylt/coro_io/coro_file.hpp" #include "ylt/coro_io/coro_io.hpp" #include "ylt/coro_io/io_context_pool.hpp" +#include "ylt/coro_io/load_blancer.hpp" #include "ylt/metric/system_metric.hpp" namespace cinatra { @@ -185,9 +185,9 @@ class coro_http_server { void use_metrics(bool enable_json = false, std::string url_path = "/metrics") { init_metrics(); - using root = - ylt::metric::metric_collector_t; + using root = ylt::metric::metric_collector_t< + ylt::metric::default_static_metric_manager, + ylt::metric::system_metric_manager>; set_http_handler( url_path, [enable_json](coro_http_request &req, coro_http_response &res) { @@ -216,14 +216,15 @@ class coro_http_server { throw std::invalid_argument("not config hosts yet!"); } - auto channel = std::make_shared>( - coro_io::channel::create(hosts, {.lba = type}, - weights)); + auto load_blancer = + std::make_shared>( + coro_io::load_blancer::create( + hosts, {.lba = type}, weights)); auto handler = - [this, channel, type]( + [this, load_blancer, type]( coro_http_request &req, coro_http_response &response) -> async_simple::coro::Lazy { - co_await channel->send_request( + co_await load_blancer->send_request( [this, &req, &response]( coro_http_client &client, std::string_view host) -> async_simple::coro::Lazy { @@ -255,14 +256,15 @@ class coro_http_server { throw std::invalid_argument("not config hosts yet!"); } - auto channel = std::make_shared>( - coro_io::channel::create(hosts, {.lba = type}, - weights)); + auto load_blancer = + std::make_shared>( + coro_io::load_blancer::create( + hosts, {.lba = type}, weights)); set_http_handler( url_path, - [channel](coro_http_request &req, - coro_http_response &resp) -> async_simple::coro::Lazy { + [load_blancer](coro_http_request &req, coro_http_response &resp) + -> async_simple::coro::Lazy { websocket_result result{}; while (true) { result = co_await req.get_conn()->read_websocket(); @@ -275,7 +277,7 @@ class coro_http_server { break; } - co_await channel->send_request( + co_await load_blancer->send_request( [&req, result]( coro_http_client &client, std::string_view host) -> async_simple::coro::Lazy { @@ -928,20 +930,20 @@ class coro_http_server { using namespace ylt::metric; cinatra_metric_conf::enable_metric = true; - default_metric_manager::create_metric_static( + default_static_metric_manager::instance().create_metric_static( cinatra_metric_conf::server_total_req, ""); - default_metric_manager::create_metric_static( + default_static_metric_manager::instance().create_metric_static( cinatra_metric_conf::server_failed_req, ""); - default_metric_manager::create_metric_static( + default_static_metric_manager::instance().create_metric_static( cinatra_metric_conf::server_total_recv_bytes, ""); - default_metric_manager::create_metric_static( + default_static_metric_manager::instance().create_metric_static( cinatra_metric_conf::server_total_send_bytes, ""); - default_metric_manager::create_metric_static( + default_static_metric_manager::instance().create_metric_static( cinatra_metric_conf::server_total_fd, ""); - default_metric_manager::create_metric_static( + default_static_metric_manager::instance().create_metric_static( cinatra_metric_conf::server_req_latency, "", std::vector{30, 40, 50, 60, 70, 80, 90, 100, 150}); - default_metric_manager::create_metric_static( + default_static_metric_manager::instance().create_metric_static( cinatra_metric_conf::server_read_latency, "", std::vector{3, 5, 7, 9, 13, 18, 23, 35, 50}); #if defined(__GNUC__) diff --git a/include/cinatra/metric_conf.hpp b/include/cinatra/metric_conf.hpp index 4800481a..3114fbb7 100644 --- a/include/cinatra/metric_conf.hpp +++ b/include/cinatra/metric_conf.hpp @@ -6,7 +6,9 @@ #include "ylt/metric/gauge.hpp" #include "ylt/metric/histogram.hpp" #include "ylt/metric/metric.hpp" +#include "ylt/metric/metric_manager.hpp" #include "ylt/metric/summary.hpp" +#include "ylt/metric/system_metric.hpp" namespace cinatra { struct cinatra_metric_conf { @@ -25,8 +27,9 @@ struct cinatra_metric_conf { return; } - static auto m = ylt::metric::default_metric_manager::get_metric_static< - ylt::metric::counter_t>(server_total_req); + static auto m = + ylt::metric::default_static_metric_manager::instance() + .get_metric_static(server_total_req); if (m == nullptr) { return; } @@ -37,8 +40,9 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = ylt::metric::default_metric_manager::get_metric_static< - ylt::metric::counter_t>(server_failed_req); + static auto m = + ylt::metric::default_static_metric_manager::instance() + .get_metric_static(server_failed_req); if (m == nullptr) { return; } @@ -49,8 +53,9 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = ylt::metric::default_metric_manager::get_metric_static< - ylt::metric::gauge_t>(server_total_fd); + static auto m = + ylt::metric::default_static_metric_manager::instance() + .get_metric_static(server_total_fd); if (m == nullptr) { return; } @@ -61,8 +66,9 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = ylt::metric::default_metric_manager::get_metric_static< - ylt::metric::gauge_t>(server_total_fd); + static auto m = + ylt::metric::default_static_metric_manager::instance() + .get_metric_static(server_total_fd); if (m == nullptr) { return; } @@ -73,8 +79,9 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = ylt::metric::default_metric_manager::get_metric_static< - ylt::metric::counter_t>(server_total_recv_bytes); + static auto m = + ylt::metric::default_static_metric_manager::instance() + .get_metric_static(server_total_recv_bytes); if (m == nullptr) { return; } @@ -85,8 +92,9 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = ylt::metric::default_metric_manager::get_metric_static< - ylt::metric::counter_t>(server_total_send_bytes); + static auto m = + ylt::metric::default_static_metric_manager::instance() + .get_metric_static(server_total_send_bytes); if (m == nullptr) { return; } @@ -97,8 +105,9 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = ylt::metric::default_metric_manager::get_metric_static< - ylt::metric::histogram_t>(server_req_latency); + static auto m = + ylt::metric::default_static_metric_manager::instance() + .get_metric_static(server_req_latency); if (m == nullptr) { return; } @@ -109,8 +118,9 @@ struct cinatra_metric_conf { if (!enable_metric) { return; } - static auto m = ylt::metric::default_metric_manager::get_metric_static< - ylt::metric::histogram_t>(server_read_latency); + static auto m = + ylt::metric::default_static_metric_manager::instance() + .get_metric_static(server_read_latency); if (m == nullptr) { return; } diff --git a/include/cinatra/ylt/coro_io/client_pool.hpp b/include/cinatra/ylt/coro_io/client_pool.hpp index 89af0393..37a76839 100644 --- a/include/cinatra/ylt/coro_io/client_pool.hpp +++ b/include/cinatra/ylt/coro_io/client_pool.hpp @@ -54,7 +54,7 @@ template class client_pools; template -class channel; +class load_blancer; template @@ -112,6 +112,20 @@ class client_pool : public std::enable_shared_from_this< return std::chrono::milliseconds{static_cast(e(r) * ms.count())}; } + static async_simple::coro::Lazy> + reconnect_impl(std::unique_ptr& client, + std::shared_ptr& self) { + auto pre_time_point = std::chrono::steady_clock::now(); + auto result = co_await client->connect(self->host_name_); + bool ok = client_t::is_ok(result); + auto post_time_point = std::chrono::steady_clock::now(); + auto cost_time = std::chrono::duration_cast( + post_time_point - pre_time_point); + CINATRA_LOG_TRACE << "reconnect client{" << client.get() << "} cost time: " + << cost_time / std::chrono::milliseconds{1} << "ms"; + co_return std::pair{ok, cost_time}; + } + static async_simple::coro::Lazy reconnect( std::unique_ptr& client, std::weak_ptr watcher) { using namespace std::chrono_literals; @@ -123,13 +137,7 @@ class client_pool : public std::enable_shared_from_this< << client->get_port() << "}, try count:" << i + 1 << "max retry limit:" << self->pool_config_.connect_retry_count; - auto pre_time_point = std::chrono::steady_clock::now(); - bool ok = client_t::is_ok(co_await client->connect(self->host_name_)); - auto post_time_point = std::chrono::steady_clock::now(); - auto cost_time = post_time_point - pre_time_point; - CINATRA_LOG_TRACE << "reconnect client{" << client.get() - << "} cost time: " - << cost_time / std::chrono::milliseconds{1} << "ms"; + auto [ok, cost_time] = co_await reconnect_impl(client, self); if (ok) { CINATRA_LOG_TRACE << "reconnect client{" << client.get() << "} success"; co_return; @@ -148,9 +156,70 @@ class client_pool : public std::enable_shared_from_this< CINATRA_LOG_WARNING << "reconnect client{" << client.get() << "},host:{" << client->get_host() << ":" << client->get_port() << "} out of max limit, stop retry. connect failed"; + alive_detect(client->get_config(), std::move(self)).start([](auto&&) { + }); client = nullptr; } + static async_simple::coro::Lazy alive_detect( + const typename client_t::config& client_config, + std::weak_ptr watcher) { + std::shared_ptr self = watcher.lock(); + using namespace std::chrono_literals; + if (self && self->pool_config_.host_alive_detect_duration.count() != 0 && + self->free_client_count() == 0) { + bool expected = true; + if (!self->is_alive_.compare_exchange_strong( + expected, false)) { // other alive detect coroutine is running. + co_return; + } + if (self->free_client_count() > 0) { // recheck for multi-thread + self->is_alive_ = true; + co_return; + } + auto executor = self->io_context_pool_.get_executor(); + auto client = std::make_unique(*executor); + if (!client->init_config(client_config)) + AS_UNLIKELY { + CINATRA_LOG_ERROR + << "Init client config failed in host alive detect. That " + "is not expected."; + co_return; + } + while (true) { + auto [ok, cost_time] = co_await reconnect_impl(client, self); + if (ok) { + CINATRA_LOG_TRACE << "reconnect client{" << client.get() + << "} success. stop alive detect."; + self->collect_free_client(std::move(client)); + self->is_alive_ = + true; /*if client close(), we still mark it as alive*/ + co_return; + } + if (self->is_alive_) { + CINATRA_LOG_TRACE << "client pool is aliving, stop connect client {" + << client.get() << "} for alive detect"; + co_return; + } + CINATRA_LOG_TRACE << "reconnect client{" << client.get() + << "} failed. continue alive detect."; + auto wait_time = rand_time( + (self->pool_config_.host_alive_detect_duration - cost_time) / 1ms * + 1ms); + self = nullptr; + if (wait_time.count() > 0) { + co_await coro_io::sleep_for(wait_time, &client->get_executor()); + } + self = watcher.lock(); + if (self->is_alive_) { + CINATRA_LOG_TRACE << "client pool is aliving, stop connect client {" + << client.get() << "} for alive detect"; + co_return; + } + } + } + } + async_simple::coro::Lazy> get_client( const typename client_t::config& client_config) { std::unique_ptr client; @@ -211,6 +280,7 @@ class client_pool : public std::enable_shared_from_this< enqueue(short_connect_clients_, std::move(client), pool_config_.short_connect_idle_timeout); } + is_alive_ = true; } else { CINATRA_LOG_TRACE << "client{" << client.get() @@ -248,6 +318,8 @@ class client_pool : public std::enable_shared_from_this< std::chrono::milliseconds reconnect_wait_time{1000}; std::chrono::milliseconds idle_timeout{30000}; std::chrono::milliseconds short_connect_idle_timeout{1000}; + std::chrono::milliseconds host_alive_detect_duration{ + 30000}; /* zero means wont detect */ typename client_t::config client_config; }; @@ -315,6 +387,12 @@ class client_pool : public std::enable_shared_from_this< std::size_t free_client_count() const noexcept { return free_clients_.size() + short_connect_clients_.size(); } + /** + * @brief if host may not useable now. + * + * @return bool + */ + bool is_alive() const noexcept { return is_alive_; } /** * @brief approx connection of client pools @@ -330,7 +408,7 @@ class client_pool : public std::enable_shared_from_this< friend class client_pools; template - friend class channel; + friend class load_blancer; template async_simple::coro::Lazy> send_request( @@ -371,6 +449,7 @@ class client_pool : public std::enable_shared_from_this< std::string host_name_; pool_config pool_config_; io_context_pool_t& io_context_pool_; + std::atomic is_alive_ = true; }; template class client_queue { moodycamel::ConcurrentQueue queue_[2]; diff --git a/include/cinatra/ylt/coro_io/channel.hpp b/include/cinatra/ylt/coro_io/load_blancer.hpp similarity index 81% rename from include/cinatra/ylt/coro_io/channel.hpp rename to include/cinatra/ylt/coro_io/load_blancer.hpp index a7962c97..15e7fa13 100644 --- a/include/cinatra/ylt/coro_io/channel.hpp +++ b/include/cinatra/ylt/coro_io/load_blancer.hpp @@ -32,15 +32,15 @@ enum class load_blance_algorithm { }; template -class channel { +class load_blancer { using client_pool_t = client_pool; using client_pools_t = client_pools; public: - struct channel_config { + struct load_blancer_config { typename client_pool_t::pool_config pool_config; load_blance_algorithm lba = load_blance_algorithm::RR; - ~channel_config(){}; + ~load_blancer_config(){}; }; private: @@ -48,9 +48,10 @@ class channel { std::unique_ptr> index = std::make_unique>(); async_simple::coro::Lazy> operator()( - const channel& channel) { + const load_blancer& load_blancer) { auto i = index->fetch_add(1, std::memory_order_relaxed); - co_return channel.client_pools_[i % channel.client_pools_.size()]; + co_return load_blancer + .client_pools_[i % load_blancer.client_pools_.size()]; } }; @@ -84,14 +85,15 @@ class channel { } async_simple::coro::Lazy> operator()( - const channel& channel) { + const load_blancer& load_blancer) { int selected = select_host_with_weight_round_robin(); if (selected == -1) { selected = 0; } wrr_current_ = selected; - co_return channel.client_pools_[selected % channel.client_pools_.size()]; + co_return load_blancer + .client_pools_[selected % load_blancer.client_pools_.size()]; } private: @@ -138,27 +140,27 @@ class channel { struct RandomLoadBlancer { async_simple::coro::Lazy> operator()( - const channel& channel) { + const load_blancer& load_blancer) { static thread_local std::default_random_engine e(std::time(nullptr)); std::uniform_int_distribution rnd{std::size_t{0}, - channel.client_pools_.size() - 1}; - co_return channel.client_pools_[rnd(e)]; + load_blancer.client_pools_.size() - 1}; + co_return load_blancer.client_pools_[rnd(e)]; } }; - channel() = default; + load_blancer() = default; public: - channel(channel&& o) + load_blancer(load_blancer&& o) : config_(std::move(o.config_)), lb_worker(std::move(o.lb_worker)), client_pools_(std::move(o.client_pools_)){}; - channel& operator=(channel&& o) { + load_blancer& operator=(load_blancer&& o) { this->config_ = std::move(o.config_); this->lb_worker = std::move(o.lb_worker); this->client_pools_ = std::move(o.client_pools_); } - channel(const channel& o) = delete; - channel& operator=(const channel& o) = delete; + load_blancer(const load_blancer& o) = delete; + load_blancer& operator=(const load_blancer& o) = delete; auto send_request(auto op, typename client_t::config& config) -> decltype(std::declval().send_request(std::move(op), @@ -166,11 +168,14 @@ class channel { config)) { std::shared_ptr client_pool; if (client_pools_.size() > 1) { - client_pool = co_await std::visit( - [this](auto& worker) { - return worker(*this); - }, - lb_worker); + int cnt = 0; + do { + client_pool = co_await std::visit( + [this](auto& worker) { + return worker(*this); + }, + lb_worker); + } while (!client_pool->is_alive() && ++cnt <= size() * 2); } else { client_pool = client_pools_[0]; @@ -182,18 +187,19 @@ class channel { return send_request(std::move(op), config_.pool_config.client_config); } - static channel create(const std::vector& hosts, - const channel_config& config = {}, - const std::vector& weights = {}, - client_pools_t& client_pools = - g_clients_pool()) { - channel ch; + static load_blancer create( + const std::vector& hosts, + const load_blancer_config& config = {}, + const std::vector& weights = {}, + client_pools_t& client_pools = + g_clients_pool()) { + load_blancer ch; ch.init(hosts, config, weights, client_pools); return ch; } /** - * @brief return the channel's hosts size. + * @brief return the load_blancer's hosts size. * * @return std::size_t */ @@ -201,7 +207,7 @@ class channel { private: void init(const std::vector& hosts, - const channel_config& config, const std::vector& weights, + const load_blancer_config& config, const std::vector& weights, client_pools_t& client_pools) { config_ = config; client_pools_.reserve(hosts.size()); @@ -227,7 +233,7 @@ class channel { } return; } - channel_config config_; + load_blancer_config config_; std::variant lb_worker; std::vector> client_pools_; }; diff --git a/include/cinatra/ylt/metric/counter.hpp b/include/cinatra/ylt/metric/counter.hpp index 59429705..750154b2 100644 --- a/include/cinatra/ylt/metric/counter.hpp +++ b/include/cinatra/ylt/metric/counter.hpp @@ -1,16 +1,20 @@ #pragma once +#include +#include #include #include +#include #include "metric.hpp" +#include "thread_local_value.hpp" namespace ylt::metric { enum class op_type_t { INC, DEC, SET }; #ifdef CINATRA_ENABLE_METRIC_JSON struct json_counter_metric_t { - std::unordered_multimap labels; - int64_t value; + std::map labels; + std::variant value; }; REFLECTION(json_counter_metric_t, labels, value); struct json_counter_t { @@ -22,288 +26,493 @@ struct json_counter_t { REFLECTION(json_counter_t, name, help, type, metrics); #endif -class counter_t : public metric_t { +template +inline void set_value(T &label_val, value_type value, op_type_t type) { + switch (type) { + case op_type_t::INC: { +#ifdef __APPLE__ + if constexpr (std::is_floating_point_v) { + mac_os_atomic_fetch_add(&label_val, value); + } + else { + label_val += value; + } +#else + label_val += value; +#endif + } break; + case op_type_t::DEC: +#ifdef __APPLE__ + if constexpr (std::is_floating_point_v) { + mac_os_atomic_fetch_sub(&label_val, value); + } + else { + label_val -= value; + } +#else + label_val -= value; +#endif + break; + case op_type_t::SET: + label_val = value; + break; + } +} + +template +class basic_static_counter : public static_metric { public: - // default, no labels, only contains an atomic value. - counter_t(std::string name, std::string help) - : metric_t(MetricType::Counter, std::move(name), std::move(help)) { - use_atomic_ = true; - g_user_metric_count++; + // static counter, no labels, only contains an atomic value. + basic_static_counter(std::string name, std::string help, + size_t dupli_count = 2) + : static_metric(MetricType::Counter, std::move(name), std::move(help)) { + init_thread_local(dupli_count); } - // static labels value, contains a map with atomic value. - counter_t(std::string name, std::string help, - std::map labels) - : metric_t(MetricType::Counter, std::move(name), std::move(help), - std::move(labels)) { - atomic_value_map_.emplace(labels_value_, 0); - g_user_metric_count++; - use_atomic_ = true; + // static counter, contains a static labels with atomic value. + basic_static_counter(std::string name, std::string help, + std::map labels, + uint32_t dupli_count = 2) + : static_metric(MetricType::Counter, std::move(name), std::move(help), + std::move(labels)) { + init_thread_local(dupli_count); } - // dynamic labels value - counter_t(std::string name, std::string help, - std::vector labels_name) - : metric_t(MetricType::Counter, std::move(name), std::move(help), - std::move(labels_name)) { + void init_thread_local(uint32_t dupli_count) { + if (dupli_count > 0) { + dupli_count_ = dupli_count; + default_label_value_ = {dupli_count}; + } + g_user_metric_count++; } - virtual ~counter_t() { g_user_metric_count--; } + virtual ~basic_static_counter() { g_user_metric_count--; } - double value() { return default_label_value_; } + void inc(value_type val = 1) { + if (val <= 0) { + return; + } - double value(const std::vector &labels_value) { - if (use_atomic_) { - double val = atomic_value_map_[labels_value]; - return val; +#ifdef __APPLE__ + if constexpr (std::is_floating_point_v) { + mac_os_atomic_fetch_add(&default_label_value_.local_value(), val); } else { - std::lock_guard lock(mtx_); - return value_map_[labels_value]; + default_label_value_.inc(val); } +#else + default_label_value_.inc(val); +#endif } - metric_hash_map value_map() override { - metric_hash_map map; - if (use_atomic_) { - map = {atomic_value_map_.begin(), atomic_value_map_.end()}; - } - else { - std::lock_guard lock(mtx_); - map = value_map_; + value_type update(value_type value) { + if (!has_change_) [[unlikely]] { + has_change_ = true; } - return map; + return default_label_value_.update(value); } - void serialize(std::string &str) override { - if (labels_name_.empty()) { - if (default_label_value_ == 0) { - return; - } - serialize_head(str); - serialize_default_label(str); - return; - } + value_type reset() { return default_label_value_.reset(); } - auto map = value_map(); - if (map.empty()) { + value_type value() { return default_label_value_.value(); } + + void serialize(std::string &str) override { + auto value = default_label_value_.value(); + if (value == 0 && !has_change_) { return; } - std::string value_str; - serialize_map(map, value_str); - if (!value_str.empty()) { - serialize_head(str); - str.append(value_str); - } + serialize_head(str); + serialize_default_label(str, value); } #ifdef CINATRA_ENABLE_METRIC_JSON void serialize_to_json(std::string &str) override { - std::string s; - if (labels_name_.empty()) { - if (default_label_value_ == 0) { - return; - } - json_counter_t counter{name_, help_, std::string(metric_name())}; - int64_t value = default_label_value_; - counter.metrics.push_back({{}, value}); - iguana::to_json(counter, str); + if (default_label_value_.value() == 0) { return; } - auto map = value_map(); json_counter_t counter{name_, help_, std::string(metric_name())}; - to_json(counter, map, str); + auto value = default_label_value_.value(); + counter.metrics.push_back({static_labels_, value}); + iguana::to_json(counter, str); } +#endif - template - void to_json(json_counter_t &counter, T &map, std::string &str) { - for (auto &[k, v] : map) { - if (v == 0) { - continue; - } - json_counter_metric_t metric; - size_t index = 0; - for (auto &label_value : k) { - metric.labels.emplace(labels_name_[index++], label_value); - } - metric.value = (int64_t)v; - counter.metrics.push_back(std::move(metric)); + protected: + void serialize_default_label(std::string &str, value_type value) { + str.append(name_); + if (labels_name_.empty()) { + str.append(" "); } - if (!counter.metrics.empty()) { - iguana::to_json(counter, str); + else { + str.append("{"); + build_string(str, labels_name_, labels_value_); + str.append("} "); } + + str.append(std::to_string(value)); + + str.append("\n"); } -#endif - void inc(double val = 1) { - if (val < 0) { - throw std::invalid_argument("the value is less than zero"); + void build_string(std::string &str, const std::vector &v1, + const std::vector &v2) { + for (size_t i = 0; i < v1.size(); i++) { + str.append(v1[i]).append("=\"").append(v2[i]).append("\"").append(","); } + str.pop_back(); + } -#ifdef __APPLE__ - mac_os_atomic_fetch_add(&default_label_value_, val); -#else - default_label_value_ += val; -#endif + thread_local_value default_label_value_; + uint32_t dupli_count_ = 2; + bool has_change_ = false; +}; + +template +struct array_hash { + size_t operator()(const Key &arr) const { + unsigned int seed = 131; + unsigned int hash = 0; + + for (const auto &str : arr) { + for (auto ch : str) { + hash = hash * seed + ch; + } + } + + return (hash & 0x7FFFFFFF); + } +}; + +using counter_t = basic_static_counter; +using counter_d = basic_static_counter; + +template +using dynamic_metric_hash_map = std::unordered_map>; + +template +class basic_dynamic_counter : public dynamic_metric { + public: + // dynamic labels value + basic_dynamic_counter(std::string name, std::string help, + std::array labels_name, + size_t dupli_count = 2) + : dynamic_metric(MetricType::Counter, std::move(name), std::move(help), + std::move(labels_name)), + dupli_count_(dupli_count) { + g_user_metric_count++; } - void inc(const std::vector &labels_value, double value = 1) { + virtual ~basic_dynamic_counter() { g_user_metric_count--; } + + void inc(const std::array &labels_value, + value_type value = 1) { if (value == 0) { return; } - validate(labels_value, value); - if (use_atomic_) { - if (labels_value != labels_value_) { - throw std::invalid_argument( - "the given labels_value is not match with origin labels_value"); + std::unique_lock lock(mtx_); + if (value_map_.size() > ylt_label_capacity) { + return; + } + auto [it, r] = value_map_.try_emplace( + labels_value, thread_local_value(dupli_count_)); + lock.unlock(); + if (r) { + g_user_metric_label_count->local_value()++; + if (ylt_label_max_age.count()) { + it->second.set_created_time(std::chrono::system_clock::now()); + } + } + set_value(it->second.local_value(), value, op_type_t::INC); + } + + value_type update(const std::array &labels_value, + value_type value) { + std::unique_lock lock(mtx_); + if (value_map_.size() > ylt_label_capacity) { + return value_type{}; + } + if (!has_change_) [[unlikely]] + has_change_ = true; + auto [it, r] = value_map_.try_emplace( + labels_value, thread_local_value(dupli_count_)); + lock.unlock(); + if (r) { + g_user_metric_label_count->local_value()++; + if (ylt_label_max_age.count()) { + it->second.set_created_time(std::chrono::system_clock::now()); } - set_value(atomic_value_map_[labels_value], value, op_type_t::INC); } - else { + return it->second.update(value); + } + + value_type value(const std::array &labels_value) { + std::lock_guard lock(mtx_); + if (auto it = value_map_.find(labels_value); it != value_map_.end()) { + return it->second.value(); + } + + return value_type{}; + } + + value_type reset() { + value_type val = {}; + + std::lock_guard lock(mtx_); + for (auto &[key, t] : value_map_) { + val += t.reset(); + } + + return val; + } + + dynamic_metric_hash_map, + thread_local_value> + value_map() { + [[maybe_unused]] bool has_change = false; + return value_map(has_change); + } + + dynamic_metric_hash_map, + thread_local_value> + value_map(bool &has_change) { + dynamic_metric_hash_map, + thread_local_value> + map; + { std::lock_guard lock(mtx_); - stat_metric(labels_value); - set_value(value_map_[labels_value], value, op_type_t::INC); + map = value_map_; + has_change = has_change_; } + + return map; } - void stat_metric(const std::vector &labels_value) { - if (!value_map_.contains(labels_value)) { - g_user_metric_labels->inc(); + size_t label_value_count() override { + std::lock_guard lock(mtx_); + return value_map_.size(); + } + + void clean_expired_label() override { + if (ylt_label_max_age.count() == 0) { + return; } + + auto now = std::chrono::system_clock::now(); + std::lock_guard lock(mtx_); + std::erase_if(value_map_, [&now](auto &pair) mutable { + bool r = std::chrono::duration_cast( + now - pair.second.get_created_time()) + .count() >= ylt_label_max_age.count(); + return r; + }); } - void update(double value) { default_label_value_ = value; } + void remove_label_value( + const std::map &labels) override { + { + std::lock_guard lock(mtx_); + if (value_map_.empty()) { + return; + } + } - void update(const std::vector &labels_value, double value) { - if (labels_value.empty() || labels_name_.size() != labels_value.size()) { - throw std::invalid_argument( - "the number of labels_value name and labels_value is not match"); + const auto &labels_name = this->labels_name(); + if (labels.size() > labels_name.size()) { + return; } - if (use_atomic_) { - if (labels_value != labels_value_) { - throw std::invalid_argument( - "the given labels_value is not match with origin labels_value"); + + if (labels.size() == labels_name.size()) { + std::vector label_value; + for (auto &lb_name : labels_name) { + if (auto i = labels.find(lb_name); i != labels.end()) { + label_value.push_back(i->second); + } } - set_value(atomic_value_map_[labels_value], value, op_type_t::SET); + + std::lock_guard lock(mtx_); + std::erase_if(value_map_, [&, this](auto &pair) { + return equal(label_value, pair.first); + }); + return; } else { + std::vector vec; + for (auto &lb_name : labels_name) { + if (auto i = labels.find(lb_name); i != labels.end()) { + vec.push_back(i->second); + } + else { + vec.push_back(""); + } + } + if (vec.empty()) { + return; + } + std::lock_guard lock(mtx_); - set_value(value_map_[labels_value], value, op_type_t::SET); + std::erase_if(value_map_, [&](auto &pair) { + auto &[arr, _] = pair; + for (size_t i = 0; i < vec.size(); i++) { + if (!vec[i].empty() && vec[i] != arr[i]) { + return false; + } + } + return true; + }); } } - metric_hash_map> &atomic_value_map() { - return atomic_value_map_; + bool has_label_value(const std::string &value) override { + [[maybe_unused]] bool has_change = false; + auto map = value_map(has_change); + for (auto &[label_value, _] : map) { + if (auto it = std::find(label_value.begin(), label_value.end(), value); + it != label_value.end()) { + return true; + } + } + + return false; } - protected: - void serialize_default_label(std::string &str) { - str.append(name_); - if (labels_name_.empty()) { - str.append(" "); + bool has_label_value(const std::regex ®ex) override { + [[maybe_unused]] bool has_change = false; + auto map = value_map(has_change); + for (auto &[label_value, _] : map) { + if (auto it = std::find_if(label_value.begin(), label_value.end(), + [&](auto &val) { + return std::regex_match(val, regex); + }); + it != label_value.end()) { + return true; + } } - if (type_ == MetricType::Counter) { - str.append(std::to_string((int64_t)default_label_value_)); + return false; + } + + bool has_label_value(const std::vector &label_value) override { + std::array arr{}; + size_t size = (std::min)((size_t)N, label_value.size()); + for (size_t i = 0; i < size; i++) { + arr[i] = label_value[i]; } - else { - str.append(std::to_string(default_label_value_)); + std::lock_guard lock(mtx_); + return value_map_.contains(arr); + } + + void serialize(std::string &str) override { + bool has_change = false; + auto map = value_map(has_change); + if (map.empty()) { + return; } - str.append("\n"); + std::string value_str; + serialize_map(map, value_str, has_change); + if (!value_str.empty()) { + serialize_head(str); + str.append(value_str); + } } +#ifdef CINATRA_ENABLE_METRIC_JSON + void serialize_to_json(std::string &str) override { + std::string s; + bool has_change = false; + auto map = value_map(has_change); + json_counter_t counter{name_, help_, std::string(metric_name())}; + to_json(counter, map, str, has_change); + } + + template + void to_json(json_counter_t &counter, T &map, std::string &str, + bool has_change) { + for (auto &[k, v] : map) { + auto val = v.value(); + if (val == 0 && !has_change) { + continue; + } + json_counter_metric_t metric; + size_t index = 0; + for (auto &label_value : k) { + metric.labels.emplace(labels_name_[index++], label_value); + } + metric.value = (int64_t)val; + counter.metrics.push_back(std::move(metric)); + } + if (!counter.metrics.empty()) { + iguana::to_json(counter, str); + } + } +#endif + + protected: template - void serialize_map(T &value_map, std::string &str) { + void serialize_map(T &value_map, std::string &str, bool has_change) { for (auto &[labels_value, value] : value_map) { - if (value == 0) { + auto val = value.value(); + if (val == 0 && !has_change) { continue; } str.append(name_); - str.append("{"); - build_string(str, labels_name_, labels_value); - str.append("} "); - - if (type_ == MetricType::Counter) { - str.append(std::to_string((int64_t)value)); + if (labels_name_.empty()) { + str.append(" "); } else { - str.append(std::to_string(value)); + str.append("{"); + build_string(str, labels_name_, labels_value); + str.append("} "); } + str.append(std::to_string(val)); + str.append("\n"); } } + template + bool equal(const std::vector &v, const std::array &a) { + if (v.size() != N) + return false; + + return std::equal(v.begin(), v.end(), a.begin()); + } + void build_string(std::string &str, const std::vector &v1, - const std::vector &v2) { + const auto &v2) { for (size_t i = 0; i < v1.size(); i++) { str.append(v1[i]).append("=\"").append(v2[i]).append("\"").append(","); } str.pop_back(); } - void validate(const std::vector &labels_value, double value) { - if (value < 0) { - throw std::invalid_argument("the value is less than zero"); - } - if (labels_value.empty() || labels_name_.size() != labels_value.size()) { - throw std::invalid_argument( - "the number of labels_value name and labels_value is not match"); - } - } + std::mutex mtx_; + dynamic_metric_hash_map, + thread_local_value> + value_map_; + size_t dupli_count_ = 2; + bool has_change_ = false; +}; - template - void set_value(T &label_val, double value, op_type_t type) { - switch (type) { - case op_type_t::INC: { -#ifdef __APPLE__ - if constexpr (is_atomic) { - mac_os_atomic_fetch_add(&label_val, value); - } - else { - label_val += value; - } -#else - if constexpr (is_atomic) { - label_val.fetch_add(value, std::memory_order_relaxed); - } - else { - label_val += value; - } -#endif - } break; - case op_type_t::DEC: -#ifdef __APPLE__ - if constexpr (is_atomic) { - mac_os_atomic_fetch_sub(&label_val, value); - } - else { - label_val -= value; - } -#else - if constexpr (is_atomic) { - label_val.fetch_sub(value, std::memory_order_relaxed); - } - else { - label_val -= value; - } -#endif - break; - case op_type_t::SET: - label_val = value; - break; - } - } +using dynamic_counter_1t = basic_dynamic_counter; +using dynamic_counter_1d = basic_dynamic_counter; - metric_hash_map> atomic_value_map_; - std::atomic default_label_value_ = 0; +using dynamic_counter_2t = basic_dynamic_counter; +using dynamic_counter_2d = basic_dynamic_counter; +using dynamic_counter_t = dynamic_counter_2t; +using dynamic_counter_d = dynamic_counter_2d; - std::mutex mtx_; - metric_hash_map value_map_; -}; +using dynamic_counter_3t = basic_dynamic_counter; +using dynamic_counter_3d = basic_dynamic_counter; + +using dynamic_counter_4t = basic_dynamic_counter; +using dynamic_counter_4d = basic_dynamic_counter; + +using dynamic_counter_5t = basic_dynamic_counter; +using dynamic_counter_5d = basic_dynamic_counter; } // namespace ylt::metric \ No newline at end of file diff --git a/include/cinatra/ylt/metric/detail/ckms_quantiles.hpp b/include/cinatra/ylt/metric/detail/ckms_quantiles.hpp index fd42a8c2..b56b6167 100644 --- a/include/cinatra/ylt/metric/detail/ckms_quantiles.hpp +++ b/include/cinatra/ylt/metric/detail/ckms_quantiles.hpp @@ -117,7 +117,7 @@ class CKMSQuantiles { std::size_t idx = 0; std::size_t item = idx++; - for (std::size_t i = start; i < buffer_count_; ++i) { + for (uint16_t i = start; i < buffer_count_; ++i) { double v = buffer_[i]; while (idx < sample_.size() && sample_[item].value < v) { item = idx++; @@ -167,9 +167,9 @@ class CKMSQuantiles { private: const std::reference_wrapper> quantiles_; - std::size_t count_; + uint16_t count_; std::vector sample_; - std::array buffer_; - std::size_t buffer_count_; + std::array buffer_; + uint16_t buffer_count_; }; } // namespace ylt::metric \ No newline at end of file diff --git a/include/cinatra/ylt/metric/detail/time_window_quantiles.hpp b/include/cinatra/ylt/metric/detail/time_window_quantiles.hpp index 6b81179f..07debac1 100644 --- a/include/cinatra/ylt/metric/detail/time_window_quantiles.hpp +++ b/include/cinatra/ylt/metric/detail/time_window_quantiles.hpp @@ -8,7 +8,7 @@ class TimeWindowQuantiles { public: TimeWindowQuantiles(const std::vector& quantiles, - Clock::duration max_age_seconds, int age_buckets) + Clock::duration max_age_seconds, uint16_t age_buckets) : quantiles_(quantiles), ckms_quantiles_(age_buckets, CKMSQuantiles(quantiles_)), current_bucket_(0), @@ -44,7 +44,7 @@ class TimeWindowQuantiles { const std::vector& quantiles_; mutable std::vector ckms_quantiles_; - mutable std::size_t current_bucket_; + mutable uint16_t current_bucket_; mutable Clock::time_point last_rotation_; const Clock::duration rotation_interval_; diff --git a/include/cinatra/ylt/metric/gauge.hpp b/include/cinatra/ylt/metric/gauge.hpp index 6261ce07..50d678c1 100644 --- a/include/cinatra/ylt/metric/gauge.hpp +++ b/include/cinatra/ylt/metric/gauge.hpp @@ -4,50 +4,107 @@ #include "counter.hpp" namespace ylt::metric { -class gauge_t : public counter_t { + +template +class basic_static_gauge : public basic_static_counter { + using metric_t::set_metric_type; + using basic_static_counter::default_label_value_; + using metric_t::labels_value_; + using basic_static_counter::dupli_count_; + using basic_static_counter::has_change_; + public: - gauge_t(std::string name, std::string help) - : counter_t(std::move(name), std::move(help)) { - set_metric_type(MetricType::Gauge); - } - gauge_t(std::string name, std::string help, - std::vector labels_name) - : counter_t(std::move(name), std::move(help), std::move(labels_name)) { + basic_static_gauge(std::string name, std::string help, size_t dupli_count = 2) + : basic_static_counter(std::move(name), std::move(help), + dupli_count) { set_metric_type(MetricType::Gauge); } - gauge_t(std::string name, std::string help, - std::map labels) - : counter_t(std::move(name), std::move(help), std::move(labels)) { + basic_static_gauge(std::string name, std::string help, + std::map labels, + size_t dupli_count = 2) + : basic_static_counter(std::move(name), std::move(help), + std::move(labels), dupli_count) { set_metric_type(MetricType::Gauge); } - void dec(double value = 1) { + void dec(value_type value = 1) { + if (!has_change_) [[unlikely]] { + has_change_ = true; + } #ifdef __APPLE__ - mac_os_atomic_fetch_sub(&default_label_value_, value); + if constexpr (std::is_floating_point_v) { + mac_os_atomic_fetch_sub(&default_label_value_.local_value(), value); + } + else { + default_label_value_.dec(value); + } #else - default_label_value_ -= value; + default_label_value_.dec(value); #endif } +}; +using gauge_t = basic_static_gauge; +using gauge_d = basic_static_gauge; + +template +class basic_dynamic_gauge : public basic_dynamic_counter { + using metric_t::set_metric_type; + using basic_dynamic_counter::value_map_; + using basic_dynamic_counter::mtx_; + using basic_dynamic_counter::dupli_count_; + using basic_dynamic_counter::has_change_; + + public: + basic_dynamic_gauge(std::string name, std::string help, + std::array labels_name, + size_t dupli_count = 2) + : basic_dynamic_counter(std::move(name), std::move(help), + std::move(labels_name), + dupli_count) { + set_metric_type(MetricType::Gauge); + } - void dec(const std::vector& labels_value, double value = 1) { + void dec(const std::array& labels_value, + value_type value = 1) { if (value == 0) { return; } - validate(labels_value, value); - if (use_atomic_) { - if (labels_value != labels_value_) { - throw std::invalid_argument( - "the given labels_value is not match with origin labels_value"); - } - set_value(atomic_value_map_[labels_value], value, op_type_t::DEC); + std::unique_lock lock(mtx_); + if (value_map_.size() > ylt_label_capacity) { + return; } - else { - std::lock_guard lock(mtx_); - stat_metric(labels_value); - set_value(value_map_[labels_value], value, op_type_t::DEC); + if (!has_change_) [[unlikely]] + has_change_ = true; + auto [it, r] = value_map_.try_emplace( + labels_value, thread_local_value(dupli_count_)); + lock.unlock(); + if (r) { + g_user_metric_label_count->local_value()++; + if (ylt_label_max_age.count()) { + it->second.set_created_time(std::chrono::system_clock::now()); + } } + + set_value(it->second.local_value(), value, op_type_t::DEC); } }; + +using dynamic_gauge_1t = basic_dynamic_gauge; +using dynamic_gauge_1d = basic_dynamic_gauge; + +using dynamic_gauge_t = basic_dynamic_gauge; +using dynamic_gauge_d = basic_dynamic_gauge; +using dynamic_gauge_2t = dynamic_gauge_t; +using dynamic_gauge_2d = dynamic_gauge_d; + +using dynamic_gauge_3t = basic_dynamic_gauge; +using dynamic_gauge_3d = basic_dynamic_gauge; + +using dynamic_gauge_4t = basic_dynamic_gauge; +using dynamic_gauge_4d = basic_dynamic_gauge; + +using dynamic_gauge_5t = basic_dynamic_gauge; +using dynamic_gauge_5d = basic_dynamic_gauge; } // namespace ylt::metric \ No newline at end of file diff --git a/include/cinatra/ylt/metric/histogram.hpp b/include/cinatra/ylt/metric/histogram.hpp index 69f8c3cb..4b4fe140 100644 --- a/include/cinatra/ylt/metric/histogram.hpp +++ b/include/cinatra/ylt/metric/histogram.hpp @@ -26,63 +26,28 @@ struct json_histogram_t { REFLECTION(json_histogram_t, name, help, type, metrics); #endif -class histogram_t : public metric_t { +template +class basic_static_histogram : public static_metric { public: - histogram_t(std::string name, std::string help, std::vector buckets) - : bucket_boundaries_(buckets), - metric_t(MetricType::Histogram, std::move(name), std::move(help)), - sum_(std::make_shared("", "")) { - if (!is_strict_sorted(begin(bucket_boundaries_), end(bucket_boundaries_))) { - throw std::invalid_argument("Bucket Boundaries must be strictly sorted"); - } - - g_user_metric_count++; - - for (size_t i = 0; i < buckets.size() + 1; i++) { - bucket_counts_.push_back(std::make_shared("", "")); - } - use_atomic_ = true; + basic_static_histogram(std::string name, std::string help, + std::vector buckets, size_t dupli_count = 2) + : bucket_boundaries_(std::move(buckets)), + static_metric(MetricType::Histogram, std::move(name), std::move(help)), + sum_(std::make_shared("", "", dupli_count)) { + init_bucket_counter(dupli_count, bucket_boundaries_.size()); } - histogram_t(std::string name, std::string help, std::vector buckets, - std::vector labels_name) - : bucket_boundaries_(buckets), - metric_t(MetricType::Histogram, name, help, labels_name), - sum_(std::make_shared(name, help, labels_name)) { - if (!is_strict_sorted(begin(bucket_boundaries_), end(bucket_boundaries_))) { - throw std::invalid_argument("Bucket Boundaries must be strictly sorted"); - } - - g_user_metric_count++; - - for (size_t i = 0; i < buckets.size() + 1; i++) { - bucket_counts_.push_back( - std::make_shared(name, help, labels_name)); - } + basic_static_histogram(std::string name, std::string help, + std::vector buckets, + std::map labels, + size_t dupli_count = 2) + : bucket_boundaries_(std::move(buckets)), + static_metric(MetricType::Histogram, name, help, labels), + sum_(std::make_shared("", "", dupli_count)) { + init_bucket_counter(dupli_count, bucket_boundaries_.size()); } - histogram_t(std::string name, std::string help, std::vector buckets, - std::map labels) - : bucket_boundaries_(buckets), - metric_t(MetricType::Histogram, name, help, labels), - sum_(std::make_shared(name, help, labels)) { - if (!is_strict_sorted(begin(bucket_boundaries_), end(bucket_boundaries_))) { - throw std::invalid_argument("Bucket Boundaries must be strictly sorted"); - } - - g_user_metric_count++; - - for (size_t i = 0; i < buckets.size() + 1; i++) { - bucket_counts_.push_back(std::make_shared(name, help, labels)); - } - use_atomic_ = true; - } - - void observe(double value) { - if (!use_atomic_ || !labels_name_.empty()) { - throw std::invalid_argument("not a default label metric"); - } - + void observe(value_type value) { const auto bucket_index = static_cast( std::distance(bucket_boundaries_.begin(), std::lower_bound(bucket_boundaries_.begin(), @@ -91,39 +56,27 @@ class histogram_t : public metric_t { bucket_counts_[bucket_index]->inc(); } - void observe(const std::vector &labels_value, double value) { - if (sum_->labels_name().empty()) { - throw std::invalid_argument("not a label metric"); - } - - const auto bucket_index = static_cast( - std::distance(bucket_boundaries_.begin(), - std::lower_bound(bucket_boundaries_.begin(), - bucket_boundaries_.end(), value))); - sum_->inc(labels_value, value); - bucket_counts_[bucket_index]->inc(labels_value); - } - auto get_bucket_counts() { return bucket_counts_; } - metric_hash_map value_map() override { return sum_->value_map(); } - void serialize(std::string &str) override { - if (!sum_->labels_name().empty()) { - serialize_with_labels(str); - return; - } + auto val = sum_->value(); - if (sum_->value() == 0) { + if (val == 0) { return; } serialize_head(str); - double count = 0; + value_type count = 0; auto bucket_counts = get_bucket_counts(); for (size_t i = 0; i < bucket_counts.size(); i++) { auto counter = bucket_counts[i]; str.append(name_).append("_bucket{"); + + if (!labels_name_.empty()) { + build_label_string(str, labels_name_, labels_value_); + str.append(","); + } + if (i == bucket_boundaries_.size()) { str.append("le=\"").append("+Inf").append("\"} "); } @@ -138,10 +91,7 @@ class histogram_t : public metric_t { str.append("\n"); } - str.append(name_) - .append("_sum ") - .append(std::to_string(sum_->value())) - .append("\n"); + str.append(name_).append("_sum ").append(std::to_string(val)).append("\n"); str.append(name_) .append("_count ") @@ -151,18 +101,14 @@ class histogram_t : public metric_t { #ifdef CINATRA_ENABLE_METRIC_JSON void serialize_to_json(std::string &str) override { - if (!sum_->labels_name().empty()) { - serialize_to_json_with_labels(str); - return; - } - - if (sum_->value() == 0) { + auto val = sum_->value(); + if (val == 0) { return; } json_histogram_t hist{name_, help_, std::string(metric_name())}; - double count = 0; + value_type count = 0; auto bucket_counts = get_bucket_counts(); json_histogram_metric_t metric{}; for (size_t i = 0; i < bucket_counts.size(); i++) { @@ -180,7 +126,11 @@ class histogram_t : public metric_t { } } metric.count = (int64_t)count; - metric.sum = sum_->value(); + metric.sum = val; + + for (size_t i = 0; i < labels_value_.size(); i++) { + metric.labels[labels_name_[i]] = labels_value_[i]; + } hist.metrics.push_back(std::move(metric)); @@ -189,6 +139,15 @@ class histogram_t : public metric_t { #endif private: + void init_bucket_counter(size_t dupli_count, size_t bucket_size) { + g_user_metric_count++; + + for (size_t i = 0; i < bucket_size + 1; i++) { + bucket_counts_.push_back( + std::make_shared("", "", dupli_count)); + } + } + template bool is_strict_sorted(ForwardIterator first, ForwardIterator last) { return std::adjacent_find(first, last, @@ -196,25 +155,80 @@ class histogram_t : public metric_t { ForwardIterator>::value_type>()) == last; } - void serialize_with_labels(std::string &str) { + std::vector bucket_boundaries_; + std::vector> bucket_counts_; // readonly + std::shared_ptr sum_; +}; +using histogram_t = basic_static_histogram; +using histogram_d = basic_static_histogram; + +template +class basic_dynamic_histogram : public dynamic_metric { + public: + basic_dynamic_histogram(std::string name, std::string help, + std::vector buckets, + std::array labels_name, + size_t dupli_count = 2) + : bucket_boundaries_(buckets), + dynamic_metric(MetricType::Histogram, name, help, labels_name), + sum_(std::make_shared>( + name, help, labels_name, dupli_count)) { + g_user_metric_count++; + + for (size_t i = 0; i < buckets.size() + 1; i++) { + bucket_counts_.push_back( + std::make_shared>( + name, help, labels_name, dupli_count)); + } + } + + void observe(const std::array &labels_value, + value_type value) { + const auto bucket_index = static_cast( + std::distance(bucket_boundaries_.begin(), + std::lower_bound(bucket_boundaries_.begin(), + bucket_boundaries_.end(), value))); + sum_->inc(labels_value, value); + bucket_counts_[bucket_index]->inc(labels_value); + } + + auto get_bucket_counts() { return bucket_counts_; } + + bool has_label_value(const std::string &label_val) override { + return sum_->has_label_value(label_val); + } + + bool has_label_value(const std::regex ®ex) override { + return sum_->has_label_value(regex); + } + + bool has_label_value(const std::vector &label_value) override { + return sum_->has_label_value(label_value); + } + + void serialize(std::string &str) override { auto value_map = sum_->value_map(); if (value_map.empty()) { return; } + serialize_head(str); + std::string value_str; auto bucket_counts = get_bucket_counts(); for (auto &[labels_value, value] : value_map) { - if (value == 0) { + if (value.value() == 0) { continue; } - double count = 0; + value_type count = 0; for (size_t i = 0; i < bucket_counts.size(); i++) { auto counter = bucket_counts[i]; value_str.append(name_).append("_bucket{"); - build_label_string(str, sum_->labels_name(), labels_value); - value_str.append(","); + if (!labels_name_.empty()) { + build_label_string(value_str, labels_name_, labels_value); + value_str.append(","); + } if (i == bucket_boundaries_.size()) { value_str.append("le=\"").append("+Inf").append("\"} "); @@ -234,7 +248,6 @@ class histogram_t : public metric_t { return; } - serialize_head(str); str.append(value_str); str.append(name_); @@ -242,12 +255,7 @@ class histogram_t : public metric_t { build_label_string(str, sum_->labels_name(), labels_value); str.append("} "); - if (type_ == MetricType::Counter) { - str.append(std::to_string((int64_t)value)); - } - else { - str.append(std::to_string(value)); - } + str.append(std::to_string(value.value())); str.append("\n"); str.append(name_).append("_count{"); @@ -259,7 +267,7 @@ class histogram_t : public metric_t { } #ifdef CINATRA_ENABLE_METRIC_JSON - void serialize_to_json_with_labels(std::string &str) { + void serialize_to_json(std::string &str) override { auto value_map = sum_->value_map(); if (value_map.empty()) { return; @@ -269,7 +277,7 @@ class histogram_t : public metric_t { auto bucket_counts = get_bucket_counts(); for (auto &[labels_value, value] : value_map) { - if (value == 0) { + if (value.value() == 0) { continue; } @@ -305,8 +313,34 @@ class histogram_t : public metric_t { } #endif + private: + template + bool is_strict_sorted(ForwardIterator first, ForwardIterator last) { + return std::adjacent_find(first, last, + std::greater_equal::value_type>()) == last; + } + std::vector bucket_boundaries_; - std::vector> bucket_counts_; // readonly - std::shared_ptr sum_; + std::vector>> + bucket_counts_; // readonly + std::shared_ptr> sum_; }; + +using dynamic_histogram_1t = basic_dynamic_histogram; +using dynamic_histogram_1d = basic_dynamic_histogram; + +using dynamic_histogram_2t = basic_dynamic_histogram; +using dynamic_histogram_2d = basic_dynamic_histogram; +using dynamic_histogram_t = dynamic_histogram_2t; +using dynamic_histogram_d = dynamic_histogram_2d; + +using dynamic_histogram_3t = basic_dynamic_histogram; +using dynamic_histogram_3d = basic_dynamic_histogram; + +using dynamic_histogram_4t = basic_dynamic_histogram; +using dynamic_histogram_4d = basic_dynamic_histogram; + +using dynamic_histogram_5t = basic_dynamic_histogram; +using dynamic_histogram_5d = basic_dynamic_histogram; } // namespace ylt::metric \ No newline at end of file diff --git a/include/cinatra/ylt/metric/metric.hpp b/include/cinatra/ylt/metric/metric.hpp index bfd3de1d..f37de8ca 100644 --- a/include/cinatra/ylt/metric/metric.hpp +++ b/include/cinatra/ylt/metric/metric.hpp @@ -15,6 +15,7 @@ #include "async_simple/coro/Lazy.h" #include "async_simple/coro/SyncAwait.h" #include "cinatra/cinatra_log_wrapper.hpp" +#include "thread_local_value.hpp" #if __has_include("ylt/coro_io/coro_io.hpp") #include "ylt/coro_io/coro_io.hpp" #else @@ -45,27 +46,27 @@ enum class MetricType { struct metric_filter_options { std::optional name_regex{}; std::optional label_regex{}; + std::optional label_value_regex{}; bool is_white = true; }; -struct vector_hash { - size_t operator()(const std::vector& vec) const { - unsigned int seed = 131; - unsigned int hash = 0; - - for (const auto& str : vec) { - for (auto ch : str) { - hash = hash * seed + ch; - } - } - - return (hash & 0x7FFFFFFF); - } -}; +#ifdef __APPLE__ +inline double mac_os_atomic_fetch_add(std::atomic* obj, double arg) { + double v; + do { + v = obj->load(); + } while (!std::atomic_compare_exchange_weak(obj, &v, v + arg)); + return v; +} -template -using metric_hash_map = - std::unordered_map, T, vector_hash>; +inline double mac_os_atomic_fetch_sub(std::atomic* obj, double arg) { + double v; + do { + v = obj->load(); + } while (!std::atomic_compare_exchange_weak(obj, &v, v - arg)); + return v; +} +#endif class metric_t { public: @@ -75,10 +76,14 @@ class metric_t { name_(std::move(name)), help_(std::move(help)), metric_created_time_(std::chrono::system_clock::now()) {} + + template metric_t(MetricType type, std::string name, std::string help, - std::vector labels_name) + std::array labels_name) : metric_t(type, std::move(name), std::move(help)) { - labels_name_ = std::move(labels_name); + for (size_t i = 0; i < N; i++) { + labels_name_.push_back(std::move(labels_name[i])); + } } metric_t(MetricType type, std::string name, std::string help, @@ -94,6 +99,8 @@ class metric_t { std::string_view name() { return name_; } + const std::string& str_name() { return name_; } + std::string_view help() { return help_; } MetricType metric_type() { return type_; } @@ -122,7 +129,36 @@ class metric_t { return static_labels_; } - virtual metric_hash_map value_map() { return {}; } + virtual size_t label_value_count() { return 0; } + + virtual bool has_label_value(const std::string& label_value) { + return std::find(labels_value_.begin(), labels_value_.end(), label_value) != + labels_value_.end(); + } + + virtual void clean_expired_label() {} + + virtual bool has_label_value(const std::vector& label_value) { + return labels_value_ == label_value; + } + virtual bool has_label_value(const std::regex& regex) { + auto it = std::find_if(labels_value_.begin(), labels_value_.end(), + [&](auto& value) { + return std::regex_match(value, regex); + }); + + return it != labels_value_.end(); + } + bool has_label_name(const std::vector& label_name) { + return labels_name_ == label_name; + } + bool has_label_name(const std::string& label_name) { + return std::find(labels_name_.begin(), labels_name_.end(), label_name) != + labels_name_.end(); + } + + virtual void remove_label_value( + const std::map& labels) {} virtual void serialize(std::string& str) {} @@ -143,8 +179,6 @@ class metric_t { } #endif - bool is_atomic() const { return use_atomic_; } - template T* as() { return dynamic_cast(this); @@ -163,7 +197,7 @@ class metric_t { void build_label_string(std::string& str, const std::vector& label_name, - const std::vector& label_value) { + const auto& label_value) { for (size_t i = 0; i < label_name.size(); i++) { str.append(label_name[i]) .append("=\"") @@ -174,521 +208,45 @@ class metric_t { str.pop_back(); } -#ifdef __APPLE__ - double mac_os_atomic_fetch_add(std::atomic* obj, double arg) { - double v; - do { - v = obj->load(); - } while (!std::atomic_compare_exchange_weak(obj, &v, v + arg)); - return v; - } - - double mac_os_atomic_fetch_sub(std::atomic* obj, double arg) { - double v; - do { - v = obj->load(); - } while (!std::atomic_compare_exchange_weak(obj, &v, v - arg)); - return v; - } -#endif - MetricType type_ = MetricType::Nil; std::string name_; std::string help_; std::map static_labels_; std::vector labels_name_; // read only std::vector labels_value_; // read only - bool use_atomic_ = false; std::chrono::system_clock::time_point metric_created_time_{}; }; -template -struct metric_manager_t; +class static_metric : public metric_t { + using metric_t::metric_t; +}; -struct ylt_system_tag_t {}; -using system_metric_manager = metric_manager_t; +class dynamic_metric : public metric_t { + using metric_t::metric_t; +}; -class counter_t; -inline auto g_user_metric_labels = - std::make_shared("ylt_user_metric_labels", ""); -inline auto g_summary_failed_count = - std::make_shared("ylt_summary_failed_count", ""); +inline auto g_user_metric_label_count = new thread_local_value(2); +inline std::atomic g_summary_failed_count = 0; inline std::atomic g_user_metric_count = 0; inline std::atomic ylt_metric_capacity = 10000000; +inline int64_t ylt_label_capacity = 20000000; + +inline std::chrono::seconds ylt_label_max_age{0}; +inline std::chrono::seconds ylt_label_check_expire_duration{0}; inline void set_metric_capacity(int64_t max_count) { ylt_metric_capacity = max_count; } -template -struct metric_manager_t { - struct null_mutex_t { - void lock() {} - void unlock() {} - }; - - // create and register metric - template - static std::shared_ptr create_metric_static(const std::string& name, - const std::string& help, - Args&&... args) { - auto m = std::make_shared(name, help, std::forward(args)...); - bool r = register_metric_static(m); - if (!r) { - return nullptr; - } - return m; - } - - template - static std::shared_ptr create_metric_dynamic(const std::string& name, - const std::string& help, - Args&&... args) { - auto m = std::make_shared(name, help, std::forward(args)...); - bool r = register_metric_dynamic(m); - if (!r) { - return nullptr; - } - return m; - } - - static bool register_metric_static(std::shared_ptr metric) { - return register_metric_impl(metric); - } - - static bool register_metric_dynamic(std::shared_ptr metric) { - return register_metric_impl(metric); - } - - static bool remove_metric_static(const std::string& name) { - return remove_metric_impl(name); - } - - static bool remove_metric_dynamic(const std::string& name) { - return remove_metric_impl(name); - } - - static bool remove_metric_dynamic(std::shared_ptr metric) { - return remove_metric_impl(std::string(metric->name())); - } - - static void remove_metric_dynamic(const std::vector& names) { - if (names.empty()) { - return; - } - auto lock = get_lock(); - for (auto& name : names) { - metric_map_.erase(name); - } - } - - static void remove_metric_dynamic( - std::vector> metrics) { - if (metrics.empty()) { - return; - } - auto lock = get_lock(); - for (auto& metric : metrics) { - metric_map_.erase(std::string(metric->name())); - } - } - - template - static bool register_metric_dynamic(Metrics... metrics) { - bool r = true; - ((void)(r && (r = register_metric_impl(metrics), true)), ...); - return r; - } - - static bool register_metric_dynamic( - std::vector> metrics) { - bool r = true; - std::vector> vec; - for (auto& metric : metrics) { - r = register_metric_impl(metric); - if (!r) { - r = false; - break; - } - - vec.push_back(metric); - } - - if (!r) { - remove_metric_dynamic(vec); - } - - return r; - } - - template - static bool register_metric_static(Metrics... metrics) { - bool r = true; - ((void)(r && (r = register_metric_impl(metrics), true)), ...); - return r; - } - - static auto get_metrics() { - if (need_lock_) { - return collect(); - } - else { - return collect(); - } - } - - static auto metric_map_static() { return metric_map_impl(); } - static auto metric_map_dynamic() { return metric_map_impl(); } - - static size_t metric_count_static() { return metric_count_impl(); } - - static size_t metric_count_dynamic() { return metric_count_impl(); } - - static std::vector metric_keys_static() { - return metric_keys_impl(); - } - - static std::vector metric_keys_dynamic() { - return metric_keys_impl(); - } - - // static labels: {{"method", "GET"}, {"url", "/"}} - static std::vector> get_metric_by_labels_static( - const std::map& labels) { - std::vector> vec; - auto map = metric_map_static(); - for (auto& [name, m] : map) { - const auto& static_labels = m->get_static_labels(); - if (static_labels == labels) { - vec.push_back(m); - } - } - return vec; - } - - // static label: {"method", "GET"} - static std::vector> get_metric_by_label_static( - const std::pair& label) { - std::vector> vec; - auto map = metric_map_static(); - for (auto& [name, t] : map) { - const auto& static_labels = t->get_static_labels(); - for (const auto& pair : static_labels) { - if (pair.first == label.first && pair.second == label.second) { - vec.push_back(t); - } - } - } - return vec; - } - - // labels: {{"method", "POST"}, {"code", "200"}} - static std::vector> get_metric_by_labels_dynamic( - const std::map& labels) { - std::vector> vec; - auto map = metric_map_dynamic(); - for (auto& [name, t] : map) { - auto val_map = t->value_map(); - auto labels_name = t->labels_name(); - - for (auto& [k, v] : labels) { - if (auto it = std::find(labels_name.begin(), labels_name.end(), k); - it != labels_name.end()) { - if (auto it = std::find_if(val_map.begin(), val_map.end(), - [label_val = v](auto& pair) { - auto& key = pair.first; - return std::find(key.begin(), key.end(), - label_val) != key.end(); - }); - it != val_map.end()) { - vec.push_back(t); - } - } - } - } - - return vec; - } - - template - static std::shared_ptr get_metric_static(const std::string& name) { - auto m = get_metric_impl(name); - if (m == nullptr) { - return nullptr; - } - return std::dynamic_pointer_cast(m); - } - - template - static std::shared_ptr get_metric_dynamic(const std::string& name) { - auto m = get_metric_impl(name); - if (m == nullptr) { - return nullptr; - } - return std::dynamic_pointer_cast(m); - } - - static std::string serialize( - const std::vector>& metrics) { - std::string str; - for (auto& m : metrics) { - if (m->metric_type() == MetricType::Summary) { - async_simple::coro::syncAwait(m->serialize_async(str)); - } - else { - m->serialize(str); - } - } - - return str; - } - - static std::string serialize_static() { return serialize(collect()); } - - static std::string serialize_dynamic() { return serialize(collect()); } - -#ifdef CINATRA_ENABLE_METRIC_JSON - static std::string serialize_to_json_static() { - auto metrics = collect(); - return serialize_to_json(metrics); - } - - static std::string serialize_to_json_dynamic() { - auto metrics = collect(); - return serialize_to_json(metrics); - } - - static std::string serialize_to_json( - const std::vector>& metrics) { - if (metrics.empty()) { - return ""; - } - std::string str; - str.append("["); - for (auto& m : metrics) { - size_t start = str.size(); - if (m->metric_type() == MetricType::Summary) { - async_simple::coro::syncAwait(m->serialize_to_json_async(str)); - } - else { - m->serialize_to_json(str); - } - - if (str.size() > start) - str.append(","); - } - str.back() = ']'; - return str; - } -#endif - - static std::vector> filter_metrics_static( - const metric_filter_options& options) { - return filter_metrics(options); - } - - static std::vector> filter_metrics_dynamic( - const metric_filter_options& options) { - return filter_metrics(options); - } - - private: - template - static void check_lock() { - if (need_lock_ != need_lock) { - std::string str = "need lock "; - std::string s = need_lock_ ? "true" : "false"; - std::string r = need_lock ? "true" : "false"; - str.append(s).append(" but set as ").append(r); - throw std::invalid_argument(str); - } - } - - template - static auto get_lock() { - check_lock(); - if constexpr (need_lock) { - return std::scoped_lock(mtx_); - } - else { - return std::scoped_lock(null_mtx_); - } - } - - template - static bool register_metric_impl(std::shared_ptr metric) { - // the first time regiter_metric will set metric_manager_t lock or not lock. - // visit metric_manager_t with different lock strategy will cause throw - // exception. - std::call_once(flag_, [] { - need_lock_ = need_lock; - }); - - std::string name(metric->name()); - auto lock = get_lock(); - if (g_user_metric_count > ylt_metric_capacity) { - CINATRA_LOG_ERROR << "metric count at capacity size: " - << g_user_metric_count; - return false; - } - bool r = metric_map_.emplace(name, std::move(metric)).second; - if (!r) { - CINATRA_LOG_ERROR << "duplicate registered metric name: " << name; - } - return r; - } - - template - static size_t remove_metric_impl(const std::string& name) { - auto lock = get_lock(); - return metric_map_.erase(name); - } - - template - static auto metric_map_impl() { - auto lock = get_lock(); - return metric_map_; - } - - template - static size_t metric_count_impl() { - auto lock = get_lock(); - return metric_map_.size(); - } - - template - static std::vector metric_keys_impl() { - std::vector keys; - { - auto lock = get_lock(); - for (auto& pair : metric_map_) { - keys.push_back(pair.first); - } - } - - return keys; - } - - template - static std::shared_ptr get_metric_impl(const std::string& name) { - auto lock = get_lock(); - auto it = metric_map_.find(name); - if (it == metric_map_.end()) { - return nullptr; - } - return it->second; - } - - template - static auto collect() { - std::vector> metrics; - { - auto lock = get_lock(); - for (auto& pair : metric_map_) { - metrics.push_back(pair.second); - } - } - return metrics; - } - - static void filter_by_label_name( - std::vector>& filtered_metrics, - std::shared_ptr m, const metric_filter_options& options, - std::vector& indexs, size_t index) { - const auto& labels_name = m->labels_name(); - for (auto& label_name : labels_name) { - if (std::regex_match(label_name, *options.label_regex)) { - if (options.is_white) { - filtered_metrics.push_back(m); - } - else { - indexs.push_back(index); - } - } - } - } - - template - static std::vector> filter_metrics( - const metric_filter_options& options) { - auto metrics = collect(); - if (!options.name_regex && !options.label_regex) { - return metrics; - } - - std::vector> filtered_metrics; - std::vector indexs; - size_t index = 0; - for (auto& m : metrics) { - if (options.name_regex && !options.label_regex) { - if (std::regex_match(std::string(m->name()), *options.name_regex)) { - if (options.is_white) { - filtered_metrics.push_back(m); - } - else { - indexs.push_back(index); - } - } - } - else if (options.label_regex && !options.name_regex) { - filter_by_label_name(filtered_metrics, m, options, indexs, index); - } - else { - if (std::regex_match(std::string(m->name()), *options.name_regex)) { - filter_by_label_name(filtered_metrics, m, options, indexs, index); - } - } - index++; - } - - if (!options.is_white) { - for (size_t i : indexs) { - metrics.erase(std::next(metrics.begin(), i)); - } - return metrics; - } - - return filtered_metrics; - } - - static inline std::mutex mtx_; - static inline std::unordered_map> - metric_map_; - - static inline null_mutex_t null_mtx_; - static inline std::atomic_bool need_lock_ = true; - static inline std::once_flag flag_; -}; - -struct ylt_default_metric_tag_t {}; -using default_metric_manager = metric_manager_t; - -template -struct metric_collector_t { - static std::string serialize() { - auto vec = get_all_metrics(); - return default_metric_manager::serialize(vec); - } - -#ifdef CINATRA_ENABLE_METRIC_JSON - static std::string serialize_to_json() { - auto vec = get_all_metrics(); - return default_metric_manager::serialize_to_json(vec); - } -#endif - - static std::vector> get_all_metrics() { - std::vector> vec; - (append_vector(vec), ...); - return vec; - } +inline void set_label_capacity(int64_t max_label_count) { + ylt_label_capacity = max_label_count; +} - private: - template - static void append_vector(std::vector>& vec) { - auto v = T::get_metrics(); - vec.insert(vec.end(), v.begin(), v.end()); - } -}; +inline void set_label_max_age( + std::chrono::seconds max_age, + std::chrono::seconds check_duration = std::chrono::seconds(60 * 10)) { + ylt_label_max_age = max_age; + ylt_label_check_expire_duration = check_duration; +} } // namespace ylt::metric \ No newline at end of file diff --git a/include/cinatra/ylt/metric/metric_manager.hpp b/include/cinatra/ylt/metric/metric_manager.hpp new file mode 100644 index 00000000..8a48d5f1 --- /dev/null +++ b/include/cinatra/ylt/metric/metric_manager.hpp @@ -0,0 +1,637 @@ +#pragma once +#include +#include +#include + +#include "metric.hpp" + +namespace ylt::metric { +class manager_helper { + public: + static bool register_metric(auto& metric_map, auto metric) { + if (g_user_metric_count > ylt_metric_capacity) { + CINATRA_LOG_ERROR << "metric count at capacity size: " + << g_user_metric_count; + return false; + } + auto [it, r] = metric_map.try_emplace(metric->str_name(), metric); + if (!r) { + CINATRA_LOG_ERROR << "duplicate registered metric name: " + << metric->str_name(); + return false; + } + + return true; + } + + static std::string serialize( + const std::vector>& metrics) { + std::string str; + for (auto& m : metrics) { + if (m->metric_type() == MetricType::Summary) { + async_simple::coro::syncAwait(m->serialize_async(str)); + } + else { + m->serialize(str); + } + } + + return str; + } + +#ifdef CINATRA_ENABLE_METRIC_JSON + static std::string serialize_to_json( + const std::vector>& metrics) { + if (metrics.empty()) { + return ""; + } + std::string str; + str.append("["); + for (auto& m : metrics) { + size_t start = str.size(); + if (m->metric_type() == MetricType::Summary) { + async_simple::coro::syncAwait(m->serialize_to_json_async(str)); + } + else { + m->serialize_to_json(str); + } + + if (str.size() > start) + str.append(","); + } + + if (str.size() == 1) { + return ""; + } + + str.back() = ']'; + return str; + } +#endif + + static std::vector> filter_metrics_by_name( + auto& metrics, const std::regex& name_regex) { + std::vector> filtered_metrics; + for (auto& m : metrics) { + if (std::regex_match(m->str_name(), name_regex)) { + filtered_metrics.push_back(m); + } + } + return filtered_metrics; + } + + static std::vector> filter_metrics_by_label_name( + auto& metrics, const std::regex& label_name_regex) { + std::vector> filtered_metrics; + for (auto& m : metrics) { + const auto& labels_name = m->labels_name(); + for (auto& label_name : labels_name) { + if (std::regex_match(label_name, label_name_regex)) { + filtered_metrics.push_back(m); + } + } + } + return filtered_metrics; + } + + static std::vector> filter_metrics_by_label_value( + auto& metrics, const std::regex& label_value_regex) { + std::vector> filtered_metrics; + for (auto& m : metrics) { + if (m->has_label_value(label_value_regex)) { + filtered_metrics.push_back(m); + } + } + return filtered_metrics; + } + + static std::vector> filter_metrics( + auto& metrics, const metric_filter_options& options) { + if (!(options.name_regex || options.label_regex || + options.label_value_regex)) { + return metrics; + } + + std::vector> filtered_metrics = metrics; + if (options.name_regex) { + filtered_metrics = filter_metrics_by_name(metrics, *options.name_regex); + if (filtered_metrics.empty()) { + return {}; + } + } + + if (options.label_regex) { + filtered_metrics = + filter_metrics_by_label_name(filtered_metrics, *options.label_regex); + if (filtered_metrics.empty()) { + return {}; + } + } + + if (options.label_value_regex) { + filtered_metrics = filter_metrics_by_label_value( + filtered_metrics, *options.label_value_regex); + if (filtered_metrics.empty()) { + return {}; + } + } + + if (!options.is_white) { + for (auto& m : filtered_metrics) { + std::erase_if(metrics, [&](auto t) { + return t == m; + }); + } + return metrics; + } + + return filtered_metrics; + } + + static void filter_by_label_name( + std::vector>& filtered_metrics, + std::shared_ptr m, const metric_filter_options& options) { + const auto& labels_name = m->labels_name(); + for (auto& label_name : labels_name) { + if (std::regex_match(label_name, *options.label_regex)) { + filtered_metrics.push_back(m); + } + } + } +}; + +template +class static_metric_manager { + public: + static_metric_manager(static_metric_manager const&) = delete; + static_metric_manager(static_metric_manager&&) = delete; + static_metric_manager& operator=(static_metric_manager const&) = delete; + static_metric_manager& operator=(static_metric_manager&&) = delete; + + static static_metric_manager& instance() { + static auto* inst = new static_metric_manager(); + return *inst; + } + + template + std::pair> create_metric_static( + const std::string& name, const std::string& help, Args&&... args) { + auto m = std::make_shared(name, help, std::forward(args)...); + bool r = register_metric(m); + if (!r) { + return std::make_pair(std::make_error_code(std::errc::invalid_argument), + nullptr); + } + + return std::make_pair(std::error_code{}, m); + } + + bool register_metric(std::shared_ptr metric) { + return manager_helper::register_metric(metric_map_, metric); + } + + size_t metric_count() { return metric_map_.size(); } + + auto metric_map() { return metric_map_; } + + auto collect() { + std::vector> metrics; + + for (auto& pair : metric_map_) { + metrics.push_back(pair.second); + } + + return metrics; + } + + std::string serialize(const std::vector>& metrics) { + return manager_helper::serialize(metrics); + } + + std::string serialize_static() { + return manager_helper::serialize(collect()); + } + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string serialize_to_json_static() { + return manager_helper::serialize_to_json(collect()); + } +#endif + template + std::shared_ptr get_metric_static(const std::string& name) { + static_assert(std::is_base_of_v, + "must be dynamic metric"); + auto it = metric_map_.find(name); + if (it == metric_map_.end()) { + return nullptr; + } + return std::dynamic_pointer_cast(it->second); + } + + std::shared_ptr get_metric_by_name(const std::string& name) { + auto it = metric_map_.find(name); + if (it == metric_map_.end()) { + return nullptr; + } + + return it->second; + } + + std::vector> get_metric_by_label( + const std::map& labels) { + std::vector> metrics; + + for (auto& [key, m] : metric_map_) { + if (m->get_static_labels() == labels) { + metrics.push_back(m); + } + } + + return metrics; + } + + std::vector> filter_metrics_static( + const metric_filter_options& options) { + auto metrics = collect(); + return manager_helper::filter_metrics(metrics, options); + } + + std::vector> filter_metrics_by_label_value( + const std::regex& label_regex) { + auto metrics = collect(); + return manager_helper::filter_metrics_by_label_value(metrics, label_regex); + } + + private: + static_metric_manager() = default; + + std::unordered_map> metric_map_; +}; + +// using metric_manager_t = static_metric_manager; + +template +class dynamic_metric_manager { + public: + dynamic_metric_manager(dynamic_metric_manager const&) = delete; + dynamic_metric_manager(dynamic_metric_manager&&) = delete; + dynamic_metric_manager& operator=(dynamic_metric_manager const&) = delete; + dynamic_metric_manager& operator=(dynamic_metric_manager&&) = delete; + + static dynamic_metric_manager& instance() { + static auto* inst = new dynamic_metric_manager(); + return *inst; + } + + template + std::pair> create_metric_dynamic( + const std::string& name, const std::string& help, Args&&... args) { + auto m = std::make_shared(name, help, std::forward(args)...); + bool r = register_metric(m); + if (!r) { + return std::make_pair(std::make_error_code(std::errc::invalid_argument), + nullptr); + } + + return std::make_pair(std::error_code{}, m); + } + + bool register_metric(std::shared_ptr metric) { + std::unique_lock lock(mtx_); + return manager_helper::register_metric(metric_map_, metric); + } + + bool register_metric(std::vector> metrics) { + bool r = false; + for (auto& m : metrics) { + r = register_metric(m); + if (!r) { + break; + } + } + + return r; + } + + std::string serialize_dynamic() { + return manager_helper::serialize(collect()); + } + + std::string serialize(const std::vector>& metrics) { + return manager_helper::serialize(metrics); + } + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string serialize_to_json_dynamic() { + return manager_helper::serialize_to_json(collect()); + } +#endif + + bool remove_metric(const std::string& name) { + std::unique_lock lock(mtx_); + return metric_map_.erase(name); + } + + bool remove_metric(std::shared_ptr metric) { + if (metric == nullptr) { + return false; + } + + return remove_metric(metric->str_name()); + } + + void remove_metric(const std::vector& names) { + if (names.empty()) { + return; + } + + for (auto& name : names) { + remove_metric(name); + } + } + + void remove_metric(std::vector> metrics) { + if (metrics.empty()) { + return; + } + + for (auto& metric : metrics) { + remove_metric(metric); + } + } + + void remove_label_value(const std::map& labels) { + std::unique_lock lock(mtx_); + for (auto& [_, m] : metric_map_) { + m->remove_label_value(labels); + } + } + + void remove_metric_by_label( + const std::map& labels) { + std::unique_lock lock(mtx_); + for (auto it = metric_map_.begin(); it != metric_map_.end();) { + auto& m = it->second; + const auto& labels_name = m->labels_name(); + if (labels.size() > labels_name.size()) { + continue; + } + + if (labels.size() == labels_name.size()) { + std::vector label_value; + for (auto& lb_name : labels_name) { + if (auto i = labels.find(lb_name); i != labels.end()) { + label_value.push_back(i->second); + } + } + + std::erase_if(metric_map_, [&](auto& pair) { + return pair.second->has_label_value(label_value); + }); + if (m->has_label_value(label_value)) { + metric_map_.erase(it); + } + break; + } + else { + bool need_erase = false; + for (auto& lb_name : labels_name) { + if (auto i = labels.find(lb_name); i != labels.end()) { + if (m->has_label_value(i->second)) { + it = metric_map_.erase(it); + need_erase = true; + break; + } + } + } + + if (!need_erase) + ++it; + } + } + } + + void remove_metric_by_label_name( + const std::vector& labels_name) { + std::unique_lock lock(mtx_); + for (auto& [name, m] : metric_map_) { + if (m->labels_name() == labels_name) { + metric_map_.erase(name); + break; + } + } + } + + void remove_metric_by_label_name(std::string_view labels_name) { + std::unique_lock lock(mtx_); + for (auto it = metric_map_.cbegin(); it != metric_map_.cend();) { + auto& names = it->second->labels_name(); + if (auto sit = std::find(names.begin(), names.end(), labels_name); + sit != names.end()) { + metric_map_.erase(it++); + } + else { + ++it; + } + } + } + + size_t metric_count() { + std::unique_lock lock(mtx_); + return metric_map_.size(); + } + + auto metric_map() { + std::unique_lock lock(mtx_); + return metric_map_; + } + + auto collect() { + std::vector> metrics; + { + std::unique_lock lock(mtx_); + for (auto& pair : metric_map_) { + metrics.push_back(pair.second); + } + } + return metrics; + } + + template + std::shared_ptr get_metric_dynamic(const std::string& name) { + static_assert(std::is_base_of_v, + "must be dynamic metric"); + auto map = metric_map(); + auto it = map.find(name); + if (it == map.end()) { + return nullptr; + } + return std::dynamic_pointer_cast(it->second); + } + + std::shared_ptr get_metric_by_name(std::string_view name) { + auto map = metric_map(); + auto it = map.find(name); + if (it == map.end()) { + return nullptr; + } + + return it->second; + } + + std::vector> get_metric_by_label( + const std::vector>& labels) { + std::vector label_value; + for (auto& [k, v] : labels) { + label_value.push_back(v); + } + + return get_metric_by_label_value(label_value); + } + + std::vector> get_metric_by_label_name( + const std::vector& labels_name) { + auto map = metric_map(); + std::vector> vec; + for (auto& [name, m] : map) { + if (m->labels_name() == labels_name) { + vec.push_back(m); + } + } + return vec; + } + + std::vector> filter_metrics_dynamic( + const metric_filter_options& options) { + auto metrics = collect(); + return manager_helper::filter_metrics(metrics, options); + } + + std::vector> filter_metrics_by_label_value( + const std::regex& label_regex) { + auto metrics = collect(); + return manager_helper::filter_metrics_by_label_value(metrics, label_regex); + } + + private: + void clean_label_expired() { + executor_ = coro_io::create_io_context_pool(1); + timer_ = std::make_shared(executor_->get_executor()); + check_label_expired(timer_) + .via(executor_->get_executor()) + .start([](auto&&) { + }); + } + + async_simple::coro::Lazy check_label_expired( + std::weak_ptr weak) { + while (true) { + auto timer = weak.lock(); + if (timer == nullptr) { + co_return; + } + + timer->expires_after(ylt_label_check_expire_duration); + bool r = co_await timer->async_await(); + if (!r) { + co_return; + } + + std::unique_lock lock(mtx_); + for (auto& [_, m] : metric_map_) { + m->clean_expired_label(); + } + } + } + + dynamic_metric_manager() { + if (ylt_label_max_age.count() > 0) { + clean_label_expired(); + } + } + + std::vector> get_metric_by_label_value( + const std::vector& label_value) { + auto map = metric_map(); + std::vector> vec; + for (auto& [name, m] : map) { + if (m->has_label_value(label_value)) { + vec.push_back(m); + } + } + return vec; + } + + void remove_metric_by_label_value( + const std::vector& label_value) { + std::unique_lock lock(mtx_); + for (auto& [name, m] : metric_map_) { + if (m->has_label_value(label_value)) { + metric_map_.erase(name); + break; + } + } + } + + std::shared_mutex mtx_; + std::unordered_map> metric_map_; + std::shared_ptr timer_ = nullptr; + std::shared_ptr executor_ = nullptr; +}; + +struct ylt_default_metric_tag_t {}; +using default_static_metric_manager = + static_metric_manager; +using default_dynamiv_metric_manager = + dynamic_metric_manager; + +template +struct metric_manager_t; + +struct ylt_system_tag_t {}; +using system_metric_manager = static_metric_manager; + +template +struct metric_collector_t { + static std::string serialize() { + auto vec = get_all_metrics(); + return manager_helper::serialize(vec); + } + +#ifdef CINATRA_ENABLE_METRIC_JSON + static std::string serialize_to_json() { + auto vec = get_all_metrics(); + return manager_helper::serialize_to_json(vec); + } + + static std::string serialize_to_json( + const std::vector>& metrics) { + return manager_helper::serialize_to_json(metrics); + } +#endif + + static std::string serialize( + const std::vector>& metrics) { + return manager_helper::serialize(metrics); + } + + static std::vector> get_all_metrics() { + std::vector> vec; + (append_vector(vec), ...); + return vec; + } + + static std::vector> filter_metrics( + const metric_filter_options& options) { + auto vec = get_all_metrics(); + return manager_helper::filter_metrics(vec, options); + } + + private: + template + static void append_vector(std::vector>& vec) { + auto v = T::instance().collect(); + vec.insert(vec.end(), v.begin(), v.end()); + } +}; +} // namespace ylt::metric \ No newline at end of file diff --git a/include/cinatra/ylt/metric/summary.hpp b/include/cinatra/ylt/metric/summary.hpp index d28ba35b..9ac1a980 100644 --- a/include/cinatra/ylt/metric/summary.hpp +++ b/include/cinatra/ylt/metric/summary.hpp @@ -1,4 +1,5 @@ #pragma once +#include #include #include "detail/time_window_quantiles.hpp" @@ -27,92 +28,48 @@ struct json_summary_t { REFLECTION(json_summary_t, name, help, type, metrics); #endif -struct summary_label_sample { - std::vector labels_value; - double value; +struct block_t { + std::atomic stop_ = false; + ylt::detail::moodycamel::ConcurrentQueue sample_queue_; + std::shared_ptr quantile_values_; + std::uint64_t count_; + double sum_; }; -class summary_t : public metric_t { +class summary_t : public static_metric { public: using Quantiles = std::vector; summary_t(std::string name, std::string help, Quantiles quantiles, std::chrono::milliseconds max_age = std::chrono::seconds{60}, - int age_buckets = 5) + uint16_t age_buckets = 5) : quantiles_{std::move(quantiles)}, - metric_t(MetricType::Summary, std::move(name), std::move(help)), - max_age_(max_age), - age_buckets_(age_buckets) { - init_block(block_); - block_->quantile_values_ = - std::make_shared(quantiles_, max_age, age_buckets); - use_atomic_ = true; - g_user_metric_count++; - } - - summary_t(std::string name, std::string help, Quantiles quantiles, - std::vector labels_name, - std::chrono::milliseconds max_age = std::chrono::seconds{60}, - int age_buckets = 5) - : quantiles_{std::move(quantiles)}, - metric_t(MetricType::Summary, std::move(name), std::move(help), - std::move(labels_name)), - max_age_(max_age), - age_buckets_(age_buckets) { - init_block(labels_block_); - g_user_metric_count++; + static_metric(MetricType::Summary, std::move(name), std::move(help)) { + init_no_label(max_age, age_buckets); } summary_t(std::string name, std::string help, Quantiles quantiles, std::map static_labels, std::chrono::milliseconds max_age = std::chrono::seconds{60}, - int age_buckets = 5) + uint16_t age_buckets = 5) : quantiles_{std::move(quantiles)}, - metric_t(MetricType::Summary, std::move(name), std::move(help), - std::move(static_labels)), - max_age_(max_age), - age_buckets_(age_buckets) { - init_block(labels_block_); - labels_block_->label_quantile_values_[labels_value_] = - std::make_shared(quantiles_, max_age, age_buckets); - labels_block_->label_count_.emplace(labels_value_, 0); - labels_block_->label_sum_.emplace(labels_value_, 0); - use_atomic_ = true; - g_user_metric_count++; + static_metric(MetricType::Summary, std::move(name), std::move(help), + std::move(static_labels)) { + init_no_label(max_age, age_buckets); } ~summary_t() { if (block_) { block_->stop_ = true; } - - if (labels_block_) { - labels_block_->stop_ = true; - } } - struct block_t { - std::atomic stop_ = false; - moodycamel::ConcurrentQueue sample_queue_; - std::shared_ptr quantile_values_; - std::uint64_t count_; - double sum_; - }; - - struct labels_block_t { - std::atomic stop_ = false; - moodycamel::ConcurrentQueue sample_queue_; - metric_hash_map> - label_quantile_values_; - metric_hash_map label_count_; - metric_hash_map label_sum_; - }; - void observe(double value) { - if (!labels_name_.empty()) { - throw std::invalid_argument("not a default label metric"); + if (!has_observe_) [[unlikely]] { + has_observe_ = true; } - if (block_->sample_queue_.size_approx() >= 20000000) { - g_summary_failed_count->inc(); + int64_t max_limit = (std::min)(ylt_label_capacity, (int64_t)1000000); + if (block_->sample_queue_.size_approx() >= max_limit) { + g_summary_failed_count++; return; } block_->sample_queue_.enqueue(value); @@ -124,28 +81,6 @@ class summary_t : public metric_t { } } - void observe(std::vector labels_value, double value) { - if (labels_value.empty()) { - throw std::invalid_argument("not a label metric"); - } - if (use_atomic_) { - if (labels_value != labels_value_) { - throw std::invalid_argument("not equal with static label"); - } - } - if (labels_block_->sample_queue_.size_approx() >= 20000000) { - g_summary_failed_count->inc(); - return; - } - labels_block_->sample_queue_.enqueue({std::move(labels_value), value}); - - bool expected = false; - if (is_coro_started_.compare_exchange_strong(expected, true)) { - start(labels_block_).via(excutor_->get_executor()).start([](auto &&) { - }); - } - } - async_simple::coro::Lazy> get_rates(double &sum, uint64_t &count) { std::vector vec; @@ -166,46 +101,6 @@ class summary_t : public metric_t { co_return vec; } - async_simple::coro::Lazy> get_rates( - const std::vector &labels_value, double &sum, - uint64_t &count) { - std::vector vec; - if (quantiles_.empty()) { - co_return std::vector{}; - } - - if (use_atomic_) { - if (labels_value != labels_value_) { - throw std::invalid_argument("not equal with static label"); - } - } - - co_await coro_io::post( - [this, &vec, &sum, &count, &labels_value] { - auto it = labels_block_->label_quantile_values_.find(labels_value); - if (it == labels_block_->label_quantile_values_.end()) { - return; - } - sum = labels_block_->label_sum_[labels_value]; - count = labels_block_->label_count_[labels_value]; - for (const auto &quantile : quantiles_) { - vec.push_back(it->second->get(quantile.quantile)); - } - }, - excutor_->get_executor()); - - co_return vec; - } - - metric_hash_map value_map() override { - auto ret = async_simple::coro::syncAwait(coro_io::post( - [this] { - return labels_block_->label_sum_; - }, - excutor_->get_executor())); - return ret.value(); - } - async_simple::coro::Lazy get_sum() { auto ret = co_await coro_io::post( [this] { @@ -227,11 +122,11 @@ class summary_t : public metric_t { size_t size_approx() { return block_->sample_queue_.size_approx(); } async_simple::coro::Lazy serialize_async(std::string &str) override { - if (block_ == nullptr) { - co_await serialize_async_with_label(str); + if (quantiles_.empty()) { co_return; } - if (quantiles_.empty()) { + + if (!has_observe_) { co_return; } @@ -243,7 +138,13 @@ class summary_t : public metric_t { for (size_t i = 0; i < quantiles_.size(); i++) { str.append(name_); - str.append("{quantile=\""); + str.append("{"); + if (!labels_name_.empty()) { + build_label_string(str, labels_name_, labels_value_); + str.append(","); + } + + str.append("quantile=\""); str.append(std::to_string(quantiles_[i].quantile)).append("\"} "); str.append(std::to_string(rates[i])).append("\n"); } @@ -258,12 +159,11 @@ class summary_t : public metric_t { #ifdef CINATRA_ENABLE_METRIC_JSON async_simple::coro::Lazy serialize_to_json_async( std::string &str) override { - if (block_ == nullptr) { - co_await serialize_to_json_with_label_async(str); + if (quantiles_.empty()) { co_return; } - if (quantiles_.empty()) { + if (!has_observe_) { co_return; } @@ -275,6 +175,9 @@ class summary_t : public metric_t { json_summary_metric_t metric; for (size_t i = 0; i < quantiles_.size(); i++) { + for (size_t i = 0; i < labels_name_.size(); i++) { + metric.labels[labels_name_[i]] = labels_value_[i]; + } metric.quantiles.emplace(quantiles_[i].quantile, rates[i]); } @@ -294,9 +197,16 @@ class summary_t : public metric_t { }); } + void init_no_label(std::chrono::milliseconds max_age, uint16_t age_buckets) { + init_block(block_); + block_->quantile_values_ = + std::make_shared(quantiles_, max_age, age_buckets); + g_user_metric_count++; + } + async_simple::coro::Lazy start(std::shared_ptr block) { double sample; - size_t count = 1000000; + size_t count = 100000; while (!block->stop_) { size_t index = 0; while (block->sample_queue_.try_dequeue(sample)) { @@ -329,10 +239,137 @@ class summary_t : public metric_t { co_return; } + Quantiles quantiles_; // readonly + std::shared_ptr block_; + static inline std::shared_ptr excutor_ = + coro_io::create_io_context_pool(1); + std::atomic is_coro_started_ = false; + bool has_observe_ = false; +}; + +template +struct summary_label_sample { + std::array labels_value; + double value; +}; + +struct sum_and_count_t { + double sum; + uint64_t count; +}; + +template +struct labels_block_t { + std::atomic stop_ = false; + ylt::detail::moodycamel::ConcurrentQueue> + sample_queue_; + dynamic_metric_hash_map, + std::shared_ptr> + label_quantile_values_; + dynamic_metric_hash_map, sum_and_count_t> + sum_and_count_; +}; + +template +class basic_dynamic_summary : public dynamic_metric { + public: + using Quantiles = std::vector; + + basic_dynamic_summary( + std::string name, std::string help, Quantiles quantiles, + std::array labels_name, + std::chrono::milliseconds max_age = std::chrono::seconds{60}, + uint16_t age_buckets = 5) + : quantiles_{std::move(quantiles)}, + dynamic_metric(MetricType::Summary, std::move(name), std::move(help), + std::move(labels_name)), + max_age_(max_age), + age_buckets_(age_buckets) { + init_block(labels_block_); + g_user_metric_count++; + } + + ~basic_dynamic_summary() { + if (labels_block_) { + labels_block_->stop_ = true; + } + } + + void observe(std::array labels_value, double value) { + if (!has_observe_) [[unlikely]] { + has_observe_ = true; + } + int64_t max_limit = (std::min)(ylt_label_capacity, (int64_t)1000000); + if (labels_block_->sample_queue_.size_approx() >= max_limit) { + g_summary_failed_count++; + return; + } + labels_block_->sample_queue_.enqueue({std::move(labels_value), value}); + + bool expected = false; + if (is_coro_started_.compare_exchange_strong(expected, true)) { + start(labels_block_).via(excutor_->get_executor()).start([](auto &&) { + }); + } + } + + size_t size_approx() { return labels_block_->sample_queue_.size_approx(); } + + size_t label_value_count() override { + auto block = labels_block_; + return async_simple::coro::syncAwait(coro_io::post([block] { + return block->sum_and_count_.size(); + })) + .value(); + } + + async_simple::coro::Lazy> get_rates( + const std::array &labels_value, double &sum, + uint64_t &count) { + std::vector vec; + if (quantiles_.empty()) { + co_return std::vector{}; + } + + co_await coro_io::post( + [this, &vec, &sum, &count, &labels_value] { + auto it = labels_block_->label_quantile_values_.find(labels_value); + if (it == labels_block_->label_quantile_values_.end()) { + return; + } + sum = labels_block_->sum_and_count_[labels_value].sum; + count = labels_block_->sum_and_count_[labels_value].count; + for (const auto &quantile : quantiles_) { + vec.push_back(it->second->get(quantile.quantile)); + } + }, + excutor_->get_executor()); + + co_return vec; + } + + async_simple::coro::Lazy serialize_async(std::string &str) override { + co_await serialize_async_with_label(str); + } + +#ifdef CINATRA_ENABLE_METRIC_JSON + async_simple::coro::Lazy serialize_to_json_async( + std::string &str) override { + co_await serialize_to_json_with_label_async(str); + } +#endif + private: + template + void init_block(std::shared_ptr &block) { + block = std::make_shared(); + start(block).via(excutor_->get_executor()).start([](auto &&) { + }); + } + async_simple::coro::Lazy start( - std::shared_ptr label_block) { - summary_label_sample sample; - size_t count = 1000000; + std::shared_ptr> label_block) { + summary_label_sample sample; + size_t count = 100000; while (!label_block->stop_) { size_t index = 0; while (label_block->sample_queue_.try_dequeue(sample)) { @@ -345,8 +382,8 @@ class summary_t : public metric_t { ptr->insert(sample.value); - label_block->label_count_[sample.labels_value] += 1; - label_block->label_sum_[sample.labels_value] += sample.value; + label_block->sum_and_count_[sample.labels_value].count += 1; + label_block->sum_and_count_[sample.labels_value].sum += sample.value; index++; if (index == count) { break; @@ -379,15 +416,19 @@ class summary_t : public metric_t { co_return; } + if (!has_observe_) { + co_return; + } + serialize_head(str); auto sum_map = co_await coro_io::post( [this] { - return labels_block_->label_sum_; + return labels_block_->sum_and_count_; }, excutor_->get_executor()); - for (auto &[labels_value, sum_val] : sum_map.value()) { + for (auto &[labels_value, _] : sum_map.value()) { double sum = 0; uint64_t count = 0; auto rates = co_await get_rates(labels_value, sum, count); @@ -422,15 +463,19 @@ class summary_t : public metric_t { co_return; } + if (!has_observe_) { + co_return; + } + auto sum_map = co_await coro_io::post( [this] { - return labels_block_->label_sum_; + return labels_block_->sum_and_count_; }, excutor_->get_executor()); json_summary_t summary{name_, help_, std::string(metric_name())}; - for (auto &[labels_value, sum_val] : sum_map.value()) { + for (auto &[labels_value, _] : sum_map.value()) { json_summary_metric_t metric; double sum = 0; uint64_t count = 0; @@ -451,12 +496,19 @@ class summary_t : public metric_t { #endif Quantiles quantiles_; // readonly - std::shared_ptr block_; - std::shared_ptr labels_block_; + std::shared_ptr> labels_block_; static inline std::shared_ptr excutor_ = coro_io::create_io_context_pool(1); std::chrono::milliseconds max_age_; - int age_buckets_; + uint16_t age_buckets_; std::atomic is_coro_started_ = false; + bool has_observe_ = false; }; + +using dynamic_summary_1 = basic_dynamic_summary<1>; +using dynamic_summary_2 = basic_dynamic_summary<2>; +using dynamic_summary = dynamic_summary_2; +using dynamic_summary_3 = basic_dynamic_summary<3>; +using dynamic_summary_4 = basic_dynamic_summary<4>; +using dynamic_summary_5 = basic_dynamic_summary<5>; } // namespace ylt::metric \ No newline at end of file diff --git a/include/cinatra/ylt/metric/system_metric.hpp b/include/cinatra/ylt/metric/system_metric.hpp index 95c52c7d..8cd305f9 100644 --- a/include/cinatra/ylt/metric/system_metric.hpp +++ b/include/cinatra/ylt/metric/system_metric.hpp @@ -16,19 +16,63 @@ #include "ylt/metric/counter.hpp" #include "ylt/metric/gauge.hpp" #include "ylt/metric/metric.hpp" +#include "ylt/metric/metric_manager.hpp" #else #include "cinatra/ylt/coro_io/coro_io.hpp" #include "cinatra/ylt/coro_io/io_context_pool.hpp" #include "cinatra/ylt/metric/counter.hpp" #include "cinatra/ylt/metric/gauge.hpp" #include "cinatra/ylt/metric/metric.hpp" +#include "cinatra/ylt/metric/metric_manager.hpp" #endif -// reference: brpc/src/bvar/default_variables.cpp +// modified based on: brpc/src/bvar/default_variables.cpp namespace ylt::metric { namespace detail { +#if defined(__APPLE__) +#include + +inline int read_command_output_through_popen(std::ostream& os, + const char* cmd) { + FILE* pipe = popen(cmd, "r"); + if (pipe == NULL) { + return -1; + } + char buffer[1024]; + for (;;) { + size_t nr = fread(buffer, 1, sizeof(buffer), pipe); + if (nr != 0) { + os.write(buffer, nr); + } + if (nr != sizeof(buffer)) { + if (feof(pipe)) { + break; + } + else if (ferror(pipe)) { + break; + } + // retry; + } + } + + const int wstatus = pclose(pipe); + + if (wstatus < 0) { + return wstatus; + } + if (WIFEXITED(wstatus)) { + return WEXITSTATUS(wstatus); + } + if (WIFSIGNALED(wstatus)) { + os << "Child process was killed by signal " << WTERMSIG(wstatus); + } + errno = ECHILD; + return -1; +} +#endif + inline int64_t last_time_us = 0; inline int64_t last_sys_time_us = 0; inline int64_t last_user_time_us = 0; @@ -39,19 +83,19 @@ inline int64_t gettimeofday_us() { return now.tv_sec * 1000000L + now.tv_usec; } -inline int64_t timeval_to_microseconds(const timeval &tv) { +inline int64_t timeval_to_microseconds(const timeval& tv) { return tv.tv_sec * 1000000L + tv.tv_usec; } inline void stat_cpu() { static auto process_cpu_usage = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_cpu_usage"); static auto process_cpu_usage_system = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_cpu_usage_system"); static auto process_cpu_usage_user = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_cpu_usage_user"); rusage usage{}; @@ -90,24 +134,40 @@ inline void stat_cpu() { inline void stat_memory() { static auto process_memory_virtual = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_memory_virtual"); static auto process_memory_resident = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_memory_resident"); static auto process_memory_shared = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_memory_shared"); long virtual_size = 0; long resident = 0; long share = 0; + static long page_size = sysconf(_SC_PAGE_SIZE); + +#if defined(__APPLE__) + static pid_t pid = getpid(); + static int64_t pagesize = getpagesize(); + std::ostringstream oss; + char cmdbuf[128]; + snprintf(cmdbuf, sizeof(cmdbuf), "ps -p %ld -o rss=,vsz=", (long)pid); + if (read_command_output_through_popen(oss, cmdbuf) != 0) { + return; + } + const std::string& result = oss.str(); + if (sscanf(result.c_str(), "%ld %ld", &resident, &virtual_size) != 2) { + return; + } +#else std::ifstream file("/proc/self/statm"); if (!file) { return; } file >> virtual_size >> resident >> share; - static long page_size = sysconf(_SC_PAGE_SIZE); +#endif process_memory_virtual->update(virtual_size * page_size); process_memory_resident->update(resident * page_size); @@ -126,18 +186,21 @@ struct ProcIO { inline void stat_io() { static auto process_io_read_bytes_second = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_io_read_bytes_second"); static auto process_io_write_bytes_second = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_io_write_bytes_second"); static auto process_io_read_second = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_io_read_second"); static auto process_io_write_second = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_io_write_second"); + ProcIO s{}; +#if defined(__APPLE__) +#else auto stream_file = std::shared_ptr(fopen("/proc/self/io", "r"), [](FILE *ptr) { fclose(ptr); @@ -146,13 +209,13 @@ inline void stat_io() { return; } - ProcIO s{}; if (fscanf(stream_file.get(), "%*s %lu %*s %lu %*s %lu %*s %lu %*s %lu %*s %lu %*s %lu", &s.rchar, &s.wchar, &s.syscr, &s.syscw, &s.read_bytes, &s.write_bytes, &s.cancelled_write_bytes) != 7) { return; } +#endif process_io_read_bytes_second->update(s.rchar); process_io_write_bytes_second->update(s.wchar); @@ -162,26 +225,41 @@ inline void stat_io() { inline void stat_avg_load() { static auto system_loadavg_1m = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_system_loadavg_1m"); static auto system_loadavg_5m = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_system_loadavg_5m"); static auto system_loadavg_15m = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_system_loadavg_15m"); + + double loadavg_1m = 0; + double loadavg_5m = 0; + double loadavg_15m = 0; + +#if defined(__APPLE__) + std::ostringstream oss; + if (read_command_output_through_popen(oss, "sysctl -n vm.loadavg") != 0) { + return; + } + const std::string& result = oss.str(); + if (sscanf(result.c_str(), "{ %lf %lf %lf }", &loadavg_1m, &loadavg_5m, + &loadavg_15m) != 3) { + return; + } +#else std::ifstream file("/proc/loadavg"); if (!file) { return; } - double loadavg_1m = 0; - double loadavg_5m = 0; - double loadavg_15m = 0; file >> loadavg_1m >> loadavg_5m >> loadavg_15m; +#endif + system_loadavg_1m->update(loadavg_1m); - system_loadavg_1m->update(loadavg_5m); - system_loadavg_1m->update(loadavg_15m); + system_loadavg_5m->update(loadavg_5m); + system_loadavg_15m->update(loadavg_15m); } struct ProcStat { @@ -209,27 +287,31 @@ struct ProcStat { inline void process_status() { static auto process_uptime = - system_metric_manager::get_metric_static("ylt_process_uptime"); + system_metric_manager::instance().get_metric_static( + "ylt_process_uptime"); static auto process_priority = - system_metric_manager::get_metric_static("ylt_process_priority"); + system_metric_manager::instance().get_metric_static( + "ylt_process_priority"); static auto pid = - system_metric_manager::get_metric_static("ylt_pid"); + system_metric_manager::instance().get_metric_static("ylt_pid"); static auto ppid = - system_metric_manager::get_metric_static("ylt_ppid"); + system_metric_manager::instance().get_metric_static("ylt_ppid"); static auto pgrp = - system_metric_manager::get_metric_static("ylt_pgrp"); + system_metric_manager::instance().get_metric_static("ylt_pgrp"); static auto thread_count = - system_metric_manager::get_metric_static("ylt_thread_count"); + system_metric_manager::instance().get_metric_static( + "ylt_thread_count"); + ProcStat stat{}; +#if defined(__linux__) auto stream_file = - std::shared_ptr(fopen("/proc/self/stat", "r"), [](FILE *ptr) { + std::shared_ptr(fopen("/proc/self/stat", "r"), [](FILE* ptr) { fclose(ptr); }); if (stream_file == nullptr) { return; } - ProcStat stat{}; if (fscanf(stream_file.get(), "%d %*s %c " "%d %d %d %d %d " @@ -243,7 +325,26 @@ inline void process_status() { &stat.nice, &stat.num_threads) != 19) { return; } - +#elif defined(__APPLE__) + static pid_t proc_id = getpid(); + std::ostringstream oss; + char cmdbuf[128]; + snprintf(cmdbuf, sizeof(cmdbuf), + "ps -p %ld -o pid,ppid,pgid,sess" + ",tpgid,flags,pri,nice | tail -n1", + (long)proc_id); + if (read_command_output_through_popen(oss, cmdbuf) != 0) { + return; + } + const std::string &result = oss.str(); + if (sscanf(result.c_str(), + "%d %d %d %d" + "%d %u %ld %ld", + &stat.pid, &stat.ppid, &stat.pgrp, &stat.session, &stat.tpgid, + &stat.flags, &stat.priority, &stat.nice) != 8) { + return; + } +#endif process_uptime->inc(); process_priority->update(stat.priority); pid->update(stat.pid); @@ -254,9 +355,19 @@ inline void process_status() { inline void stat_metric() { static auto user_metric_count = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_user_metric_count"); user_metric_count->update(g_user_metric_count); + + static auto user_metric_label_count = + system_metric_manager::instance().get_metric_static( + "ylt_user_metric_labels"); + user_metric_label_count->update(g_user_metric_label_count->value()); + + static auto ylt_summary_failed_count = + system_metric_manager::instance().get_metric_static( + "ylt_summary_failed_count"); + ylt_summary_failed_count->update(g_summary_failed_count); } inline void ylt_stat() { @@ -268,7 +379,12 @@ inline void ylt_stat() { stat_metric(); } -inline void start_stat(std::shared_ptr timer) { +inline void start_stat(std::weak_ptr weak) { + auto timer = weak.lock(); + if (timer == nullptr) { + return; + } + timer->expires_after(std::chrono::seconds(1)); timer->async_wait([timer](std::error_code ec) { if (ec) { @@ -282,51 +398,56 @@ inline void start_stat(std::shared_ptr timer) { } // namespace detail inline bool start_system_metric() { - system_metric_manager::create_metric_static("ylt_process_cpu_usage", - ""); - system_metric_manager::create_metric_static( + system_metric_manager::instance().create_metric_static( + "ylt_process_cpu_usage", ""); + system_metric_manager::instance().create_metric_static( "ylt_process_cpu_usage_system", ""); - system_metric_manager::create_metric_static( + system_metric_manager::instance().create_metric_static( "ylt_process_cpu_usage_user", ""); - system_metric_manager::create_metric_static( + system_metric_manager::instance().create_metric_static( "ylt_process_memory_virtual", ""); - system_metric_manager::create_metric_static( + system_metric_manager::instance().create_metric_static( "ylt_process_memory_resident", ""); - system_metric_manager::create_metric_static( + system_metric_manager::instance().create_metric_static( "ylt_process_memory_shared", ""); - system_metric_manager::create_metric_static("ylt_process_uptime", - ""); - system_metric_manager::create_metric_static("ylt_pid", ""); - system_metric_manager::create_metric_static("ylt_ppid", ""); - system_metric_manager::create_metric_static("ylt_pgrp", ""); - system_metric_manager::create_metric_static("ylt_thread_count", ""); - system_metric_manager::create_metric_static("ylt_process_priority", - ""); - - system_metric_manager::create_metric_static("ylt_user_metric_count", - ""); - - system_metric_manager::create_metric_static("ylt_system_loadavg_1m", - ""); - system_metric_manager::create_metric_static("ylt_system_loadavg_5m", - ""); - system_metric_manager::create_metric_static("ylt_system_loadavg_15m", - ""); - - system_metric_manager::create_metric_static( + system_metric_manager::instance().create_metric_static( + "ylt_process_uptime", ""); + system_metric_manager::instance().create_metric_static("ylt_pid", + ""); + system_metric_manager::instance().create_metric_static("ylt_ppid", + ""); + system_metric_manager::instance().create_metric_static("ylt_pgrp", + ""); + system_metric_manager::instance().create_metric_static( + "ylt_thread_count", ""); + system_metric_manager::instance().create_metric_static( + "ylt_process_priority", ""); + + system_metric_manager::instance().create_metric_static( + "ylt_user_metric_count", ""); + system_metric_manager::instance().create_metric_static( + "ylt_user_metric_labels", ""); + system_metric_manager::instance().create_metric_static( + "ylt_summary_failed_count", ""); + + system_metric_manager::instance().create_metric_static( + "ylt_system_loadavg_1m", ""); + system_metric_manager::instance().create_metric_static( + "ylt_system_loadavg_5m", ""); + system_metric_manager::instance().create_metric_static( + "ylt_system_loadavg_15m", ""); + + system_metric_manager::instance().create_metric_static( "ylt_process_io_read_bytes_second", ""); - system_metric_manager::create_metric_static( + system_metric_manager::instance().create_metric_static( "ylt_process_io_write_bytes_second", ""); - system_metric_manager::create_metric_static( + system_metric_manager::instance().create_metric_static( "ylt_process_io_read_second", ""); - system_metric_manager::create_metric_static( + system_metric_manager::instance().create_metric_static( "ylt_process_io_write_second", ""); - system_metric_manager::register_metric_static(g_user_metric_labels); - system_metric_manager::register_metric_static(g_summary_failed_count); - static auto exucutor = coro_io::create_io_context_pool(1); auto timer = std::make_shared(exucutor->get_executor()); diff --git a/include/cinatra/ylt/metric/thread_local_value.hpp b/include/cinatra/ylt/metric/thread_local_value.hpp new file mode 100644 index 00000000..b73789dd --- /dev/null +++ b/include/cinatra/ylt/metric/thread_local_value.hpp @@ -0,0 +1,112 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace ylt::metric { +inline uint32_t get_round_index(uint32_t size) { + static std::atomic round = 0; + static thread_local uint32_t index = round++; + return index % size; +} +template +class thread_local_value { + public: + thread_local_value(uint32_t dupli_count = std::thread::hardware_concurrency()) + : duplicates_(dupli_count) {} + + ~thread_local_value() { + for (auto &t : duplicates_) { + if (t) { + delete t.load(); + } + } + } + + thread_local_value(const thread_local_value &other) + : duplicates_(other.duplicates_.size()) { + for (size_t i = 0; i < other.duplicates_.size(); i++) { + if (other.duplicates_[i]) { + auto ptr = + new std::atomic(other.duplicates_[i].load()->load()); + duplicates_[i] = ptr; + } + } + } + + thread_local_value &operator=(const thread_local_value &other) { + for (size_t i = 0; i < other.duplicates_.size(); i++) { + if (other.duplicates_[i]) { + auto ptr = + new std::atomic(other.duplicates_[i].load()->load()); + duplicates_[i] = ptr; + } + } + return *this; + } + + thread_local_value(thread_local_value &&other) { + duplicates_ = std::move(other.duplicates_); + } + + thread_local_value &operator=(thread_local_value &&other) { + duplicates_ = std::move(other.duplicates_); + return *this; + } + + void inc(value_type value = 1) { local_value() += value; } + + void dec(value_type value = 1) { local_value() -= value; } + + value_type update(value_type value = 1) { + value_type val = get_value(0).exchange(value); + for (size_t i = 1; i < duplicates_.size(); i++) { + if (duplicates_[i]) { + val += duplicates_[i].load()->exchange(0); + } + } + return val; + } + + value_type reset() { return update(0); } + + auto &local_value() { + auto index = get_round_index(duplicates_.size()); + return get_value(index); + } + + auto &get_value(size_t index) { + if (duplicates_[index] == nullptr) { + auto ptr = new std::atomic(0); + + std::atomic *expected = nullptr; + if (!duplicates_[index].compare_exchange_strong(expected, ptr)) { + delete ptr; + } + } + return *duplicates_[index]; + } + + value_type value() { + value_type val = 0; + for (auto &t : duplicates_) { + if (t) { + val += t.load()->load(); + } + } + return val; + } + + void set_created_time(std::chrono::system_clock::time_point tm) { + created_time_ = tm; + } + + auto get_created_time() { return created_time_; } + + private: + std::vector *>> duplicates_; + std::chrono::system_clock::time_point created_time_{}; +}; +} // namespace ylt::metric diff --git a/include/cinatra/ylt/util/concurrentqueue.h b/include/cinatra/ylt/util/concurrentqueue.h index e6ef3699..e0ca9f5f 100644 --- a/include/cinatra/ylt/util/concurrentqueue.h +++ b/include/cinatra/ylt/util/concurrentqueue.h @@ -100,7 +100,7 @@ // Platform-specific definitions of a numeric thread ID type and an invalid // value -namespace moodycamel { +namespace ylt::detail::moodycamel { namespace details { template struct thread_id_converter { @@ -109,22 +109,22 @@ struct thread_id_converter { static thread_id_hash_t prehash(thread_id_t const& x) { return x; } }; } // namespace details -} // namespace moodycamel +} // namespace ylt::detail::moodycamel #if defined(MCDBGQ_USE_RELACY) -namespace moodycamel { +namespace ylt::detail::moodycamel { namespace details { typedef std::uint32_t thread_id_t; static const thread_id_t invalid_thread_id = 0xFFFFFFFFU; static const thread_id_t invalid_thread_id2 = 0xFFFFFFFEU; static inline thread_id_t thread_id() { return rl::thread_index(); } } // namespace details -} // namespace moodycamel +} // namespace ylt::detail::moodycamel #elif defined(_WIN32) || defined(__WINDOWS__) || defined(__WIN32__) // No sense pulling in windows.h in a header, we'll manually declare the // function we use and rely on backwards-compatibility for this not to break extern "C" __declspec(dllimport) unsigned long __stdcall GetCurrentThreadId( void); -namespace moodycamel { +namespace ylt::detail::moodycamel { namespace details { static_assert(sizeof(unsigned long) == sizeof(std::uint32_t), "Expected size of unsigned long to be 32 bits on Windows"); @@ -139,11 +139,11 @@ static inline thread_id_t thread_id() { return static_cast(::GetCurrentThreadId()); } } // namespace details -} // namespace moodycamel +} // namespace ylt::detail::moodycamel #elif defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || \ (defined(__APPLE__) && TARGET_OS_IPHONE) || \ defined(MOODYCAMEL_NO_THREAD_LOCAL) -namespace moodycamel { +namespace ylt::detail::moodycamel { namespace details { static_assert(sizeof(std::thread::id) == 4 || sizeof(std::thread::id) == 8, "std::thread::id is expected to be either 4 or 8 bytes"); @@ -199,7 +199,7 @@ struct thread_id_converter { // Assume C++11 compliant compiler #define MOODYCAMEL_THREADLOCAL thread_local #endif -namespace moodycamel { +namespace ylt::detail::moodycamel { namespace details { typedef std::uintptr_t thread_id_t; static const thread_id_t invalid_thread_id = 0; // Address can't be nullptr @@ -330,7 +330,7 @@ inline thread_id_t thread_id() { #endif #endif -namespace moodycamel { +namespace ylt::detail::moodycamel { namespace details { #ifndef MOODYCAMEL_ALIGNAS // VS2013 doesn't support alignas or alignof, and align() requires a constant @@ -390,7 +390,7 @@ struct identity { #endif #endif } // namespace details -} // namespace moodycamel +} // namespace ylt::detail::moodycamel // TSAN can false report races in lock-free code. To enable TSAN to be used // from projects that use this one, we can apply per-function compile-time @@ -405,7 +405,7 @@ struct identity { #endif // TSAN // Compiler-specific likely/unlikely hints -namespace moodycamel { +namespace ylt::detail::moodycamel { namespace details { #if defined(__GNUC__) static inline bool(likely)(bool x) { return __builtin_expect((x), true); } @@ -415,13 +415,13 @@ static inline bool(likely)(bool x) { return x; } static inline bool(unlikely)(bool x) { return x; } #endif } // namespace details -} // namespace moodycamel +} // namespace ylt::detail::moodycamel #ifdef MOODYCAMEL_QUEUE_INTERNAL_DEBUG #include "internal/concurrentqueue_internal_debug.h" #endif -namespace moodycamel { +namespace ylt::detail::moodycamel { namespace details { template struct const_numeric_max { @@ -937,8 +937,8 @@ inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, template class ConcurrentQueue { public: - typedef ::moodycamel::ProducerToken producer_token_t; - typedef ::moodycamel::ConsumerToken consumer_token_t; + typedef ::ylt::detail::moodycamel::ProducerToken producer_token_t; + typedef ::ylt::detail::moodycamel::ConsumerToken consumer_token_t; typedef typename Traits::index_t index_t; typedef typename Traits::size_t size_t; @@ -4388,7 +4388,7 @@ inline void swap(typename ConcurrentQueue::ImplicitProducerKVP& a, a.swap(b); } -} // namespace moodycamel +} // namespace ylt::detail::moodycamel #if defined(_MSC_VER) && (!defined(_HAS_CXX17) || !_HAS_CXX17) #pragma warning(pop) diff --git a/tests/test_metric.cpp b/tests/test_metric.cpp index ebc0d220..52750427 100644 --- a/tests/test_metric.cpp +++ b/tests/test_metric.cpp @@ -8,15 +8,497 @@ #include "cinatra/ylt/metric/counter.hpp" #include "cinatra/ylt/metric/histogram.hpp" +#include "cinatra/ylt/metric/metric_manager.hpp" #include "cinatra/ylt/metric/summary.hpp" #include "cinatra/ylt/metric/system_metric.hpp" #include "doctest/doctest.h" using namespace ylt; using namespace ylt::metric; +struct metrc_tag {}; + +struct test_tag {}; + +TEST_CASE("serialize zero") { + counter_t c("test", ""); + gauge_t g("test1", ""); + std::string str; + c.serialize(str); + CHECK(str.empty()); + g.serialize(str); + CHECK(str.empty()); + c.inc(); + c.serialize(str); + CHECK(!str.empty()); + str.clear(); + g.inc(); + g.serialize(str); + CHECK(!str.empty()); + c.update(0); + c.serialize(str); + CHECK(!str.empty()); + str.clear(); + g.dec(); + g.serialize(str); + CHECK(!str.empty()); + str.clear(); + + dynamic_counter_1t c1("test", "", {"url"}); + c1.serialize(str); + CHECK(str.empty()); + dynamic_gauge_1t g1("test", "", {"url"}); + g1.serialize(str); + CHECK(str.empty()); + c1.inc({"/test"}); + c1.serialize(str); + CHECK(!str.empty()); + str.clear(); + g1.inc({"/test"}); + g1.serialize(str); + CHECK(!str.empty()); + str.clear(); + + c1.update({"/test"}, 0); + c1.serialize(str); + CHECK(!str.empty()); + str.clear(); + + g1.dec({"/test"}); + g1.serialize(str); + CHECK(!str.empty()); + str.clear(); + +#ifdef CINATRA_ENABLE_METRIC_JSON + c1.serialize_to_json(str); + CHECK(!str.empty()); + str.clear(); + g1.serialize_to_json(str); + CHECK(!str.empty()); + str.clear(); +#endif + + histogram_t h("test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}); + h.serialize(str); + CHECK(str.empty()); +#ifdef CINATRA_ENABLE_METRIC_JSON + h.serialize_to_json(str); +#endif + CHECK(str.empty()); + h.observe(23); + h.serialize(str); + CHECK(!str.empty()); + str.clear(); + + std::map customMap = {}; + auto summary = std::make_shared( + "test", "help", + summary_t::Quantiles{ + {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, + customMap); + async_simple::coro::syncAwait(summary->serialize_async(str)); + CHECK(str.empty()); +#ifdef CINATRA_ENABLE_METRIC_JSON + async_simple::coro::syncAwait(summary->serialize_to_json_async(str)); + CHECK(str.empty()); +#endif + summary->observe(0); + async_simple::coro::syncAwait(summary->serialize_async(str)); + CHECK(!str.empty()); + str.clear(); +#ifdef CINATRA_ENABLE_METRIC_JSON + async_simple::coro::syncAwait(summary->serialize_to_json_async(str)); + CHECK(!str.empty()); + str.clear(); +#endif +} + +TEST_CASE("test metric manager") { + auto c = std::make_shared("test1", ""); + auto g = std::make_shared("test2", ""); + auto& inst_s = static_metric_manager::instance(); + static_metric_manager::instance().register_metric(c); + static_metric_manager::instance().register_metric(g); + auto pair = inst_s.create_metric_static("test1", ""); + CHECK(pair.first == std::errc::invalid_argument); + auto v1 = inst_s.get_metric_by_label({}); + CHECK(v1.size() == 2); + auto v2 = inst_s.get_metric_by_name("test1"); + CHECK(v2 != nullptr); + + c->inc(); + g->inc(); + + inst_s.create_metric_static( + "test_counter", "", std::map{{"url", "/"}}); + auto ms = inst_s.filter_metrics_by_label_value(std::regex("/")); + CHECK(ms.size() == 1); + + { + std::string str = inst_s.serialize_static(); + std::cout << str << "\n"; +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json = inst_s.serialize_to_json_static(); + std::cout << json << "\n"; +#endif + } + + { + metric_filter_options options; + options.name_regex = ".*test.*"; + auto v5 = inst_s.filter_metrics_static(options); + CHECK(v5.size() == 3); + options.label_regex = "url"; + auto v6 = inst_s.filter_metrics_static(options); + CHECK(v6.size() == 1); + } + + auto dc = std::make_shared( + std::string("test3"), std::string(""), + std::array{"url", "code"}); + dynamic_metric_manager::instance().register_metric(dc); + auto& inst_d = dynamic_metric_manager::instance(); + auto pair1 = inst_d.create_metric_dynamic( + std::string("test3"), std::string(""), std::array{}); + CHECK(pair1.first == std::errc::invalid_argument); + dc->inc({"/", "200"}); + + { + std::string str = inst_d.serialize_dynamic(); + std::cout << str << "\n"; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json = inst_d.serialize_to_json_dynamic(); + std::cout << json << "\n"; +#endif + + using root_manager = metric_collector_t, + dynamic_metric_manager>; + str = root_manager::serialize(); + std::cout << str << "\n"; +#ifdef CINATRA_ENABLE_METRIC_JSON + json = root_manager::serialize_to_json(); + std::cout << json << "\n"; +#endif + } + + auto v3 = inst_d.get_metric_by_label({{"url", "/"}, {"code", "200"}}); + CHECK(v3.size() == 1); + + auto v4 = inst_d.get_metric_by_label_name({"url", "code"}); + CHECK(v4.size() == 1); + + inst_d.remove_metric(dc); + CHECK(inst_d.metric_count() == 0); + inst_d.register_metric(dc); + + inst_d.remove_metric(dc->str_name()); + CHECK(inst_d.metric_count() == 0); + inst_d.register_metric(dc); + + inst_d.remove_metric(std::vector>{dc}); + CHECK(inst_d.metric_count() == 0); + inst_d.register_metric(dc); + + inst_d.remove_metric({dc->str_name()}); + CHECK(inst_d.metric_count() == 0); + inst_d.register_metric(dc); + + inst_d.remove_metric_by_label({{"code", "400"}}); + CHECK(inst_d.metric_count() == 1); + inst_d.remove_metric_by_label({{"code", "200"}}); + CHECK(inst_d.metric_count() == 0); + inst_d.register_metric(dc); + + inst_d.remove_label_value({{"code", "400"}}); + CHECK(inst_d.metric_count() == 1); + inst_d.remove_label_value({{"code", "200"}}); + CHECK(dc->label_value_count() == 0); + dc->inc({"/", "200"}); + + CHECK(dc->label_value_count() == 1); + inst_d.remove_label_value({{"url", "/"}}); + CHECK(dc->label_value_count() == 0); + dc->inc({"/", "200"}); + + CHECK(dc->label_value_count() == 1); + inst_d.remove_label_value({{"url", "/"}, {"code", "200"}}); + CHECK(dc->label_value_count() == 0); + dc->inc({"/", "200"}); + + inst_d.remove_metric_by_label_name(std::vector{"url", "code"}); + CHECK(inst_d.metric_count() == 0); + inst_d.register_metric(dc); + + inst_d.remove_metric_by_label_name("url"); + CHECK(inst_d.metric_count() == 0); + inst_d.register_metric(dc); + + inst_d.remove_metric_by_label_name("code"); + CHECK(inst_d.metric_count() == 0); + inst_d.register_metric(dc); + + auto pair2 = inst_d.create_metric_dynamic( + "test4", "", std::array{"method", "code"}); + + metric_filter_options options; + options.name_regex = ".*test.*"; + auto v5 = inst_d.filter_metrics_dynamic(options); + CHECK(v5.size() == 2); + options.label_regex = "method"; + auto v6 = inst_d.filter_metrics_dynamic(options); + CHECK(v6.size() == 1); + + options.label_value_regex = "200"; + + auto v7 = inst_d.filter_metrics_dynamic(options); + CHECK(v7.size() == 0); + + pair2.second->inc({"200"}); + auto v8 = inst_d.filter_metrics_dynamic(options); + CHECK(v8.size() == 1); +} + +TEST_CASE("test dynamic counter") { + basic_dynamic_counter c("test", "", {"url", "code"}); + c.inc({"/", "200"}); + c.inc({"/test", "200"}); + auto v1 = c.value({"/", "200"}); + auto v2 = c.value({"/test", "200"}); + CHECK(v1 == 1); + CHECK(v2 == 1); + + { + std::string str; + c.serialize(str); + std::cout << str << "\n"; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json; + c.serialize_to_json(json); + std::cout << json << "\n"; +#endif + } + + basic_dynamic_counter c1("test1", "", {}); + c1.inc({}); + auto v3 = c1.value({}); + CHECK(v3 == 1); + + { + std::string str; + c1.serialize(str); + std::cout << str << "\n"; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json; + c1.serialize_to_json(json); + std::cout << json << "\n"; +#endif + } + + basic_dynamic_gauge g("test_gauge", "", {"url"}); + g.inc({"/"}); + CHECK(g.value({"/"}) == 1); + + g.dec({"/"}); + CHECK(g.value({"/"}) == 0); + + basic_dynamic_gauge g1("test_gauge1", "", {}); + g1.inc({}); + CHECK(g1.value({}) == 1); + g1.dec({}); + CHECK(g1.value({}) == 0); + + dynamic_gauge_t g2("test_g2", "", {"url", "code"}); + g2.inc({"/", "200"}); + CHECK(g2.value({"/", "200"}) == 1); +} + +TEST_CASE("test static counter") { + basic_static_counter c("test", ""); + c.inc(); + c.inc(); + auto v = c.value(); + CHECK(v == 2); + + { + std::string str; + c.serialize(str); + std::cout << str << "\n"; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json; + c.serialize_to_json(json); + std::cout << json << "\n"; +#endif + } + + basic_static_counter c1( + "test", "", + std::map{{"method", "GET"}, {"url", "/"}}); + c1.inc(); + c1.inc(); + auto v1 = c1.value(); + CHECK(v1 == 2); + + { + std::string str; + c1.serialize(str); + std::cout << str << "\n"; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json; + c1.serialize_to_json(json); + std::cout << json << "\n"; +#endif + } + + basic_static_gauge g("test", ""); + g.inc(); + g.inc(); + auto v3 = g.value(); + CHECK(v3 == 2); + g.dec(); + CHECK(g.value() == 1); + + basic_static_gauge g1("test", "", + std::map{}); + g1.inc(); + g1.inc(); + auto v4 = g1.value(); + CHECK(v4 == 2); + g1.dec(); + CHECK(g1.value() == 1); +} + +TEST_CASE("test static histogram") { + { + histogram_t h("test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}); + h.observe(23); + auto counts = h.get_bucket_counts(); + CHECK(counts[3]->value() == 1); + h.observe(42); + CHECK(counts[3]->value() == 2); + h.observe(60); + CHECK(counts[4]->value() == 1); + h.observe(120); + CHECK(counts[5]->value() == 1); + h.observe(1); + CHECK(counts[0]->value() == 1); + + std::string str; + h.serialize(str); + std::cout << str; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json; + h.serialize_to_json(json); + std::cout << json << "\n"; +#endif + } + + { + histogram_t h( + "test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, + std::map{{"method", "GET"}, {"url", "/"}}); + h.observe(23); + auto counts = h.get_bucket_counts(); + CHECK(counts[3]->value() == 1); + h.observe(42); + CHECK(counts[3]->value() == 2); + h.observe(60); + CHECK(counts[4]->value() == 1); + h.observe(120); + CHECK(counts[5]->value() == 1); + h.observe(1); + CHECK(counts[0]->value() == 1); + + std::string str; + h.serialize(str); + std::cout << str; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json; + h.serialize_to_json(json); + std::cout << json << "\n"; +#endif + } + + { + histogram_t h( + "test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, + std::map{{"method", "GET"}, {"url", "/"}}); + + std::string str; + h.serialize(str); + std::cout << str; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string json; + h.serialize_to_json(json); + std::cout << json << "\n"; +#endif + } +} + +TEST_CASE("test dynamic histogram") { + dynamic_histogram_t h("test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, + {"method", "url"}); + h.observe({"GET", "/"}, 23); + auto counts = h.get_bucket_counts(); + CHECK(counts[3]->value({"GET", "/"}) == 1); + h.observe({"GET", "/"}, 42); + CHECK(counts[3]->value({"GET", "/"}) == 2); + h.observe({"GET", "/"}, 60); + CHECK(counts[4]->value({"GET", "/"}) == 1); + h.observe({"GET", "/"}, 120); + CHECK(counts[5]->value({"GET", "/"}) == 1); + h.observe({"GET", "/"}, 1); + CHECK(counts[0]->value({"GET", "/"}) == 1); + + h.observe({"POST", "/"}, 23); + CHECK(counts[3]->value({"POST", "/"}) == 1); + h.observe({"POST", "/"}, 42); + CHECK(counts[3]->value({"POST", "/"}) == 2); + h.observe({"POST", "/"}, 60); + CHECK(counts[4]->value({"POST", "/"}) == 1); + h.observe({"POST", "/"}, 120); + CHECK(counts[5]->value({"POST", "/"}) == 1); + h.observe({"POST", "/"}, 1); + CHECK(counts[0]->value({"POST", "/"}) == 1); + + std::string str; + h.serialize(str); + std::cout << str; + +#ifdef CINATRA_ENABLE_METRIC_JSON + std::string str_json; + h.serialize_to_json(str_json); + std::cout << str_json << "\n"; +#endif +} + +struct my_tag {}; +using my_manager = static_metric_manager; + +auto g_pair = my_manager::instance().create_metric_static( + "test_g_counter", ""); + TEST_CASE("test no lable") { { - gauge_t g{"test_gauge", "help"}; + std::map customMap = {}; + auto summary = std::make_shared( + "test", "help", + summary_t::Quantiles{ + {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, + customMap); + summary->observe(100); + } + auto g_counter = g_pair.second; + g_counter->inc(); + CHECK(g_counter->value() == 1); + { + gauge_t g{"test_gauge", "help", 10}; g.inc(); g.inc(); @@ -26,16 +508,21 @@ TEST_CASE("test no lable") { g.dec(); CHECK(g.value() == 1); - CHECK_THROWS_AS(g.dec({}, 1), std::invalid_argument); - CHECK_THROWS_AS(g.inc({}, 1), std::invalid_argument); - CHECK_THROWS_AS(g.update({}, 1), std::invalid_argument); - counter_t c{"test_counter", "help"}; + counter_t c{"test_counter", "help", 10}; c.inc(); c.inc(); std::string str1; c.serialize(str1); CHECK(str1.find("test_counter 2") != std::string::npos); + + auto r = c.reset(); + CHECK(r == 2); + CHECK(c.value() == 0); + + r = g.update(10); + CHECK(r == 1); + CHECK(g.value() == 10); } { counter_t c("get_count", "get counter"); @@ -49,10 +536,6 @@ TEST_CASE("test no lable") { CHECK(c.value() == 2); - CHECK_THROWS_AS(c.inc(-2), std::invalid_argument); - CHECK_THROWS_AS(c.inc({}, 1), std::invalid_argument); - CHECK_THROWS_AS(c.update({}, 1), std::invalid_argument); - c.update(10); CHECK(c.value() == 10); @@ -65,26 +548,22 @@ TEST_CASE("test with atomic") { counter_t c( "get_count", "get counter", std::map{{"method", "GET"}, {"url", "/"}}); - std::vector labels_value{"GET", "/"}; - c.inc(labels_value); - c.inc(labels_value, 2); - CHECK(c.value(labels_value) == 3); - CHECK_THROWS_AS(c.inc({"GET", "/test"}), std::invalid_argument); - CHECK_THROWS_AS(c.inc({"POST", "/"}), std::invalid_argument); - c.update(labels_value, 10); - CHECK(c.value(labels_value) == 10); + + c.inc(); + c.inc(2); + CHECK(c.value() == 3); + c.update(10); + CHECK(c.value() == 10); gauge_t g( "get_qps", "get qps", std::map{{"method", "GET"}, {"url", "/"}}); - g.inc(labels_value); - g.inc(labels_value, 2); - CHECK(g.value(labels_value) == 3); - CHECK_THROWS_AS(g.inc({"GET", "/test"}), std::invalid_argument); - CHECK_THROWS_AS(g.inc({"POST", "/"}), std::invalid_argument); - g.dec(labels_value); - g.dec(labels_value, 1); - CHECK(g.value(labels_value) == 1); + g.inc(); + g.inc(2); + CHECK(g.value() == 3); + g.dec(); + g.dec(1); + CHECK(g.value() == 1); std::string str; c.serialize(str); @@ -94,29 +573,47 @@ TEST_CASE("test with atomic") { std::cout << str1; CHECK(str.find("} 10") != std::string::npos); CHECK(str1.find("} 1") != std::string::npos); + + { + gauge_t g("get_qps", "get qps", + std::map{{"method", "POST"}, + {"url", "/test"}}); + g.inc(); + g.inc(); + CHECK(g.value() == 2); + CHECK(g.value() == 2); + g.dec(); + CHECK(g.value() == 1); + CHECK(g.value() == 1); + g.dec(); + CHECK(g.value() == 0); + CHECK(g.value() == 0); + } } TEST_CASE("test counter with dynamic labels value") { { - auto c = std::make_shared( - "get_count", "get counter", std::vector{"method", "code"}); + auto c = std::make_shared( + "get_count", "get counter", + std::array{"method", "code"}); CHECK(c->name() == "get_count"); - auto g = std::make_shared( - "get_count", "get counter", std::vector{"method", "code"}); + auto g = std::make_shared( + std::string("get_count"), std::string("get counter"), + std::array{"method", "code"}); CHECK(g->name() == "get_count"); CHECK(g->metric_name() == "gauge"); } { - counter_t c("get_count", "get counter", - std::vector{"method", "code"}); + dynamic_counter_t c(std::string("get_count"), std::string("get counter"), + std::array{"method", "code"}); CHECK(c.labels_name() == std::vector{"method", "code"}); c.inc({"GET", "200"}, 1); auto values = c.value_map(); - CHECK(values[{"GET", "200"}] == 1); + CHECK(values[{"GET", "200"}].value() == 1); c.inc({"GET", "200"}, 2); values = c.value_map(); - CHECK(values[{"GET", "200"}] == 3); + CHECK(values[{"GET", "200"}].value() == 3); std::string str; c.serialize(str); @@ -125,12 +622,10 @@ TEST_CASE("test counter with dynamic labels value") { CHECK(str.find("get_count{method=\"GET\",code=\"200\"} 3") != std::string::npos); - CHECK_THROWS_AS(c.inc({"GET", "200", "/"}, 2), std::invalid_argument); - c.update({"GET", "200"}, 20); std::this_thread::sleep_for(std::chrono::milliseconds(10)); values = c.value_map(); - CHECK(values[{"GET", "200"}] == 20); + CHECK(values[{"GET", "200"}].value() == 20); } } @@ -159,15 +654,15 @@ TEST_CASE("test gauge") { } { - gauge_t g("get_count", "get counter", {"method", "code", "url"}); + dynamic_gauge_3t g("get_count", "get counter", {"method", "code", "url"}); CHECK(g.labels_name() == std::vector{"method", "code", "url"}); // method, status code, url g.inc({"GET", "200", "/"}, 1); auto values = g.value_map(); - CHECK(values[{"GET", "200", "/"}] == 1); + CHECK(values[{"GET", "200", "/"}].value() == 1); g.inc({"GET", "200", "/"}, 2); values = g.value_map(); - CHECK(values[{"GET", "200", "/"}] == 3); + CHECK(values[{"GET", "200", "/"}].value() == 3); g.inc({"POST", "200", "/"}, 4); @@ -185,14 +680,12 @@ TEST_CASE("test gauge") { CHECK(str.find("get_count{method=\"GET\",code=\"200\",url=\"/\"} 3") != std::string::npos); - CHECK_THROWS_AS(g.dec({"GET", "200"}, 1), std::invalid_argument); - g.dec({"GET", "200", "/"}, 1); values = g.value_map(); - CHECK(values[{"GET", "200", "/"}] == 2); + CHECK(values[{"GET", "200", "/"}].value() == 2); g.dec({"GET", "200", "/"}, 2); values = g.value_map(); - CHECK(values[{"GET", "200", "/"}] == 0); + CHECK(values[{"GET", "200", "/"}].value() == 0); } } @@ -258,158 +751,134 @@ TEST_CASE("test summary") { TEST_CASE("test register metric") { auto c = std::make_shared(std::string("get_count"), std::string("get counter")); - default_metric_manager::register_metric_static(c); - CHECK_FALSE(default_metric_manager::register_metric_static(c)); + default_static_metric_manager::instance().register_metric(c); + CHECK_FALSE(default_static_metric_manager::instance().register_metric(c)); auto g = std::make_shared(std::string("get_guage_count"), std::string("get counter")); - default_metric_manager::register_metric_static(g); + default_static_metric_manager::instance().register_metric(g); - auto map1 = default_metric_manager::metric_map_static(); + auto map1 = default_static_metric_manager::instance().metric_map(); for (auto& [k, v] : map1) { bool r = k == "get_count" || k == "get_guage_count"; break; } - CHECK(default_metric_manager::metric_count_static() >= 2); - CHECK(default_metric_manager::metric_keys_static().size() >= 2); + CHECK(default_static_metric_manager::instance().metric_count() >= 2); c->inc(); g->inc(); - auto map = default_metric_manager::metric_map_static(); + auto map = default_static_metric_manager::instance().metric_map(); CHECK(map["get_count"]->as()->value() == 1); CHECK(map["get_guage_count"]->as()->value() == 1); - auto s = default_metric_manager::serialize_static(); + auto s = default_static_metric_manager::instance().serialize_static(); std::cout << s << "\n"; CHECK(s.find("get_count 1") != std::string::npos); CHECK(s.find("get_guage_count 1") != std::string::npos); - auto m = default_metric_manager::get_metric_static("get_count"); + auto m = + default_static_metric_manager::instance().get_metric_static( + "get_count"); CHECK(m->as()->value() == 1); auto m1 = - default_metric_manager::get_metric_static("get_guage_count"); + default_static_metric_manager::instance().get_metric_static( + "get_guage_count"); CHECK(m1->as()->value() == 1); - - { - // because the first regiter_metric is set no lock, so visit - // default_metric_manager with lock will throw. - auto c1 = std::make_shared(std::string(""), std::string("")); - CHECK_THROWS_AS(default_metric_manager::register_metric_dynamic(c1), - std::invalid_argument); - CHECK_THROWS_AS(default_metric_manager::metric_count_dynamic(), - std::invalid_argument); - CHECK_THROWS_AS(default_metric_manager::metric_keys_dynamic(), - std::invalid_argument); - CHECK_THROWS_AS(default_metric_manager::metric_map_dynamic(), - std::invalid_argument); - CHECK_THROWS_AS(default_metric_manager::get_metric_dynamic(""), - std::invalid_argument); - } } template struct test_id_t {}; TEST_CASE("test remove metric and serialize metrics") { - using metric_mgr = metric_manager_t>; - metric_mgr::create_metric_dynamic("test_counter", ""); - metric_mgr::create_metric_dynamic("test_counter2", ""); + using metric_mgr = dynamic_metric_manager>; + + metric_mgr::instance().create_metric_dynamic( + "test_counter", "", std::array{}); + metric_mgr::instance().create_metric_dynamic( + "test_counter2", "", std::array{}); - size_t count = metric_mgr::metric_count_dynamic(); + size_t count = metric_mgr::instance().metric_count(); CHECK(count == 2); - metric_mgr::remove_metric_dynamic("test_counter"); - count = metric_mgr::metric_count_dynamic(); + metric_mgr::instance().remove_metric("test_counter"); + count = metric_mgr::instance().metric_count(); CHECK(count == 1); - metric_mgr::remove_metric_dynamic("test_counter2"); - count = metric_mgr::metric_count_dynamic(); + metric_mgr::instance().remove_metric("test_counter2"); + count = metric_mgr::instance().metric_count(); CHECK(count == 0); - CHECK_THROWS_AS( - metric_mgr::create_metric_static("test_static_counter", ""), - std::invalid_argument); - - using metric_mgr2 = metric_manager_t>; - auto c = - metric_mgr2::create_metric_static("test_static_counter", ""); - auto c2 = - metric_mgr2::create_metric_static("test_static_counter2", ""); - c->inc(); - c2->inc(); + using metric_mgr2 = static_metric_manager>; + auto c = metric_mgr2::instance().create_metric_static( + "test_static_counter", ""); + auto c2 = metric_mgr2::instance().create_metric_static( + "test_static_counter2", ""); + c.second->inc(); + c2.second->inc(); #ifdef CINATRA_ENABLE_METRIC_JSON - auto s = metric_mgr2::serialize_to_json_static(); + auto s = metric_mgr2::instance().serialize_to_json_static(); std::cout << s << "\n"; - auto s1 = metric_mgr2::serialize_to_json({c, c2}); - CHECK(s.size() == s1.size()); #endif - CHECK_THROWS_AS(metric_mgr2::metric_count_dynamic(), std::invalid_argument); - count = metric_mgr2::metric_count_static(); + count = metric_mgr2::instance().metric_count(); CHECK(count == 2); - CHECK_THROWS_AS(metric_mgr2::remove_metric_dynamic("test_static_counter"), - std::invalid_argument); - - metric_mgr2::remove_metric_static("test_static_counter"); - count = metric_mgr2::metric_count_static(); - CHECK(count == 1); } TEST_CASE("test filter metrics static") { - using metric_mgr = metric_manager_t>; - auto c = metric_mgr::create_metric_static( + using metric_mgr = static_metric_manager>; + auto c = metric_mgr::instance().create_metric_static( "test_static_counter", "", std::map{{"method", "GET"}}); - auto c2 = metric_mgr::create_metric_static( + auto c2 = metric_mgr::instance().create_metric_static( "test_static_counter2", "", std::map{{"url", "/"}}); - c->inc({"GET"}); - c2->inc({"/"}); + c.second->inc(); + c2.second->inc(); metric_filter_options options; options.name_regex = ".*counter.*"; { - auto metrics = metric_mgr::filter_metrics_static(options); + auto metrics = metric_mgr::instance().filter_metrics_static(options); CHECK(metrics.size() == 2); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.find("test_static_counter") != std::string::npos); std::cout << s << "\n"; } options.label_regex = ".*ur.*"; { - auto metrics = metric_mgr::filter_metrics_static(options); + auto metrics = metric_mgr::instance().filter_metrics_static(options); CHECK(metrics.size() == 1); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.find("test_static_counter2") != std::string::npos); std::cout << s << "\n"; } options.name_regex = "no_such_name"; { - auto metrics = metric_mgr::filter_metrics_static(options); + auto metrics = metric_mgr::instance().filter_metrics_static(options); CHECK(metrics.empty()); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.empty()); } options = {}; options.label_regex = "no_such_label"; { - auto metrics = metric_mgr::filter_metrics_static(options); + auto metrics = metric_mgr::instance().filter_metrics_static(options); CHECK(metrics.empty()); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.empty()); } // don't filter options = {}; { - auto metrics = metric_mgr::filter_metrics_static(options); + auto metrics = metric_mgr::instance().filter_metrics_static(options); CHECK(metrics.size() == 2); } @@ -417,9 +886,9 @@ TEST_CASE("test filter metrics static") { options.label_regex = ".*ur.*"; options.is_white = false; { - auto metrics = metric_mgr::filter_metrics_static(options); + auto metrics = metric_mgr::instance().filter_metrics_static(options); CHECK(metrics.size() == 1); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.find("test_static_counter") != std::string::npos); CHECK(s.find("test_static_counter2") == std::string::npos); } @@ -428,9 +897,9 @@ TEST_CASE("test filter metrics static") { options.label_regex = ".*ur.*"; options.is_white = false; { - auto metrics = metric_mgr::filter_metrics_static(options); + auto metrics = metric_mgr::instance().filter_metrics_static(options); CHECK(metrics.size() == 1); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.find("test_static_counter") != std::string::npos); CHECK(s.find("method") != std::string::npos); CHECK(s.find("test_static_counter2") == std::string::npos); @@ -439,11 +908,13 @@ TEST_CASE("test filter metrics static") { } TEST_CASE("test filter metrics dynamic") { - using metric_mgr = metric_manager_t>; - auto c = metric_mgr::create_metric_dynamic( - "test_dynamic_counter", "", std::vector{{"method"}}); - auto c2 = metric_mgr::create_metric_dynamic( - "test_dynamic_counter2", "", std::vector{{"url"}}); + using metric_mgr = dynamic_metric_manager>; + auto [ec, c] = + metric_mgr::instance().create_metric_dynamic( + "test_dynamic_counter", "", std::array{"method"}); + auto [ec2, c2] = + metric_mgr::instance().create_metric_dynamic( + "test_dynamic_counter2", "", std::array{"url"}); c->inc({"GET"}); c->inc({"POST"}); c2->inc({"/"}); @@ -452,44 +923,44 @@ TEST_CASE("test filter metrics dynamic") { metric_filter_options options; options.name_regex = ".*counter.*"; { - auto metrics = metric_mgr::filter_metrics_dynamic(options); + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); CHECK(metrics.size() == 2); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.find("test_dynamic_counter") != std::string::npos); std::cout << s << "\n"; } options.label_regex = ".*ur.*"; { - auto metrics = metric_mgr::filter_metrics_dynamic(options); + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); CHECK(metrics.size() == 1); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.find("test_dynamic_counter2") != std::string::npos); std::cout << s << "\n"; } options.name_regex = "no_such_name"; { - auto metrics = metric_mgr::filter_metrics_dynamic(options); + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); CHECK(metrics.empty()); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.empty()); } options = {}; options.label_regex = "no_such_label"; { - auto metrics = metric_mgr::filter_metrics_dynamic(options); + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); CHECK(metrics.empty()); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.empty()); } // don't filter options = {}; { - auto metrics = metric_mgr::filter_metrics_dynamic(options); + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); CHECK(metrics.size() == 2); } @@ -497,9 +968,9 @@ TEST_CASE("test filter metrics dynamic") { options.label_regex = ".*ur.*"; options.is_white = false; { - auto metrics = metric_mgr::filter_metrics_dynamic(options); + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); CHECK(metrics.size() == 1); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.find("test_dynamic_counter") != std::string::npos); CHECK(s.find("test_dynamic_counter2") == std::string::npos); } @@ -508,9 +979,9 @@ TEST_CASE("test filter metrics dynamic") { options.label_regex = ".*ur.*"; options.is_white = false; { - auto metrics = metric_mgr::filter_metrics_dynamic(options); + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); CHECK(metrics.size() == 1); - auto s = metric_mgr::serialize(metrics); + auto s = manager_helper::serialize(metrics); CHECK(s.find("test_dynamic_counter") != std::string::npos); CHECK(s.find("method") != std::string::npos); CHECK(s.find("test_dynamic_counter2") == std::string::npos); @@ -519,141 +990,154 @@ TEST_CASE("test filter metrics dynamic") { } TEST_CASE("test get metric by static labels and label") { - using metric_mgr = metric_manager_t>; - metric_mgr::create_metric_static( + using metric_mgr = static_metric_manager>; + metric_mgr::instance().create_metric_static( "http_req_test", "", std::map{{"method", "GET"}, {"url", "/"}}); - metric_mgr::create_metric_static( + metric_mgr::instance().create_metric_static( "http_req_test1", "", std::map{{"method", "POST"}, {"url", "/"}}); - metric_mgr::create_metric_static( + metric_mgr::instance().create_metric_static( "http_req_test2", "", std::map{{"method", "GET"}, {"url", "/test"}}); - auto v = metric_mgr::get_metric_by_labels_static( + auto v = metric_mgr::instance().get_metric_by_label( std::map{{"method", "GET"}, {"url", "/test"}}); CHECK(v[0]->name() == "http_req_test2"); - v = metric_mgr::get_metric_by_labels_static( + v = metric_mgr::instance().get_metric_by_label( std::map{{"method", "GET"}, {"url", "/"}}); CHECK(v[0]->name() == "http_req_test"); - auto h1 = metric_mgr::create_metric_static( + auto [ec, h1] = metric_mgr::instance().create_metric_static( "http_req_static_hist", "help", std::vector{5.23, 10.54, 20.0, 50.0, 100.0}, std::map{{"method", "GET"}, {"url", "/"}}); - h1->observe({"GET", "/"}, 23); + h1->observe(23); - auto s1 = metric_mgr::create_metric_static( + std::string str1; + h1->serialize(str1); + std::cout << str1; + CHECK(str1.find("method=\"GET\",url=\"/\",le=") != std::string::npos); + + auto map = + std::map{{"method", "GET"}, {"url", "/"}}; + auto [ec1, s1] = metric_mgr::instance().create_metric_static( "http_req_static_summary", "help", summary_t::Quantiles{ {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, std::map{{"method", "GET"}, {"url", "/"}}); - s1->observe({"GET", "/"}, 23); + s1->observe(23); - auto vec = metric_mgr::get_metric_by_label_static({"method", "GET"}); - CHECK(vec.size() == 4); - - vec = metric_mgr::get_metric_by_label_static({"url", "/"}); - CHECK(vec.size() == 4); - - vec = metric_mgr::get_metric_by_label_static({"url", "/test"}); - CHECK(vec.size() == 1); - - vec = metric_mgr::get_metric_by_label_static({"method", "POST"}); - CHECK(vec.size() == 1); - - vec = metric_mgr::get_metric_by_labels_static( - std::map{{"method", "HEAD"}, {"url", "/test"}}); - CHECK(vec.empty()); - - vec = metric_mgr::get_metric_by_labels_static( - std::map{{"method", "GET"}}); - CHECK(vec.empty()); + auto vec = metric_mgr::instance().get_metric_by_label(map); + CHECK(vec.size() == 3); - vec = metric_mgr::get_metric_by_label_static({"url", "/index"}); - CHECK(vec.empty()); + { + using metric_mgr2 = static_metric_manager>; + auto [ec, s2] = metric_mgr2::instance().create_metric_static( + "http_req_static_summary2", "help", + summary_t::Quantiles{ + {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, + map); + s2->observe(23); + + auto vec = metric_mgr2::instance().get_metric_by_label(map); + CHECK(vec.size() == 1); + } } TEST_CASE("test get metric by dynamic labels") { - using metric_mgr = metric_manager_t>; - auto c = metric_mgr::create_metric_dynamic( - "http_req_static", "", std::vector{"method", "code"}); + using metric_mgr = dynamic_metric_manager>; + auto [ec, c] = + metric_mgr::instance().create_metric_dynamic( + "http_req_static", "", std::array{"method", "code"}); - auto c1 = metric_mgr::create_metric_dynamic( - "http_req_static1", "", std::vector{"method", "code"}); + auto [ec1, c1] = + metric_mgr::instance().create_metric_dynamic( + "http_req_static1", "", std::array{"method", "code"}); - auto c2 = metric_mgr::create_metric_dynamic( - "http_req_static2", "", std::vector{"method", "code"}); + auto [ec2, c2] = + metric_mgr::instance().create_metric_dynamic( + "http_req_static2", "", std::array{"method", "code"}); - auto c3 = metric_mgr::create_metric_dynamic( - "http_req_static3", "", std::vector{"method", "code"}); + auto [ec3, c3] = + metric_mgr::instance().create_metric_dynamic( + "http_req_static3", "", std::array{"method", "code"}); c->inc({"POST", "200"}); c1->inc({"GET", "200"}); c2->inc({"POST", "301"}); c3->inc({"POST", "400"}); - auto c4 = metric_mgr::create_metric_dynamic( - "http_req_static4", "", std::vector{"host", "url"}); + auto [ec4, c4] = + metric_mgr::instance().create_metric_dynamic( + "http_req_static4", "", std::array{"host", "url"}); - auto c5 = metric_mgr::create_metric_dynamic( - "http_req_static5", "", std::vector{"host", "url"}); + auto [ec5, c5] = + metric_mgr::instance().create_metric_dynamic( + "http_req_static5", "", std::array{"host", "url"}); c4->inc({"shanghai", "/"}); c5->inc({"shanghai", "/test"}); - auto vec = metric_mgr::get_metric_by_labels_dynamic({{"method", "POST"}}); + auto vec = + metric_mgr::instance().filter_metrics_by_label_value(std::regex("POST")); CHECK(vec.size() == 3); - vec = metric_mgr::get_metric_by_labels_dynamic({{"method", "GET"}}); + vec = metric_mgr::instance().filter_metrics_by_label_value(std::regex("GET")); CHECK(vec.size() == 1); - vec = metric_mgr::get_metric_by_labels_dynamic({{"host", "shanghai"}}); + vec = metric_mgr::instance().filter_metrics_by_label_value( + std::regex("shanghai")); CHECK(vec.size() == 2); - vec = metric_mgr::get_metric_by_labels_dynamic({{"url", "/"}}); + vec = metric_mgr::instance().filter_metrics_by_label_value(std::regex("/")); CHECK(vec.size() == 1); - vec = metric_mgr::get_metric_by_labels_dynamic({{"url", "/test"}}); + vec = + metric_mgr::instance().filter_metrics_by_label_value(std::regex("/test")); CHECK(vec.size() == 1); - vec = metric_mgr::get_metric_by_labels_dynamic({{"url", "/none"}}); + vec = + metric_mgr::instance().filter_metrics_by_label_value(std::regex("/none")); CHECK(vec.size() == 0); - vec = metric_mgr::get_metric_by_labels_dynamic({{"method", "HEAD"}}); + vec = + metric_mgr::instance().filter_metrics_by_label_value(std::regex("HEAD")); CHECK(vec.size() == 0); - auto h1 = metric_mgr::create_metric_dynamic( - "http_req_static_hist", "help", - std::vector{5.23, 10.54, 20.0, 50.0, 100.0}, - std::vector{"method", "url"}); + auto [ec6, h1] = + metric_mgr::instance().create_metric_dynamic( + "http_req_static_hist", "help", + std::vector{5.23, 10.54, 20.0, 50.0, 100.0}, + std::array{"method", "url"}); h1->observe({"GET", "/"}, 23); - auto s1 = metric_mgr::create_metric_dynamic( - "http_req_static_summary", "help", - summary_t::Quantiles{ - {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, - std::vector{"method", "url"}); + auto [ec7, s1] = + metric_mgr::instance().create_metric_dynamic( + "http_req_static_summary", "help", + summary_t::Quantiles{ + {0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, + std::array{"method", "url"}); s1->observe({"GET", "/"}, 23); - vec = metric_mgr::get_metric_by_labels_dynamic({{"method", "GET"}}); + vec = metric_mgr::instance().filter_metrics_by_label_value(std::regex("GET")); CHECK(vec.size() >= 2); - auto str = metric_mgr::serialize(vec); + auto str = metric_mgr::instance().serialize_dynamic(); std::cout << str; #ifdef CINATRA_ENABLE_METRIC_JSON - auto json_str = metric_mgr::serialize_to_json(vec); + auto json_str = metric_mgr::instance().serialize_to_json_dynamic(); std::cout << json_str << "\n"; #endif } TEST_CASE("test histogram serialize with dynamic labels") { - histogram_t h("test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, - std::vector{"method", "url"}); + dynamic_histogram_2t h("test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, + std::array{"method", "url"}); h.observe({"GET", "/"}, 23); auto counts = h.get_bucket_counts(); CHECK(counts[3]->value({"GET", "/"}) == 1); @@ -688,21 +1172,38 @@ TEST_CASE("test histogram serialize with dynamic labels") { #endif } +TEST_CASE("test histogram serialize with static labels default") { + histogram_t h( + "test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, + std::map{{"method", "GET"}, {"url", "/"}}); + h.observe(23); + auto counts = h.get_bucket_counts(); + CHECK(counts[3]->value() == 1); + h.observe(42); + CHECK(counts[3]->value() == 2); + h.observe(60); + CHECK(counts[4]->value() == 1); + h.observe(120); + CHECK(counts[5]->value() == 1); + h.observe(1); + CHECK(counts[0]->value() == 1); +} + TEST_CASE("test histogram serialize with static labels") { histogram_t h( "test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, std::map{{"method", "GET"}, {"url", "/"}}); - h.observe({"GET", "/"}, 23); + h.observe(23); auto counts = h.get_bucket_counts(); - CHECK(counts[3]->value({"GET", "/"}) == 1); - h.observe({"GET", "/"}, 42); - CHECK(counts[3]->value({"GET", "/"}) == 2); - h.observe({"GET", "/"}, 60); - CHECK(counts[4]->value({"GET", "/"}) == 1); - h.observe({"GET", "/"}, 120); - CHECK(counts[5]->value({"GET", "/"}) == 1); - h.observe({"GET", "/"}, 1); - CHECK(counts[0]->value({"GET", "/"}) == 1); + CHECK(counts[3]->value() == 1); + h.observe(42); + CHECK(counts[3]->value() == 2); + h.observe(60); + CHECK(counts[4]->value() == 1); + h.observe(120); + CHECK(counts[5]->value() == 1); + h.observe(1); + CHECK(counts[0]->value() == 1); std::string str; h.serialize(str); @@ -716,10 +1217,11 @@ TEST_CASE("test histogram serialize with static labels") { } TEST_CASE("test summary with dynamic labels") { - summary_t summary{"test_summary", - "summary help", - {{0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, - std::vector{"method", "url"}}; + basic_dynamic_summary<2> summary{ + "test_summary", + "summary help", + {{0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}, + {"method", "url"}}; std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<> distr(1, 100); @@ -757,19 +1259,19 @@ TEST_CASE("test summary with static labels") { std::mt19937 gen(rd()); std::uniform_int_distribution<> distr(1, 100); for (int i = 0; i < 50; i++) { - summary.observe({"GET", "/"}, distr(gen)); + summary.observe(distr(gen)); } std::this_thread::sleep_for(std::chrono::milliseconds(100)); - CHECK_THROWS_AS(summary.observe({"POST", "/"}, 1), std::invalid_argument); - double sum; uint64_t count; - auto rates = async_simple::coro::syncAwait( - summary.get_rates({"GET", "/"}, sum, count)); + auto rates = async_simple::coro::syncAwait(summary.get_rates(sum, count)); std::cout << rates.size() << "\n"; + auto rates1 = async_simple::coro::syncAwait(summary.get_rates(sum, count)); + CHECK(rates == rates1); + std::string str; async_simple::coro::syncAwait(summary.serialize_async(str)); std::cout << str; @@ -784,9 +1286,9 @@ TEST_CASE("test summary with static labels") { TEST_CASE("test serialize with emptry metrics") { std::string s1; - auto h1 = std::make_shared( + auto h1 = std::make_shared( "get_count2", "help", std::vector{5.23, 10.54, 20.0, 50.0, 100.0}, - std::vector{"method"}); + std::array{"method"}); h1->serialize(s1); CHECK(s1.empty()); #ifdef CINATRA_ENABLE_METRIC_JSON @@ -833,9 +1335,9 @@ TEST_CASE("test serialize with emptry metrics") { CHECK(s1.empty()); #endif - auto c3 = std::make_shared(std::string("get_count"), - std::string("get counter"), - std::vector{"method"}); + auto c3 = std::make_shared( + std::string("get_count"), std::string("get counter"), + std::array{"method"}); c3->serialize(s1); CHECK(s1.empty()); #ifdef CINATRA_ENABLE_METRIC_JSON @@ -881,7 +1383,7 @@ TEST_CASE("test serialize with emptry metrics") { { std::string str; - c2->inc({"GET"}); + c2->inc(); c2->serialize(str); CHECK(!str.empty()); str.clear(); @@ -893,7 +1395,7 @@ TEST_CASE("test serialize with emptry metrics") { { std::string str; - c3->inc({"GET"}); + c3->inc({"POST"}); c3->serialize(str); CHECK(!str.empty()); str.clear(); @@ -905,30 +1407,35 @@ TEST_CASE("test serialize with emptry metrics") { } TEST_CASE("test serialize with multiple threads") { - auto c = std::make_shared(std::string("get_count"), - std::string("get counter"), - std::vector{"method"}); - auto g = std::make_shared(std::string("get_count1"), - std::string("get counter"), - std::vector{"method"}); + { + dynamic_histogram_d h("test", "help", {5.23, 10.54, 20.0, 50.0, 100.0}, + std::array{"url", "code"}); + h.observe({"/", "code"}, 23); + } + auto c = std::make_shared( + std::string("get_count"), std::string("get counter"), + std::array{"method"}); + auto g = std::make_shared( + std::string("get_count1"), std::string("get counter"), + std::array{"method"}); - auto h1 = std::make_shared( + auto h1 = std::make_shared( "get_count2", "help", std::vector{5.23, 10.54, 20.0, 50.0, 100.0}, - std::vector{"method"}); + std::array{"method"}); - auto c1 = std::make_shared(std::string("get_count3"), - std::string("get counter"), - std::vector{"method"}); + auto c1 = std::make_shared( + std::string("get_count3"), std::string("get counter"), + std::array{"method"}); - using test_metric_manager = metric_manager_t>; + using test_metric_manager = dynamic_metric_manager>; - test_metric_manager::register_metric_dynamic(c, g, h1, c1); + test_metric_manager::instance().register_metric({c, g, h1, c1}); c->inc({"POST"}, 1); g->inc({"GET"}, 1); h1->observe({"HEAD"}, 1); - auto s = test_metric_manager::serialize_dynamic(); + auto s = test_metric_manager::instance().serialize_dynamic(); std::cout << s; CHECK(!s.empty()); CHECK(s.find("get_count") != std::string::npos); @@ -937,7 +1444,7 @@ TEST_CASE("test serialize with multiple threads") { CHECK(s.find("get_count3") == std::string::npos); #ifdef CINATRA_ENABLE_METRIC_JSON - auto json = test_metric_manager::serialize_to_json_dynamic(); + auto json = test_metric_manager::instance().serialize_to_json_dynamic(); std::cout << json << "\n"; CHECK(!json.empty()); CHECK(json.find("get_count") != std::string::npos); @@ -949,22 +1456,23 @@ TEST_CASE("test serialize with multiple threads") { #if defined(__GNUC__) TEST_CASE("test system metric") { start_system_metric(); - detail::ylt_stat(); + metric::detail::ylt_stat(); - auto s = system_metric_manager::serialize_static(); + auto s = system_metric_manager::instance().serialize_static(); std::cout << s; CHECK(!s.empty()); #ifdef CINATRA_ENABLE_METRIC_JSON - auto json = system_metric_manager::serialize_to_json_static(); + auto json = system_metric_manager::instance().serialize_to_json_static(); std::cout << json << "\n"; CHECK(!json.empty()); #endif - using metric_manager = metric_manager_t>; - auto c = metric_manager::create_metric_dynamic("test_qps", ""); - c->inc(42); - using root = metric_collector_t>; + auto c = metric_manager::instance().create_metric_dynamic( + std::string("test_qps"), "", std::array{"url"}); + c.second->inc({"/"}, 42); + using root = metric_collector_t; s.clear(); s = root::serialize(); @@ -978,73 +1486,163 @@ TEST_CASE("test system metric") { CHECK(!json.empty()); #endif } -#endif TEST_CASE("test metric capacity") { std::cout << g_user_metric_count << "\n"; - using test_metric_manager = metric_manager_t>; + using test_metric_manager = dynamic_metric_manager>; set_metric_capacity(g_user_metric_count + 2); - auto c = test_metric_manager::create_metric_dynamic("counter", ""); - CHECK(c != nullptr); + auto c = + test_metric_manager::instance().create_metric_dynamic( + std::string("counter"), "", std::array{}); + CHECK(c.second != nullptr); auto c1 = - test_metric_manager::create_metric_dynamic("counter1", ""); - CHECK(c1 != nullptr); + test_metric_manager::instance().create_metric_dynamic( + std::string("counter1"), "", std::array{}); + CHECK(c1.second != nullptr); auto c2 = - test_metric_manager::create_metric_dynamic("counter2", ""); - CHECK(c2 == nullptr); + test_metric_manager::instance().create_metric_dynamic( + std::string("counter2"), "", std::array{}); + CHECK(c2.second == nullptr); set_metric_capacity(10000000); auto process_memory_resident = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_memory_resident"); std::cout << (int64_t)process_memory_resident->value() << "\n"; auto process_memory_virtual = - system_metric_manager::get_metric_static( + system_metric_manager::instance().get_metric_static( "ylt_process_memory_virtual"); std::cout << (int64_t)process_memory_virtual->value() << "\n"; } +#endif TEST_CASE("test remove dynamic metric") { - using test_metric_manager = metric_manager_t>; - auto c = test_metric_manager::create_metric_dynamic("counter", ""); - CHECK(c != nullptr); - auto c1 = - test_metric_manager::create_metric_dynamic("counter1", ""); - CHECK(c1 != nullptr); - auto c2 = - test_metric_manager::create_metric_dynamic("counter2", ""); - CHECK(c2 != nullptr); - - test_metric_manager::remove_metric_dynamic(c); - CHECK(test_metric_manager::metric_count_dynamic() == 2); - test_metric_manager::remove_metric_dynamic(c1); - CHECK(test_metric_manager::metric_count_dynamic() == 1); - test_metric_manager::remove_metric_dynamic(c2); - CHECK(test_metric_manager::metric_count_dynamic() == 0); - - test_metric_manager::register_metric_dynamic(c, c1, c2); - CHECK(test_metric_manager::metric_count_dynamic() == 3); - test_metric_manager::remove_metric_dynamic("counter"); - CHECK(test_metric_manager::metric_count_dynamic() == 2); - test_metric_manager::remove_metric_dynamic( + using test_metric_manager = dynamic_metric_manager>; + auto pair = + test_metric_manager::instance().create_metric_dynamic( + std::string("counter"), "", std::array{}); + CHECK(pair.second != nullptr); + auto pair1 = + test_metric_manager::instance().create_metric_dynamic( + std::string("counter1"), "", std::array{}); + CHECK(pair1.second != nullptr); + auto pair2 = + test_metric_manager::instance().create_metric_dynamic( + std::string("counter2"), "", std::array{}); + CHECK(pair2.second != nullptr); + + auto c = pair.second; + auto c1 = pair1.second; + auto c2 = pair2.second; + + test_metric_manager::instance().remove_metric(c); + CHECK(test_metric_manager::instance().metric_count() == 2); + test_metric_manager::instance().remove_metric(c1); + CHECK(test_metric_manager::instance().metric_count() == 1); + test_metric_manager::instance().remove_metric(c2); + CHECK(test_metric_manager::instance().metric_count() == 0); + + test_metric_manager::instance().register_metric({c, c1, c2}); + CHECK(test_metric_manager::instance().metric_count() == 3); + test_metric_manager::instance().remove_metric("counter"); + CHECK(test_metric_manager::instance().metric_count() == 2); + test_metric_manager::instance().remove_metric( std::vector{"counter1", "counter2"}); - CHECK(test_metric_manager::metric_count_dynamic() == 0); - - test_metric_manager::register_metric_dynamic( - std::vector>{c, c1, c2}); - CHECK(test_metric_manager::metric_count_dynamic() == 3); - test_metric_manager::remove_metric_dynamic({c1, c2}); - CHECK(test_metric_manager::metric_count_dynamic() == 1); - auto r = test_metric_manager::register_metric_dynamic( - std::vector>{c, c1}); + CHECK(test_metric_manager::instance().metric_count() == 0); + + test_metric_manager::instance().register_metric({c, c1, c2}); + CHECK(test_metric_manager::instance().metric_count() == 3); + test_metric_manager::instance().remove_metric({c1, c2}); + CHECK(test_metric_manager::instance().metric_count() == 1); + auto r = test_metric_manager::instance().register_metric({c, c1}); CHECK(!r); - CHECK(test_metric_manager::metric_count_dynamic() == 1); + CHECK(test_metric_manager::instance().metric_count() == 1); - r = test_metric_manager::register_metric_dynamic( - std::vector>{c1, c}); + r = test_metric_manager::instance().register_metric({c1, c}); CHECK(!r); - CHECK(test_metric_manager::metric_count_dynamic() == 1); + CHECK(test_metric_manager::instance().metric_count() == 2); +} + +TEST_CASE("testFilterMetricsDynamicWithMultiLabel") { + using metric_mgr = dynamic_metric_manager>; + auto [ec, c] = + metric_mgr::instance().create_metric_dynamic( + "test_dynamic_counter", "", + std::array{"method", "bucket"}); + auto [ec2, c2] = + metric_mgr::instance().create_metric_dynamic( + "test_dynamic_counter2", "", + std::array{"url", "bucket"}); + c->inc({"GET", "bucket1"}); + c->inc({"POST", "bucket2"}); + c2->inc({"/", "bucket1"}); + c2->inc({"/test", "bucket3"}); + + auto counter = metric_mgr::instance().get_metric_dynamic( + "test_dynamic_counter"); + CHECK(counter != nullptr); + + metric_filter_options options; + options.name_regex = ".*counter.*"; + { + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); + CHECK(metrics.size() == 2); + + auto s = metric_mgr::instance().serialize(metrics); + CHECK(s.find("test_dynamic_counter") != std::string::npos); + std::cout << s << "\n"; + } + + options.label_regex = "bucket"; + { + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); + CHECK(metrics.size() == 2); + auto s = metric_mgr::instance().serialize(metrics); + CHECK(s.find("test_dynamic_counter2") != std::string::npos); + std::cout << s << "\n"; + } + + options.label_regex = "method"; + { + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); + CHECK(metrics.size() == 1); + auto s = metric_mgr::instance().serialize(metrics); + CHECK(s.find("test_dynamic_counter") != std::string::npos); + std::cout << s << "\n"; + } + + options.label_regex = "url"; + { + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); + CHECK(metrics.size() == 1); + auto s = metric_mgr::instance().serialize(metrics); + CHECK(s.find("test_dynamic_counter2") != std::string::npos); + std::cout << s << "\n"; + } + + // black + options.label_regex = ".*bucket.*"; + options.is_white = false; + { + auto metrics = metric_mgr::instance().filter_metrics_dynamic(options); + CHECK(metrics.size() == 0); + } +} + +TEST_CASE("test metric manager clean expired label") { + set_label_max_age(std::chrono::seconds(1), std::chrono::seconds(1)); + auto& inst = dynamic_metric_manager::instance(); + auto pair = inst.create_metric_dynamic( + std::string("some_counter"), "", std::array{"url"}); + auto c = pair.second; + c->inc({"/"}); + c->inc({"/test"}); + CHECK(c->label_value_count() == 2); + std::this_thread::sleep_for(std::chrono::seconds(2)); + c->inc({"/index"}); + size_t count = c->label_value_count(); + CHECK(count == 1); } DOCTEST_MSVC_SUPPRESS_WARNING_WITH_PUSH(4007)